ff148490b3
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-5 Source: Original Platform
15707 lines
579 KiB
JSON
15707 lines
579 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 200,
|
|
"global_step": 681,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"KL/chosen_KL_mean": 0.00527191162109375,
|
|
"KL/mean": 0.016706019639968872,
|
|
"KL/rejected_KL_mean": 0.028141021728515625,
|
|
"KL/std": 0.272699236869812,
|
|
"epoch": 0.0014684287812041115,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.02287006378173828,
|
|
"fcm_dpo/q_t": 0.5005706548690796,
|
|
"grad_norm": 83.51212310791016,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.4974287748336792,
|
|
"logits/rejected": -0.43299180269241333,
|
|
"logps/chosen": -50.1435661315918,
|
|
"logps/ref_chosen": -50.14883804321289,
|
|
"logps/ref_rejected": -74.1280517578125,
|
|
"logps/rejected": -74.09991455078125,
|
|
"loss": 1.389,
|
|
"margin_dpo/margin_mean": -0.02287048101425171,
|
|
"margin_dpo/margin_std": 0.41920793056488037,
|
|
"step": 1
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.03498649597167969,
|
|
"KL/mean": -0.00212840735912323,
|
|
"KL/rejected_KL_mean": 0.030735015869140625,
|
|
"KL/std": 0.24797174334526062,
|
|
"epoch": 0.002936857562408223,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.06572261452674866,
|
|
"fcm_dpo/q_t": 0.5016425848007202,
|
|
"grad_norm": 72.20186614990234,
|
|
"learning_rate": 7.246376811594203e-09,
|
|
"logits/chosen": -0.49536412954330444,
|
|
"logits/rejected": -0.4594460427761078,
|
|
"logps/chosen": -52.65568923950195,
|
|
"logps/ref_chosen": -52.620704650878906,
|
|
"logps/ref_rejected": -75.30413818359375,
|
|
"logps/rejected": -75.27340698242188,
|
|
"loss": 1.3932,
|
|
"margin_dpo/margin_mean": -0.06572240591049194,
|
|
"margin_dpo/margin_std": 0.35048407316207886,
|
|
"step": 2
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.00826263427734375,
|
|
"KL/mean": 0.004124656319618225,
|
|
"KL/rejected_KL_mean": 0.016513824462890625,
|
|
"KL/std": 0.2530524432659149,
|
|
"epoch": 0.004405286343612335,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.02477547526359558,
|
|
"fcm_dpo/q_t": 0.5006192922592163,
|
|
"grad_norm": 70.89527130126953,
|
|
"learning_rate": 1.4492753623188406e-08,
|
|
"logits/chosen": -0.4815559387207031,
|
|
"logits/rejected": -0.4421369135379791,
|
|
"logps/chosen": -60.98986053466797,
|
|
"logps/ref_chosen": -60.981597900390625,
|
|
"logps/ref_rejected": -68.67259216308594,
|
|
"logps/rejected": -68.65608215332031,
|
|
"loss": 1.3891,
|
|
"margin_dpo/margin_mean": -0.02477523684501648,
|
|
"margin_dpo/margin_std": 0.3734402656555176,
|
|
"step": 3
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.02429962158203125,
|
|
"KL/mean": 0.015571162104606628,
|
|
"KL/rejected_KL_mean": 0.05544281005859375,
|
|
"KL/std": 0.252570241689682,
|
|
"epoch": 0.005873715124816446,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.07974502444267273,
|
|
"fcm_dpo/q_t": 0.50199294090271,
|
|
"grad_norm": 72.09413146972656,
|
|
"learning_rate": 2.1739130434782606e-08,
|
|
"logits/chosen": -0.4686494469642639,
|
|
"logits/rejected": -0.44108062982559204,
|
|
"logps/chosen": -56.79201126098633,
|
|
"logps/ref_chosen": -56.7677116394043,
|
|
"logps/ref_rejected": -86.64710998535156,
|
|
"logps/rejected": -86.59166717529297,
|
|
"loss": 1.3946,
|
|
"margin_dpo/margin_mean": -0.079745352268219,
|
|
"margin_dpo/margin_std": 0.35485363006591797,
|
|
"step": 4
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.0070819854736328125,
|
|
"KL/mean": 0.02031552791595459,
|
|
"KL/rejected_KL_mean": 0.0335540771484375,
|
|
"KL/std": 0.24011048674583435,
|
|
"epoch": 0.007342143906020558,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.0264662504196167,
|
|
"fcm_dpo/q_t": 0.5006614923477173,
|
|
"grad_norm": 89.8433609008789,
|
|
"learning_rate": 2.898550724637681e-08,
|
|
"logits/chosen": -0.5143770575523376,
|
|
"logits/rejected": -0.47061315178871155,
|
|
"logps/chosen": -53.852294921875,
|
|
"logps/ref_chosen": -53.859375,
|
|
"logps/ref_rejected": -84.14918518066406,
|
|
"logps/rejected": -84.11563110351562,
|
|
"loss": 1.3892,
|
|
"margin_dpo/margin_mean": -0.026465564966201782,
|
|
"margin_dpo/margin_std": 0.3306843340396881,
|
|
"step": 5
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.03646278381347656,
|
|
"KL/mean": -0.02894607186317444,
|
|
"KL/rejected_KL_mean": -0.021427154541015625,
|
|
"KL/std": 0.2451232373714447,
|
|
"epoch": 0.00881057268722467,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.015035033226013184,
|
|
"fcm_dpo/q_t": 0.5003755688667297,
|
|
"grad_norm": 92.09574127197266,
|
|
"learning_rate": 3.6231884057971014e-08,
|
|
"logits/chosen": -0.522139310836792,
|
|
"logits/rejected": -0.4815322160720825,
|
|
"logps/chosen": -63.0439453125,
|
|
"logps/ref_chosen": -63.007484436035156,
|
|
"logps/ref_rejected": -92.64534759521484,
|
|
"logps/rejected": -92.66677856445312,
|
|
"loss": 1.3881,
|
|
"margin_dpo/margin_mean": -0.015034288167953491,
|
|
"margin_dpo/margin_std": 0.3625510036945343,
|
|
"step": 6
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.0068035125732421875,
|
|
"KL/mean": 0.020816445350646973,
|
|
"KL/rejected_KL_mean": 0.03482818603515625,
|
|
"KL/std": 0.2857266068458557,
|
|
"epoch": 0.010279001468428781,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.028023719787597656,
|
|
"fcm_dpo/q_t": 0.5007004141807556,
|
|
"grad_norm": 82.25199890136719,
|
|
"learning_rate": 4.347826086956521e-08,
|
|
"logits/chosen": -0.5038982629776001,
|
|
"logits/rejected": -0.4701980650424957,
|
|
"logps/chosen": -57.76801681518555,
|
|
"logps/ref_chosen": -57.774818420410156,
|
|
"logps/ref_rejected": -103.92059326171875,
|
|
"logps/rejected": -103.8857650756836,
|
|
"loss": 1.3896,
|
|
"margin_dpo/margin_mean": -0.02802315354347229,
|
|
"margin_dpo/margin_std": 0.42090797424316406,
|
|
"step": 7
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.03360557556152344,
|
|
"KL/mean": 0.027956083416938782,
|
|
"KL/rejected_KL_mean": 0.022308349609375,
|
|
"KL/std": 0.2712041437625885,
|
|
"epoch": 0.011747430249632892,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.011296361684799194,
|
|
"fcm_dpo/q_t": 0.4997172951698303,
|
|
"grad_norm": 78.32466888427734,
|
|
"learning_rate": 5.0724637681159424e-08,
|
|
"logits/chosen": -0.5075632333755493,
|
|
"logits/rejected": -0.4815378189086914,
|
|
"logps/chosen": -58.682430267333984,
|
|
"logps/ref_chosen": -58.716033935546875,
|
|
"logps/ref_rejected": -79.3114242553711,
|
|
"logps/rejected": -79.28911590576172,
|
|
"loss": 1.3855,
|
|
"margin_dpo/margin_mean": 0.01129680871963501,
|
|
"margin_dpo/margin_std": 0.3612467646598816,
|
|
"step": 8
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.0057525634765625,
|
|
"KL/mean": -0.018807604908943176,
|
|
"KL/rejected_KL_mean": -0.031864166259765625,
|
|
"KL/std": 0.26702290773391724,
|
|
"epoch": 0.013215859030837005,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.026109427213668823,
|
|
"fcm_dpo/q_t": 0.499347448348999,
|
|
"grad_norm": 84.89402770996094,
|
|
"learning_rate": 5.797101449275362e-08,
|
|
"logits/chosen": -0.5175442099571228,
|
|
"logits/rejected": -0.4723869264125824,
|
|
"logps/chosen": -69.87259674072266,
|
|
"logps/ref_chosen": -69.8668441772461,
|
|
"logps/ref_rejected": -99.6026611328125,
|
|
"logps/rejected": -99.63452911376953,
|
|
"loss": 1.384,
|
|
"margin_dpo/margin_mean": 0.02610918879508972,
|
|
"margin_dpo/margin_std": 0.359569787979126,
|
|
"step": 9
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.0016269683837890625,
|
|
"KL/mean": 0.005393713712692261,
|
|
"KL/rejected_KL_mean": 0.00916290283203125,
|
|
"KL/std": 0.2506978213787079,
|
|
"epoch": 0.014684287812041116,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.007536411285400391,
|
|
"fcm_dpo/q_t": 0.5001886487007141,
|
|
"grad_norm": 70.9793930053711,
|
|
"learning_rate": 6.521739130434782e-08,
|
|
"logits/chosen": -0.4903263449668884,
|
|
"logits/rejected": -0.446901798248291,
|
|
"logps/chosen": -48.356056213378906,
|
|
"logps/ref_chosen": -48.35768508911133,
|
|
"logps/ref_rejected": -80.37206268310547,
|
|
"logps/rejected": -80.36289978027344,
|
|
"loss": 1.3874,
|
|
"margin_dpo/margin_mean": -0.007535874843597412,
|
|
"margin_dpo/margin_std": 0.35438239574432373,
|
|
"step": 10
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.045078277587890625,
|
|
"KL/mean": 0.014331206679344177,
|
|
"KL/rejected_KL_mean": -0.016414642333984375,
|
|
"KL/std": 0.236942321062088,
|
|
"epoch": 0.016152716593245228,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.06148645281791687,
|
|
"fcm_dpo/q_t": 0.49846333265304565,
|
|
"grad_norm": 68.05949401855469,
|
|
"learning_rate": 7.246376811594203e-08,
|
|
"logits/chosen": -0.45714545249938965,
|
|
"logits/rejected": -0.43192940950393677,
|
|
"logps/chosen": -52.971778869628906,
|
|
"logps/ref_chosen": -53.01685333251953,
|
|
"logps/ref_rejected": -87.78038024902344,
|
|
"logps/rejected": -87.79679107666016,
|
|
"loss": 1.3804,
|
|
"margin_dpo/margin_mean": 0.06148698925971985,
|
|
"margin_dpo/margin_std": 0.33037588000297546,
|
|
"step": 11
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.022512435913085938,
|
|
"KL/mean": -0.03226548433303833,
|
|
"KL/rejected_KL_mean": -0.04201507568359375,
|
|
"KL/std": 0.2767062187194824,
|
|
"epoch": 0.01762114537444934,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.019503533840179443,
|
|
"fcm_dpo/q_t": 0.4995127022266388,
|
|
"grad_norm": 96.18541717529297,
|
|
"learning_rate": 7.971014492753623e-08,
|
|
"logits/chosen": -0.5195867419242859,
|
|
"logits/rejected": -0.4828483462333679,
|
|
"logps/chosen": -61.82794952392578,
|
|
"logps/ref_chosen": -61.80543518066406,
|
|
"logps/ref_rejected": -104.8582763671875,
|
|
"logps/rejected": -104.90028381347656,
|
|
"loss": 1.3846,
|
|
"margin_dpo/margin_mean": 0.019502699375152588,
|
|
"margin_dpo/margin_std": 0.3449036180973053,
|
|
"step": 12
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.00475311279296875,
|
|
"KL/mean": -0.012506544589996338,
|
|
"KL/rejected_KL_mean": -0.020259857177734375,
|
|
"KL/std": 0.26460498571395874,
|
|
"epoch": 0.01908957415565345,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.015512943267822266,
|
|
"fcm_dpo/q_t": 0.49961256980895996,
|
|
"grad_norm": 79.2775650024414,
|
|
"learning_rate": 8.695652173913042e-08,
|
|
"logits/chosen": -0.48537132143974304,
|
|
"logits/rejected": -0.45758724212646484,
|
|
"logps/chosen": -64.26510620117188,
|
|
"logps/ref_chosen": -64.2603530883789,
|
|
"logps/ref_rejected": -87.20307922363281,
|
|
"logps/rejected": -87.22334289550781,
|
|
"loss": 1.3851,
|
|
"margin_dpo/margin_mean": 0.015512913465499878,
|
|
"margin_dpo/margin_std": 0.339786171913147,
|
|
"step": 13
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.0053424835205078125,
|
|
"KL/mean": -0.026964426040649414,
|
|
"KL/rejected_KL_mean": -0.04859161376953125,
|
|
"KL/std": 0.2796820104122162,
|
|
"epoch": 0.020558002936857563,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.04325270652770996,
|
|
"fcm_dpo/q_t": 0.49891969561576843,
|
|
"grad_norm": 85.09868621826172,
|
|
"learning_rate": 9.420289855072464e-08,
|
|
"logits/chosen": -0.49180495738983154,
|
|
"logits/rejected": -0.45296943187713623,
|
|
"logps/chosen": -58.11555480957031,
|
|
"logps/ref_chosen": -58.11021041870117,
|
|
"logps/ref_rejected": -104.04708099365234,
|
|
"logps/rejected": -104.09567260742188,
|
|
"loss": 1.3825,
|
|
"margin_dpo/margin_mean": 0.043252259492874146,
|
|
"margin_dpo/margin_std": 0.44487419724464417,
|
|
"step": 14
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.014326095581054688,
|
|
"KL/mean": -0.034481316804885864,
|
|
"KL/rejected_KL_mean": -0.054637908935546875,
|
|
"KL/std": 0.24156004190444946,
|
|
"epoch": 0.022026431718061675,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.04031383991241455,
|
|
"fcm_dpo/q_t": 0.4989916980266571,
|
|
"grad_norm": 64.1568374633789,
|
|
"learning_rate": 1.0144927536231885e-07,
|
|
"logits/chosen": -0.5022023916244507,
|
|
"logits/rejected": -0.4839800298213959,
|
|
"logps/chosen": -56.98123550415039,
|
|
"logps/ref_chosen": -56.96691131591797,
|
|
"logps/ref_rejected": -80.80863952636719,
|
|
"logps/rejected": -80.86328125,
|
|
"loss": 1.3826,
|
|
"margin_dpo/margin_mean": 0.04031360149383545,
|
|
"margin_dpo/margin_std": 0.3656350076198578,
|
|
"step": 15
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.029695510864257812,
|
|
"KL/mean": -0.008761048316955566,
|
|
"KL/rejected_KL_mean": -0.047214508056640625,
|
|
"KL/std": 0.2751133441925049,
|
|
"epoch": 0.023494860499265784,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.07690972089767456,
|
|
"fcm_dpo/q_t": 0.49807754158973694,
|
|
"grad_norm": 83.51917266845703,
|
|
"learning_rate": 1.0869565217391303e-07,
|
|
"logits/chosen": -0.5214604139328003,
|
|
"logits/rejected": -0.47995007038116455,
|
|
"logps/chosen": -61.71019744873047,
|
|
"logps/ref_chosen": -61.739891052246094,
|
|
"logps/ref_rejected": -84.36947631835938,
|
|
"logps/rejected": -84.41668701171875,
|
|
"loss": 1.3791,
|
|
"margin_dpo/margin_mean": 0.07690989971160889,
|
|
"margin_dpo/margin_std": 0.42422112822532654,
|
|
"step": 16
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.06725311279296875,
|
|
"KL/mean": 0.007791772484779358,
|
|
"KL/rejected_KL_mean": -0.051666259765625,
|
|
"KL/std": 0.29778245091438293,
|
|
"epoch": 0.024963289280469897,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.1189166009426117,
|
|
"fcm_dpo/q_t": 0.4970288872718811,
|
|
"grad_norm": 78.13703155517578,
|
|
"learning_rate": 1.1594202898550725e-07,
|
|
"logits/chosen": -0.5090387463569641,
|
|
"logits/rejected": -0.4729403853416443,
|
|
"logps/chosen": -67.64308166503906,
|
|
"logps/ref_chosen": -67.71033477783203,
|
|
"logps/ref_rejected": -85.37865447998047,
|
|
"logps/rejected": -85.4303207397461,
|
|
"loss": 1.3749,
|
|
"margin_dpo/margin_mean": 0.11891645193099976,
|
|
"margin_dpo/margin_std": 0.415335476398468,
|
|
"step": 17
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.0026035308837890625,
|
|
"KL/mean": -0.012814626097679138,
|
|
"KL/rejected_KL_mean": -0.0230255126953125,
|
|
"KL/std": 0.2600899636745453,
|
|
"epoch": 0.02643171806167401,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.020421653985977173,
|
|
"fcm_dpo/q_t": 0.4994891285896301,
|
|
"grad_norm": 82.34599304199219,
|
|
"learning_rate": 1.2318840579710146e-07,
|
|
"logits/chosen": -0.48533254861831665,
|
|
"logits/rejected": -0.4290851950645447,
|
|
"logps/chosen": -47.742095947265625,
|
|
"logps/ref_chosen": -47.7394905090332,
|
|
"logps/ref_rejected": -75.4722900390625,
|
|
"logps/rejected": -75.49531555175781,
|
|
"loss": 1.3846,
|
|
"margin_dpo/margin_mean": 0.020421594381332397,
|
|
"margin_dpo/margin_std": 0.32879841327667236,
|
|
"step": 18
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.024383544921875,
|
|
"KL/mean": 0.0039770156145095825,
|
|
"KL/rejected_KL_mean": -0.01642608642578125,
|
|
"KL/std": 0.23556864261627197,
|
|
"epoch": 0.027900146842878122,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.040806323289871216,
|
|
"fcm_dpo/q_t": 0.49897968769073486,
|
|
"grad_norm": 74.32623291015625,
|
|
"learning_rate": 1.3043478260869563e-07,
|
|
"logits/chosen": -0.4715813398361206,
|
|
"logits/rejected": -0.4200263023376465,
|
|
"logps/chosen": -70.18097686767578,
|
|
"logps/ref_chosen": -70.20536041259766,
|
|
"logps/ref_rejected": -89.7575912475586,
|
|
"logps/rejected": -89.77401733398438,
|
|
"loss": 1.3825,
|
|
"margin_dpo/margin_mean": 0.0408073365688324,
|
|
"margin_dpo/margin_std": 0.30957865715026855,
|
|
"step": 19
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.03499412536621094,
|
|
"KL/mean": -0.04548324644565582,
|
|
"KL/rejected_KL_mean": -0.05596923828125,
|
|
"KL/std": 0.2744811177253723,
|
|
"epoch": 0.02936857562408223,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.020983070135116577,
|
|
"fcm_dpo/q_t": 0.4994752109050751,
|
|
"grad_norm": 74.43611145019531,
|
|
"learning_rate": 1.3768115942028986e-07,
|
|
"logits/chosen": -0.5649498105049133,
|
|
"logits/rejected": -0.5099019408226013,
|
|
"logps/chosen": -50.838233947753906,
|
|
"logps/ref_chosen": -50.80324172973633,
|
|
"logps/ref_rejected": -78.82334899902344,
|
|
"logps/rejected": -78.87931823730469,
|
|
"loss": 1.3846,
|
|
"margin_dpo/margin_mean": 0.020983099937438965,
|
|
"margin_dpo/margin_std": 0.3876810669898987,
|
|
"step": 20
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.050243377685546875,
|
|
"KL/mean": -0.018013283610343933,
|
|
"KL/rejected_KL_mean": -0.08626937866210938,
|
|
"KL/std": 0.3195314407348633,
|
|
"epoch": 0.030837004405286344,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.1365136206150055,
|
|
"fcm_dpo/q_t": 0.49658873677253723,
|
|
"grad_norm": 77.88671875,
|
|
"learning_rate": 1.4492753623188405e-07,
|
|
"logits/chosen": -0.5110548138618469,
|
|
"logits/rejected": -0.48853152990341187,
|
|
"logps/chosen": -50.01277160644531,
|
|
"logps/ref_chosen": -50.063018798828125,
|
|
"logps/ref_rejected": -77.86878967285156,
|
|
"logps/rejected": -77.95506286621094,
|
|
"loss": 1.3731,
|
|
"margin_dpo/margin_mean": 0.13651368021965027,
|
|
"margin_dpo/margin_std": 0.4034786820411682,
|
|
"step": 21
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.012243270874023438,
|
|
"KL/mean": -0.08835446834564209,
|
|
"KL/rejected_KL_mean": -0.16446685791015625,
|
|
"KL/std": 0.28480368852615356,
|
|
"epoch": 0.032305433186490456,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.1522234082221985,
|
|
"fcm_dpo/q_t": 0.49619609117507935,
|
|
"grad_norm": 84.3863525390625,
|
|
"learning_rate": 1.5217391304347825e-07,
|
|
"logits/chosen": -0.505510151386261,
|
|
"logits/rejected": -0.462830126285553,
|
|
"logps/chosen": -59.06987762451172,
|
|
"logps/ref_chosen": -59.05763626098633,
|
|
"logps/ref_rejected": -97.50466918945312,
|
|
"logps/rejected": -97.66914367675781,
|
|
"loss": 1.3716,
|
|
"margin_dpo/margin_mean": 0.1522233486175537,
|
|
"margin_dpo/margin_std": 0.4398460388183594,
|
|
"step": 22
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.016481399536132812,
|
|
"KL/mean": -0.06710737943649292,
|
|
"KL/rejected_KL_mean": -0.1506977081298828,
|
|
"KL/std": 0.340972900390625,
|
|
"epoch": 0.033773861967694566,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.16717880964279175,
|
|
"fcm_dpo/q_t": 0.49582529067993164,
|
|
"grad_norm": 80.40861511230469,
|
|
"learning_rate": 1.5942028985507245e-07,
|
|
"logits/chosen": -0.5021919012069702,
|
|
"logits/rejected": -0.48034122586250305,
|
|
"logps/chosen": -60.061214447021484,
|
|
"logps/ref_chosen": -60.07769775390625,
|
|
"logps/ref_rejected": -81.13955688476562,
|
|
"logps/rejected": -81.29025268554688,
|
|
"loss": 1.3703,
|
|
"margin_dpo/margin_mean": 0.16717904806137085,
|
|
"margin_dpo/margin_std": 0.5109284520149231,
|
|
"step": 23
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.0286712646484375,
|
|
"KL/mean": -0.07983805984258652,
|
|
"KL/rejected_KL_mean": -0.18835067749023438,
|
|
"KL/std": 0.29715216159820557,
|
|
"epoch": 0.03524229074889868,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.2170248180627823,
|
|
"fcm_dpo/q_t": 0.49457818269729614,
|
|
"grad_norm": 83.2780990600586,
|
|
"learning_rate": 1.6666666666666665e-07,
|
|
"logits/chosen": -0.5041535496711731,
|
|
"logits/rejected": -0.48770707845687866,
|
|
"logps/chosen": -44.26236343383789,
|
|
"logps/ref_chosen": -44.29103469848633,
|
|
"logps/ref_rejected": -99.12521362304688,
|
|
"logps/rejected": -99.31356811523438,
|
|
"loss": 1.3652,
|
|
"margin_dpo/margin_mean": 0.21702483296394348,
|
|
"margin_dpo/margin_std": 0.4385562539100647,
|
|
"step": 24
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.02964019775390625,
|
|
"KL/mean": -0.08741536736488342,
|
|
"KL/rejected_KL_mean": -0.2044677734375,
|
|
"KL/std": 0.34008604288101196,
|
|
"epoch": 0.03671071953010279,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.23411154747009277,
|
|
"fcm_dpo/q_t": 0.4941510558128357,
|
|
"grad_norm": 74.14248657226562,
|
|
"learning_rate": 1.7391304347826085e-07,
|
|
"logits/chosen": -0.4980379641056061,
|
|
"logits/rejected": -0.4681099057197571,
|
|
"logps/chosen": -52.50741195678711,
|
|
"logps/ref_chosen": -52.537052154541016,
|
|
"logps/ref_rejected": -89.34219360351562,
|
|
"logps/rejected": -89.54666137695312,
|
|
"loss": 1.3635,
|
|
"margin_dpo/margin_mean": 0.234111487865448,
|
|
"margin_dpo/margin_std": 0.41896557807922363,
|
|
"step": 25
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.05152320861816406,
|
|
"KL/mean": -0.12674298882484436,
|
|
"KL/rejected_KL_mean": -0.3050079345703125,
|
|
"KL/std": 0.4253169298171997,
|
|
"epoch": 0.0381791483113069,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.3565349578857422,
|
|
"fcm_dpo/q_t": 0.49109458923339844,
|
|
"grad_norm": 87.5750732421875,
|
|
"learning_rate": 1.8115942028985507e-07,
|
|
"logits/chosen": -0.5453453063964844,
|
|
"logits/rejected": -0.513872504234314,
|
|
"logps/chosen": -53.87128448486328,
|
|
"logps/ref_chosen": -53.92280578613281,
|
|
"logps/ref_rejected": -103.35971069335938,
|
|
"logps/rejected": -103.66471862792969,
|
|
"loss": 1.3516,
|
|
"margin_dpo/margin_mean": 0.3565359115600586,
|
|
"margin_dpo/margin_std": 0.4943789839744568,
|
|
"step": 26
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.11599159240722656,
|
|
"KL/mean": -0.13826510310173035,
|
|
"KL/rejected_KL_mean": -0.39252471923828125,
|
|
"KL/std": 0.476246178150177,
|
|
"epoch": 0.039647577092511016,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.5085150599479675,
|
|
"fcm_dpo/q_t": 0.48730653524398804,
|
|
"grad_norm": 93.57472229003906,
|
|
"learning_rate": 1.8840579710144927e-07,
|
|
"logits/chosen": -0.5514802932739258,
|
|
"logits/rejected": -0.5156171321868896,
|
|
"logps/chosen": -42.78253936767578,
|
|
"logps/ref_chosen": -42.898529052734375,
|
|
"logps/ref_rejected": -98.72419738769531,
|
|
"logps/rejected": -99.11672973632812,
|
|
"loss": 1.3369,
|
|
"margin_dpo/margin_mean": 0.5085144639015198,
|
|
"margin_dpo/margin_std": 0.5824718475341797,
|
|
"step": 27
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.011579513549804688,
|
|
"KL/mean": -0.13822337985038757,
|
|
"KL/rejected_KL_mean": -0.2880287170410156,
|
|
"KL/std": 0.42481285333633423,
|
|
"epoch": 0.041116005873715125,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.2996072769165039,
|
|
"fcm_dpo/q_t": 0.49251770973205566,
|
|
"grad_norm": 75.85013580322266,
|
|
"learning_rate": 1.9565217391304347e-07,
|
|
"logits/chosen": -0.497283399105072,
|
|
"logits/rejected": -0.4410708546638489,
|
|
"logps/chosen": -60.544921875,
|
|
"logps/ref_chosen": -60.55650329589844,
|
|
"logps/ref_rejected": -91.40111541748047,
|
|
"logps/rejected": -91.68914794921875,
|
|
"loss": 1.3574,
|
|
"margin_dpo/margin_mean": 0.2996074855327606,
|
|
"margin_dpo/margin_std": 0.562934160232544,
|
|
"step": 28
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.12349700927734375,
|
|
"KL/mean": -0.16272687911987305,
|
|
"KL/rejected_KL_mean": -0.4489555358886719,
|
|
"KL/std": 0.4961986243724823,
|
|
"epoch": 0.042584434654919234,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.5724496841430664,
|
|
"fcm_dpo/q_t": 0.4857064485549927,
|
|
"grad_norm": 90.49112701416016,
|
|
"learning_rate": 2.028985507246377e-07,
|
|
"logits/chosen": -0.5566602945327759,
|
|
"logits/rejected": -0.5104827880859375,
|
|
"logps/chosen": -57.684288024902344,
|
|
"logps/ref_chosen": -57.80778503417969,
|
|
"logps/ref_rejected": -97.39434814453125,
|
|
"logps/rejected": -97.84330749511719,
|
|
"loss": 1.3307,
|
|
"margin_dpo/margin_mean": 0.5724506378173828,
|
|
"margin_dpo/margin_std": 0.5618862509727478,
|
|
"step": 29
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.11771583557128906,
|
|
"KL/mean": -0.2122490406036377,
|
|
"KL/rejected_KL_mean": -0.5422172546386719,
|
|
"KL/std": 0.5805710554122925,
|
|
"epoch": 0.04405286343612335,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.659930944442749,
|
|
"fcm_dpo/q_t": 0.4835302233695984,
|
|
"grad_norm": 87.53767395019531,
|
|
"learning_rate": 2.1014492753623187e-07,
|
|
"logits/chosen": -0.4962325990200043,
|
|
"logits/rejected": -0.4656328558921814,
|
|
"logps/chosen": -52.45965576171875,
|
|
"logps/ref_chosen": -52.577369689941406,
|
|
"logps/ref_rejected": -98.48920440673828,
|
|
"logps/rejected": -99.03141784667969,
|
|
"loss": 1.3225,
|
|
"margin_dpo/margin_mean": 0.659930944442749,
|
|
"margin_dpo/margin_std": 0.6480692625045776,
|
|
"step": 30
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.10221672058105469,
|
|
"KL/mean": -0.15314733982086182,
|
|
"KL/rejected_KL_mean": -0.4085121154785156,
|
|
"KL/std": 0.5654332637786865,
|
|
"epoch": 0.04552129221732746,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.510732889175415,
|
|
"fcm_dpo/q_t": 0.48725682497024536,
|
|
"grad_norm": 67.43456268310547,
|
|
"learning_rate": 2.1739130434782607e-07,
|
|
"logits/chosen": -0.49099212884902954,
|
|
"logits/rejected": -0.44460946321487427,
|
|
"logps/chosen": -63.70470428466797,
|
|
"logps/ref_chosen": -63.806922912597656,
|
|
"logps/ref_rejected": -72.89400482177734,
|
|
"logps/rejected": -73.30252075195312,
|
|
"loss": 1.3371,
|
|
"margin_dpo/margin_mean": 0.5107325315475464,
|
|
"margin_dpo/margin_std": 0.6857659220695496,
|
|
"step": 31
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.2014923095703125,
|
|
"KL/mean": -0.19277532398700714,
|
|
"KL/rejected_KL_mean": -0.5870437622070312,
|
|
"KL/std": 0.6862952709197998,
|
|
"epoch": 0.04698972099853157,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.7885334491729736,
|
|
"fcm_dpo/q_t": 0.4803431034088135,
|
|
"grad_norm": 80.95833587646484,
|
|
"learning_rate": 2.2463768115942027e-07,
|
|
"logits/chosen": -0.5271800756454468,
|
|
"logits/rejected": -0.4867057204246521,
|
|
"logps/chosen": -62.53803253173828,
|
|
"logps/ref_chosen": -62.739524841308594,
|
|
"logps/ref_rejected": -89.3175048828125,
|
|
"logps/rejected": -89.904541015625,
|
|
"loss": 1.3106,
|
|
"margin_dpo/margin_mean": 0.7885338664054871,
|
|
"margin_dpo/margin_std": 0.8146342635154724,
|
|
"step": 32
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.13482093811035156,
|
|
"KL/mean": -0.1859203577041626,
|
|
"KL/rejected_KL_mean": -0.5066604614257812,
|
|
"KL/std": 0.5469956994056702,
|
|
"epoch": 0.048458149779735685,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.6414769291877747,
|
|
"fcm_dpo/q_t": 0.4839860796928406,
|
|
"grad_norm": 72.35101318359375,
|
|
"learning_rate": 2.318840579710145e-07,
|
|
"logits/chosen": -0.5072885751724243,
|
|
"logits/rejected": -0.481620728969574,
|
|
"logps/chosen": -53.12615203857422,
|
|
"logps/ref_chosen": -53.26097106933594,
|
|
"logps/ref_rejected": -87.8851318359375,
|
|
"logps/rejected": -88.39179992675781,
|
|
"loss": 1.3242,
|
|
"margin_dpo/margin_mean": 0.6414770483970642,
|
|
"margin_dpo/margin_std": 0.6191001534461975,
|
|
"step": 33
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.08164787292480469,
|
|
"KL/mean": -0.32943296432495117,
|
|
"KL/rejected_KL_mean": -0.7405166625976562,
|
|
"KL/std": 0.7699615359306335,
|
|
"epoch": 0.049926578560939794,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.8221615552902222,
|
|
"fcm_dpo/q_t": 0.47951143980026245,
|
|
"grad_norm": 77.51274871826172,
|
|
"learning_rate": 2.391304347826087e-07,
|
|
"logits/chosen": -0.48901137709617615,
|
|
"logits/rejected": -0.4716091752052307,
|
|
"logps/chosen": -50.735679626464844,
|
|
"logps/ref_chosen": -50.81732940673828,
|
|
"logps/ref_rejected": -101.92184448242188,
|
|
"logps/rejected": -102.66236114501953,
|
|
"loss": 1.3079,
|
|
"margin_dpo/margin_mean": 0.8221613764762878,
|
|
"margin_dpo/margin_std": 0.9267702102661133,
|
|
"step": 34
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.11441421508789062,
|
|
"KL/mean": -0.5115079879760742,
|
|
"KL/rejected_KL_mean": -1.1374282836914062,
|
|
"KL/std": 1.0209949016571045,
|
|
"epoch": 0.0513950073421439,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.2518417835235596,
|
|
"fcm_dpo/q_t": 0.46891528367996216,
|
|
"grad_norm": 82.27989196777344,
|
|
"learning_rate": 2.463768115942029e-07,
|
|
"logits/chosen": -0.5277581214904785,
|
|
"logits/rejected": -0.49088025093078613,
|
|
"logps/chosen": -50.91007614135742,
|
|
"logps/ref_chosen": -51.02449035644531,
|
|
"logps/ref_rejected": -106.82443237304688,
|
|
"logps/rejected": -107.96185302734375,
|
|
"loss": 1.2684,
|
|
"margin_dpo/margin_mean": 1.2518424987792969,
|
|
"margin_dpo/margin_std": 1.1583997011184692,
|
|
"step": 35
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.054615020751953125,
|
|
"KL/mean": -0.5534474849700928,
|
|
"KL/rejected_KL_mean": -1.1615104675292969,
|
|
"KL/std": 1.1053463220596313,
|
|
"epoch": 0.05286343612334802,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.2161287069320679,
|
|
"fcm_dpo/q_t": 0.469775915145874,
|
|
"grad_norm": 72.23912048339844,
|
|
"learning_rate": 2.536231884057971e-07,
|
|
"logits/chosen": -0.5657041072845459,
|
|
"logits/rejected": -0.5297294855117798,
|
|
"logps/chosen": -51.93687438964844,
|
|
"logps/ref_chosen": -51.991493225097656,
|
|
"logps/ref_rejected": -86.0406265258789,
|
|
"logps/rejected": -87.20213317871094,
|
|
"loss": 1.2726,
|
|
"margin_dpo/margin_mean": 1.2161275148391724,
|
|
"margin_dpo/margin_std": 1.2665972709655762,
|
|
"step": 36
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.014173507690429688,
|
|
"KL/mean": -0.5154328942298889,
|
|
"KL/rejected_KL_mean": -1.0450401306152344,
|
|
"KL/std": 1.0978405475616455,
|
|
"epoch": 0.05433186490455213,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.059217929840088,
|
|
"fcm_dpo/q_t": 0.4737340807914734,
|
|
"grad_norm": 61.976402282714844,
|
|
"learning_rate": 2.6086956521739126e-07,
|
|
"logits/chosen": -0.5021863579750061,
|
|
"logits/rejected": -0.4578307271003723,
|
|
"logps/chosen": -62.79293441772461,
|
|
"logps/ref_chosen": -62.807106018066406,
|
|
"logps/ref_rejected": -77.89507293701172,
|
|
"logps/rejected": -78.94010925292969,
|
|
"loss": 1.288,
|
|
"margin_dpo/margin_mean": 1.0592180490493774,
|
|
"margin_dpo/margin_std": 1.3723002672195435,
|
|
"step": 37
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.17836570739746094,
|
|
"KL/mean": -0.5307787656784058,
|
|
"KL/rejected_KL_mean": -1.2399253845214844,
|
|
"KL/std": 1.3880457878112793,
|
|
"epoch": 0.055800293685756244,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.418290615081787,
|
|
"fcm_dpo/q_t": 0.4649973213672638,
|
|
"grad_norm": 69.40721893310547,
|
|
"learning_rate": 2.681159420289855e-07,
|
|
"logits/chosen": -0.5123181343078613,
|
|
"logits/rejected": -0.47957277297973633,
|
|
"logps/chosen": -48.212154388427734,
|
|
"logps/ref_chosen": -48.39051818847656,
|
|
"logps/ref_rejected": -97.91244506835938,
|
|
"logps/rejected": -99.1523666381836,
|
|
"loss": 1.2564,
|
|
"margin_dpo/margin_mean": 1.4182908535003662,
|
|
"margin_dpo/margin_std": 1.6710336208343506,
|
|
"step": 38
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.11879920959472656,
|
|
"KL/mean": -0.7346929311752319,
|
|
"KL/rejected_KL_mean": -1.5881805419921875,
|
|
"KL/std": 1.3292229175567627,
|
|
"epoch": 0.05726872246696035,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.7069776058197021,
|
|
"fcm_dpo/q_t": 0.4576621651649475,
|
|
"grad_norm": 74.3014907836914,
|
|
"learning_rate": 2.753623188405797e-07,
|
|
"logits/chosen": -0.5710403919219971,
|
|
"logits/rejected": -0.5320708155632019,
|
|
"logps/chosen": -50.631675720214844,
|
|
"logps/ref_chosen": -50.75047302246094,
|
|
"logps/ref_rejected": -78.56951141357422,
|
|
"logps/rejected": -80.1576919555664,
|
|
"loss": 1.228,
|
|
"margin_dpo/margin_mean": 1.7069778442382812,
|
|
"margin_dpo/margin_std": 1.4111273288726807,
|
|
"step": 39
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.2220172882080078,
|
|
"KL/mean": -0.5618829131126404,
|
|
"KL/rejected_KL_mean": -1.3457870483398438,
|
|
"KL/std": 1.41432523727417,
|
|
"epoch": 0.05873715124816446,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.5678019523620605,
|
|
"fcm_dpo/q_t": 0.46130138635635376,
|
|
"grad_norm": 60.054290771484375,
|
|
"learning_rate": 2.8260869565217386e-07,
|
|
"logits/chosen": -0.507063627243042,
|
|
"logits/rejected": -0.4761931002140045,
|
|
"logps/chosen": -57.7630500793457,
|
|
"logps/ref_chosen": -57.985069274902344,
|
|
"logps/ref_rejected": -74.3000717163086,
|
|
"logps/rejected": -75.64585876464844,
|
|
"loss": 1.2432,
|
|
"margin_dpo/margin_mean": 1.5678024291992188,
|
|
"margin_dpo/margin_std": 1.709043264389038,
|
|
"step": 40
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.0051975250244140625,
|
|
"KL/mean": -0.9326056838035583,
|
|
"KL/rejected_KL_mean": -1.8704071044921875,
|
|
"KL/std": 1.854329228401184,
|
|
"epoch": 0.06020558002936858,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.8756054639816284,
|
|
"fcm_dpo/q_t": 0.4538491666316986,
|
|
"grad_norm": 68.05853271484375,
|
|
"learning_rate": 2.898550724637681e-07,
|
|
"logits/chosen": -0.540554940700531,
|
|
"logits/rejected": -0.5041079521179199,
|
|
"logps/chosen": -62.69062042236328,
|
|
"logps/ref_chosen": -62.69581604003906,
|
|
"logps/ref_rejected": -97.02352905273438,
|
|
"logps/rejected": -98.89393615722656,
|
|
"loss": 1.2179,
|
|
"margin_dpo/margin_mean": 1.8756061792373657,
|
|
"margin_dpo/margin_std": 2.0067355632781982,
|
|
"step": 41
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.21923828125,
|
|
"KL/mean": -1.0403838157653809,
|
|
"KL/rejected_KL_mean": -2.300006866455078,
|
|
"KL/std": 2.130916118621826,
|
|
"epoch": 0.06167400881057269,
|
|
"fcm_dpo/beta": 0.1539303958415985,
|
|
"fcm_dpo/delta": -0.01988767832517624,
|
|
"fcm_dpo/margin": 2.5192437171936035,
|
|
"fcm_dpo/q_t": 0.4016793966293335,
|
|
"grad_norm": 98.06798553466797,
|
|
"learning_rate": 2.971014492753623e-07,
|
|
"logits/chosen": -0.5261090993881226,
|
|
"logits/rejected": -0.4792720377445221,
|
|
"logps/chosen": -58.747188568115234,
|
|
"logps/ref_chosen": -58.966426849365234,
|
|
"logps/ref_rejected": -109.90837097167969,
|
|
"logps/rejected": -112.20838165283203,
|
|
"loss": 1.0939,
|
|
"margin_dpo/margin_mean": 2.5192432403564453,
|
|
"margin_dpo/margin_std": 2.3735432624816895,
|
|
"step": 42
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.5401420593261719,
|
|
"KL/mean": -0.6934725046157837,
|
|
"KL/rejected_KL_mean": -1.927093505859375,
|
|
"KL/std": 1.8467273712158203,
|
|
"epoch": 0.0631424375917768,
|
|
"fcm_dpo/beta": 0.18102575838565826,
|
|
"fcm_dpo/delta": 0.06920016556978226,
|
|
"fcm_dpo/margin": 2.4672341346740723,
|
|
"fcm_dpo/q_t": 0.4202408194541931,
|
|
"grad_norm": 109.3116455078125,
|
|
"learning_rate": 3.043478260869565e-07,
|
|
"logits/chosen": -0.540740966796875,
|
|
"logits/rejected": -0.5158405303955078,
|
|
"logps/chosen": -53.61585235595703,
|
|
"logps/ref_chosen": -54.15599822998047,
|
|
"logps/ref_rejected": -96.48019409179688,
|
|
"logps/rejected": -98.40728759765625,
|
|
"loss": 1.0181,
|
|
"margin_dpo/margin_mean": 2.4672341346740723,
|
|
"margin_dpo/margin_std": 1.9857051372528076,
|
|
"step": 43
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.1969776153564453,
|
|
"KL/mean": -1.1819499731063843,
|
|
"KL/rejected_KL_mean": -2.5608787536621094,
|
|
"KL/std": 2.232034206390381,
|
|
"epoch": 0.06461086637298091,
|
|
"fcm_dpo/beta": 0.14869627356529236,
|
|
"fcm_dpo/delta": 0.008696533739566803,
|
|
"fcm_dpo/margin": 2.7578556537628174,
|
|
"fcm_dpo/q_t": 0.40634626150131226,
|
|
"grad_norm": 103.40689849853516,
|
|
"learning_rate": 3.115942028985507e-07,
|
|
"logits/chosen": -0.49857497215270996,
|
|
"logits/rejected": -0.47800567746162415,
|
|
"logps/chosen": -49.88152313232422,
|
|
"logps/ref_chosen": -50.07849884033203,
|
|
"logps/ref_rejected": -108.78376007080078,
|
|
"logps/rejected": -111.34463500976562,
|
|
"loss": 1.0466,
|
|
"margin_dpo/margin_mean": 2.7578554153442383,
|
|
"margin_dpo/margin_std": 2.197627067565918,
|
|
"step": 44
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.1326732635498047,
|
|
"KL/mean": -1.014523983001709,
|
|
"KL/rejected_KL_mean": -2.1617279052734375,
|
|
"KL/std": 2.0959479808807373,
|
|
"epoch": 0.06607929515418502,
|
|
"fcm_dpo/beta": 0.15769031643867493,
|
|
"fcm_dpo/delta": -0.06119377911090851,
|
|
"fcm_dpo/margin": 2.2944016456604004,
|
|
"fcm_dpo/q_t": 0.39686933159828186,
|
|
"grad_norm": 85.24789428710938,
|
|
"learning_rate": 3.188405797101449e-07,
|
|
"logits/chosen": -0.4741653800010681,
|
|
"logits/rejected": -0.46142202615737915,
|
|
"logps/chosen": -48.282257080078125,
|
|
"logps/ref_chosen": -48.4149284362793,
|
|
"logps/ref_rejected": -77.93643188476562,
|
|
"logps/rejected": -80.09815979003906,
|
|
"loss": 1.1099,
|
|
"margin_dpo/margin_mean": 2.2944014072418213,
|
|
"margin_dpo/margin_std": 2.6267547607421875,
|
|
"step": 45
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.2033252716064453,
|
|
"KL/mean": -1.2960678339004517,
|
|
"KL/rejected_KL_mean": -2.795459747314453,
|
|
"KL/std": 2.6904854774475098,
|
|
"epoch": 0.06754772393538913,
|
|
"fcm_dpo/beta": 0.1267249435186386,
|
|
"fcm_dpo/delta": -0.04247567802667618,
|
|
"fcm_dpo/margin": 2.9987845420837402,
|
|
"fcm_dpo/q_t": 0.39998096227645874,
|
|
"grad_norm": 75.10869598388672,
|
|
"learning_rate": 3.260869565217391e-07,
|
|
"logits/chosen": -0.5231410264968872,
|
|
"logits/rejected": -0.4719325304031372,
|
|
"logps/chosen": -55.79610061645508,
|
|
"logps/ref_chosen": -55.999427795410156,
|
|
"logps/ref_rejected": -95.652587890625,
|
|
"logps/rejected": -98.44804382324219,
|
|
"loss": 1.1114,
|
|
"margin_dpo/margin_mean": 2.9987847805023193,
|
|
"margin_dpo/margin_std": 3.3123412132263184,
|
|
"step": 46
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.4382801055908203,
|
|
"KL/mean": -1.083209753036499,
|
|
"KL/rejected_KL_mean": -2.6046981811523438,
|
|
"KL/std": 2.6264405250549316,
|
|
"epoch": 0.06901615271659324,
|
|
"fcm_dpo/beta": 0.10618115961551666,
|
|
"fcm_dpo/delta": 0.0855509415268898,
|
|
"fcm_dpo/margin": 3.042980670928955,
|
|
"fcm_dpo/q_t": 0.452394038438797,
|
|
"grad_norm": 67.91606140136719,
|
|
"learning_rate": 3.333333333333333e-07,
|
|
"logits/chosen": -0.5615835189819336,
|
|
"logits/rejected": -0.5081955194473267,
|
|
"logps/chosen": -57.48780059814453,
|
|
"logps/ref_chosen": -57.92607879638672,
|
|
"logps/ref_rejected": -94.67920684814453,
|
|
"logps/rejected": -97.28390502929688,
|
|
"loss": 1.0978,
|
|
"margin_dpo/margin_mean": 3.0429811477661133,
|
|
"margin_dpo/margin_std": 2.6995935440063477,
|
|
"step": 47
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.11672401428222656,
|
|
"KL/mean": -1.4517210721969604,
|
|
"KL/rejected_KL_mean": -3.0201644897460938,
|
|
"KL/std": 2.6644673347473145,
|
|
"epoch": 0.07048458149779736,
|
|
"fcm_dpo/beta": 0.07081333547830582,
|
|
"fcm_dpo/delta": -0.07440280169248581,
|
|
"fcm_dpo/margin": 3.1368894577026367,
|
|
"fcm_dpo/q_t": 0.41475388407707214,
|
|
"grad_norm": 55.53757095336914,
|
|
"learning_rate": 3.4057971014492755e-07,
|
|
"logits/chosen": -0.5801494121551514,
|
|
"logits/rejected": -0.5212501287460327,
|
|
"logps/chosen": -57.07135009765625,
|
|
"logps/ref_chosen": -57.188072204589844,
|
|
"logps/ref_rejected": -88.0166015625,
|
|
"logps/rejected": -91.0367660522461,
|
|
"loss": 1.1888,
|
|
"margin_dpo/margin_mean": 3.1368894577026367,
|
|
"margin_dpo/margin_std": 2.9127197265625,
|
|
"step": 48
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.3498554229736328,
|
|
"KL/mean": -1.5747132301330566,
|
|
"KL/rejected_KL_mean": -3.499279022216797,
|
|
"KL/std": 3.408785581588745,
|
|
"epoch": 0.07195301027900147,
|
|
"fcm_dpo/beta": 0.09813052415847778,
|
|
"fcm_dpo/delta": -0.0769207701086998,
|
|
"fcm_dpo/margin": 3.8491296768188477,
|
|
"fcm_dpo/q_t": 0.3957645893096924,
|
|
"grad_norm": 59.666378021240234,
|
|
"learning_rate": 3.478260869565217e-07,
|
|
"logits/chosen": -0.5373194813728333,
|
|
"logits/rejected": -0.47734540700912476,
|
|
"logps/chosen": -61.33541488647461,
|
|
"logps/ref_chosen": -61.685272216796875,
|
|
"logps/ref_rejected": -83.76747131347656,
|
|
"logps/rejected": -87.26675415039062,
|
|
"loss": 1.1176,
|
|
"margin_dpo/margin_mean": 3.8491291999816895,
|
|
"margin_dpo/margin_std": 3.9004533290863037,
|
|
"step": 49
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.15655517578125,
|
|
"KL/mean": -2.1865463256835938,
|
|
"KL/rejected_KL_mean": -4.216529846191406,
|
|
"KL/std": 3.433018445968628,
|
|
"epoch": 0.07342143906020558,
|
|
"fcm_dpo/beta": 0.03281348943710327,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 4.059974193572998,
|
|
"fcm_dpo/q_t": 0.4669080376625061,
|
|
"grad_norm": 26.208717346191406,
|
|
"learning_rate": 3.5507246376811595e-07,
|
|
"logits/chosen": -0.5582753419876099,
|
|
"logits/rejected": -0.522718608379364,
|
|
"logps/chosen": -58.88069152832031,
|
|
"logps/ref_chosen": -58.72413635253906,
|
|
"logps/ref_rejected": -96.35814666748047,
|
|
"logps/rejected": -100.57467651367188,
|
|
"loss": 1.2617,
|
|
"margin_dpo/margin_mean": 4.059974193572998,
|
|
"margin_dpo/margin_std": 3.9510397911071777,
|
|
"step": 50
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.26395416259765625,
|
|
"KL/mean": -2.266573429107666,
|
|
"KL/rejected_KL_mean": -4.269195556640625,
|
|
"KL/std": 4.127612113952637,
|
|
"epoch": 0.07488986784140969,
|
|
"fcm_dpo/beta": 0.03281348943710327,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 4.005241870880127,
|
|
"fcm_dpo/q_t": 0.46752673387527466,
|
|
"grad_norm": 23.76943588256836,
|
|
"learning_rate": 3.6231884057971015e-07,
|
|
"logits/chosen": -0.5171994566917419,
|
|
"logits/rejected": -0.4839722514152527,
|
|
"logps/chosen": -61.63762283325195,
|
|
"logps/ref_chosen": -61.3736686706543,
|
|
"logps/ref_rejected": -76.00199890136719,
|
|
"logps/rejected": -80.27119445800781,
|
|
"loss": 1.2662,
|
|
"margin_dpo/margin_mean": 4.005241394042969,
|
|
"margin_dpo/margin_std": 5.0708160400390625,
|
|
"step": 51
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": 0.34140777587890625,
|
|
"KL/mean": -2.696107864379883,
|
|
"KL/rejected_KL_mean": -5.7336273193359375,
|
|
"KL/std": 4.7038187980651855,
|
|
"epoch": 0.0763582966226138,
|
|
"fcm_dpo/beta": 0.03281348943710327,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 6.075035572052002,
|
|
"fcm_dpo/q_t": 0.4507228136062622,
|
|
"grad_norm": 28.817222595214844,
|
|
"learning_rate": 3.695652173913043e-07,
|
|
"logits/chosen": -0.5797228217124939,
|
|
"logits/rejected": -0.525948166847229,
|
|
"logps/chosen": -51.995948791503906,
|
|
"logps/ref_chosen": -52.33735656738281,
|
|
"logps/ref_rejected": -79.97391510009766,
|
|
"logps/rejected": -85.7075424194336,
|
|
"loss": 1.2035,
|
|
"margin_dpo/margin_mean": 6.07503604888916,
|
|
"margin_dpo/margin_std": 4.994811058044434,
|
|
"step": 52
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.22272109985351562,
|
|
"KL/mean": -3.3271031379699707,
|
|
"KL/rejected_KL_mean": -6.431484222412109,
|
|
"KL/std": 5.386918544769287,
|
|
"epoch": 0.07782672540381791,
|
|
"fcm_dpo/beta": 0.05016179382801056,
|
|
"fcm_dpo/delta": 0.07214375585317612,
|
|
"fcm_dpo/margin": 6.208770275115967,
|
|
"fcm_dpo/q_t": 0.4501815438270569,
|
|
"grad_norm": 38.668609619140625,
|
|
"learning_rate": 3.7681159420289855e-07,
|
|
"logits/chosen": -0.6235509514808655,
|
|
"logits/rejected": -0.6023184061050415,
|
|
"logps/chosen": -53.53737258911133,
|
|
"logps/ref_chosen": -53.31465148925781,
|
|
"logps/ref_rejected": -91.78359985351562,
|
|
"logps/rejected": -98.21508026123047,
|
|
"loss": 1.1228,
|
|
"margin_dpo/margin_mean": 6.208770275115967,
|
|
"margin_dpo/margin_std": 6.1717681884765625,
|
|
"step": 53
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.36040306091308594,
|
|
"KL/mean": -3.0783324241638184,
|
|
"KL/rejected_KL_mean": -5.796260833740234,
|
|
"KL/std": 5.118019104003906,
|
|
"epoch": 0.07929515418502203,
|
|
"fcm_dpo/beta": 0.06915527582168579,
|
|
"fcm_dpo/delta": -0.035718172788619995,
|
|
"fcm_dpo/margin": 5.435856819152832,
|
|
"fcm_dpo/q_t": 0.3983124792575836,
|
|
"grad_norm": 43.57950210571289,
|
|
"learning_rate": 3.8405797101449274e-07,
|
|
"logits/chosen": -0.6051186323165894,
|
|
"logits/rejected": -0.5521863102912903,
|
|
"logps/chosen": -51.04906463623047,
|
|
"logps/ref_chosen": -50.68865966796875,
|
|
"logps/ref_rejected": -91.71539306640625,
|
|
"logps/rejected": -97.51165771484375,
|
|
"loss": 1.0854,
|
|
"margin_dpo/margin_mean": 5.435855865478516,
|
|
"margin_dpo/margin_std": 5.120929718017578,
|
|
"step": 54
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.9818630218505859,
|
|
"KL/mean": -4.163016319274902,
|
|
"KL/rejected_KL_mean": -7.344165802001953,
|
|
"KL/std": 6.130636215209961,
|
|
"epoch": 0.08076358296622614,
|
|
"fcm_dpo/beta": 0.05893651396036148,
|
|
"fcm_dpo/delta": -0.0688391923904419,
|
|
"fcm_dpo/margin": 6.362303733825684,
|
|
"fcm_dpo/q_t": 0.3950488865375519,
|
|
"grad_norm": 41.72751998901367,
|
|
"learning_rate": 3.9130434782608694e-07,
|
|
"logits/chosen": -0.6603301167488098,
|
|
"logits/rejected": -0.5991765260696411,
|
|
"logps/chosen": -63.59709930419922,
|
|
"logps/ref_chosen": -62.615234375,
|
|
"logps/ref_rejected": -88.99349975585938,
|
|
"logps/rejected": -96.3376693725586,
|
|
"loss": 1.1394,
|
|
"margin_dpo/margin_mean": 6.362303733825684,
|
|
"margin_dpo/margin_std": 7.427134037017822,
|
|
"step": 55
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.7029590606689453,
|
|
"KL/mean": -3.7865757942199707,
|
|
"KL/rejected_KL_mean": -6.870197296142578,
|
|
"KL/std": 6.062340259552002,
|
|
"epoch": 0.08223201174743025,
|
|
"fcm_dpo/beta": 0.02372913435101509,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 6.167237758636475,
|
|
"fcm_dpo/q_t": 0.46382689476013184,
|
|
"grad_norm": 18.926555633544922,
|
|
"learning_rate": 3.9855072463768114e-07,
|
|
"logits/chosen": -0.6277616024017334,
|
|
"logits/rejected": -0.5860629677772522,
|
|
"logps/chosen": -58.63568878173828,
|
|
"logps/ref_chosen": -57.9327278137207,
|
|
"logps/ref_rejected": -94.1744384765625,
|
|
"logps/rejected": -101.04463195800781,
|
|
"loss": 1.2529,
|
|
"margin_dpo/margin_mean": 6.167238235473633,
|
|
"margin_dpo/margin_std": 7.341178894042969,
|
|
"step": 56
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -0.7325782775878906,
|
|
"KL/mean": -4.221752166748047,
|
|
"KL/rejected_KL_mean": -7.710926055908203,
|
|
"KL/std": 6.029708385467529,
|
|
"epoch": 0.08370044052863436,
|
|
"fcm_dpo/beta": 0.02372913435101509,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 6.978342056274414,
|
|
"fcm_dpo/q_t": 0.458979070186615,
|
|
"grad_norm": 21.024066925048828,
|
|
"learning_rate": 4.057971014492754e-07,
|
|
"logits/chosen": -0.6242918968200684,
|
|
"logits/rejected": -0.5983684062957764,
|
|
"logps/chosen": -71.22785949707031,
|
|
"logps/ref_chosen": -70.49528503417969,
|
|
"logps/ref_rejected": -95.56546020507812,
|
|
"logps/rejected": -103.27638244628906,
|
|
"loss": 1.2339,
|
|
"margin_dpo/margin_mean": 6.978342533111572,
|
|
"margin_dpo/margin_std": 6.642031669616699,
|
|
"step": 57
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1.0038623809814453,
|
|
"KL/mean": -4.862548351287842,
|
|
"KL/rejected_KL_mean": -8.721233367919922,
|
|
"KL/std": 6.818406105041504,
|
|
"epoch": 0.08516886930983847,
|
|
"fcm_dpo/beta": 0.04264631122350693,
|
|
"fcm_dpo/delta": 0.09533653408288956,
|
|
"fcm_dpo/margin": 7.717365264892578,
|
|
"fcm_dpo/q_t": 0.4548690915107727,
|
|
"grad_norm": 33.23288345336914,
|
|
"learning_rate": 4.1304347826086954e-07,
|
|
"logits/chosen": -0.6018198132514954,
|
|
"logits/rejected": -0.5264607071876526,
|
|
"logps/chosen": -63.136802673339844,
|
|
"logps/ref_chosen": -62.13294219970703,
|
|
"logps/ref_rejected": -84.61729431152344,
|
|
"logps/rejected": -93.33853149414062,
|
|
"loss": 1.108,
|
|
"margin_dpo/margin_mean": 7.7173662185668945,
|
|
"margin_dpo/margin_std": 8.156654357910156,
|
|
"step": 58
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1.4942893981933594,
|
|
"KL/mean": -5.754292011260986,
|
|
"KL/rejected_KL_mean": -10.014289855957031,
|
|
"KL/std": 7.55443000793457,
|
|
"epoch": 0.08663729809104258,
|
|
"fcm_dpo/beta": 0.01908048428595066,
|
|
"fcm_dpo/delta": -0.1171402782201767,
|
|
"fcm_dpo/margin": 8.52000617980957,
|
|
"fcm_dpo/q_t": 0.41208070516586304,
|
|
"grad_norm": 19.520845413208008,
|
|
"learning_rate": 4.2028985507246374e-07,
|
|
"logits/chosen": -0.6208063364028931,
|
|
"logits/rejected": -0.580247163772583,
|
|
"logps/chosen": -53.426815032958984,
|
|
"logps/ref_chosen": -51.932525634765625,
|
|
"logps/ref_rejected": -88.88520050048828,
|
|
"logps/rejected": -98.89949035644531,
|
|
"loss": 1.2374,
|
|
"margin_dpo/margin_mean": 8.520007133483887,
|
|
"margin_dpo/margin_std": 8.690988540649414,
|
|
"step": 59
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -2.7874374389648438,
|
|
"KL/mean": -6.149212837219238,
|
|
"KL/rejected_KL_mean": -9.510990142822266,
|
|
"KL/std": 6.844639778137207,
|
|
"epoch": 0.0881057268722467,
|
|
"fcm_dpo/beta": 0.01908048428595066,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 6.723544597625732,
|
|
"fcm_dpo/q_t": 0.4681331217288971,
|
|
"grad_norm": 17.15147590637207,
|
|
"learning_rate": 4.2753623188405794e-07,
|
|
"logits/chosen": -0.6261638402938843,
|
|
"logits/rejected": -0.5673079490661621,
|
|
"logps/chosen": -63.7296257019043,
|
|
"logps/ref_chosen": -60.94218826293945,
|
|
"logps/ref_rejected": -85.39340209960938,
|
|
"logps/rejected": -94.90438842773438,
|
|
"loss": 1.2677,
|
|
"margin_dpo/margin_mean": 6.723544597625732,
|
|
"margin_dpo/margin_std": 7.779609680175781,
|
|
"step": 60
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1.5964698791503906,
|
|
"KL/mean": -5.815464973449707,
|
|
"KL/rejected_KL_mean": -10.034461975097656,
|
|
"KL/std": 9.321011543273926,
|
|
"epoch": 0.08957415565345081,
|
|
"fcm_dpo/beta": 0.01908048428595066,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 8.437992095947266,
|
|
"fcm_dpo/q_t": 0.4607029855251312,
|
|
"grad_norm": 17.691282272338867,
|
|
"learning_rate": 4.3478260869565214e-07,
|
|
"logits/chosen": -0.6190700531005859,
|
|
"logits/rejected": -0.585500955581665,
|
|
"logps/chosen": -62.22999572753906,
|
|
"logps/ref_chosen": -60.633522033691406,
|
|
"logps/ref_rejected": -89.85249328613281,
|
|
"logps/rejected": -99.886962890625,
|
|
"loss": 1.244,
|
|
"margin_dpo/margin_mean": 8.437992095947266,
|
|
"margin_dpo/margin_std": 11.62002182006836,
|
|
"step": 61
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1.81982421875,
|
|
"KL/mean": -5.002796173095703,
|
|
"KL/rejected_KL_mean": -8.185768127441406,
|
|
"KL/std": 6.878681182861328,
|
|
"epoch": 0.09104258443465492,
|
|
"fcm_dpo/beta": 0.01908048428595066,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 6.365945816040039,
|
|
"fcm_dpo/q_t": 0.4699386656284332,
|
|
"grad_norm": 14.713135719299316,
|
|
"learning_rate": 4.420289855072464e-07,
|
|
"logits/chosen": -0.609377384185791,
|
|
"logits/rejected": -0.5751521587371826,
|
|
"logps/chosen": -57.97059631347656,
|
|
"logps/ref_chosen": -56.15077209472656,
|
|
"logps/ref_rejected": -75.56619262695312,
|
|
"logps/rejected": -83.75196838378906,
|
|
"loss": 1.2752,
|
|
"margin_dpo/margin_mean": 6.365945816040039,
|
|
"margin_dpo/margin_std": 8.420263290405273,
|
|
"step": 62
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -3.0295848846435547,
|
|
"KL/mean": -7.3545379638671875,
|
|
"KL/rejected_KL_mean": -11.679492950439453,
|
|
"KL/std": 8.834741592407227,
|
|
"epoch": 0.09251101321585903,
|
|
"fcm_dpo/beta": 0.01908048428595066,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 8.649913787841797,
|
|
"fcm_dpo/q_t": 0.45918020606040955,
|
|
"grad_norm": 17.89116668701172,
|
|
"learning_rate": 4.4927536231884053e-07,
|
|
"logits/chosen": -0.624936044216156,
|
|
"logits/rejected": -0.5820035338401794,
|
|
"logps/chosen": -76.1769790649414,
|
|
"logps/ref_chosen": -73.14739227294922,
|
|
"logps/ref_rejected": -97.61006164550781,
|
|
"logps/rejected": -109.28955078125,
|
|
"loss": 1.236,
|
|
"margin_dpo/margin_mean": 8.64991283416748,
|
|
"margin_dpo/margin_std": 9.18691635131836,
|
|
"step": 63
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1.2771644592285156,
|
|
"KL/mean": -6.465435981750488,
|
|
"KL/rejected_KL_mean": -11.653709411621094,
|
|
"KL/std": 9.661531448364258,
|
|
"epoch": 0.09397944199706314,
|
|
"fcm_dpo/beta": 0.05195356905460358,
|
|
"fcm_dpo/delta": 0.12018927931785583,
|
|
"fcm_dpo/margin": 10.376541137695312,
|
|
"fcm_dpo/q_t": 0.43298691511154175,
|
|
"grad_norm": 37.171104431152344,
|
|
"learning_rate": 4.5652173913043473e-07,
|
|
"logits/chosen": -0.5985631942749023,
|
|
"logits/rejected": -0.5670138597488403,
|
|
"logps/chosen": -55.27576446533203,
|
|
"logps/ref_chosen": -53.998600006103516,
|
|
"logps/ref_rejected": -93.53019714355469,
|
|
"logps/rejected": -105.18391418457031,
|
|
"loss": 1.0256,
|
|
"margin_dpo/margin_mean": 10.376540184020996,
|
|
"margin_dpo/margin_std": 11.894201278686523,
|
|
"step": 64
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -3.6957435607910156,
|
|
"KL/mean": -8.960607528686523,
|
|
"KL/rejected_KL_mean": -14.225475311279297,
|
|
"KL/std": 10.946447372436523,
|
|
"epoch": 0.09544787077826726,
|
|
"fcm_dpo/beta": 0.009888127446174622,
|
|
"fcm_dpo/delta": -0.18592241406440735,
|
|
"fcm_dpo/margin": 10.529730796813965,
|
|
"fcm_dpo/q_t": 0.4064898192882538,
|
|
"grad_norm": 11.372929573059082,
|
|
"learning_rate": 4.63768115942029e-07,
|
|
"logits/chosen": -0.6771352887153625,
|
|
"logits/rejected": -0.6651996374130249,
|
|
"logps/chosen": -68.53174591064453,
|
|
"logps/ref_chosen": -64.83599853515625,
|
|
"logps/ref_rejected": -109.94645690917969,
|
|
"logps/rejected": -124.17193603515625,
|
|
"loss": 1.2885,
|
|
"margin_dpo/margin_mean": 10.529730796813965,
|
|
"margin_dpo/margin_std": 12.0299072265625,
|
|
"step": 65
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -3.5717506408691406,
|
|
"KL/mean": -8.375314712524414,
|
|
"KL/rejected_KL_mean": -13.178871154785156,
|
|
"KL/std": 9.874279975891113,
|
|
"epoch": 0.09691629955947137,
|
|
"fcm_dpo/beta": 0.009888127446174622,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 9.607120513916016,
|
|
"fcm_dpo/q_t": 0.4763941168785095,
|
|
"grad_norm": 9.938729286193848,
|
|
"learning_rate": 4.7101449275362313e-07,
|
|
"logits/chosen": -0.6698400974273682,
|
|
"logits/rejected": -0.6391260623931885,
|
|
"logps/chosen": -55.01527404785156,
|
|
"logps/ref_chosen": -51.44352722167969,
|
|
"logps/ref_rejected": -75.63629913330078,
|
|
"logps/rejected": -88.81517028808594,
|
|
"loss": 1.2972,
|
|
"margin_dpo/margin_mean": 9.607121467590332,
|
|
"margin_dpo/margin_std": 11.98110580444336,
|
|
"step": 66
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -3.3190994262695312,
|
|
"KL/mean": -8.360406875610352,
|
|
"KL/rejected_KL_mean": -13.401721954345703,
|
|
"KL/std": 10.303428649902344,
|
|
"epoch": 0.09838472834067548,
|
|
"fcm_dpo/beta": 0.009888127446174622,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 10.082622528076172,
|
|
"fcm_dpo/q_t": 0.47523707151412964,
|
|
"grad_norm": 10.002849578857422,
|
|
"learning_rate": 4.782608695652174e-07,
|
|
"logits/chosen": -0.6043215394020081,
|
|
"logits/rejected": -0.5609872341156006,
|
|
"logps/chosen": -62.65990447998047,
|
|
"logps/ref_chosen": -59.34080505371094,
|
|
"logps/ref_rejected": -72.78728485107422,
|
|
"logps/rejected": -86.18901062011719,
|
|
"loss": 1.293,
|
|
"margin_dpo/margin_mean": 10.082620620727539,
|
|
"margin_dpo/margin_std": 12.531867027282715,
|
|
"step": 67
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -4.522434234619141,
|
|
"KL/mean": -9.367167472839355,
|
|
"KL/rejected_KL_mean": -14.211898803710938,
|
|
"KL/std": 9.465312957763672,
|
|
"epoch": 0.09985315712187959,
|
|
"fcm_dpo/beta": 0.009888127446174622,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 9.689462661743164,
|
|
"fcm_dpo/q_t": 0.47615760564804077,
|
|
"grad_norm": 9.184582710266113,
|
|
"learning_rate": 4.855072463768116e-07,
|
|
"logits/chosen": -0.6481267213821411,
|
|
"logits/rejected": -0.5857020020484924,
|
|
"logps/chosen": -69.728271484375,
|
|
"logps/ref_chosen": -65.2058334350586,
|
|
"logps/ref_rejected": -77.20724487304688,
|
|
"logps/rejected": -91.41914367675781,
|
|
"loss": 1.2957,
|
|
"margin_dpo/margin_mean": 9.689462661743164,
|
|
"margin_dpo/margin_std": 10.790483474731445,
|
|
"step": 68
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -5.772527694702148,
|
|
"KL/mean": -12.4307222366333,
|
|
"KL/rejected_KL_mean": -19.088916778564453,
|
|
"KL/std": 12.551191329956055,
|
|
"epoch": 0.1013215859030837,
|
|
"fcm_dpo/beta": 0.009888127446174622,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 13.31639289855957,
|
|
"fcm_dpo/q_t": 0.4673622250556946,
|
|
"grad_norm": 11.88070297241211,
|
|
"learning_rate": 4.927536231884058e-07,
|
|
"logits/chosen": -0.633673906326294,
|
|
"logits/rejected": -0.6116066575050354,
|
|
"logps/chosen": -65.59176635742188,
|
|
"logps/ref_chosen": -59.81924057006836,
|
|
"logps/ref_rejected": -103.38886260986328,
|
|
"logps/rejected": -122.477783203125,
|
|
"loss": 1.264,
|
|
"margin_dpo/margin_mean": 13.31639289855957,
|
|
"margin_dpo/margin_std": 14.175312042236328,
|
|
"step": 69
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -8.516521453857422,
|
|
"KL/mean": -15.706306457519531,
|
|
"KL/rejected_KL_mean": -22.89609146118164,
|
|
"KL/std": 16.06771469116211,
|
|
"epoch": 0.1027900146842878,
|
|
"fcm_dpo/beta": 0.009888127446174622,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 14.37956428527832,
|
|
"fcm_dpo/q_t": 0.46483179926872253,
|
|
"grad_norm": 11.995932579040527,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": -0.6291612386703491,
|
|
"logits/rejected": -0.5962782502174377,
|
|
"logps/chosen": -70.44715881347656,
|
|
"logps/ref_chosen": -61.930641174316406,
|
|
"logps/ref_rejected": -91.06078338623047,
|
|
"logps/rejected": -113.95687866210938,
|
|
"loss": 1.2564,
|
|
"margin_dpo/margin_mean": 14.37956428527832,
|
|
"margin_dpo/margin_std": 17.146224975585938,
|
|
"step": 70
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -9.071426391601562,
|
|
"KL/mean": -17.838401794433594,
|
|
"KL/rejected_KL_mean": -26.605384826660156,
|
|
"KL/std": 17.67050552368164,
|
|
"epoch": 0.10425844346549193,
|
|
"fcm_dpo/beta": 0.009888127446174622,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 17.53395652770996,
|
|
"fcm_dpo/q_t": 0.4574003219604492,
|
|
"grad_norm": 13.351109504699707,
|
|
"learning_rate": 4.999967061337492e-07,
|
|
"logits/chosen": -0.6778059601783752,
|
|
"logits/rejected": -0.6437849998474121,
|
|
"logps/chosen": -70.82176208496094,
|
|
"logps/ref_chosen": -61.750335693359375,
|
|
"logps/ref_rejected": -97.33662414550781,
|
|
"logps/rejected": -123.94200134277344,
|
|
"loss": 1.2301,
|
|
"margin_dpo/margin_mean": 17.533954620361328,
|
|
"margin_dpo/margin_std": 19.450382232666016,
|
|
"step": 71
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -10.510194778442383,
|
|
"KL/mean": -19.48761749267578,
|
|
"KL/rejected_KL_mean": -28.465042114257812,
|
|
"KL/std": 18.958513259887695,
|
|
"epoch": 0.10572687224669604,
|
|
"fcm_dpo/beta": 0.009888127446174622,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 17.954853057861328,
|
|
"fcm_dpo/q_t": 0.456373393535614,
|
|
"grad_norm": 12.942930221557617,
|
|
"learning_rate": 4.999868246217933e-07,
|
|
"logits/chosen": -0.6576621532440186,
|
|
"logits/rejected": -0.6210486888885498,
|
|
"logps/chosen": -76.56360626220703,
|
|
"logps/ref_chosen": -66.05341339111328,
|
|
"logps/ref_rejected": -95.2869873046875,
|
|
"logps/rejected": -123.75202941894531,
|
|
"loss": 1.2293,
|
|
"margin_dpo/margin_mean": 17.954853057861328,
|
|
"margin_dpo/margin_std": 23.021011352539062,
|
|
"step": 72
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -12.810209274291992,
|
|
"KL/mean": -21.97817611694336,
|
|
"KL/rejected_KL_mean": -31.14615249633789,
|
|
"KL/std": 24.339290618896484,
|
|
"epoch": 0.10719530102790015,
|
|
"fcm_dpo/beta": 0.009888127446174622,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 18.335941314697266,
|
|
"fcm_dpo/q_t": 0.45616060495376587,
|
|
"grad_norm": 13.09295654296875,
|
|
"learning_rate": 4.999703557245192e-07,
|
|
"logits/chosen": -0.6870651245117188,
|
|
"logits/rejected": -0.6481956243515015,
|
|
"logps/chosen": -79.06648254394531,
|
|
"logps/ref_chosen": -66.25627136230469,
|
|
"logps/ref_rejected": -90.45613098144531,
|
|
"logps/rejected": -121.60228729248047,
|
|
"loss": 1.2364,
|
|
"margin_dpo/margin_mean": 18.335941314697266,
|
|
"margin_dpo/margin_std": 31.158466339111328,
|
|
"step": 73
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -13.415903091430664,
|
|
"KL/mean": -23.937971115112305,
|
|
"KL/rejected_KL_mean": -34.46004104614258,
|
|
"KL/std": 27.361499786376953,
|
|
"epoch": 0.10866372980910426,
|
|
"fcm_dpo/beta": 0.019809015095233917,
|
|
"fcm_dpo/delta": 0.04002254828810692,
|
|
"fcm_dpo/margin": 21.04413414001465,
|
|
"fcm_dpo/q_t": 0.42780494689941406,
|
|
"grad_norm": 22.95852279663086,
|
|
"learning_rate": 4.999472998758977e-07,
|
|
"logits/chosen": -0.6472772359848022,
|
|
"logits/rejected": -0.6437745690345764,
|
|
"logps/chosen": -66.84078979492188,
|
|
"logps/ref_chosen": -53.42488098144531,
|
|
"logps/ref_rejected": -95.94693756103516,
|
|
"logps/rejected": -130.406982421875,
|
|
"loss": 1.1006,
|
|
"margin_dpo/margin_mean": 21.04413604736328,
|
|
"margin_dpo/margin_std": 34.61186218261719,
|
|
"step": 74
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -13.621763229370117,
|
|
"KL/mean": -28.440505981445312,
|
|
"KL/rejected_KL_mean": -43.259239196777344,
|
|
"KL/std": 28.2391357421875,
|
|
"epoch": 0.11013215859030837,
|
|
"fcm_dpo/beta": 0.011893379501998425,
|
|
"fcm_dpo/delta": -0.11567549407482147,
|
|
"fcm_dpo/margin": 29.637487411499023,
|
|
"fcm_dpo/q_t": 0.3860425055027008,
|
|
"grad_norm": 16.972978591918945,
|
|
"learning_rate": 4.999176576834721e-07,
|
|
"logits/chosen": -0.6723306179046631,
|
|
"logits/rejected": -0.6689744591712952,
|
|
"logps/chosen": -65.48342895507812,
|
|
"logps/ref_chosen": -51.861663818359375,
|
|
"logps/ref_rejected": -111.25398254394531,
|
|
"logps/rejected": -154.51321411132812,
|
|
"loss": 1.1449,
|
|
"margin_dpo/margin_mean": 29.63748550415039,
|
|
"margin_dpo/margin_std": 32.814430236816406,
|
|
"step": 75
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -16.395811080932617,
|
|
"KL/mean": -25.452667236328125,
|
|
"KL/rejected_KL_mean": -34.509517669677734,
|
|
"KL/std": 23.012611389160156,
|
|
"epoch": 0.11160058737151249,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 18.113712310791016,
|
|
"fcm_dpo/q_t": 0.4790630340576172,
|
|
"grad_norm": 7.068286895751953,
|
|
"learning_rate": 4.998814299283415e-07,
|
|
"logits/chosen": -0.7196544408798218,
|
|
"logits/rejected": -0.6837272644042969,
|
|
"logps/chosen": -69.66184997558594,
|
|
"logps/ref_chosen": -53.26603698730469,
|
|
"logps/ref_rejected": -78.21662902832031,
|
|
"logps/rejected": -112.72615051269531,
|
|
"loss": 1.3076,
|
|
"margin_dpo/margin_mean": 18.113712310791016,
|
|
"margin_dpo/margin_std": 25.670862197875977,
|
|
"step": 76
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -15.274223327636719,
|
|
"KL/mean": -30.40639877319336,
|
|
"KL/rejected_KL_mean": -45.53857421875,
|
|
"KL/std": 31.36872100830078,
|
|
"epoch": 0.1130690161527166,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 30.26435089111328,
|
|
"fcm_dpo/q_t": 0.4651873707771301,
|
|
"grad_norm": 9.03381633758545,
|
|
"learning_rate": 4.998386175651409e-07,
|
|
"logits/chosen": -0.6737496256828308,
|
|
"logits/rejected": -0.639689564704895,
|
|
"logps/chosen": -73.37090301513672,
|
|
"logps/ref_chosen": -58.0966796875,
|
|
"logps/ref_rejected": -93.77361297607422,
|
|
"logps/rejected": -139.31219482421875,
|
|
"loss": 1.2578,
|
|
"margin_dpo/margin_mean": 30.26434898376465,
|
|
"margin_dpo/margin_std": 35.96830749511719,
|
|
"step": 77
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -15.677688598632812,
|
|
"KL/mean": -26.735885620117188,
|
|
"KL/rejected_KL_mean": -37.79408264160156,
|
|
"KL/std": 26.46023941040039,
|
|
"epoch": 0.1145374449339207,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 22.116390228271484,
|
|
"fcm_dpo/q_t": 0.4745638370513916,
|
|
"grad_norm": 7.281389236450195,
|
|
"learning_rate": 4.997892217220159e-07,
|
|
"logits/chosen": -0.644652247428894,
|
|
"logits/rejected": -0.6242761015892029,
|
|
"logps/chosen": -71.29147338867188,
|
|
"logps/ref_chosen": -55.61378479003906,
|
|
"logps/ref_rejected": -84.93436431884766,
|
|
"logps/rejected": -122.72844696044922,
|
|
"loss": 1.2917,
|
|
"margin_dpo/margin_mean": 22.116392135620117,
|
|
"margin_dpo/margin_std": 31.19441795349121,
|
|
"step": 78
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -16.822900772094727,
|
|
"KL/mean": -28.855121612548828,
|
|
"KL/rejected_KL_mean": -40.8873405456543,
|
|
"KL/std": 29.233537673950195,
|
|
"epoch": 0.11600587371512482,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 24.064435958862305,
|
|
"fcm_dpo/q_t": 0.4723590612411499,
|
|
"grad_norm": 7.421467304229736,
|
|
"learning_rate": 4.997332437005931e-07,
|
|
"logits/chosen": -0.6622911095619202,
|
|
"logits/rejected": -0.6371140480041504,
|
|
"logps/chosen": -72.27339172363281,
|
|
"logps/ref_chosen": -55.45048522949219,
|
|
"logps/ref_rejected": -87.64756774902344,
|
|
"logps/rejected": -128.534912109375,
|
|
"loss": 1.2846,
|
|
"margin_dpo/margin_mean": 24.064434051513672,
|
|
"margin_dpo/margin_std": 35.614994049072266,
|
|
"step": 79
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -21.039852142333984,
|
|
"KL/mean": -33.44929504394531,
|
|
"KL/rejected_KL_mean": -45.858741760253906,
|
|
"KL/std": 33.63664245605469,
|
|
"epoch": 0.11747430249632893,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 24.818889617919922,
|
|
"fcm_dpo/q_t": 0.4716973304748535,
|
|
"grad_norm": 8.430150032043457,
|
|
"learning_rate": 4.996706849759452e-07,
|
|
"logits/chosen": -0.7373731732368469,
|
|
"logits/rejected": -0.7053723335266113,
|
|
"logps/chosen": -79.55914306640625,
|
|
"logps/ref_chosen": -58.519290924072266,
|
|
"logps/ref_rejected": -87.54750061035156,
|
|
"logps/rejected": -133.40625,
|
|
"loss": 1.2836,
|
|
"margin_dpo/margin_mean": 24.818889617919922,
|
|
"margin_dpo/margin_std": 39.66154098510742,
|
|
"step": 80
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -21.12224578857422,
|
|
"KL/mean": -38.02497863769531,
|
|
"KL/rejected_KL_mean": -54.927711486816406,
|
|
"KL/std": 39.90003204345703,
|
|
"epoch": 0.11894273127753303,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 33.80546188354492,
|
|
"fcm_dpo/q_t": 0.4615294337272644,
|
|
"grad_norm": 9.538393020629883,
|
|
"learning_rate": 4.996015471965529e-07,
|
|
"logits/chosen": -0.6799561977386475,
|
|
"logits/rejected": -0.6571604013442993,
|
|
"logps/chosen": -87.57111358642578,
|
|
"logps/ref_chosen": -66.44886779785156,
|
|
"logps/ref_rejected": -129.66270446777344,
|
|
"logps/rejected": -184.59042358398438,
|
|
"loss": 1.2478,
|
|
"margin_dpo/margin_mean": 33.80546188354492,
|
|
"margin_dpo/margin_std": 47.89176940917969,
|
|
"step": 81
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -24.305784225463867,
|
|
"KL/mean": -37.62807846069336,
|
|
"KL/rejected_KL_mean": -50.950374603271484,
|
|
"KL/std": 36.823760986328125,
|
|
"epoch": 0.12041116005873716,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 26.64459228515625,
|
|
"fcm_dpo/q_t": 0.4697510600090027,
|
|
"grad_norm": 8.230402946472168,
|
|
"learning_rate": 4.995258321842611e-07,
|
|
"logits/chosen": -0.6198301315307617,
|
|
"logits/rejected": -0.6155990958213806,
|
|
"logps/chosen": -76.53816986083984,
|
|
"logps/ref_chosen": -52.232383728027344,
|
|
"logps/ref_rejected": -90.74325561523438,
|
|
"logps/rejected": -141.69363403320312,
|
|
"loss": 1.2812,
|
|
"margin_dpo/margin_mean": 26.644596099853516,
|
|
"margin_dpo/margin_std": 51.493133544921875,
|
|
"step": 82
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -25.275197982788086,
|
|
"KL/mean": -42.2493782043457,
|
|
"KL/rejected_KL_mean": -59.22356033325195,
|
|
"KL/std": 38.86623001098633,
|
|
"epoch": 0.12187958883994127,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 33.9483642578125,
|
|
"fcm_dpo/q_t": 0.4612438678741455,
|
|
"grad_norm": 9.907917022705078,
|
|
"learning_rate": 4.994435419342304e-07,
|
|
"logits/chosen": -0.6601589322090149,
|
|
"logits/rejected": -0.6394397616386414,
|
|
"logps/chosen": -81.10258483886719,
|
|
"logps/ref_chosen": -55.82738494873047,
|
|
"logps/ref_rejected": -103.71589660644531,
|
|
"logps/rejected": -162.939453125,
|
|
"loss": 1.2466,
|
|
"margin_dpo/margin_mean": 33.948368072509766,
|
|
"margin_dpo/margin_std": 46.71687698364258,
|
|
"step": 83
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -26.676267623901367,
|
|
"KL/mean": -39.81716537475586,
|
|
"KL/rejected_KL_mean": -52.958065032958984,
|
|
"KL/std": 34.16858673095703,
|
|
"epoch": 0.12334801762114538,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 26.28179359436035,
|
|
"fcm_dpo/q_t": 0.46975940465927124,
|
|
"grad_norm": 8.419811248779297,
|
|
"learning_rate": 4.993546786148857e-07,
|
|
"logits/chosen": -0.6571387052536011,
|
|
"logits/rejected": -0.6230216026306152,
|
|
"logps/chosen": -93.85243225097656,
|
|
"logps/ref_chosen": -67.1761703491211,
|
|
"logps/ref_rejected": -87.29859924316406,
|
|
"logps/rejected": -140.2566680908203,
|
|
"loss": 1.2748,
|
|
"margin_dpo/margin_mean": 26.281795501708984,
|
|
"margin_dpo/margin_std": 33.972564697265625,
|
|
"step": 84
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -26.707151412963867,
|
|
"KL/mean": -40.551788330078125,
|
|
"KL/rejected_KL_mean": -54.396427154541016,
|
|
"KL/std": 34.13534927368164,
|
|
"epoch": 0.12481644640234948,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 27.689273834228516,
|
|
"fcm_dpo/q_t": 0.4682667851448059,
|
|
"grad_norm": 8.280872344970703,
|
|
"learning_rate": 4.992592445678582e-07,
|
|
"logits/chosen": -0.6289738416671753,
|
|
"logits/rejected": -0.5984049439430237,
|
|
"logps/chosen": -85.11376953125,
|
|
"logps/ref_chosen": -58.4066162109375,
|
|
"logps/ref_rejected": -78.63880157470703,
|
|
"logps/rejected": -133.0352325439453,
|
|
"loss": 1.2717,
|
|
"margin_dpo/margin_mean": 27.689273834228516,
|
|
"margin_dpo/margin_std": 41.85002136230469,
|
|
"step": 85
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -36.066551208496094,
|
|
"KL/mean": -50.90142822265625,
|
|
"KL/rejected_KL_mean": -65.7363052368164,
|
|
"KL/std": 51.3672981262207,
|
|
"epoch": 0.1262848751835536,
|
|
"fcm_dpo/beta": 0.004640421364456415,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 29.669748306274414,
|
|
"fcm_dpo/q_t": 0.46719542145729065,
|
|
"grad_norm": 9.010835647583008,
|
|
"learning_rate": 4.991572423079235e-07,
|
|
"logits/chosen": -0.6406770348548889,
|
|
"logits/rejected": -0.6358238458633423,
|
|
"logps/chosen": -92.20401000976562,
|
|
"logps/ref_chosen": -56.13746643066406,
|
|
"logps/ref_rejected": -88.12165069580078,
|
|
"logps/rejected": -153.8579559326172,
|
|
"loss": 1.2809,
|
|
"margin_dpo/margin_mean": 29.669748306274414,
|
|
"margin_dpo/margin_std": 72.3070068359375,
|
|
"step": 86
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -35.95440673828125,
|
|
"KL/mean": -55.21514129638672,
|
|
"KL/rejected_KL_mean": -74.47587585449219,
|
|
"KL/std": 52.28978729248047,
|
|
"epoch": 0.1277533039647577,
|
|
"fcm_dpo/beta": 0.012798674404621124,
|
|
"fcm_dpo/delta": 0.10860809683799744,
|
|
"fcm_dpo/margin": 38.52145767211914,
|
|
"fcm_dpo/q_t": 0.433593213558197,
|
|
"grad_norm": 24.902225494384766,
|
|
"learning_rate": 4.990486745229364e-07,
|
|
"logits/chosen": -0.6695064306259155,
|
|
"logits/rejected": -0.6587918996810913,
|
|
"logps/chosen": -91.59049987792969,
|
|
"logps/ref_chosen": -55.63609313964844,
|
|
"logps/ref_rejected": -95.46757507324219,
|
|
"logps/rejected": -169.94345092773438,
|
|
"loss": 1.1243,
|
|
"margin_dpo/margin_mean": 38.52145767211914,
|
|
"margin_dpo/margin_std": 65.65547943115234,
|
|
"step": 87
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -46.075992584228516,
|
|
"KL/mean": -60.8734245300293,
|
|
"KL/rejected_KL_mean": -75.67085266113281,
|
|
"KL/std": 57.557559967041016,
|
|
"epoch": 0.12922173274596183,
|
|
"fcm_dpo/beta": 0.007393650244921446,
|
|
"fcm_dpo/delta": -0.062026459723711014,
|
|
"fcm_dpo/margin": 29.5948543548584,
|
|
"fcm_dpo/q_t": 0.4339485168457031,
|
|
"grad_norm": 14.445233345031738,
|
|
"learning_rate": 4.989335440737586e-07,
|
|
"logits/chosen": -0.6514161825180054,
|
|
"logits/rejected": -0.6543829441070557,
|
|
"logps/chosen": -119.74714660644531,
|
|
"logps/ref_chosen": -73.67115020751953,
|
|
"logps/ref_rejected": -106.70849609375,
|
|
"logps/rejected": -182.3793487548828,
|
|
"loss": 1.2408,
|
|
"margin_dpo/margin_mean": 29.594852447509766,
|
|
"margin_dpo/margin_std": 64.25787353515625,
|
|
"step": 88
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -32.12773132324219,
|
|
"KL/mean": -48.858558654785156,
|
|
"KL/rejected_KL_mean": -65.5893783569336,
|
|
"KL/std": 51.16180419921875,
|
|
"epoch": 0.13069016152716592,
|
|
"fcm_dpo/beta": 0.009811213240027428,
|
|
"fcm_dpo/delta": 0.05031702667474747,
|
|
"fcm_dpo/margin": 33.46165466308594,
|
|
"fcm_dpo/q_t": 0.44110527634620667,
|
|
"grad_norm": 17.186080932617188,
|
|
"learning_rate": 4.988118539941847e-07,
|
|
"logits/chosen": -0.6749956607818604,
|
|
"logits/rejected": -0.6535379886627197,
|
|
"logps/chosen": -92.75264739990234,
|
|
"logps/ref_chosen": -60.624916076660156,
|
|
"logps/ref_rejected": -82.08354949951172,
|
|
"logps/rejected": -147.6729278564453,
|
|
"loss": 1.1533,
|
|
"margin_dpo/margin_mean": 33.46165466308594,
|
|
"margin_dpo/margin_std": 60.91192626953125,
|
|
"step": 89
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -36.43550109863281,
|
|
"KL/mean": -62.53802490234375,
|
|
"KL/rejected_KL_mean": -88.64056396484375,
|
|
"KL/std": 68.16754913330078,
|
|
"epoch": 0.13215859030837004,
|
|
"fcm_dpo/beta": 0.0063932533375918865,
|
|
"fcm_dpo/delta": -0.19979318976402283,
|
|
"fcm_dpo/margin": 52.205047607421875,
|
|
"fcm_dpo/q_t": 0.38456064462661743,
|
|
"grad_norm": 15.716353416442871,
|
|
"learning_rate": 4.986836074908615e-07,
|
|
"logits/chosen": -0.6015010476112366,
|
|
"logits/rejected": -0.6259626150131226,
|
|
"logps/chosen": -89.72080993652344,
|
|
"logps/ref_chosen": -53.285308837890625,
|
|
"logps/ref_rejected": -111.54470825195312,
|
|
"logps/rejected": -200.18527221679688,
|
|
"loss": 1.2609,
|
|
"margin_dpo/margin_mean": 52.20505142211914,
|
|
"margin_dpo/margin_std": 86.0552978515625,
|
|
"step": 90
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -40.36619567871094,
|
|
"KL/mean": -57.59088897705078,
|
|
"KL/rejected_KL_mean": -74.81558227539062,
|
|
"KL/std": 51.144317626953125,
|
|
"epoch": 0.13362701908957417,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 34.44939041137695,
|
|
"fcm_dpo/q_t": 0.4857920706272125,
|
|
"grad_norm": 3.791431188583374,
|
|
"learning_rate": 4.985488079432037e-07,
|
|
"logits/chosen": -0.6380863189697266,
|
|
"logits/rejected": -0.6169267892837524,
|
|
"logps/chosen": -102.16915130615234,
|
|
"logps/ref_chosen": -61.802955627441406,
|
|
"logps/ref_rejected": -87.87395477294922,
|
|
"logps/rejected": -162.68954467773438,
|
|
"loss": 1.3334,
|
|
"margin_dpo/margin_mean": 34.44939041137695,
|
|
"margin_dpo/margin_std": 70.18351745605469,
|
|
"step": 91
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -35.530731201171875,
|
|
"KL/mean": -54.40737533569336,
|
|
"KL/rejected_KL_mean": -73.28401184082031,
|
|
"KL/std": 54.11292266845703,
|
|
"epoch": 0.13509544787077826,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 37.7532844543457,
|
|
"fcm_dpo/q_t": 0.48441988229751587,
|
|
"grad_norm": 4.081094264984131,
|
|
"learning_rate": 4.984074589033043e-07,
|
|
"logits/chosen": -0.6290682554244995,
|
|
"logits/rejected": -0.6120110750198364,
|
|
"logps/chosen": -87.17150115966797,
|
|
"logps/ref_chosen": -51.640769958496094,
|
|
"logps/ref_rejected": -77.88117980957031,
|
|
"logps/rejected": -151.16519165039062,
|
|
"loss": 1.3277,
|
|
"margin_dpo/margin_mean": 37.7532844543457,
|
|
"margin_dpo/margin_std": 65.40171813964844,
|
|
"step": 92
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -39.78628158569336,
|
|
"KL/mean": -58.02899169921875,
|
|
"KL/rejected_KL_mean": -76.27169799804688,
|
|
"KL/std": 48.12828063964844,
|
|
"epoch": 0.13656387665198239,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 36.48542404174805,
|
|
"fcm_dpo/q_t": 0.48493102192878723,
|
|
"grad_norm": 4.032928466796875,
|
|
"learning_rate": 4.982595640958425e-07,
|
|
"logits/chosen": -0.6539796590805054,
|
|
"logits/rejected": -0.6243857145309448,
|
|
"logps/chosen": -92.31552124023438,
|
|
"logps/ref_chosen": -52.529239654541016,
|
|
"logps/ref_rejected": -77.16075134277344,
|
|
"logps/rejected": -153.4324493408203,
|
|
"loss": 1.329,
|
|
"margin_dpo/margin_mean": 36.48542404174805,
|
|
"margin_dpo/margin_std": 57.570045471191406,
|
|
"step": 93
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -43.512413024902344,
|
|
"KL/mean": -66.17471313476562,
|
|
"KL/rejected_KL_mean": -88.83700561523438,
|
|
"KL/std": 59.194549560546875,
|
|
"epoch": 0.13803230543318648,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 45.32459259033203,
|
|
"fcm_dpo/q_t": 0.4812876582145691,
|
|
"grad_norm": 4.497649192810059,
|
|
"learning_rate": 4.98105127417984e-07,
|
|
"logits/chosen": -0.5847138166427612,
|
|
"logits/rejected": -0.5839489698410034,
|
|
"logps/chosen": -104.73502349853516,
|
|
"logps/ref_chosen": -61.22261047363281,
|
|
"logps/ref_rejected": -99.59902954101562,
|
|
"logps/rejected": -188.43603515625,
|
|
"loss": 1.3155,
|
|
"margin_dpo/margin_mean": 45.32460021972656,
|
|
"margin_dpo/margin_std": 64.99853515625,
|
|
"step": 94
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -43.288063049316406,
|
|
"KL/mean": -61.76789855957031,
|
|
"KL/rejected_KL_mean": -80.24773406982422,
|
|
"KL/std": 63.553306579589844,
|
|
"epoch": 0.1395007342143906,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 36.95966720581055,
|
|
"fcm_dpo/q_t": 0.48472627997398376,
|
|
"grad_norm": 4.214771270751953,
|
|
"learning_rate": 4.979441529392784e-07,
|
|
"logits/chosen": -0.6216604113578796,
|
|
"logits/rejected": -0.6044141054153442,
|
|
"logps/chosen": -95.81170654296875,
|
|
"logps/ref_chosen": -52.523643493652344,
|
|
"logps/ref_rejected": -75.8803482055664,
|
|
"logps/rejected": -156.12808227539062,
|
|
"loss": 1.3292,
|
|
"margin_dpo/margin_mean": 36.95966339111328,
|
|
"margin_dpo/margin_std": 67.57984161376953,
|
|
"step": 95
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -46.38825225830078,
|
|
"KL/mean": -72.26142883300781,
|
|
"KL/rejected_KL_mean": -98.13460540771484,
|
|
"KL/std": 66.60676574707031,
|
|
"epoch": 0.14096916299559473,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 51.74634552001953,
|
|
"fcm_dpo/q_t": 0.4787074327468872,
|
|
"grad_norm": 4.915516376495361,
|
|
"learning_rate": 4.977766449015534e-07,
|
|
"logits/chosen": -0.5953140258789062,
|
|
"logits/rejected": -0.5802059173583984,
|
|
"logps/chosen": -108.54522705078125,
|
|
"logps/ref_chosen": -62.15697479248047,
|
|
"logps/ref_rejected": -96.59601593017578,
|
|
"logps/rejected": -194.73062133789062,
|
|
"loss": 1.3066,
|
|
"margin_dpo/margin_mean": 51.74634552001953,
|
|
"margin_dpo/margin_std": 77.54536437988281,
|
|
"step": 96
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -47.02070999145508,
|
|
"KL/mean": -65.5418701171875,
|
|
"KL/rejected_KL_mean": -84.06302642822266,
|
|
"KL/std": 60.283546447753906,
|
|
"epoch": 0.14243759177679882,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 37.042320251464844,
|
|
"fcm_dpo/q_t": 0.4846979081630707,
|
|
"grad_norm": 4.432089328765869,
|
|
"learning_rate": 4.976026077188012e-07,
|
|
"logits/chosen": -0.4999476671218872,
|
|
"logits/rejected": -0.4703494906425476,
|
|
"logps/chosen": -101.66707611083984,
|
|
"logps/ref_chosen": -54.646366119384766,
|
|
"logps/ref_rejected": -76.96475219726562,
|
|
"logps/rejected": -161.02777099609375,
|
|
"loss": 1.3285,
|
|
"margin_dpo/margin_mean": 37.042320251464844,
|
|
"margin_dpo/margin_std": 60.82837677001953,
|
|
"step": 97
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -63.49620056152344,
|
|
"KL/mean": -85.60096740722656,
|
|
"KL/rejected_KL_mean": -107.70573425292969,
|
|
"KL/std": 73.20948028564453,
|
|
"epoch": 0.14390602055800295,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 44.20954132080078,
|
|
"fcm_dpo/q_t": 0.4817187786102295,
|
|
"grad_norm": 5.483908653259277,
|
|
"learning_rate": 4.974220459770639e-07,
|
|
"logits/chosen": -0.5312040448188782,
|
|
"logits/rejected": -0.5291551947593689,
|
|
"logps/chosen": -128.75482177734375,
|
|
"logps/ref_chosen": -65.25862884521484,
|
|
"logps/ref_rejected": -96.5274887084961,
|
|
"logps/rejected": -204.23321533203125,
|
|
"loss": 1.3201,
|
|
"margin_dpo/margin_mean": 44.20954132080078,
|
|
"margin_dpo/margin_std": 89.74884033203125,
|
|
"step": 98
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -47.63540267944336,
|
|
"KL/mean": -77.33531188964844,
|
|
"KL/rejected_KL_mean": -107.03522491455078,
|
|
"KL/std": 73.14799499511719,
|
|
"epoch": 0.14537444933920704,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 59.39982604980469,
|
|
"fcm_dpo/q_t": 0.47561943531036377,
|
|
"grad_norm": 5.031144618988037,
|
|
"learning_rate": 4.972349644343108e-07,
|
|
"logits/chosen": -0.5268571376800537,
|
|
"logits/rejected": -0.5339376926422119,
|
|
"logps/chosen": -93.27388763427734,
|
|
"logps/ref_chosen": -45.638484954833984,
|
|
"logps/ref_rejected": -86.43793487548828,
|
|
"logps/rejected": -193.47315979003906,
|
|
"loss": 1.2962,
|
|
"margin_dpo/margin_mean": 59.39982604980469,
|
|
"margin_dpo/margin_std": 93.68307495117188,
|
|
"step": 99
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -58.90423583984375,
|
|
"KL/mean": -70.80164337158203,
|
|
"KL/rejected_KL_mean": -82.69905090332031,
|
|
"KL/std": 66.31571960449219,
|
|
"epoch": 0.14684287812041116,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 23.79481315612793,
|
|
"fcm_dpo/q_t": 0.49016737937927246,
|
|
"grad_norm": 4.631414413452148,
|
|
"learning_rate": 4.970413680203148e-07,
|
|
"logits/chosen": -0.5172683596611023,
|
|
"logits/rejected": -0.4957023859024048,
|
|
"logps/chosen": -116.49821472167969,
|
|
"logps/ref_chosen": -57.59397888183594,
|
|
"logps/ref_rejected": -74.06021118164062,
|
|
"logps/rejected": -156.75926208496094,
|
|
"loss": 1.3523,
|
|
"margin_dpo/margin_mean": 23.794811248779297,
|
|
"margin_dpo/margin_std": 82.77680206298828,
|
|
"step": 100
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -70.8927230834961,
|
|
"KL/mean": -88.97613525390625,
|
|
"KL/rejected_KL_mean": -107.05953979492188,
|
|
"KL/std": 81.61466979980469,
|
|
"epoch": 0.14831130690161526,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 36.166805267333984,
|
|
"fcm_dpo/q_t": 0.48512861132621765,
|
|
"grad_norm": 4.663822650909424,
|
|
"learning_rate": 4.968412618365215e-07,
|
|
"logits/chosen": -0.5123007297515869,
|
|
"logits/rejected": -0.4988871216773987,
|
|
"logps/chosen": -132.5415802001953,
|
|
"logps/ref_chosen": -61.64885330200195,
|
|
"logps/ref_rejected": -83.18968200683594,
|
|
"logps/rejected": -190.24923706054688,
|
|
"loss": 1.3335,
|
|
"margin_dpo/margin_mean": 36.16680908203125,
|
|
"margin_dpo/margin_std": 94.42303466796875,
|
|
"step": 101
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -74.67913818359375,
|
|
"KL/mean": -87.16828918457031,
|
|
"KL/rejected_KL_mean": -99.65742492675781,
|
|
"KL/std": 79.29058837890625,
|
|
"epoch": 0.14977973568281938,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 24.978286743164062,
|
|
"fcm_dpo/q_t": 0.489687979221344,
|
|
"grad_norm": 5.5214524269104,
|
|
"learning_rate": 4.966346511559149e-07,
|
|
"logits/chosen": -0.5297523140907288,
|
|
"logits/rejected": -0.5086390972137451,
|
|
"logps/chosen": -138.7580108642578,
|
|
"logps/ref_chosen": -64.0788803100586,
|
|
"logps/ref_rejected": -68.18707275390625,
|
|
"logps/rejected": -167.84451293945312,
|
|
"loss": 1.352,
|
|
"margin_dpo/margin_mean": 24.978286743164062,
|
|
"margin_dpo/margin_std": 96.62683868408203,
|
|
"step": 102
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -69.34735870361328,
|
|
"KL/mean": -98.80657958984375,
|
|
"KL/rejected_KL_mean": -128.26580810546875,
|
|
"KL/std": 92.38734436035156,
|
|
"epoch": 0.1512481644640235,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 58.918426513671875,
|
|
"fcm_dpo/q_t": 0.47593387961387634,
|
|
"grad_norm": 5.950850963592529,
|
|
"learning_rate": 4.964215414228785e-07,
|
|
"logits/chosen": -0.46111953258514404,
|
|
"logits/rejected": -0.441642165184021,
|
|
"logps/chosen": -130.64663696289062,
|
|
"logps/ref_chosen": -61.299278259277344,
|
|
"logps/ref_rejected": -93.57270812988281,
|
|
"logps/rejected": -221.8385009765625,
|
|
"loss": 1.3007,
|
|
"margin_dpo/margin_mean": 58.918426513671875,
|
|
"margin_dpo/margin_std": 119.41735076904297,
|
|
"step": 103
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -67.7513198852539,
|
|
"KL/mean": -96.42362976074219,
|
|
"KL/rejected_KL_mean": -125.09593200683594,
|
|
"KL/std": 93.0230712890625,
|
|
"epoch": 0.1527165932452276,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 57.34462356567383,
|
|
"fcm_dpo/q_t": 0.476635605096817,
|
|
"grad_norm": 8.443946838378906,
|
|
"learning_rate": 4.96201938253052e-07,
|
|
"logits/chosen": -0.4595398008823395,
|
|
"logits/rejected": -0.4493861794471741,
|
|
"logps/chosen": -122.12409210205078,
|
|
"logps/ref_chosen": -54.372772216796875,
|
|
"logps/ref_rejected": -89.5647201538086,
|
|
"logps/rejected": -214.66064453125,
|
|
"loss": 1.3031,
|
|
"margin_dpo/margin_mean": 57.344627380371094,
|
|
"margin_dpo/margin_std": 118.23411560058594,
|
|
"step": 104
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -68.69507598876953,
|
|
"KL/mean": -111.2301254272461,
|
|
"KL/rejected_KL_mean": -153.76516723632812,
|
|
"KL/std": 86.79656219482422,
|
|
"epoch": 0.15418502202643172,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 85.07008361816406,
|
|
"fcm_dpo/q_t": 0.46504706144332886,
|
|
"grad_norm": 6.229670524597168,
|
|
"learning_rate": 4.959758474331832e-07,
|
|
"logits/chosen": -0.4372522830963135,
|
|
"logits/rejected": -0.4284954369068146,
|
|
"logps/chosen": -123.33402252197266,
|
|
"logps/ref_chosen": -54.638946533203125,
|
|
"logps/ref_rejected": -97.97351837158203,
|
|
"logps/rejected": -251.7386932373047,
|
|
"loss": 1.2576,
|
|
"margin_dpo/margin_mean": 85.07008361816406,
|
|
"margin_dpo/margin_std": 102.4740219116211,
|
|
"step": 105
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -80.03727722167969,
|
|
"KL/mean": -107.10738372802734,
|
|
"KL/rejected_KL_mean": -134.177490234375,
|
|
"KL/std": 90.39563751220703,
|
|
"epoch": 0.15565345080763582,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 54.140220642089844,
|
|
"fcm_dpo/q_t": 0.47773557901382446,
|
|
"grad_norm": 5.816026210784912,
|
|
"learning_rate": 4.957432749209755e-07,
|
|
"logits/chosen": -0.38607245683670044,
|
|
"logits/rejected": -0.3913835883140564,
|
|
"logps/chosen": -134.87017822265625,
|
|
"logps/ref_chosen": -54.83289337158203,
|
|
"logps/ref_rejected": -85.22461700439453,
|
|
"logps/rejected": -219.40211486816406,
|
|
"loss": 1.3052,
|
|
"margin_dpo/margin_mean": 54.14021682739258,
|
|
"margin_dpo/margin_std": 96.50797271728516,
|
|
"step": 106
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -96.32496643066406,
|
|
"KL/mean": -126.12309265136719,
|
|
"KL/rejected_KL_mean": -155.92123413085938,
|
|
"KL/std": 109.03052520751953,
|
|
"epoch": 0.15712187958883994,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 59.59624481201172,
|
|
"fcm_dpo/q_t": 0.47559940814971924,
|
|
"grad_norm": 6.986702919006348,
|
|
"learning_rate": 4.955042268449307e-07,
|
|
"logits/chosen": -0.42685896158218384,
|
|
"logits/rejected": -0.4117262363433838,
|
|
"logps/chosen": -166.03277587890625,
|
|
"logps/ref_chosen": -69.70780944824219,
|
|
"logps/ref_rejected": -94.73950958251953,
|
|
"logps/rejected": -250.66073608398438,
|
|
"loss": 1.2995,
|
|
"margin_dpo/margin_mean": 59.596248626708984,
|
|
"margin_dpo/margin_std": 115.8310775756836,
|
|
"step": 107
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -79.45794677734375,
|
|
"KL/mean": -121.89816284179688,
|
|
"KL/rejected_KL_mean": -164.33836364746094,
|
|
"KL/std": 121.62930297851562,
|
|
"epoch": 0.15859030837004406,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 84.88043212890625,
|
|
"fcm_dpo/q_t": 0.4657972455024719,
|
|
"grad_norm": 8.013676643371582,
|
|
"learning_rate": 4.952587095041881e-07,
|
|
"logits/chosen": -0.36991098523139954,
|
|
"logits/rejected": -0.36751043796539307,
|
|
"logps/chosen": -135.46783447265625,
|
|
"logps/ref_chosen": -56.0098876953125,
|
|
"logps/ref_rejected": -95.79601287841797,
|
|
"logps/rejected": -260.1343994140625,
|
|
"loss": 1.2671,
|
|
"margin_dpo/margin_mean": 84.88043212890625,
|
|
"margin_dpo/margin_std": 156.81539916992188,
|
|
"step": 108
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -85.31330871582031,
|
|
"KL/mean": -126.01399230957031,
|
|
"KL/rejected_KL_mean": -166.71469116210938,
|
|
"KL/std": 120.20221710205078,
|
|
"epoch": 0.16005873715124816,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 81.40138244628906,
|
|
"fcm_dpo/q_t": 0.4666248559951782,
|
|
"grad_norm": 6.836396217346191,
|
|
"learning_rate": 4.95006729368358e-07,
|
|
"logits/chosen": -0.36495447158813477,
|
|
"logits/rejected": -0.3560246229171753,
|
|
"logps/chosen": -148.19879150390625,
|
|
"logps/ref_chosen": -62.88549041748047,
|
|
"logps/ref_rejected": -98.68573760986328,
|
|
"logps/rejected": -265.4004211425781,
|
|
"loss": 1.2734,
|
|
"margin_dpo/margin_mean": 81.40138244628906,
|
|
"margin_dpo/margin_std": 159.874755859375,
|
|
"step": 109
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -91.46940612792969,
|
|
"KL/mean": -125.52250671386719,
|
|
"KL/rejected_KL_mean": -159.5756072998047,
|
|
"KL/std": 117.68600463867188,
|
|
"epoch": 0.16152716593245228,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 68.10619354248047,
|
|
"fcm_dpo/q_t": 0.4723784029483795,
|
|
"grad_norm": 8.115654945373535,
|
|
"learning_rate": 4.947482930773511e-07,
|
|
"logits/chosen": -0.32437682151794434,
|
|
"logits/rejected": -0.3229568600654602,
|
|
"logps/chosen": -150.22308349609375,
|
|
"logps/ref_chosen": -58.753684997558594,
|
|
"logps/ref_rejected": -79.75001525878906,
|
|
"logps/rejected": -239.32562255859375,
|
|
"loss": 1.2933,
|
|
"margin_dpo/margin_mean": 68.10619354248047,
|
|
"margin_dpo/margin_std": 149.91207885742188,
|
|
"step": 110
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -101.22291564941406,
|
|
"KL/mean": -143.97872924804688,
|
|
"KL/rejected_KL_mean": -186.7345733642578,
|
|
"KL/std": 137.13291931152344,
|
|
"epoch": 0.16299559471365638,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 85.51166534423828,
|
|
"fcm_dpo/q_t": 0.46532922983169556,
|
|
"grad_norm": 7.223021507263184,
|
|
"learning_rate": 4.944834074412042e-07,
|
|
"logits/chosen": -0.33777564764022827,
|
|
"logits/rejected": -0.33077672123908997,
|
|
"logps/chosen": -169.84701538085938,
|
|
"logps/ref_chosen": -68.62410736083984,
|
|
"logps/ref_rejected": -98.42886352539062,
|
|
"logps/rejected": -285.1634521484375,
|
|
"loss": 1.2733,
|
|
"margin_dpo/margin_mean": 85.51167297363281,
|
|
"margin_dpo/margin_std": 186.15402221679688,
|
|
"step": 111
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -90.35836029052734,
|
|
"KL/mean": -112.38624572753906,
|
|
"KL/rejected_KL_mean": -134.4141387939453,
|
|
"KL/std": 86.84256744384766,
|
|
"epoch": 0.1644640234948605,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 44.05577087402344,
|
|
"fcm_dpo/q_t": 0.4818727374076843,
|
|
"grad_norm": 7.099332809448242,
|
|
"learning_rate": 4.942120794399002e-07,
|
|
"logits/chosen": -0.2616153955459595,
|
|
"logits/rejected": -0.2658056318759918,
|
|
"logps/chosen": -140.60800170898438,
|
|
"logps/ref_chosen": -50.24964141845703,
|
|
"logps/ref_rejected": -64.77442932128906,
|
|
"logps/rejected": -199.18856811523438,
|
|
"loss": 1.3241,
|
|
"margin_dpo/margin_mean": 44.05577087402344,
|
|
"margin_dpo/margin_std": 115.68550872802734,
|
|
"step": 112
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -115.47476196289062,
|
|
"KL/mean": -136.68405151367188,
|
|
"KL/rejected_KL_mean": -157.8933563232422,
|
|
"KL/std": 115.38925170898438,
|
|
"epoch": 0.16593245227606462,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 42.41858673095703,
|
|
"fcm_dpo/q_t": 0.4827752709388733,
|
|
"grad_norm": 6.795632839202881,
|
|
"learning_rate": 4.939343162231841e-07,
|
|
"logits/chosen": -0.28555989265441895,
|
|
"logits/rejected": -0.27538594603538513,
|
|
"logps/chosen": -182.18771362304688,
|
|
"logps/ref_chosen": -66.71295166015625,
|
|
"logps/ref_rejected": -77.96870422363281,
|
|
"logps/rejected": -235.862060546875,
|
|
"loss": 1.327,
|
|
"margin_dpo/margin_mean": 42.41858673095703,
|
|
"margin_dpo/margin_std": 120.11697387695312,
|
|
"step": 113
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -96.08192443847656,
|
|
"KL/mean": -136.51783752441406,
|
|
"KL/rejected_KL_mean": -176.95375061035156,
|
|
"KL/std": 128.7473602294922,
|
|
"epoch": 0.16740088105726872,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 80.87183380126953,
|
|
"fcm_dpo/q_t": 0.4677523374557495,
|
|
"grad_norm": 7.049834251403809,
|
|
"learning_rate": 4.936501251103751e-07,
|
|
"logits/chosen": -0.2732029855251312,
|
|
"logits/rejected": -0.2635844945907593,
|
|
"logps/chosen": -153.86700439453125,
|
|
"logps/ref_chosen": -57.78507995605469,
|
|
"logps/ref_rejected": -87.10966491699219,
|
|
"logps/rejected": -264.06341552734375,
|
|
"loss": 1.2745,
|
|
"margin_dpo/margin_mean": 80.87183380126953,
|
|
"margin_dpo/margin_std": 159.99867248535156,
|
|
"step": 114
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -116.97596740722656,
|
|
"KL/mean": -145.7581787109375,
|
|
"KL/rejected_KL_mean": -174.54037475585938,
|
|
"KL/std": 134.91941833496094,
|
|
"epoch": 0.16886930983847284,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 57.56439971923828,
|
|
"fcm_dpo/q_t": 0.4768410921096802,
|
|
"grad_norm": 6.489831924438477,
|
|
"learning_rate": 4.933595135901732e-07,
|
|
"logits/chosen": -0.31962960958480835,
|
|
"logits/rejected": -0.32673972845077515,
|
|
"logps/chosen": -182.55862426757812,
|
|
"logps/ref_chosen": -65.5826416015625,
|
|
"logps/ref_rejected": -98.56552124023438,
|
|
"logps/rejected": -273.10589599609375,
|
|
"loss": 1.3161,
|
|
"margin_dpo/margin_mean": 57.56439971923828,
|
|
"margin_dpo/margin_std": 182.9209442138672,
|
|
"step": 115
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -81.60614013671875,
|
|
"KL/mean": -113.20507049560547,
|
|
"KL/rejected_KL_mean": -144.80401611328125,
|
|
"KL/std": 99.23644256591797,
|
|
"epoch": 0.17033773861967694,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 63.19786834716797,
|
|
"fcm_dpo/q_t": 0.47415468096733093,
|
|
"grad_norm": 7.067198276519775,
|
|
"learning_rate": 4.930624893204624e-07,
|
|
"logits/chosen": -0.31514549255371094,
|
|
"logits/rejected": -0.32795295119285583,
|
|
"logps/chosen": -133.00645446777344,
|
|
"logps/ref_chosen": -51.40031433105469,
|
|
"logps/ref_rejected": -80.5218505859375,
|
|
"logps/rejected": -225.32586669921875,
|
|
"loss": 1.2933,
|
|
"margin_dpo/margin_mean": 63.19786071777344,
|
|
"margin_dpo/margin_std": 112.28499603271484,
|
|
"step": 116
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -113.90751647949219,
|
|
"KL/mean": -129.44996643066406,
|
|
"KL/rejected_KL_mean": -144.99240112304688,
|
|
"KL/std": 108.0283203125,
|
|
"epoch": 0.17180616740088106,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 31.08489227294922,
|
|
"fcm_dpo/q_t": 0.4872492551803589,
|
|
"grad_norm": 7.091765880584717,
|
|
"learning_rate": 4.927590601281083e-07,
|
|
"logits/chosen": -0.3172186017036438,
|
|
"logits/rejected": -0.3100709021091461,
|
|
"logps/chosen": -183.2059326171875,
|
|
"logps/ref_chosen": -69.29840850830078,
|
|
"logps/ref_rejected": -66.583984375,
|
|
"logps/rejected": -211.57638549804688,
|
|
"loss": 1.3476,
|
|
"margin_dpo/margin_mean": 31.084890365600586,
|
|
"margin_dpo/margin_std": 132.21383666992188,
|
|
"step": 117
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -81.68537902832031,
|
|
"KL/mean": -112.67160034179688,
|
|
"KL/rejected_KL_mean": -143.6578369140625,
|
|
"KL/std": 100.91627502441406,
|
|
"epoch": 0.17327459618208516,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 61.97246551513672,
|
|
"fcm_dpo/q_t": 0.4746900796890259,
|
|
"grad_norm": 6.520020008087158,
|
|
"learning_rate": 4.924492340087524e-07,
|
|
"logits/chosen": -0.3794967532157898,
|
|
"logits/rejected": -0.37046414613723755,
|
|
"logps/chosen": -137.32635498046875,
|
|
"logps/ref_chosen": -55.6409797668457,
|
|
"logps/ref_rejected": -75.66905975341797,
|
|
"logps/rejected": -219.32688903808594,
|
|
"loss": 1.2958,
|
|
"margin_dpo/margin_mean": 61.97245788574219,
|
|
"margin_dpo/margin_std": 119.08011627197266,
|
|
"step": 118
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -119.08765411376953,
|
|
"KL/mean": -146.5293426513672,
|
|
"KL/rejected_KL_mean": -173.97103881835938,
|
|
"KL/std": 110.69290924072266,
|
|
"epoch": 0.17474302496328928,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 54.88337326049805,
|
|
"fcm_dpo/q_t": 0.4776170551776886,
|
|
"grad_norm": 6.680598735809326,
|
|
"learning_rate": 4.92133019126601e-07,
|
|
"logits/chosen": -0.3600790500640869,
|
|
"logits/rejected": -0.36749690771102905,
|
|
"logps/chosen": -192.59783935546875,
|
|
"logps/ref_chosen": -73.51019287109375,
|
|
"logps/ref_rejected": -102.977294921875,
|
|
"logps/rejected": -276.9483337402344,
|
|
"loss": 1.3114,
|
|
"margin_dpo/margin_mean": 54.88337326049805,
|
|
"margin_dpo/margin_std": 140.7128448486328,
|
|
"step": 119
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -117.65304565429688,
|
|
"KL/mean": -159.9397735595703,
|
|
"KL/rejected_KL_mean": -202.22650146484375,
|
|
"KL/std": 132.9193115234375,
|
|
"epoch": 0.1762114537444934,
|
|
"fcm_dpo/beta": 0.0016584109980612993,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 84.57344055175781,
|
|
"fcm_dpo/q_t": 0.46578913927078247,
|
|
"grad_norm": 8.834454536437988,
|
|
"learning_rate": 4.918104238142103e-07,
|
|
"logits/chosen": -0.3894355893135071,
|
|
"logits/rejected": -0.38755276799201965,
|
|
"logps/chosen": -194.4338836669922,
|
|
"logps/ref_chosen": -76.78083801269531,
|
|
"logps/ref_rejected": -108.02374267578125,
|
|
"logps/rejected": -310.250244140625,
|
|
"loss": 1.2665,
|
|
"margin_dpo/margin_mean": 84.57344055175781,
|
|
"margin_dpo/margin_std": 151.76153564453125,
|
|
"step": 120
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -108.66337585449219,
|
|
"KL/mean": -158.3170928955078,
|
|
"KL/rejected_KL_mean": -207.9708251953125,
|
|
"KL/std": 133.39573669433594,
|
|
"epoch": 0.1776798825256975,
|
|
"fcm_dpo/beta": 0.005431335419416428,
|
|
"fcm_dpo/delta": 0.13696809113025665,
|
|
"fcm_dpo/margin": 99.30744934082031,
|
|
"fcm_dpo/q_t": 0.43542176485061646,
|
|
"grad_norm": 21.185283660888672,
|
|
"learning_rate": 4.91481456572267e-07,
|
|
"logits/chosen": -0.3457489609718323,
|
|
"logits/rejected": -0.348524272441864,
|
|
"logps/chosen": -170.45327758789062,
|
|
"logps/ref_chosen": -61.789894104003906,
|
|
"logps/ref_rejected": -109.99456787109375,
|
|
"logps/rejected": -317.96539306640625,
|
|
"loss": 1.096,
|
|
"margin_dpo/margin_mean": 99.30744934082031,
|
|
"margin_dpo/margin_std": 157.79302978515625,
|
|
"step": 121
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -86.52001953125,
|
|
"KL/mean": -148.9921112060547,
|
|
"KL/rejected_KL_mean": -211.46420288085938,
|
|
"KL/std": 132.5182342529297,
|
|
"epoch": 0.17914831130690162,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": -0.16569742560386658,
|
|
"fcm_dpo/margin": 124.94419860839844,
|
|
"fcm_dpo/q_t": 0.40711045265197754,
|
|
"grad_norm": 7.291841983795166,
|
|
"learning_rate": 4.911461260693638e-07,
|
|
"logits/chosen": -0.30201756954193115,
|
|
"logits/rejected": -0.31801754236221313,
|
|
"logps/chosen": -133.42222595214844,
|
|
"logps/ref_chosen": -46.9022102355957,
|
|
"logps/ref_rejected": -106.71418762207031,
|
|
"logps/rejected": -318.17840576171875,
|
|
"loss": 1.2453,
|
|
"margin_dpo/margin_mean": 124.9441909790039,
|
|
"margin_dpo/margin_std": 147.9832305908203,
|
|
"step": 122
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -110.82725524902344,
|
|
"KL/mean": -149.17156982421875,
|
|
"KL/rejected_KL_mean": -187.515869140625,
|
|
"KL/std": 125.73814392089844,
|
|
"epoch": 0.18061674008810572,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 76.68860626220703,
|
|
"fcm_dpo/q_t": 0.4764896631240845,
|
|
"grad_norm": 5.018327236175537,
|
|
"learning_rate": 4.908044411417711e-07,
|
|
"logits/chosen": -0.2894030809402466,
|
|
"logits/rejected": -0.29247063398361206,
|
|
"logps/chosen": -172.1658935546875,
|
|
"logps/ref_chosen": -61.33863830566406,
|
|
"logps/ref_rejected": -87.775390625,
|
|
"logps/rejected": -275.291259765625,
|
|
"loss": 1.3046,
|
|
"margin_dpo/margin_mean": 76.68860626220703,
|
|
"margin_dpo/margin_std": 171.46743774414062,
|
|
"step": 123
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -133.59381103515625,
|
|
"KL/mean": -191.1050262451172,
|
|
"KL/rejected_KL_mean": -248.61622619628906,
|
|
"KL/std": 179.1632080078125,
|
|
"epoch": 0.18208516886930984,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 115.02241516113281,
|
|
"fcm_dpo/q_t": 0.46520543098449707,
|
|
"grad_norm": 5.619283676147461,
|
|
"learning_rate": 4.904564107932048e-07,
|
|
"logits/chosen": -0.25618571043014526,
|
|
"logits/rejected": -0.2648266553878784,
|
|
"logps/chosen": -205.04214477539062,
|
|
"logps/ref_chosen": -71.44833374023438,
|
|
"logps/ref_rejected": -117.58056640625,
|
|
"logps/rejected": -366.19677734375,
|
|
"loss": 1.269,
|
|
"margin_dpo/margin_mean": 115.02241516113281,
|
|
"margin_dpo/margin_std": 230.38388061523438,
|
|
"step": 124
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -101.74983215332031,
|
|
"KL/mean": -149.4581298828125,
|
|
"KL/rejected_KL_mean": -197.16644287109375,
|
|
"KL/std": 146.47589111328125,
|
|
"epoch": 0.18355359765051396,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 95.41659545898438,
|
|
"fcm_dpo/q_t": 0.4708002507686615,
|
|
"grad_norm": 5.07564640045166,
|
|
"learning_rate": 4.90102044194588e-07,
|
|
"logits/chosen": -0.18893632292747498,
|
|
"logits/rejected": -0.20313698053359985,
|
|
"logps/chosen": -151.88677978515625,
|
|
"logps/ref_chosen": -50.136940002441406,
|
|
"logps/ref_rejected": -83.98861694335938,
|
|
"logps/rejected": -281.1550598144531,
|
|
"loss": 1.2842,
|
|
"margin_dpo/margin_mean": 95.41659545898438,
|
|
"margin_dpo/margin_std": 181.84890747070312,
|
|
"step": 125
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -110.52033996582031,
|
|
"KL/mean": -162.48785400390625,
|
|
"KL/rejected_KL_mean": -214.45535278320312,
|
|
"KL/std": 144.8026123046875,
|
|
"epoch": 0.18502202643171806,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 103.93502807617188,
|
|
"fcm_dpo/q_t": 0.46845173835754395,
|
|
"grad_norm": 5.675121307373047,
|
|
"learning_rate": 4.897413506838102e-07,
|
|
"logits/chosen": -0.2591179609298706,
|
|
"logits/rejected": -0.27047020196914673,
|
|
"logps/chosen": -166.18740844726562,
|
|
"logps/ref_chosen": -55.66706848144531,
|
|
"logps/ref_rejected": -98.1297607421875,
|
|
"logps/rejected": -312.5851135253906,
|
|
"loss": 1.2746,
|
|
"margin_dpo/margin_mean": 103.93502807617188,
|
|
"margin_dpo/margin_std": 185.01034545898438,
|
|
"step": 126
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -114.12653350830078,
|
|
"KL/mean": -142.77590942382812,
|
|
"KL/rejected_KL_mean": -171.42526245117188,
|
|
"KL/std": 126.23076629638672,
|
|
"epoch": 0.18649045521292218,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 57.298728942871094,
|
|
"fcm_dpo/q_t": 0.4821869730949402,
|
|
"grad_norm": 5.324838638305664,
|
|
"learning_rate": 4.89374339765481e-07,
|
|
"logits/chosen": -0.2520139515399933,
|
|
"logits/rejected": -0.25741541385650635,
|
|
"logps/chosen": -170.68121337890625,
|
|
"logps/ref_chosen": -56.55467987060547,
|
|
"logps/ref_rejected": -76.7957763671875,
|
|
"logps/rejected": -248.22103881835938,
|
|
"loss": 1.3258,
|
|
"margin_dpo/margin_mean": 57.298728942871094,
|
|
"margin_dpo/margin_std": 157.20663452148438,
|
|
"step": 127
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -117.71072387695312,
|
|
"KL/mean": -154.01956176757812,
|
|
"KL/rejected_KL_mean": -190.32843017578125,
|
|
"KL/std": 155.84530639648438,
|
|
"epoch": 0.18795888399412627,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 72.61770629882812,
|
|
"fcm_dpo/q_t": 0.4778425097465515,
|
|
"grad_norm": 6.066002368927002,
|
|
"learning_rate": 4.890010211106795e-07,
|
|
"logits/chosen": -0.23010198771953583,
|
|
"logits/rejected": -0.24306002259254456,
|
|
"logps/chosen": -175.83168029785156,
|
|
"logps/ref_chosen": -58.12095642089844,
|
|
"logps/ref_rejected": -76.43896484375,
|
|
"logps/rejected": -266.76739501953125,
|
|
"loss": 1.3139,
|
|
"margin_dpo/margin_mean": 72.61770629882812,
|
|
"margin_dpo/margin_std": 201.6669158935547,
|
|
"step": 128
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -156.19540405273438,
|
|
"KL/mean": -194.95504760742188,
|
|
"KL/rejected_KL_mean": -233.71466064453125,
|
|
"KL/std": 197.4521484375,
|
|
"epoch": 0.1894273127753304,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 77.51925659179688,
|
|
"fcm_dpo/q_t": 0.47747713327407837,
|
|
"grad_norm": 6.777254581451416,
|
|
"learning_rate": 4.88621404556699e-07,
|
|
"logits/chosen": -0.2056041806936264,
|
|
"logits/rejected": -0.2097330093383789,
|
|
"logps/chosen": -223.11178588867188,
|
|
"logps/ref_chosen": -66.91637420654297,
|
|
"logps/ref_rejected": -96.6422119140625,
|
|
"logps/rejected": -330.35687255859375,
|
|
"loss": 1.3193,
|
|
"margin_dpo/margin_mean": 77.51925659179688,
|
|
"margin_dpo/margin_std": 269.54656982421875,
|
|
"step": 129
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -108.04925537109375,
|
|
"KL/mean": -162.80902099609375,
|
|
"KL/rejected_KL_mean": -217.56878662109375,
|
|
"KL/std": 142.10467529296875,
|
|
"epoch": 0.19089574155653452,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 109.51951599121094,
|
|
"fcm_dpo/q_t": 0.4666077494621277,
|
|
"grad_norm": 7.042036533355713,
|
|
"learning_rate": 4.882355001067891e-07,
|
|
"logits/chosen": -0.15569573640823364,
|
|
"logits/rejected": -0.15476679801940918,
|
|
"logps/chosen": -152.7161102294922,
|
|
"logps/ref_chosen": -44.66685104370117,
|
|
"logps/ref_rejected": -82.78165435791016,
|
|
"logps/rejected": -300.3504333496094,
|
|
"loss": 1.2689,
|
|
"margin_dpo/margin_mean": 109.51950073242188,
|
|
"margin_dpo/margin_std": 188.17529296875,
|
|
"step": 130
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -112.39668273925781,
|
|
"KL/mean": -174.34365844726562,
|
|
"KL/rejected_KL_mean": -236.29067993164062,
|
|
"KL/std": 177.99996948242188,
|
|
"epoch": 0.19236417033773862,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 123.89396667480469,
|
|
"fcm_dpo/q_t": 0.46196436882019043,
|
|
"grad_norm": 7.167151927947998,
|
|
"learning_rate": 4.878433179298909e-07,
|
|
"logits/chosen": -0.15293475985527039,
|
|
"logits/rejected": -0.15630626678466797,
|
|
"logps/chosen": -157.32127380371094,
|
|
"logps/ref_chosen": -44.924591064453125,
|
|
"logps/ref_rejected": -88.44401550292969,
|
|
"logps/rejected": -324.73468017578125,
|
|
"loss": 1.2519,
|
|
"margin_dpo/margin_mean": 123.89398193359375,
|
|
"margin_dpo/margin_std": 189.91864013671875,
|
|
"step": 131
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -142.0668487548828,
|
|
"KL/mean": -188.7615966796875,
|
|
"KL/rejected_KL_mean": -235.45635986328125,
|
|
"KL/std": 182.57955932617188,
|
|
"epoch": 0.19383259911894274,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 93.3895034790039,
|
|
"fcm_dpo/q_t": 0.47222959995269775,
|
|
"grad_norm": 6.9561357498168945,
|
|
"learning_rate": 4.874448683603694e-07,
|
|
"logits/chosen": -0.1588452309370041,
|
|
"logits/rejected": -0.1891343593597412,
|
|
"logps/chosen": -201.06793212890625,
|
|
"logps/ref_chosen": -59.00108337402344,
|
|
"logps/ref_rejected": -87.89215087890625,
|
|
"logps/rejected": -323.3485107421875,
|
|
"loss": 1.295,
|
|
"margin_dpo/margin_mean": 93.3895034790039,
|
|
"margin_dpo/margin_std": 238.1549530029297,
|
|
"step": 132
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -163.40499877929688,
|
|
"KL/mean": -203.98348999023438,
|
|
"KL/rejected_KL_mean": -244.56201171875,
|
|
"KL/std": 153.83062744140625,
|
|
"epoch": 0.19530102790014683,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 81.15701293945312,
|
|
"fcm_dpo/q_t": 0.47538745403289795,
|
|
"grad_norm": 7.251523494720459,
|
|
"learning_rate": 4.870401618977415e-07,
|
|
"logits/chosen": -0.15137168765068054,
|
|
"logits/rejected": -0.1648947298526764,
|
|
"logps/chosen": -230.00949096679688,
|
|
"logps/ref_chosen": -66.60449981689453,
|
|
"logps/ref_rejected": -96.33355712890625,
|
|
"logps/rejected": -340.89556884765625,
|
|
"loss": 1.3037,
|
|
"margin_dpo/margin_mean": 81.15701293945312,
|
|
"margin_dpo/margin_std": 203.13357543945312,
|
|
"step": 133
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -126.16668701171875,
|
|
"KL/mean": -169.11993408203125,
|
|
"KL/rejected_KL_mean": -212.0731964111328,
|
|
"KL/std": 131.4796600341797,
|
|
"epoch": 0.19676945668135096,
|
|
"fcm_dpo/beta": 0.0012442920124158263,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 85.90650939941406,
|
|
"fcm_dpo/q_t": 0.47373300790786743,
|
|
"grad_norm": 7.264800071716309,
|
|
"learning_rate": 4.866292092063986e-07,
|
|
"logits/chosen": -0.1166238859295845,
|
|
"logits/rejected": -0.13339810073375702,
|
|
"logps/chosen": -178.23594665527344,
|
|
"logps/ref_chosen": -52.06925582885742,
|
|
"logps/ref_rejected": -87.6545181274414,
|
|
"logps/rejected": -299.72772216796875,
|
|
"loss": 1.2925,
|
|
"margin_dpo/margin_mean": 85.9065170288086,
|
|
"margin_dpo/margin_std": 161.20672607421875,
|
|
"step": 134
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -130.97235107421875,
|
|
"KL/mean": -212.07064819335938,
|
|
"KL/rejected_KL_mean": -293.1689453125,
|
|
"KL/std": 209.9551544189453,
|
|
"epoch": 0.19823788546255505,
|
|
"fcm_dpo/beta": 0.0025665950961411,
|
|
"fcm_dpo/delta": 0.04211168363690376,
|
|
"fcm_dpo/margin": 162.19659423828125,
|
|
"fcm_dpo/q_t": 0.42933958768844604,
|
|
"grad_norm": 16.52530288696289,
|
|
"learning_rate": 4.862120211153265e-07,
|
|
"logits/chosen": -0.09394662082195282,
|
|
"logits/rejected": -0.10758954286575317,
|
|
"logps/chosen": -181.32620239257812,
|
|
"logps/ref_chosen": -50.353858947753906,
|
|
"logps/ref_rejected": -115.97975158691406,
|
|
"logps/rejected": -409.148681640625,
|
|
"loss": 1.1038,
|
|
"margin_dpo/margin_mean": 162.1966094970703,
|
|
"margin_dpo/margin_std": 266.60357666015625,
|
|
"step": 135
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -160.09901428222656,
|
|
"KL/mean": -211.46200561523438,
|
|
"KL/rejected_KL_mean": -262.8249816894531,
|
|
"KL/std": 210.24295043945312,
|
|
"epoch": 0.19970631424375918,
|
|
"fcm_dpo/beta": 0.0018958807922899723,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 102.72596740722656,
|
|
"fcm_dpo/q_t": 0.45904627442359924,
|
|
"grad_norm": 9.262153625488281,
|
|
"learning_rate": 4.857886086178193e-07,
|
|
"logits/chosen": -0.1536802351474762,
|
|
"logits/rejected": -0.16803759336471558,
|
|
"logps/chosen": -225.17152404785156,
|
|
"logps/ref_chosen": -65.072509765625,
|
|
"logps/ref_rejected": -96.32122802734375,
|
|
"logps/rejected": -359.1462097167969,
|
|
"loss": 1.2667,
|
|
"margin_dpo/margin_mean": 102.72596740722656,
|
|
"margin_dpo/margin_std": 286.5235595703125,
|
|
"step": 136
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -135.36795043945312,
|
|
"KL/mean": -214.21568298339844,
|
|
"KL/rejected_KL_mean": -293.06341552734375,
|
|
"KL/std": 230.4893798828125,
|
|
"epoch": 0.2011747430249633,
|
|
"fcm_dpo/beta": 0.002469564788043499,
|
|
"fcm_dpo/delta": -0.016328319907188416,
|
|
"fcm_dpo/margin": 157.69546508789062,
|
|
"fcm_dpo/q_t": 0.4195204973220825,
|
|
"grad_norm": 12.261474609375,
|
|
"learning_rate": 4.853589828711902e-07,
|
|
"logits/chosen": -0.10306895524263382,
|
|
"logits/rejected": -0.11605556309223175,
|
|
"logps/chosen": -184.1270751953125,
|
|
"logps/ref_chosen": -48.759117126464844,
|
|
"logps/ref_rejected": -113.86376953125,
|
|
"logps/rejected": -406.92718505859375,
|
|
"loss": 1.141,
|
|
"margin_dpo/margin_mean": 157.69546508789062,
|
|
"margin_dpo/margin_std": 293.899658203125,
|
|
"step": 137
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -148.42919921875,
|
|
"KL/mean": -201.90768432617188,
|
|
"KL/rejected_KL_mean": -255.3861846923828,
|
|
"KL/std": 171.12557983398438,
|
|
"epoch": 0.2026431718061674,
|
|
"fcm_dpo/beta": 0.0016102675581350923,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 106.95697021484375,
|
|
"fcm_dpo/q_t": 0.4581053853034973,
|
|
"grad_norm": 9.790092468261719,
|
|
"learning_rate": 4.849231551964771e-07,
|
|
"logits/chosen": -0.11189040541648865,
|
|
"logits/rejected": -0.11567579954862595,
|
|
"logps/chosen": -208.9488525390625,
|
|
"logps/ref_chosen": -60.519649505615234,
|
|
"logps/ref_rejected": -93.19694519042969,
|
|
"logps/rejected": -348.5831298828125,
|
|
"loss": 1.2412,
|
|
"margin_dpo/margin_mean": 106.95697021484375,
|
|
"margin_dpo/margin_std": 175.25364685058594,
|
|
"step": 138
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -127.69628143310547,
|
|
"KL/mean": -184.2217559814453,
|
|
"KL/rejected_KL_mean": -240.7472381591797,
|
|
"KL/std": 150.19712829589844,
|
|
"epoch": 0.20411160058737152,
|
|
"fcm_dpo/beta": 0.0016102675581350923,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 113.05096435546875,
|
|
"fcm_dpo/q_t": 0.4562827944755554,
|
|
"grad_norm": 9.610363006591797,
|
|
"learning_rate": 4.844811370781446e-07,
|
|
"logits/chosen": -0.11106328666210175,
|
|
"logits/rejected": -0.11769875884056091,
|
|
"logps/chosen": -174.58766174316406,
|
|
"logps/ref_chosen": -46.89138412475586,
|
|
"logps/ref_rejected": -79.72798156738281,
|
|
"logps/rejected": -320.4752197265625,
|
|
"loss": 1.2391,
|
|
"margin_dpo/margin_mean": 113.05096435546875,
|
|
"margin_dpo/margin_std": 206.3931427001953,
|
|
"step": 139
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -158.94659423828125,
|
|
"KL/mean": -203.65492248535156,
|
|
"KL/rejected_KL_mean": -248.36328125,
|
|
"KL/std": 166.0042724609375,
|
|
"epoch": 0.2055800293685756,
|
|
"fcm_dpo/beta": 0.0016102675581350923,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 89.41668701171875,
|
|
"fcm_dpo/q_t": 0.46475258469581604,
|
|
"grad_norm": 9.001358032226562,
|
|
"learning_rate": 4.840329401637809e-07,
|
|
"logits/chosen": -0.11058294773101807,
|
|
"logits/rejected": -0.12042254209518433,
|
|
"logps/chosen": -217.92129516601562,
|
|
"logps/ref_chosen": -58.97471618652344,
|
|
"logps/ref_rejected": -83.28410339355469,
|
|
"logps/rejected": -331.64739990234375,
|
|
"loss": 1.2763,
|
|
"margin_dpo/margin_mean": 89.41668701171875,
|
|
"margin_dpo/margin_std": 212.3289794921875,
|
|
"step": 140
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -191.18380737304688,
|
|
"KL/mean": -233.52825927734375,
|
|
"KL/rejected_KL_mean": -275.8727111816406,
|
|
"KL/std": 238.5574188232422,
|
|
"epoch": 0.20704845814977973,
|
|
"fcm_dpo/beta": 0.0016102675581350923,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 84.68890380859375,
|
|
"fcm_dpo/q_t": 0.4663596749305725,
|
|
"grad_norm": 15.454120635986328,
|
|
"learning_rate": 4.83578576263792e-07,
|
|
"logits/chosen": -0.10689105093479156,
|
|
"logits/rejected": -0.12749376893043518,
|
|
"logps/chosen": -266.25946044921875,
|
|
"logps/ref_chosen": -75.07566833496094,
|
|
"logps/ref_rejected": -98.1922607421875,
|
|
"logps/rejected": -374.0649719238281,
|
|
"loss": 1.3095,
|
|
"margin_dpo/margin_mean": 84.68890380859375,
|
|
"margin_dpo/margin_std": 295.63360595703125,
|
|
"step": 141
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -161.26834106445312,
|
|
"KL/mean": -219.98394775390625,
|
|
"KL/rejected_KL_mean": -278.6995849609375,
|
|
"KL/std": 205.99090576171875,
|
|
"epoch": 0.20851688693098386,
|
|
"fcm_dpo/beta": 0.002624105429276824,
|
|
"fcm_dpo/delta": 0.08150181919336319,
|
|
"fcm_dpo/margin": 117.43124389648438,
|
|
"fcm_dpo/q_t": 0.4544960558414459,
|
|
"grad_norm": 13.912692070007324,
|
|
"learning_rate": 4.83118057351089e-07,
|
|
"logits/chosen": -0.09879723191261292,
|
|
"logits/rejected": -0.11309005320072174,
|
|
"logps/chosen": -219.2962646484375,
|
|
"logps/ref_chosen": -58.027931213378906,
|
|
"logps/ref_rejected": -94.58222961425781,
|
|
"logps/rejected": -373.28179931640625,
|
|
"loss": 1.1618,
|
|
"margin_dpo/margin_mean": 117.43125915527344,
|
|
"margin_dpo/margin_std": 238.91827392578125,
|
|
"step": 142
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -157.7050323486328,
|
|
"KL/mean": -193.36447143554688,
|
|
"KL/rejected_KL_mean": -229.02392578125,
|
|
"KL/std": 160.72268676757812,
|
|
"epoch": 0.20998531571218795,
|
|
"fcm_dpo/beta": 0.004260506015270948,
|
|
"fcm_dpo/delta": 0.015796946361660957,
|
|
"fcm_dpo/margin": 71.31889343261719,
|
|
"fcm_dpo/q_t": 0.4452533721923828,
|
|
"grad_norm": 27.70894432067871,
|
|
"learning_rate": 4.826513955607734e-07,
|
|
"logits/chosen": -0.09957142174243927,
|
|
"logits/rejected": -0.11266869306564331,
|
|
"logps/chosen": -215.30148315429688,
|
|
"logps/ref_chosen": -57.59645080566406,
|
|
"logps/ref_rejected": -78.99957275390625,
|
|
"logps/rejected": -308.02349853515625,
|
|
"loss": 1.2612,
|
|
"margin_dpo/margin_mean": 71.31889343261719,
|
|
"margin_dpo/margin_std": 196.57472229003906,
|
|
"step": 143
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -155.84344482421875,
|
|
"KL/mean": -195.27426147460938,
|
|
"KL/rejected_KL_mean": -234.705078125,
|
|
"KL/std": 159.63229370117188,
|
|
"epoch": 0.21145374449339208,
|
|
"fcm_dpo/beta": 0.004793780855834484,
|
|
"fcm_dpo/delta": -0.03707936033606529,
|
|
"fcm_dpo/margin": 78.86161804199219,
|
|
"fcm_dpo/q_t": 0.4113781452178955,
|
|
"grad_norm": 32.351070404052734,
|
|
"learning_rate": 4.821786031898176e-07,
|
|
"logits/chosen": -0.12348122149705887,
|
|
"logits/rejected": -0.15042133629322052,
|
|
"logps/chosen": -215.74981689453125,
|
|
"logps/ref_chosen": -59.90636444091797,
|
|
"logps/ref_rejected": -82.00025939941406,
|
|
"logps/rejected": -316.705322265625,
|
|
"loss": 1.2729,
|
|
"margin_dpo/margin_mean": 78.86161804199219,
|
|
"margin_dpo/margin_std": 176.07415771484375,
|
|
"step": 144
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -133.75552368164062,
|
|
"KL/mean": -175.75448608398438,
|
|
"KL/rejected_KL_mean": -217.75347900390625,
|
|
"KL/std": 131.30337524414062,
|
|
"epoch": 0.21292217327459617,
|
|
"fcm_dpo/beta": 0.004883944522589445,
|
|
"fcm_dpo/delta": 0.020339418202638626,
|
|
"fcm_dpo/margin": 83.99797058105469,
|
|
"fcm_dpo/q_t": 0.41977953910827637,
|
|
"grad_norm": 38.0810661315918,
|
|
"learning_rate": 4.816996926967401e-07,
|
|
"logits/chosen": -0.21709679067134857,
|
|
"logits/rejected": -0.23458248376846313,
|
|
"logps/chosen": -190.35618591308594,
|
|
"logps/ref_chosen": -56.60066604614258,
|
|
"logps/ref_rejected": -77.86631774902344,
|
|
"logps/rejected": -295.61981201171875,
|
|
"loss": 1.1677,
|
|
"margin_dpo/margin_mean": 83.99796295166016,
|
|
"margin_dpo/margin_std": 166.22833251953125,
|
|
"step": 145
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -174.54449462890625,
|
|
"KL/mean": -203.33346557617188,
|
|
"KL/rejected_KL_mean": -232.12242126464844,
|
|
"KL/std": 144.7928009033203,
|
|
"epoch": 0.2143906020558003,
|
|
"fcm_dpo/beta": 0.009946160949766636,
|
|
"fcm_dpo/delta": 0.1288416087627411,
|
|
"fcm_dpo/margin": 57.57792663574219,
|
|
"fcm_dpo/q_t": 0.4425297677516937,
|
|
"grad_norm": 173.9798583984375,
|
|
"learning_rate": 4.812146767012779e-07,
|
|
"logits/chosen": -0.14733245968818665,
|
|
"logits/rejected": -0.1549091786146164,
|
|
"logps/chosen": -240.54495239257812,
|
|
"logps/ref_chosen": -66.00045013427734,
|
|
"logps/ref_rejected": -81.70278930664062,
|
|
"logps/rejected": -313.8251953125,
|
|
"loss": 1.6518,
|
|
"margin_dpo/margin_mean": 57.57793426513672,
|
|
"margin_dpo/margin_std": 177.08001708984375,
|
|
"step": 146
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -131.95578002929688,
|
|
"KL/mean": -173.01547241210938,
|
|
"KL/rejected_KL_mean": -214.07516479492188,
|
|
"KL/std": 144.98574829101562,
|
|
"epoch": 0.21585903083700442,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": -0.4109819829463959,
|
|
"fcm_dpo/margin": 82.11940002441406,
|
|
"fcm_dpo/q_t": 0.42657575011253357,
|
|
"grad_norm": 1.725198745727539,
|
|
"learning_rate": 4.807235679840536e-07,
|
|
"logits/chosen": -0.20732438564300537,
|
|
"logits/rejected": -0.22212302684783936,
|
|
"logps/chosen": -185.36126708984375,
|
|
"logps/ref_chosen": -53.405487060546875,
|
|
"logps/ref_rejected": -71.39060974121094,
|
|
"logps/rejected": -285.4657897949219,
|
|
"loss": 1.3691,
|
|
"margin_dpo/margin_mean": 82.11940002441406,
|
|
"margin_dpo/margin_std": 182.35836791992188,
|
|
"step": 147
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -140.5421905517578,
|
|
"KL/mean": -184.202392578125,
|
|
"KL/rejected_KL_mean": -227.8626251220703,
|
|
"KL/std": 168.81472778320312,
|
|
"epoch": 0.2173274596182085,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 87.32041931152344,
|
|
"fcm_dpo/q_t": 0.4953227639198303,
|
|
"grad_norm": 2.0062618255615234,
|
|
"learning_rate": 4.802263794862384e-07,
|
|
"logits/chosen": -0.23836994171142578,
|
|
"logits/rejected": -0.24250321090221405,
|
|
"logps/chosen": -205.47927856445312,
|
|
"logps/ref_chosen": -64.93708038330078,
|
|
"logps/ref_rejected": -103.09384155273438,
|
|
"logps/rejected": -330.95648193359375,
|
|
"loss": 1.3681,
|
|
"margin_dpo/margin_mean": 87.32041931152344,
|
|
"margin_dpo/margin_std": 185.8931884765625,
|
|
"step": 148
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -124.4274673461914,
|
|
"KL/mean": -176.48355102539062,
|
|
"KL/rejected_KL_mean": -228.5396270751953,
|
|
"KL/std": 142.7633514404297,
|
|
"epoch": 0.21879588839941264,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 104.11214447021484,
|
|
"fcm_dpo/q_t": 0.4944196343421936,
|
|
"grad_norm": 1.6070046424865723,
|
|
"learning_rate": 4.797231243092118e-07,
|
|
"logits/chosen": -0.20803114771842957,
|
|
"logits/rejected": -0.21203464269638062,
|
|
"logps/chosen": -182.90122985839844,
|
|
"logps/ref_chosen": -58.47376251220703,
|
|
"logps/ref_rejected": -99.31474304199219,
|
|
"logps/rejected": -327.8543701171875,
|
|
"loss": 1.3645,
|
|
"margin_dpo/margin_mean": 104.11215209960938,
|
|
"margin_dpo/margin_std": 176.35842895507812,
|
|
"step": 149
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -107.64643096923828,
|
|
"KL/mean": -164.7757568359375,
|
|
"KL/rejected_KL_mean": -221.9050750732422,
|
|
"KL/std": 159.32635498046875,
|
|
"epoch": 0.22026431718061673,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 114.25865173339844,
|
|
"fcm_dpo/q_t": 0.49387815594673157,
|
|
"grad_norm": 1.4557676315307617,
|
|
"learning_rate": 4.792138157142157e-07,
|
|
"logits/chosen": -0.2157842516899109,
|
|
"logits/rejected": -0.2243994176387787,
|
|
"logps/chosen": -153.35223388671875,
|
|
"logps/ref_chosen": -45.705810546875,
|
|
"logps/ref_rejected": -83.34759521484375,
|
|
"logps/rejected": -305.252685546875,
|
|
"loss": 1.3624,
|
|
"margin_dpo/margin_mean": 114.2586441040039,
|
|
"margin_dpo/margin_std": 193.01649475097656,
|
|
"step": 150
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -149.71823120117188,
|
|
"KL/mean": -199.18092346191406,
|
|
"KL/rejected_KL_mean": -248.64361572265625,
|
|
"KL/std": 174.56417846679688,
|
|
"epoch": 0.22173274596182085,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 98.92537689208984,
|
|
"fcm_dpo/q_t": 0.49469929933547974,
|
|
"grad_norm": 1.555479884147644,
|
|
"learning_rate": 4.786984671220053e-07,
|
|
"logits/chosen": -0.2548075318336487,
|
|
"logits/rejected": -0.2567106783390045,
|
|
"logps/chosen": -220.2890625,
|
|
"logps/ref_chosen": -70.57083129882812,
|
|
"logps/ref_rejected": -100.46382141113281,
|
|
"logps/rejected": -349.107421875,
|
|
"loss": 1.3657,
|
|
"margin_dpo/margin_mean": 98.92536926269531,
|
|
"margin_dpo/margin_std": 200.89501953125,
|
|
"step": 151
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -130.66839599609375,
|
|
"KL/mean": -196.2487335205078,
|
|
"KL/rejected_KL_mean": -261.8291015625,
|
|
"KL/std": 162.598876953125,
|
|
"epoch": 0.22320117474302498,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 131.16065979003906,
|
|
"fcm_dpo/q_t": 0.4929710030555725,
|
|
"grad_norm": 1.7390542030334473,
|
|
"learning_rate": 4.78177092112495e-07,
|
|
"logits/chosen": -0.22812870144844055,
|
|
"logits/rejected": -0.23715990781784058,
|
|
"logps/chosen": -190.83279418945312,
|
|
"logps/ref_chosen": -60.16438674926758,
|
|
"logps/ref_rejected": -106.14045715332031,
|
|
"logps/rejected": -367.96954345703125,
|
|
"loss": 1.3588,
|
|
"margin_dpo/margin_mean": 131.16067504882812,
|
|
"margin_dpo/margin_std": 200.10365295410156,
|
|
"step": 152
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -126.16310119628906,
|
|
"KL/mean": -180.94070434570312,
|
|
"KL/rejected_KL_mean": -235.71829223632812,
|
|
"KL/std": 180.3782501220703,
|
|
"epoch": 0.22466960352422907,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 109.55519104003906,
|
|
"fcm_dpo/q_t": 0.49413180351257324,
|
|
"grad_norm": 1.530712366104126,
|
|
"learning_rate": 4.776497044244016e-07,
|
|
"logits/chosen": -0.21236395835876465,
|
|
"logits/rejected": -0.22474971413612366,
|
|
"logps/chosen": -182.47837829589844,
|
|
"logps/ref_chosen": -56.315277099609375,
|
|
"logps/ref_rejected": -85.65583801269531,
|
|
"logps/rejected": -321.3741149902344,
|
|
"loss": 1.3636,
|
|
"margin_dpo/margin_mean": 109.55519104003906,
|
|
"margin_dpo/margin_std": 232.18374633789062,
|
|
"step": 153
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -158.0811767578125,
|
|
"KL/mean": -203.92166137695312,
|
|
"KL/rejected_KL_mean": -249.7621307373047,
|
|
"KL/std": 173.6383514404297,
|
|
"epoch": 0.2261380323054332,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 91.68096923828125,
|
|
"fcm_dpo/q_t": 0.4950861930847168,
|
|
"grad_norm": 1.4504088163375854,
|
|
"learning_rate": 4.771163179548808e-07,
|
|
"logits/chosen": -0.21836550533771515,
|
|
"logits/rejected": -0.23400922119617462,
|
|
"logps/chosen": -220.82374572753906,
|
|
"logps/ref_chosen": -62.74256896972656,
|
|
"logps/ref_rejected": -104.24420166015625,
|
|
"logps/rejected": -354.00634765625,
|
|
"loss": 1.3674,
|
|
"margin_dpo/margin_mean": 91.68096923828125,
|
|
"margin_dpo/margin_std": 240.62171936035156,
|
|
"step": 154
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -142.25665283203125,
|
|
"KL/mean": -183.7818145751953,
|
|
"KL/rejected_KL_mean": -225.3070068359375,
|
|
"KL/std": 151.2476348876953,
|
|
"epoch": 0.2276064610866373,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 83.05033874511719,
|
|
"fcm_dpo/q_t": 0.49554821848869324,
|
|
"grad_norm": 1.3787976503372192,
|
|
"learning_rate": 4.7657694675916247e-07,
|
|
"logits/chosen": -0.23564405739307404,
|
|
"logits/rejected": -0.23982316255569458,
|
|
"logps/chosen": -202.9098358154297,
|
|
"logps/ref_chosen": -60.65318298339844,
|
|
"logps/ref_rejected": -77.49220275878906,
|
|
"logps/rejected": -302.7991943359375,
|
|
"loss": 1.369,
|
|
"margin_dpo/margin_mean": 83.05035400390625,
|
|
"margin_dpo/margin_std": 198.95108032226562,
|
|
"step": 155
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -190.4660186767578,
|
|
"KL/mean": -208.18328857421875,
|
|
"KL/rejected_KL_mean": -225.90054321289062,
|
|
"KL/std": 172.94052124023438,
|
|
"epoch": 0.2290748898678414,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 35.43452453613281,
|
|
"fcm_dpo/q_t": 0.49810007214546204,
|
|
"grad_norm": 1.2596914768218994,
|
|
"learning_rate": 4.7603160505017893e-07,
|
|
"logits/chosen": -0.19964438676834106,
|
|
"logits/rejected": -0.22770291566848755,
|
|
"logps/chosen": -259.9578857421875,
|
|
"logps/ref_chosen": -69.49188232421875,
|
|
"logps/ref_rejected": -77.16929626464844,
|
|
"logps/rejected": -303.06982421875,
|
|
"loss": 1.3793,
|
|
"margin_dpo/margin_mean": 35.43452453613281,
|
|
"margin_dpo/margin_std": 223.39340209960938,
|
|
"step": 156
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -160.28289794921875,
|
|
"KL/mean": -230.226806640625,
|
|
"KL/rejected_KL_mean": -300.17071533203125,
|
|
"KL/std": 185.30099487304688,
|
|
"epoch": 0.2305433186490455,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 139.88784790039062,
|
|
"fcm_dpo/q_t": 0.49250328540802,
|
|
"grad_norm": 1.719684362411499,
|
|
"learning_rate": 4.7548030719819154e-07,
|
|
"logits/chosen": -0.09867404401302338,
|
|
"logits/rejected": -0.11982227861881256,
|
|
"logps/chosen": -221.65133666992188,
|
|
"logps/ref_chosen": -61.368438720703125,
|
|
"logps/ref_rejected": -107.64636993408203,
|
|
"logps/rejected": -407.8171081542969,
|
|
"loss": 1.3572,
|
|
"margin_dpo/margin_mean": 139.88784790039062,
|
|
"margin_dpo/margin_std": 224.976318359375,
|
|
"step": 157
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -170.56492614746094,
|
|
"KL/mean": -253.34765625,
|
|
"KL/rejected_KL_mean": -336.1304016113281,
|
|
"KL/std": 241.12974548339844,
|
|
"epoch": 0.23201174743024963,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 165.5655059814453,
|
|
"fcm_dpo/q_t": 0.491136372089386,
|
|
"grad_norm": 2.193638324737549,
|
|
"learning_rate": 4.7492306773041136e-07,
|
|
"logits/chosen": -0.13828858733177185,
|
|
"logits/rejected": -0.16126753389835358,
|
|
"logps/chosen": -228.17784118652344,
|
|
"logps/ref_chosen": -57.612918853759766,
|
|
"logps/ref_rejected": -113.6946792602539,
|
|
"logps/rejected": -449.8250732421875,
|
|
"loss": 1.3521,
|
|
"margin_dpo/margin_mean": 165.5655059814453,
|
|
"margin_dpo/margin_std": 297.131591796875,
|
|
"step": 158
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -226.7890167236328,
|
|
"KL/mean": -262.0523681640625,
|
|
"KL/rejected_KL_mean": -297.31573486328125,
|
|
"KL/std": 231.03848266601562,
|
|
"epoch": 0.23348017621145375,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 70.52671813964844,
|
|
"fcm_dpo/q_t": 0.49621930718421936,
|
|
"grad_norm": 1.778730869293213,
|
|
"learning_rate": 4.743599013306165e-07,
|
|
"logits/chosen": -0.11859369277954102,
|
|
"logits/rejected": -0.12011769413948059,
|
|
"logps/chosen": -308.349365234375,
|
|
"logps/ref_chosen": -81.56034851074219,
|
|
"logps/ref_rejected": -88.89871215820312,
|
|
"logps/rejected": -386.21441650390625,
|
|
"loss": 1.372,
|
|
"margin_dpo/margin_mean": 70.52671813964844,
|
|
"margin_dpo/margin_std": 254.50335693359375,
|
|
"step": 159
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -208.1883544921875,
|
|
"KL/mean": -273.8866271972656,
|
|
"KL/rejected_KL_mean": -339.5848693847656,
|
|
"KL/std": 218.83474731445312,
|
|
"epoch": 0.23494860499265785,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 131.39654541015625,
|
|
"fcm_dpo/q_t": 0.4929630160331726,
|
|
"grad_norm": 1.8856486082077026,
|
|
"learning_rate": 4.737908228387656e-07,
|
|
"logits/chosen": -0.12426799535751343,
|
|
"logits/rejected": -0.14046964049339294,
|
|
"logps/chosen": -273.91925048828125,
|
|
"logps/ref_chosen": -65.73088073730469,
|
|
"logps/ref_rejected": -97.21781921386719,
|
|
"logps/rejected": -436.80267333984375,
|
|
"loss": 1.3592,
|
|
"margin_dpo/margin_mean": 131.3965301513672,
|
|
"margin_dpo/margin_std": 274.5325927734375,
|
|
"step": 160
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -171.75802612304688,
|
|
"KL/mean": -228.2024688720703,
|
|
"KL/rejected_KL_mean": -284.64691162109375,
|
|
"KL/std": 182.69235229492188,
|
|
"epoch": 0.23641703377386197,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 112.88887786865234,
|
|
"fcm_dpo/q_t": 0.4939511716365814,
|
|
"grad_norm": 1.585429310798645,
|
|
"learning_rate": 4.7321584725060594e-07,
|
|
"logits/chosen": -0.12636083364486694,
|
|
"logits/rejected": -0.1504145711660385,
|
|
"logps/chosen": -224.1945037841797,
|
|
"logps/ref_chosen": -52.43647003173828,
|
|
"logps/ref_rejected": -83.43095397949219,
|
|
"logps/rejected": -368.0778503417969,
|
|
"loss": 1.3628,
|
|
"margin_dpo/margin_mean": 112.88887786865234,
|
|
"margin_dpo/margin_std": 216.60546875,
|
|
"step": 161
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -180.044921875,
|
|
"KL/mean": -228.87269592285156,
|
|
"KL/rejected_KL_mean": -277.700439453125,
|
|
"KL/std": 198.00787353515625,
|
|
"epoch": 0.23788546255506607,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 97.65554809570312,
|
|
"fcm_dpo/q_t": 0.49477049708366394,
|
|
"grad_norm": 1.6253687143325806,
|
|
"learning_rate": 4.7263498971727905e-07,
|
|
"logits/chosen": -0.20148800313472748,
|
|
"logits/rejected": -0.2152554988861084,
|
|
"logps/chosen": -242.655517578125,
|
|
"logps/ref_chosen": -62.6105842590332,
|
|
"logps/ref_rejected": -89.39057922363281,
|
|
"logps/rejected": -367.091064453125,
|
|
"loss": 1.3662,
|
|
"margin_dpo/margin_mean": 97.65554809570312,
|
|
"margin_dpo/margin_std": 248.9880828857422,
|
|
"step": 162
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -198.3851776123047,
|
|
"KL/mean": -241.20689392089844,
|
|
"KL/rejected_KL_mean": -284.0285949707031,
|
|
"KL/std": 196.6535186767578,
|
|
"epoch": 0.2393538913362702,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 85.64341735839844,
|
|
"fcm_dpo/q_t": 0.4954114258289337,
|
|
"grad_norm": 1.5751463174819946,
|
|
"learning_rate": 4.720482655449212e-07,
|
|
"logits/chosen": -0.11102645099163055,
|
|
"logits/rejected": -0.11732280254364014,
|
|
"logps/chosen": -253.4068145751953,
|
|
"logps/ref_chosen": -55.021629333496094,
|
|
"logps/ref_rejected": -75.418212890625,
|
|
"logps/rejected": -359.44683837890625,
|
|
"loss": 1.3687,
|
|
"margin_dpo/margin_mean": 85.64341735839844,
|
|
"margin_dpo/margin_std": 241.90493774414062,
|
|
"step": 163
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -186.4625244140625,
|
|
"KL/mean": -250.75245666503906,
|
|
"KL/rejected_KL_mean": -315.0423583984375,
|
|
"KL/std": 190.93157958984375,
|
|
"epoch": 0.24082232011747431,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 128.579833984375,
|
|
"fcm_dpo/q_t": 0.49310967326164246,
|
|
"grad_norm": 1.862160325050354,
|
|
"learning_rate": 4.714556901942599e-07,
|
|
"logits/chosen": -0.07840365171432495,
|
|
"logits/rejected": -0.09606952965259552,
|
|
"logps/chosen": -242.10321044921875,
|
|
"logps/ref_chosen": -55.64066696166992,
|
|
"logps/ref_rejected": -79.66463470458984,
|
|
"logps/rejected": -394.7070007324219,
|
|
"loss": 1.3595,
|
|
"margin_dpo/margin_mean": 128.579833984375,
|
|
"margin_dpo/margin_std": 218.310302734375,
|
|
"step": 164
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -206.51309204101562,
|
|
"KL/mean": -245.36529541015625,
|
|
"KL/rejected_KL_mean": -284.2174987792969,
|
|
"KL/std": 193.79730224609375,
|
|
"epoch": 0.2422907488986784,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 77.70442199707031,
|
|
"fcm_dpo/q_t": 0.49583813548088074,
|
|
"grad_norm": 1.5955755710601807,
|
|
"learning_rate": 4.708572792802069e-07,
|
|
"logits/chosen": -0.10220310091972351,
|
|
"logits/rejected": -0.1038433313369751,
|
|
"logps/chosen": -267.82379150390625,
|
|
"logps/ref_chosen": -61.310691833496094,
|
|
"logps/ref_rejected": -73.67060852050781,
|
|
"logps/rejected": -357.88812255859375,
|
|
"loss": 1.3706,
|
|
"margin_dpo/margin_mean": 77.70442199707031,
|
|
"margin_dpo/margin_std": 272.9258117675781,
|
|
"step": 165
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -183.43780517578125,
|
|
"KL/mean": -275.9177551269531,
|
|
"KL/rejected_KL_mean": -368.397705078125,
|
|
"KL/std": 286.80517578125,
|
|
"epoch": 0.24375917767988253,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 184.95986938476562,
|
|
"fcm_dpo/q_t": 0.4901168942451477,
|
|
"grad_norm": 2.0007615089416504,
|
|
"learning_rate": 4.702530485714461e-07,
|
|
"logits/chosen": -0.11677154153585434,
|
|
"logits/rejected": -0.1256922483444214,
|
|
"logps/chosen": -234.42141723632812,
|
|
"logps/ref_chosen": -50.98360061645508,
|
|
"logps/ref_rejected": -98.09512329101562,
|
|
"logps/rejected": -466.4927978515625,
|
|
"loss": 1.3487,
|
|
"margin_dpo/margin_mean": 184.95986938476562,
|
|
"margin_dpo/margin_std": 381.46697998046875,
|
|
"step": 166
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -194.1734619140625,
|
|
"KL/mean": -280.3365173339844,
|
|
"KL/rejected_KL_mean": -366.4996032714844,
|
|
"KL/std": 239.65505981445312,
|
|
"epoch": 0.24522760646108663,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 172.326171875,
|
|
"fcm_dpo/q_t": 0.49077457189559937,
|
|
"grad_norm": 2.102647542953491,
|
|
"learning_rate": 4.6964301399001877e-07,
|
|
"logits/chosen": -0.1024477556347847,
|
|
"logits/rejected": -0.11726316809654236,
|
|
"logps/chosen": -244.59754943847656,
|
|
"logps/ref_chosen": -50.424095153808594,
|
|
"logps/ref_rejected": -96.03042602539062,
|
|
"logps/rejected": -462.530029296875,
|
|
"loss": 1.3508,
|
|
"margin_dpo/margin_mean": 172.32615661621094,
|
|
"margin_dpo/margin_std": 293.95623779296875,
|
|
"step": 167
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -213.13294982910156,
|
|
"KL/mean": -268.35931396484375,
|
|
"KL/rejected_KL_mean": -323.58563232421875,
|
|
"KL/std": 203.27450561523438,
|
|
"epoch": 0.24669603524229075,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 110.45269775390625,
|
|
"fcm_dpo/q_t": 0.49408331513404846,
|
|
"grad_norm": 1.7750434875488281,
|
|
"learning_rate": 4.690271916109034e-07,
|
|
"logits/chosen": -0.03718934208154678,
|
|
"logits/rejected": -0.04697761684656143,
|
|
"logps/chosen": -262.59576416015625,
|
|
"logps/ref_chosen": -49.462825775146484,
|
|
"logps/ref_rejected": -75.30855560302734,
|
|
"logps/rejected": -398.8941650390625,
|
|
"loss": 1.3635,
|
|
"margin_dpo/margin_mean": 110.45269775390625,
|
|
"margin_dpo/margin_std": 252.35952758789062,
|
|
"step": 168
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -244.06619262695312,
|
|
"KL/mean": -290.67816162109375,
|
|
"KL/rejected_KL_mean": -337.2901306152344,
|
|
"KL/std": 225.60400390625,
|
|
"epoch": 0.24816446402349487,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 93.22395324707031,
|
|
"fcm_dpo/q_t": 0.4950127601623535,
|
|
"grad_norm": 2.7092039585113525,
|
|
"learning_rate": 4.6840559766159235e-07,
|
|
"logits/chosen": -0.061293989419937134,
|
|
"logits/rejected": -0.06470146030187607,
|
|
"logps/chosen": -303.86962890625,
|
|
"logps/ref_chosen": -59.803443908691406,
|
|
"logps/ref_rejected": -83.34574890136719,
|
|
"logps/rejected": -420.6358947753906,
|
|
"loss": 1.3676,
|
|
"margin_dpo/margin_mean": 93.22394561767578,
|
|
"margin_dpo/margin_std": 315.66082763671875,
|
|
"step": 169
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -223.83151245117188,
|
|
"KL/mean": -274.68341064453125,
|
|
"KL/rejected_KL_mean": -325.53533935546875,
|
|
"KL/std": 236.1355743408203,
|
|
"epoch": 0.24963289280469897,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 101.70378112792969,
|
|
"fcm_dpo/q_t": 0.49454981088638306,
|
|
"grad_norm": 2.4471933841705322,
|
|
"learning_rate": 4.6777824852166437e-07,
|
|
"logits/chosen": -0.06102032959461212,
|
|
"logits/rejected": -0.07576321065425873,
|
|
"logps/chosen": -273.30328369140625,
|
|
"logps/ref_chosen": -49.471771240234375,
|
|
"logps/ref_rejected": -75.91734313964844,
|
|
"logps/rejected": -401.45269775390625,
|
|
"loss": 1.3656,
|
|
"margin_dpo/margin_mean": 101.70378112792969,
|
|
"margin_dpo/margin_std": 283.95513916015625,
|
|
"step": 170
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -346.0281066894531,
|
|
"KL/mean": -386.1025390625,
|
|
"KL/rejected_KL_mean": -426.177001953125,
|
|
"KL/std": 327.9516296386719,
|
|
"epoch": 0.2511013215859031,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 80.14892578125,
|
|
"fcm_dpo/q_t": 0.4957178831100464,
|
|
"grad_norm": 4.012329578399658,
|
|
"learning_rate": 4.6714516072235273e-07,
|
|
"logits/chosen": -0.04151609167456627,
|
|
"logits/rejected": -0.0675613209605217,
|
|
"logps/chosen": -430.52740478515625,
|
|
"logps/ref_chosen": -84.49931335449219,
|
|
"logps/ref_rejected": -109.38209533691406,
|
|
"logps/rejected": -535.55908203125,
|
|
"loss": 1.3712,
|
|
"margin_dpo/margin_mean": 80.14892578125,
|
|
"margin_dpo/margin_std": 414.32763671875,
|
|
"step": 171
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -278.4682312011719,
|
|
"KL/mean": -318.6019287109375,
|
|
"KL/rejected_KL_mean": -358.73565673828125,
|
|
"KL/std": 296.55731201171875,
|
|
"epoch": 0.2525697503671072,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 80.26742553710938,
|
|
"fcm_dpo/q_t": 0.49569272994995117,
|
|
"grad_norm": 2.9481098651885986,
|
|
"learning_rate": 4.6650635094610966e-07,
|
|
"logits/chosen": -0.057665470987558365,
|
|
"logits/rejected": -0.08217429369688034,
|
|
"logps/chosen": -347.12213134765625,
|
|
"logps/ref_chosen": -68.65391540527344,
|
|
"logps/ref_rejected": -85.43667602539062,
|
|
"logps/rejected": -444.1723327636719,
|
|
"loss": 1.3703,
|
|
"margin_dpo/margin_mean": 80.26742553710938,
|
|
"margin_dpo/margin_std": 307.19091796875,
|
|
"step": 172
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -234.19972229003906,
|
|
"KL/mean": -281.9803771972656,
|
|
"KL/rejected_KL_mean": -329.7610168457031,
|
|
"KL/std": 252.82623291015625,
|
|
"epoch": 0.2540381791483113,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 95.56130981445312,
|
|
"fcm_dpo/q_t": 0.4948866665363312,
|
|
"grad_norm": 1.8302026987075806,
|
|
"learning_rate": 4.6586183602616687e-07,
|
|
"logits/chosen": -0.0890713781118393,
|
|
"logits/rejected": -0.1072927713394165,
|
|
"logps/chosen": -297.2506103515625,
|
|
"logps/ref_chosen": -63.050880432128906,
|
|
"logps/ref_rejected": -78.68392181396484,
|
|
"logps/rejected": -408.4449462890625,
|
|
"loss": 1.3668,
|
|
"margin_dpo/margin_mean": 95.5613021850586,
|
|
"margin_dpo/margin_std": 271.940673828125,
|
|
"step": 173
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -216.39028930664062,
|
|
"KL/mean": -295.0453796386719,
|
|
"KL/rejected_KL_mean": -373.700439453125,
|
|
"KL/std": 275.541748046875,
|
|
"epoch": 0.2555066079295154,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 157.31015014648438,
|
|
"fcm_dpo/q_t": 0.49158352613449097,
|
|
"grad_norm": 3.099987268447876,
|
|
"learning_rate": 4.652116329460919e-07,
|
|
"logits/chosen": -0.02464154362678528,
|
|
"logits/rejected": -0.03820271044969559,
|
|
"logps/chosen": -269.7532653808594,
|
|
"logps/ref_chosen": -53.36296844482422,
|
|
"logps/ref_rejected": -101.91120910644531,
|
|
"logps/rejected": -475.61163330078125,
|
|
"loss": 1.3543,
|
|
"margin_dpo/margin_mean": 157.31015014648438,
|
|
"margin_dpo/margin_std": 358.8941650390625,
|
|
"step": 174
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -168.59716796875,
|
|
"KL/mean": -259.63287353515625,
|
|
"KL/rejected_KL_mean": -350.6685485839844,
|
|
"KL/std": 224.63267517089844,
|
|
"epoch": 0.25697503671071953,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 182.07138061523438,
|
|
"fcm_dpo/q_t": 0.4902472496032715,
|
|
"grad_norm": 3.9441075325012207,
|
|
"learning_rate": 4.645557588393406e-07,
|
|
"logits/chosen": -0.058357544243335724,
|
|
"logits/rejected": -0.0670979917049408,
|
|
"logps/chosen": -214.01492309570312,
|
|
"logps/ref_chosen": -45.417762756347656,
|
|
"logps/ref_rejected": -89.50579833984375,
|
|
"logps/rejected": -440.1743469238281,
|
|
"loss": 1.3484,
|
|
"margin_dpo/margin_mean": 182.07138061523438,
|
|
"margin_dpo/margin_std": 263.89202880859375,
|
|
"step": 175
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -193.49896240234375,
|
|
"KL/mean": -283.0251159667969,
|
|
"KL/rejected_KL_mean": -372.55126953125,
|
|
"KL/std": 272.05682373046875,
|
|
"epoch": 0.25844346549192365,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 179.05230712890625,
|
|
"fcm_dpo/q_t": 0.49042147397994995,
|
|
"grad_norm": 3.722703456878662,
|
|
"learning_rate": 4.638942309888058e-07,
|
|
"logits/chosen": -0.07522699236869812,
|
|
"logits/rejected": -0.08893322944641113,
|
|
"logps/chosen": -243.95181274414062,
|
|
"logps/ref_chosen": -50.452842712402344,
|
|
"logps/ref_rejected": -95.5589599609375,
|
|
"logps/rejected": -468.1102294921875,
|
|
"loss": 1.3495,
|
|
"margin_dpo/margin_mean": 179.05230712890625,
|
|
"margin_dpo/margin_std": 324.3958435058594,
|
|
"step": 176
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -248.76412963867188,
|
|
"KL/mean": -327.17547607421875,
|
|
"KL/rejected_KL_mean": -405.5867919921875,
|
|
"KL/std": 315.8409423828125,
|
|
"epoch": 0.2599118942731278,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 156.8226318359375,
|
|
"fcm_dpo/q_t": 0.4916095733642578,
|
|
"grad_norm": 2.4874935150146484,
|
|
"learning_rate": 4.6322706682636137e-07,
|
|
"logits/chosen": -0.09781675040721893,
|
|
"logits/rejected": -0.12420766055583954,
|
|
"logps/chosen": -309.9805908203125,
|
|
"logps/ref_chosen": -61.216468811035156,
|
|
"logps/ref_rejected": -95.89378356933594,
|
|
"logps/rejected": -501.4805908203125,
|
|
"loss": 1.3544,
|
|
"margin_dpo/margin_mean": 156.82264709472656,
|
|
"margin_dpo/margin_std": 351.21734619140625,
|
|
"step": 177
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -260.517578125,
|
|
"KL/mean": -355.90155029296875,
|
|
"KL/rejected_KL_mean": -451.2855224609375,
|
|
"KL/std": 332.3189697265625,
|
|
"epoch": 0.26138032305433184,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 190.7679443359375,
|
|
"fcm_dpo/q_t": 0.48978862166404724,
|
|
"grad_norm": 3.6528143882751465,
|
|
"learning_rate": 4.6255428393240354e-07,
|
|
"logits/chosen": -0.004009534604847431,
|
|
"logits/rejected": -0.022813618183135986,
|
|
"logps/chosen": -318.7823486328125,
|
|
"logps/ref_chosen": -58.26478958129883,
|
|
"logps/ref_rejected": -105.3653335571289,
|
|
"logps/rejected": -556.65087890625,
|
|
"loss": 1.3472,
|
|
"margin_dpo/margin_mean": 190.7679443359375,
|
|
"margin_dpo/margin_std": 349.21771240234375,
|
|
"step": 178
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -270.78173828125,
|
|
"KL/mean": -333.99163818359375,
|
|
"KL/rejected_KL_mean": -397.20159912109375,
|
|
"KL/std": 320.001708984375,
|
|
"epoch": 0.26284875183553597,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 126.41985321044922,
|
|
"fcm_dpo/q_t": 0.4932493567466736,
|
|
"grad_norm": 2.433837652206421,
|
|
"learning_rate": 4.6187590003538724e-07,
|
|
"logits/chosen": -0.06824183464050293,
|
|
"logits/rejected": -0.09712938964366913,
|
|
"logps/chosen": -331.84002685546875,
|
|
"logps/ref_chosen": -61.05832290649414,
|
|
"logps/ref_rejected": -90.52782440185547,
|
|
"logps/rejected": -487.72943115234375,
|
|
"loss": 1.3614,
|
|
"margin_dpo/margin_mean": 126.41983795166016,
|
|
"margin_dpo/margin_std": 422.2001037597656,
|
|
"step": 179
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -251.91395568847656,
|
|
"KL/mean": -345.0408935546875,
|
|
"KL/rejected_KL_mean": -438.16778564453125,
|
|
"KL/std": 260.55181884765625,
|
|
"epoch": 0.2643171806167401,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 186.25384521484375,
|
|
"fcm_dpo/q_t": 0.4900313913822174,
|
|
"grad_norm": 2.2225091457366943,
|
|
"learning_rate": 4.611919330113591e-07,
|
|
"logits/chosen": -0.046570487320423126,
|
|
"logits/rejected": -0.06946319341659546,
|
|
"logps/chosen": -306.2566833496094,
|
|
"logps/ref_chosen": -54.34272003173828,
|
|
"logps/ref_rejected": -98.21183776855469,
|
|
"logps/rejected": -536.379638671875,
|
|
"loss": 1.3482,
|
|
"margin_dpo/margin_mean": 186.25384521484375,
|
|
"margin_dpo/margin_std": 353.3677978515625,
|
|
"step": 180
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -218.62420654296875,
|
|
"KL/mean": -267.3025207519531,
|
|
"KL/rejected_KL_mean": -315.9808654785156,
|
|
"KL/std": 229.04489135742188,
|
|
"epoch": 0.2657856093979442,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 97.35667419433594,
|
|
"fcm_dpo/q_t": 0.49478650093078613,
|
|
"grad_norm": 2.206993341445923,
|
|
"learning_rate": 4.605024008834863e-07,
|
|
"logits/chosen": -0.05855263024568558,
|
|
"logits/rejected": -0.06639339029788971,
|
|
"logps/chosen": -273.6246643066406,
|
|
"logps/ref_chosen": -55.000457763671875,
|
|
"logps/ref_rejected": -61.656166076660156,
|
|
"logps/rejected": -377.63702392578125,
|
|
"loss": 1.3666,
|
|
"margin_dpo/margin_mean": 97.35667419433594,
|
|
"margin_dpo/margin_std": 312.8726806640625,
|
|
"step": 181
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -229.5802001953125,
|
|
"KL/mean": -339.115966796875,
|
|
"KL/rejected_KL_mean": -448.6517333984375,
|
|
"KL/std": 309.89312744140625,
|
|
"epoch": 0.26725403817914833,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 219.07156372070312,
|
|
"fcm_dpo/q_t": 0.48828235268592834,
|
|
"grad_norm": 2.526320695877075,
|
|
"learning_rate": 4.598073218215817e-07,
|
|
"logits/chosen": -0.07598879933357239,
|
|
"logits/rejected": -0.09682799130678177,
|
|
"logps/chosen": -270.68804931640625,
|
|
"logps/ref_chosen": -41.107852935791016,
|
|
"logps/ref_rejected": -89.5215835571289,
|
|
"logps/rejected": -538.17333984375,
|
|
"loss": 1.3417,
|
|
"margin_dpo/margin_mean": 219.07154846191406,
|
|
"margin_dpo/margin_std": 401.08624267578125,
|
|
"step": 182
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -323.77984619140625,
|
|
"KL/mean": -366.191162109375,
|
|
"KL/rejected_KL_mean": -408.6024475097656,
|
|
"KL/std": 279.89532470703125,
|
|
"epoch": 0.2687224669603524,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 84.82255554199219,
|
|
"fcm_dpo/q_t": 0.49546507000923157,
|
|
"grad_norm": 3.5669703483581543,
|
|
"learning_rate": 4.5910671414162484e-07,
|
|
"logits/chosen": -0.05871109664440155,
|
|
"logits/rejected": -0.08482161164283752,
|
|
"logps/chosen": -381.304443359375,
|
|
"logps/ref_chosen": -57.52456283569336,
|
|
"logps/ref_rejected": -75.97572326660156,
|
|
"logps/rejected": -484.57818603515625,
|
|
"loss": 1.3695,
|
|
"margin_dpo/margin_mean": 84.82254791259766,
|
|
"margin_dpo/margin_std": 320.90704345703125,
|
|
"step": 183
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -315.3003845214844,
|
|
"KL/mean": -351.2978515625,
|
|
"KL/rejected_KL_mean": -387.2952575683594,
|
|
"KL/std": 260.4176025390625,
|
|
"epoch": 0.2701908957415565,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 71.99484252929688,
|
|
"fcm_dpo/q_t": 0.4961445927619934,
|
|
"grad_norm": 2.970120668411255,
|
|
"learning_rate": 4.5840059630527985e-07,
|
|
"logits/chosen": -0.059889525175094604,
|
|
"logits/rejected": -0.0684148371219635,
|
|
"logps/chosen": -373.8453369140625,
|
|
"logps/ref_chosen": -58.544952392578125,
|
|
"logps/ref_rejected": -76.63406372070312,
|
|
"logps/rejected": -463.9293212890625,
|
|
"loss": 1.3723,
|
|
"margin_dpo/margin_mean": 71.9948501586914,
|
|
"margin_dpo/margin_std": 333.66949462890625,
|
|
"step": 184
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -327.66729736328125,
|
|
"KL/mean": -346.7569580078125,
|
|
"KL/rejected_KL_mean": -365.84661865234375,
|
|
"KL/std": 300.96337890625,
|
|
"epoch": 0.27165932452276065,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 38.17930221557617,
|
|
"fcm_dpo/q_t": 0.49796029925346375,
|
|
"grad_norm": 3.1760153770446777,
|
|
"learning_rate": 4.5768898691940836e-07,
|
|
"logits/chosen": -0.02878168597817421,
|
|
"logits/rejected": -0.03691637143492699,
|
|
"logps/chosen": -389.69317626953125,
|
|
"logps/ref_chosen": -62.025848388671875,
|
|
"logps/ref_rejected": -73.7625961303711,
|
|
"logps/rejected": -439.6092224121094,
|
|
"loss": 1.3798,
|
|
"margin_dpo/margin_mean": 38.17930221557617,
|
|
"margin_dpo/margin_std": 370.1015625,
|
|
"step": 185
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -320.67669677734375,
|
|
"KL/mean": -373.763427734375,
|
|
"KL/rejected_KL_mean": -426.8501892089844,
|
|
"KL/std": 282.6938171386719,
|
|
"epoch": 0.27312775330396477,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 106.17350769042969,
|
|
"fcm_dpo/q_t": 0.4943085312843323,
|
|
"grad_norm": 2.898653745651245,
|
|
"learning_rate": 4.5697190473557947e-07,
|
|
"logits/chosen": -0.1143377274274826,
|
|
"logits/rejected": -0.15146398544311523,
|
|
"logps/chosen": -390.0301513671875,
|
|
"logps/ref_chosen": -69.35346984863281,
|
|
"logps/ref_rejected": -88.07244873046875,
|
|
"logps/rejected": -514.9226684570312,
|
|
"loss": 1.3653,
|
|
"margin_dpo/margin_mean": 106.17350769042969,
|
|
"margin_dpo/margin_std": 373.8057556152344,
|
|
"step": 186
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -235.82952880859375,
|
|
"KL/mean": -303.52093505859375,
|
|
"KL/rejected_KL_mean": -371.21234130859375,
|
|
"KL/std": 257.0897521972656,
|
|
"epoch": 0.2745961820851689,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 135.3828125,
|
|
"fcm_dpo/q_t": 0.4927484393119812,
|
|
"grad_norm": 2.4242630004882812,
|
|
"learning_rate": 4.5624936864957555e-07,
|
|
"logits/chosen": -0.0784909650683403,
|
|
"logits/rejected": -0.09007595479488373,
|
|
"logps/chosen": -288.58599853515625,
|
|
"logps/ref_chosen": -52.7564582824707,
|
|
"logps/ref_rejected": -81.96910095214844,
|
|
"logps/rejected": -453.18145751953125,
|
|
"loss": 1.3583,
|
|
"margin_dpo/margin_mean": 135.38282775878906,
|
|
"margin_dpo/margin_std": 272.33575439453125,
|
|
"step": 187
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -211.16748046875,
|
|
"KL/mean": -291.66912841796875,
|
|
"KL/rejected_KL_mean": -372.1707763671875,
|
|
"KL/std": 238.9845733642578,
|
|
"epoch": 0.27606461086637296,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 161.00326538085938,
|
|
"fcm_dpo/q_t": 0.4913794994354248,
|
|
"grad_norm": 2.7519357204437256,
|
|
"learning_rate": 4.5552139770089454e-07,
|
|
"logits/chosen": -0.10255219042301178,
|
|
"logits/rejected": -0.13276180624961853,
|
|
"logps/chosen": -260.58294677734375,
|
|
"logps/ref_chosen": -49.415489196777344,
|
|
"logps/ref_rejected": -89.54043579101562,
|
|
"logps/rejected": -461.711181640625,
|
|
"loss": 1.353,
|
|
"margin_dpo/margin_mean": 161.00326538085938,
|
|
"margin_dpo/margin_std": 281.9280090332031,
|
|
"step": 188
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -245.26499938964844,
|
|
"KL/mean": -297.78826904296875,
|
|
"KL/rejected_KL_mean": -350.3115234375,
|
|
"KL/std": 250.982666015625,
|
|
"epoch": 0.2775330396475771,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 105.0465087890625,
|
|
"fcm_dpo/q_t": 0.49437761306762695,
|
|
"grad_norm": 3.0305230617523193,
|
|
"learning_rate": 4.5478801107224794e-07,
|
|
"logits/chosen": -0.13245689868927002,
|
|
"logits/rejected": -0.15994009375572205,
|
|
"logps/chosen": -297.6639709472656,
|
|
"logps/ref_chosen": -52.39896011352539,
|
|
"logps/ref_rejected": -72.16735076904297,
|
|
"logps/rejected": -422.4788513183594,
|
|
"loss": 1.3651,
|
|
"margin_dpo/margin_mean": 105.0465087890625,
|
|
"margin_dpo/margin_std": 324.1693115234375,
|
|
"step": 189
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -259.7969970703125,
|
|
"KL/mean": -342.51513671875,
|
|
"KL/rejected_KL_mean": -425.2333068847656,
|
|
"KL/std": 296.94696044921875,
|
|
"epoch": 0.2790014684287812,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 165.436279296875,
|
|
"fcm_dpo/q_t": 0.4911511242389679,
|
|
"grad_norm": 2.5856246948242188,
|
|
"learning_rate": 4.5404922808905543e-07,
|
|
"logits/chosen": -0.1271246075630188,
|
|
"logits/rejected": -0.15945151448249817,
|
|
"logps/chosen": -324.4801025390625,
|
|
"logps/ref_chosen": -64.68305969238281,
|
|
"logps/ref_rejected": -102.55052185058594,
|
|
"logps/rejected": -527.7838134765625,
|
|
"loss": 1.3527,
|
|
"margin_dpo/margin_mean": 165.436279296875,
|
|
"margin_dpo/margin_std": 360.4592590332031,
|
|
"step": 190
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -253.1461181640625,
|
|
"KL/mean": -364.49249267578125,
|
|
"KL/rejected_KL_mean": -475.8388977050781,
|
|
"KL/std": 322.4438171386719,
|
|
"epoch": 0.28046989720998533,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 222.6928253173828,
|
|
"fcm_dpo/q_t": 0.4880927801132202,
|
|
"grad_norm": 3.680899143218994,
|
|
"learning_rate": 4.5330506821893565e-07,
|
|
"logits/chosen": -0.12386652827262878,
|
|
"logits/rejected": -0.11776319891214371,
|
|
"logps/chosen": -321.80499267578125,
|
|
"logps/ref_chosen": -68.65887451171875,
|
|
"logps/ref_rejected": -110.1396713256836,
|
|
"logps/rejected": -585.9785766601562,
|
|
"loss": 1.3408,
|
|
"margin_dpo/margin_mean": 222.6928253173828,
|
|
"margin_dpo/margin_std": 386.87347412109375,
|
|
"step": 191
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -300.694091796875,
|
|
"KL/mean": -368.8135986328125,
|
|
"KL/rejected_KL_mean": -436.93310546875,
|
|
"KL/std": 301.52886962890625,
|
|
"epoch": 0.28193832599118945,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 136.23902893066406,
|
|
"fcm_dpo/q_t": 0.49271297454833984,
|
|
"grad_norm": 2.103487253189087,
|
|
"learning_rate": 4.5255555107119336e-07,
|
|
"logits/chosen": -0.09601673483848572,
|
|
"logits/rejected": -0.10303403437137604,
|
|
"logps/chosen": -370.4210205078125,
|
|
"logps/ref_chosen": -69.72691345214844,
|
|
"logps/ref_rejected": -103.32135009765625,
|
|
"logps/rejected": -540.2544555664062,
|
|
"loss": 1.359,
|
|
"margin_dpo/margin_mean": 136.23902893066406,
|
|
"margin_dpo/margin_std": 378.644287109375,
|
|
"step": 192
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -283.2509460449219,
|
|
"KL/mean": -325.3600769042969,
|
|
"KL/rejected_KL_mean": -367.46923828125,
|
|
"KL/std": 314.67529296875,
|
|
"epoch": 0.2834067547723935,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 84.21826171875,
|
|
"fcm_dpo/q_t": 0.49550098180770874,
|
|
"grad_norm": 2.3068125247955322,
|
|
"learning_rate": 4.5180069639630236e-07,
|
|
"logits/chosen": -0.07964509725570679,
|
|
"logits/rejected": -0.1134941577911377,
|
|
"logps/chosen": -343.44146728515625,
|
|
"logps/ref_chosen": -60.19049835205078,
|
|
"logps/ref_rejected": -76.40755462646484,
|
|
"logps/rejected": -443.87677001953125,
|
|
"loss": 1.3702,
|
|
"margin_dpo/margin_mean": 84.21826934814453,
|
|
"margin_dpo/margin_std": 401.8240966796875,
|
|
"step": 193
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -178.2296905517578,
|
|
"KL/mean": -237.00930786132812,
|
|
"KL/rejected_KL_mean": -295.7889099121094,
|
|
"KL/std": 197.4080810546875,
|
|
"epoch": 0.28487518355359764,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 117.55923461914062,
|
|
"fcm_dpo/q_t": 0.49369940161705017,
|
|
"grad_norm": 1.7572847604751587,
|
|
"learning_rate": 4.510405240853854e-07,
|
|
"logits/chosen": 0.0028687757439911366,
|
|
"logits/rejected": -0.012111449614167213,
|
|
"logps/chosen": -216.070068359375,
|
|
"logps/ref_chosen": -37.84037399291992,
|
|
"logps/ref_rejected": -60.684783935546875,
|
|
"logps/rejected": -356.47369384765625,
|
|
"loss": 1.3618,
|
|
"margin_dpo/margin_mean": 117.55921936035156,
|
|
"margin_dpo/margin_std": 219.46182250976562,
|
|
"step": 194
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -269.02020263671875,
|
|
"KL/mean": -363.59765625,
|
|
"KL/rejected_KL_mean": -458.17510986328125,
|
|
"KL/std": 306.4468994140625,
|
|
"epoch": 0.28634361233480177,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 189.15487670898438,
|
|
"fcm_dpo/q_t": 0.4898764491081238,
|
|
"grad_norm": 3.175015687942505,
|
|
"learning_rate": 4.5027505416968985e-07,
|
|
"logits/chosen": -0.10875138640403748,
|
|
"logits/rejected": -0.11520682275295258,
|
|
"logps/chosen": -323.9117736816406,
|
|
"logps/ref_chosen": -54.891571044921875,
|
|
"logps/ref_rejected": -96.77095794677734,
|
|
"logps/rejected": -554.946044921875,
|
|
"loss": 1.3473,
|
|
"margin_dpo/margin_mean": 189.15487670898438,
|
|
"margin_dpo/margin_std": 321.9908752441406,
|
|
"step": 195
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -243.0733642578125,
|
|
"KL/mean": -316.908447265625,
|
|
"KL/rejected_KL_mean": -390.7435302734375,
|
|
"KL/std": 265.1428527832031,
|
|
"epoch": 0.2878120411160059,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 147.670166015625,
|
|
"fcm_dpo/q_t": 0.4920935332775116,
|
|
"grad_norm": 2.1525228023529053,
|
|
"learning_rate": 4.495043068200599e-07,
|
|
"logits/chosen": -0.10881956666707993,
|
|
"logits/rejected": -0.12770894169807434,
|
|
"logps/chosen": -296.318603515625,
|
|
"logps/ref_chosen": -53.245243072509766,
|
|
"logps/ref_rejected": -76.05294799804688,
|
|
"logps/rejected": -466.7964782714844,
|
|
"loss": 1.3564,
|
|
"margin_dpo/margin_mean": 147.670166015625,
|
|
"margin_dpo/margin_std": 344.3466491699219,
|
|
"step": 196
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -274.3104248046875,
|
|
"KL/mean": -325.2745361328125,
|
|
"KL/rejected_KL_mean": -376.23858642578125,
|
|
"KL/std": 251.0189971923828,
|
|
"epoch": 0.28928046989721,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 101.92816925048828,
|
|
"fcm_dpo/q_t": 0.4945385158061981,
|
|
"grad_norm": 2.172877073287964,
|
|
"learning_rate": 4.4872830234640493e-07,
|
|
"logits/chosen": -0.10666961967945099,
|
|
"logits/rejected": -0.11786352097988129,
|
|
"logps/chosen": -334.73077392578125,
|
|
"logps/ref_chosen": -60.42033386230469,
|
|
"logps/ref_rejected": -77.20890808105469,
|
|
"logps/rejected": -453.447509765625,
|
|
"loss": 1.3655,
|
|
"margin_dpo/margin_mean": 101.92816162109375,
|
|
"margin_dpo/margin_std": 281.21978759765625,
|
|
"step": 197
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -272.42474365234375,
|
|
"KL/mean": -371.7640380859375,
|
|
"KL/rejected_KL_mean": -471.1033020019531,
|
|
"KL/std": 313.407470703125,
|
|
"epoch": 0.2907488986784141,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 198.67855834960938,
|
|
"fcm_dpo/q_t": 0.4893642067909241,
|
|
"grad_norm": 2.680528402328491,
|
|
"learning_rate": 4.479470611971645e-07,
|
|
"logits/chosen": -0.13605228066444397,
|
|
"logits/rejected": -0.1457493156194687,
|
|
"logps/chosen": -327.4609375,
|
|
"logps/ref_chosen": -55.03618621826172,
|
|
"logps/ref_rejected": -97.24325561523438,
|
|
"logps/rejected": -568.3465576171875,
|
|
"loss": 1.3453,
|
|
"margin_dpo/margin_mean": 198.67855834960938,
|
|
"margin_dpo/margin_std": 319.24285888671875,
|
|
"step": 198
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -274.304443359375,
|
|
"KL/mean": -358.85919189453125,
|
|
"KL/rejected_KL_mean": -443.41387939453125,
|
|
"KL/std": 275.95819091796875,
|
|
"epoch": 0.2922173274596182,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 169.10943603515625,
|
|
"fcm_dpo/q_t": 0.4909524619579315,
|
|
"grad_norm": 3.292473077774048,
|
|
"learning_rate": 4.471606039587695e-07,
|
|
"logits/chosen": -0.14171649515628815,
|
|
"logits/rejected": -0.13244497776031494,
|
|
"logps/chosen": -331.13330078125,
|
|
"logps/ref_chosen": -56.828826904296875,
|
|
"logps/ref_rejected": -84.64820861816406,
|
|
"logps/rejected": -528.0621337890625,
|
|
"loss": 1.352,
|
|
"margin_dpo/margin_mean": 169.10943603515625,
|
|
"margin_dpo/margin_std": 374.6290283203125,
|
|
"step": 199
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -292.3767395019531,
|
|
"KL/mean": -373.58990478515625,
|
|
"KL/rejected_KL_mean": -454.8030090332031,
|
|
"KL/std": 326.7904052734375,
|
|
"epoch": 0.2936857562408223,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 162.42630004882812,
|
|
"fcm_dpo/q_t": 0.49131280183792114,
|
|
"grad_norm": 3.315680980682373,
|
|
"learning_rate": 4.4636895135509966e-07,
|
|
"logits/chosen": -0.07947467267513275,
|
|
"logits/rejected": -0.10722815245389938,
|
|
"logps/chosen": -345.44378662109375,
|
|
"logps/ref_chosen": -53.06706237792969,
|
|
"logps/ref_rejected": -80.60843658447266,
|
|
"logps/rejected": -535.4114379882812,
|
|
"loss": 1.3542,
|
|
"margin_dpo/margin_mean": 162.42628479003906,
|
|
"margin_dpo/margin_std": 453.01910400390625,
|
|
"step": 200
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -343.0718688964844,
|
|
"KL/mean": -435.6768493652344,
|
|
"KL/rejected_KL_mean": -528.2818603515625,
|
|
"KL/std": 393.9222717285156,
|
|
"epoch": 0.29515418502202645,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 185.20999145507812,
|
|
"fcm_dpo/q_t": 0.49011439085006714,
|
|
"grad_norm": 3.324786424636841,
|
|
"learning_rate": 4.455721242469372e-07,
|
|
"logits/chosen": -0.23055091500282288,
|
|
"logits/rejected": -0.24232345819473267,
|
|
"logps/chosen": -418.4740905761719,
|
|
"logps/ref_chosen": -75.4022216796875,
|
|
"logps/ref_rejected": -114.80821990966797,
|
|
"logps/rejected": -643.090087890625,
|
|
"loss": 1.3498,
|
|
"margin_dpo/margin_mean": 185.20999145507812,
|
|
"margin_dpo/margin_std": 481.95977783203125,
|
|
"step": 201
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -312.4244689941406,
|
|
"KL/mean": -377.79998779296875,
|
|
"KL/rejected_KL_mean": -443.1754455566406,
|
|
"KL/std": 326.81707763671875,
|
|
"epoch": 0.2966226138032305,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 130.75100708007812,
|
|
"fcm_dpo/q_t": 0.4930059015750885,
|
|
"grad_norm": 3.468919515609741,
|
|
"learning_rate": 4.4477014363141755e-07,
|
|
"logits/chosen": -0.1302487850189209,
|
|
"logits/rejected": -0.14801092445850372,
|
|
"logps/chosen": -362.5257873535156,
|
|
"logps/ref_chosen": -50.101318359375,
|
|
"logps/ref_rejected": -86.98503112792969,
|
|
"logps/rejected": -530.1604614257812,
|
|
"loss": 1.3601,
|
|
"margin_dpo/margin_mean": 130.75100708007812,
|
|
"margin_dpo/margin_std": 375.7841796875,
|
|
"step": 202
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -355.20648193359375,
|
|
"KL/mean": -438.63665771484375,
|
|
"KL/rejected_KL_mean": -522.0668334960938,
|
|
"KL/std": 384.030029296875,
|
|
"epoch": 0.29809104258443464,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 166.86033630371094,
|
|
"fcm_dpo/q_t": 0.491105854511261,
|
|
"grad_norm": 3.1627941131591797,
|
|
"learning_rate": 4.439630306414758e-07,
|
|
"logits/chosen": -0.12041978538036346,
|
|
"logits/rejected": -0.12802743911743164,
|
|
"logps/chosen": -415.816162109375,
|
|
"logps/ref_chosen": -60.60969543457031,
|
|
"logps/ref_rejected": -85.89596557617188,
|
|
"logps/rejected": -607.9627685546875,
|
|
"loss": 1.3532,
|
|
"margin_dpo/margin_mean": 166.86032104492188,
|
|
"margin_dpo/margin_std": 459.39349365234375,
|
|
"step": 203
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -420.5950622558594,
|
|
"KL/mean": -458.009033203125,
|
|
"KL/rejected_KL_mean": -495.42303466796875,
|
|
"KL/std": 421.4627990722656,
|
|
"epoch": 0.29955947136563876,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 74.8279800415039,
|
|
"fcm_dpo/q_t": 0.49598783254623413,
|
|
"grad_norm": 4.85085391998291,
|
|
"learning_rate": 4.431508065452897e-07,
|
|
"logits/chosen": -0.22811344265937805,
|
|
"logits/rejected": -0.23437625169754028,
|
|
"logps/chosen": -500.760009765625,
|
|
"logps/ref_chosen": -80.16496276855469,
|
|
"logps/ref_rejected": -87.69590759277344,
|
|
"logps/rejected": -583.1189575195312,
|
|
"loss": 1.3732,
|
|
"margin_dpo/margin_mean": 74.8279800415039,
|
|
"margin_dpo/margin_std": 487.329833984375,
|
|
"step": 204
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -364.5575256347656,
|
|
"KL/mean": -444.526611328125,
|
|
"KL/rejected_KL_mean": -524.4957275390625,
|
|
"KL/std": 428.46484375,
|
|
"epoch": 0.3010279001468429,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 159.93820190429688,
|
|
"fcm_dpo/q_t": 0.49142158031463623,
|
|
"grad_norm": 4.245858192443848,
|
|
"learning_rate": 4.4233349274571974e-07,
|
|
"logits/chosen": -0.1302354335784912,
|
|
"logits/rejected": -0.14217376708984375,
|
|
"logps/chosen": -423.9422607421875,
|
|
"logps/ref_chosen": -59.384735107421875,
|
|
"logps/ref_rejected": -85.12505340576172,
|
|
"logps/rejected": -609.6207885742188,
|
|
"loss": 1.3552,
|
|
"margin_dpo/margin_mean": 159.93820190429688,
|
|
"margin_dpo/margin_std": 488.643310546875,
|
|
"step": 205
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -275.8797607421875,
|
|
"KL/mean": -371.9986572265625,
|
|
"KL/rejected_KL_mean": -468.1175842285156,
|
|
"KL/std": 284.67950439453125,
|
|
"epoch": 0.302496328928047,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 192.2378387451172,
|
|
"fcm_dpo/q_t": 0.489697128534317,
|
|
"grad_norm": 3.0303680896759033,
|
|
"learning_rate": 4.415111107797445e-07,
|
|
"logits/chosen": -0.15259331464767456,
|
|
"logits/rejected": -0.15699225664138794,
|
|
"logps/chosen": -322.84423828125,
|
|
"logps/ref_chosen": -46.964500427246094,
|
|
"logps/ref_rejected": -98.9534912109375,
|
|
"logps/rejected": -567.071044921875,
|
|
"loss": 1.3468,
|
|
"margin_dpo/margin_mean": 192.2378387451172,
|
|
"margin_dpo/margin_std": 329.2159423828125,
|
|
"step": 206
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -274.165283203125,
|
|
"KL/mean": -373.14715576171875,
|
|
"KL/rejected_KL_mean": -472.1290283203125,
|
|
"KL/std": 336.2535095214844,
|
|
"epoch": 0.3039647577092511,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 197.9637451171875,
|
|
"fcm_dpo/q_t": 0.4894152283668518,
|
|
"grad_norm": 4.557210445404053,
|
|
"learning_rate": 4.4068368231789365e-07,
|
|
"logits/chosen": -0.20802870392799377,
|
|
"logits/rejected": -0.2097472846508026,
|
|
"logps/chosen": -330.2215576171875,
|
|
"logps/ref_chosen": -56.05625915527344,
|
|
"logps/ref_rejected": -84.44779968261719,
|
|
"logps/rejected": -556.5768432617188,
|
|
"loss": 1.3464,
|
|
"margin_dpo/margin_mean": 197.9637451171875,
|
|
"margin_dpo/margin_std": 424.3826904296875,
|
|
"step": 207
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -368.54052734375,
|
|
"KL/mean": -451.637451171875,
|
|
"KL/rejected_KL_mean": -534.734375,
|
|
"KL/std": 326.3774108886719,
|
|
"epoch": 0.3054331864904552,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 166.19386291503906,
|
|
"fcm_dpo/q_t": 0.49110543727874756,
|
|
"grad_norm": 2.7596354484558105,
|
|
"learning_rate": 4.398512291636768e-07,
|
|
"logits/chosen": -0.20875459909439087,
|
|
"logits/rejected": -0.21936002373695374,
|
|
"logps/chosen": -435.6081237792969,
|
|
"logps/ref_chosen": -67.06761169433594,
|
|
"logps/ref_rejected": -94.28689575195312,
|
|
"logps/rejected": -629.0213012695312,
|
|
"loss": 1.353,
|
|
"margin_dpo/margin_mean": 166.19387817382812,
|
|
"margin_dpo/margin_std": 422.05780029296875,
|
|
"step": 208
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -301.03485107421875,
|
|
"KL/mean": -364.5048828125,
|
|
"KL/rejected_KL_mean": -427.9748840332031,
|
|
"KL/std": 285.0235900878906,
|
|
"epoch": 0.3069016152716593,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 126.94000244140625,
|
|
"fcm_dpo/q_t": 0.4932047724723816,
|
|
"grad_norm": 2.5512523651123047,
|
|
"learning_rate": 4.3901377325300857e-07,
|
|
"logits/chosen": -0.1063951924443245,
|
|
"logits/rejected": -0.124468132853508,
|
|
"logps/chosen": -357.216552734375,
|
|
"logps/ref_chosen": -56.18169403076172,
|
|
"logps/ref_rejected": -80.94152069091797,
|
|
"logps/rejected": -508.9163818359375,
|
|
"loss": 1.3607,
|
|
"margin_dpo/margin_mean": 126.94000244140625,
|
|
"margin_dpo/margin_std": 349.48199462890625,
|
|
"step": 209
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -243.17129516601562,
|
|
"KL/mean": -323.7019958496094,
|
|
"KL/rejected_KL_mean": -404.2326965332031,
|
|
"KL/std": 286.4044189453125,
|
|
"epoch": 0.30837004405286345,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 161.06141662597656,
|
|
"fcm_dpo/q_t": 0.49137187004089355,
|
|
"grad_norm": 3.259232997894287,
|
|
"learning_rate": 4.381713366536311e-07,
|
|
"logits/chosen": -0.1291487067937851,
|
|
"logits/rejected": -0.14884674549102783,
|
|
"logps/chosen": -289.5430908203125,
|
|
"logps/ref_chosen": -46.371822357177734,
|
|
"logps/ref_rejected": -76.68162536621094,
|
|
"logps/rejected": -480.914306640625,
|
|
"loss": 1.3535,
|
|
"margin_dpo/margin_mean": 161.06143188476562,
|
|
"margin_dpo/margin_std": 353.78619384765625,
|
|
"step": 210
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -375.2530517578125,
|
|
"KL/mean": -432.86187744140625,
|
|
"KL/rejected_KL_mean": -490.470703125,
|
|
"KL/std": 338.1985778808594,
|
|
"epoch": 0.30983847283406757,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 115.21764373779297,
|
|
"fcm_dpo/q_t": 0.4938422441482544,
|
|
"grad_norm": 2.629580497741699,
|
|
"learning_rate": 4.373239415645323e-07,
|
|
"logits/chosen": -0.13434600830078125,
|
|
"logits/rejected": -0.1492466777563095,
|
|
"logps/chosen": -454.1854248046875,
|
|
"logps/ref_chosen": -78.93235778808594,
|
|
"logps/ref_rejected": -86.82098388671875,
|
|
"logps/rejected": -577.2916870117188,
|
|
"loss": 1.3638,
|
|
"margin_dpo/margin_mean": 115.21763610839844,
|
|
"margin_dpo/margin_std": 410.43426513671875,
|
|
"step": 211
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -300.54364013671875,
|
|
"KL/mean": -434.3825988769531,
|
|
"KL/rejected_KL_mean": -568.2215576171875,
|
|
"KL/std": 372.66168212890625,
|
|
"epoch": 0.31130690161527164,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 267.6778869628906,
|
|
"fcm_dpo/q_t": 0.4856899082660675,
|
|
"grad_norm": 5.687210559844971,
|
|
"learning_rate": 4.3647161031536086e-07,
|
|
"logits/chosen": -0.11962936818599701,
|
|
"logits/rejected": -0.1340862214565277,
|
|
"logps/chosen": -358.74066162109375,
|
|
"logps/ref_chosen": -58.19701385498047,
|
|
"logps/ref_rejected": -103.05785369873047,
|
|
"logps/rejected": -671.2794189453125,
|
|
"loss": 1.332,
|
|
"margin_dpo/margin_mean": 267.6778869628906,
|
|
"margin_dpo/margin_std": 430.5390625,
|
|
"step": 212
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -320.9078063964844,
|
|
"KL/mean": -405.0869140625,
|
|
"KL/rejected_KL_mean": -489.2659912109375,
|
|
"KL/std": 353.5803527832031,
|
|
"epoch": 0.31277533039647576,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 168.35821533203125,
|
|
"fcm_dpo/q_t": 0.4909900426864624,
|
|
"grad_norm": 3.844770908355713,
|
|
"learning_rate": 4.3561436536583774e-07,
|
|
"logits/chosen": -0.09740199148654938,
|
|
"logits/rejected": -0.12570348381996155,
|
|
"logps/chosen": -388.4205322265625,
|
|
"logps/ref_chosen": -67.51271057128906,
|
|
"logps/ref_rejected": -93.91471862792969,
|
|
"logps/rejected": -583.1807250976562,
|
|
"loss": 1.3531,
|
|
"margin_dpo/margin_mean": 168.35821533203125,
|
|
"margin_dpo/margin_std": 470.94073486328125,
|
|
"step": 213
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -238.504638671875,
|
|
"KL/mean": -338.5084228515625,
|
|
"KL/rejected_KL_mean": -438.51214599609375,
|
|
"KL/std": 269.5102233886719,
|
|
"epoch": 0.3142437591776799,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 200.00753784179688,
|
|
"fcm_dpo/q_t": 0.4892929792404175,
|
|
"grad_norm": 3.0564281940460205,
|
|
"learning_rate": 4.3475222930516473e-07,
|
|
"logits/chosen": -0.06811121106147766,
|
|
"logits/rejected": -0.07607707381248474,
|
|
"logps/chosen": -280.1095275878906,
|
|
"logps/ref_chosen": -41.604888916015625,
|
|
"logps/ref_rejected": -77.51741027832031,
|
|
"logps/rejected": -516.029541015625,
|
|
"loss": 1.3451,
|
|
"margin_dpo/margin_mean": 200.0075225830078,
|
|
"margin_dpo/margin_std": 330.0015869140625,
|
|
"step": 214
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -308.419677734375,
|
|
"KL/mean": -413.72235107421875,
|
|
"KL/rejected_KL_mean": -519.0250854492188,
|
|
"KL/std": 338.5547790527344,
|
|
"epoch": 0.315712187958884,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 210.60540771484375,
|
|
"fcm_dpo/q_t": 0.4887344241142273,
|
|
"grad_norm": 4.060998916625977,
|
|
"learning_rate": 4.3388522485142885e-07,
|
|
"logits/chosen": -0.1189073771238327,
|
|
"logits/rejected": -0.13333210349082947,
|
|
"logps/chosen": -361.69891357421875,
|
|
"logps/ref_chosen": -53.279266357421875,
|
|
"logps/ref_rejected": -89.96464538574219,
|
|
"logps/rejected": -608.98974609375,
|
|
"loss": 1.3432,
|
|
"margin_dpo/margin_mean": 210.60540771484375,
|
|
"margin_dpo/margin_std": 367.251708984375,
|
|
"step": 215
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -300.3120422363281,
|
|
"KL/mean": -386.34326171875,
|
|
"KL/rejected_KL_mean": -472.37445068359375,
|
|
"KL/std": 297.8331298828125,
|
|
"epoch": 0.31718061674008813,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 172.0624237060547,
|
|
"fcm_dpo/q_t": 0.49078911542892456,
|
|
"grad_norm": 3.7745511531829834,
|
|
"learning_rate": 4.330133748510036e-07,
|
|
"logits/chosen": -0.12054896354675293,
|
|
"logits/rejected": -0.12499400973320007,
|
|
"logps/chosen": -349.1998291015625,
|
|
"logps/ref_chosen": -48.887794494628906,
|
|
"logps/ref_rejected": -77.19892883300781,
|
|
"logps/rejected": -549.5733642578125,
|
|
"loss": 1.3514,
|
|
"margin_dpo/margin_mean": 172.0624237060547,
|
|
"margin_dpo/margin_std": 384.0893249511719,
|
|
"step": 216
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -347.5061340332031,
|
|
"KL/mean": -462.15380859375,
|
|
"KL/rejected_KL_mean": -576.801513671875,
|
|
"KL/std": 366.660888671875,
|
|
"epoch": 0.3186490455212922,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 229.29534912109375,
|
|
"fcm_dpo/q_t": 0.48774290084838867,
|
|
"grad_norm": 3.4250800609588623,
|
|
"learning_rate": 4.3213670227794757e-07,
|
|
"logits/chosen": -0.09066063165664673,
|
|
"logits/rejected": -0.10337048768997192,
|
|
"logps/chosen": -397.3514404296875,
|
|
"logps/ref_chosen": -49.845306396484375,
|
|
"logps/ref_rejected": -100.07832336425781,
|
|
"logps/rejected": -676.8798217773438,
|
|
"loss": 1.3402,
|
|
"margin_dpo/margin_mean": 229.29534912109375,
|
|
"margin_dpo/margin_std": 456.92340087890625,
|
|
"step": 217
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -365.1001892089844,
|
|
"KL/mean": -444.54656982421875,
|
|
"KL/rejected_KL_mean": -523.9929809570312,
|
|
"KL/std": 365.8412170410156,
|
|
"epoch": 0.3201174743024963,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 158.892822265625,
|
|
"fcm_dpo/q_t": 0.4915262460708618,
|
|
"grad_norm": 3.7529687881469727,
|
|
"learning_rate": 4.3125523023339815e-07,
|
|
"logits/chosen": -0.14683587849140167,
|
|
"logits/rejected": -0.16048407554626465,
|
|
"logps/chosen": -423.6768798828125,
|
|
"logps/ref_chosen": -58.576683044433594,
|
|
"logps/ref_rejected": -87.84639739990234,
|
|
"logps/rejected": -611.83935546875,
|
|
"loss": 1.3551,
|
|
"margin_dpo/margin_mean": 158.892822265625,
|
|
"margin_dpo/margin_std": 475.37939453125,
|
|
"step": 218
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -386.04595947265625,
|
|
"KL/mean": -442.3204650878906,
|
|
"KL/rejected_KL_mean": -498.594970703125,
|
|
"KL/std": 393.7286376953125,
|
|
"epoch": 0.32158590308370044,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 112.54901123046875,
|
|
"fcm_dpo/q_t": 0.4940031170845032,
|
|
"grad_norm": 4.430355548858643,
|
|
"learning_rate": 4.303689819449636e-07,
|
|
"logits/chosen": -0.19574600458145142,
|
|
"logits/rejected": -0.21589338779449463,
|
|
"logps/chosen": -447.12982177734375,
|
|
"logps/ref_chosen": -61.083858489990234,
|
|
"logps/ref_rejected": -85.83042907714844,
|
|
"logps/rejected": -584.4254150390625,
|
|
"loss": 1.3655,
|
|
"margin_dpo/margin_mean": 112.54901885986328,
|
|
"margin_dpo/margin_std": 510.53802490234375,
|
|
"step": 219
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -407.1309814453125,
|
|
"KL/mean": -457.27777099609375,
|
|
"KL/rejected_KL_mean": -507.4245910644531,
|
|
"KL/std": 324.55487060546875,
|
|
"epoch": 0.32305433186490456,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 100.29361724853516,
|
|
"fcm_dpo/q_t": 0.49463126063346863,
|
|
"grad_norm": 3.9129955768585205,
|
|
"learning_rate": 4.2947798076611047e-07,
|
|
"logits/chosen": -0.22126300632953644,
|
|
"logits/rejected": -0.21606583893299103,
|
|
"logps/chosen": -477.16229248046875,
|
|
"logps/ref_chosen": -70.03128051757812,
|
|
"logps/ref_rejected": -87.68551635742188,
|
|
"logps/rejected": -595.110107421875,
|
|
"loss": 1.3665,
|
|
"margin_dpo/margin_mean": 100.29361724853516,
|
|
"margin_dpo/margin_std": 378.6761169433594,
|
|
"step": 220
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -290.21429443359375,
|
|
"KL/mean": -445.29681396484375,
|
|
"KL/rejected_KL_mean": -600.3792724609375,
|
|
"KL/std": 374.867919921875,
|
|
"epoch": 0.3245227606461087,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 310.1649475097656,
|
|
"fcm_dpo/q_t": 0.4834199547767639,
|
|
"grad_norm": 7.014521598815918,
|
|
"learning_rate": 4.285822501755485e-07,
|
|
"logits/chosen": -0.23672357201576233,
|
|
"logits/rejected": -0.24645334482192993,
|
|
"logps/chosen": -342.3690185546875,
|
|
"logps/ref_chosen": -52.15470886230469,
|
|
"logps/ref_rejected": -106.46768188476562,
|
|
"logps/rejected": -706.846923828125,
|
|
"loss": 1.323,
|
|
"margin_dpo/margin_mean": 310.1649475097656,
|
|
"margin_dpo/margin_std": 433.8343811035156,
|
|
"step": 221
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -317.06378173828125,
|
|
"KL/mean": -416.68621826171875,
|
|
"KL/rejected_KL_mean": -516.3086547851562,
|
|
"KL/std": 329.75885009765625,
|
|
"epoch": 0.32599118942731276,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 199.24484252929688,
|
|
"fcm_dpo/q_t": 0.48933765292167664,
|
|
"grad_norm": 3.5604019165039062,
|
|
"learning_rate": 4.276818137766118e-07,
|
|
"logits/chosen": -0.24718379974365234,
|
|
"logits/rejected": -0.27352237701416016,
|
|
"logps/chosen": -378.0348815917969,
|
|
"logps/ref_chosen": -60.971099853515625,
|
|
"logps/ref_rejected": -100.00115203857422,
|
|
"logps/rejected": -616.309814453125,
|
|
"loss": 1.3459,
|
|
"margin_dpo/margin_mean": 199.24484252929688,
|
|
"margin_dpo/margin_std": 405.140869140625,
|
|
"step": 222
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -326.71197509765625,
|
|
"KL/mean": -409.00048828125,
|
|
"KL/rejected_KL_mean": -491.2890319824219,
|
|
"KL/std": 325.3358154296875,
|
|
"epoch": 0.3274596182085169,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 164.57704162597656,
|
|
"fcm_dpo/q_t": 0.49121803045272827,
|
|
"grad_norm": 3.8082237243652344,
|
|
"learning_rate": 4.2677669529663686e-07,
|
|
"logits/chosen": -0.13181817531585693,
|
|
"logits/rejected": -0.14403775334358215,
|
|
"logps/chosen": -379.3525390625,
|
|
"logps/ref_chosen": -52.64057540893555,
|
|
"logps/ref_rejected": -82.82502746582031,
|
|
"logps/rejected": -574.1140747070312,
|
|
"loss": 1.3537,
|
|
"margin_dpo/margin_mean": 164.57704162597656,
|
|
"margin_dpo/margin_std": 439.234619140625,
|
|
"step": 223
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -307.269287109375,
|
|
"KL/mean": -398.96954345703125,
|
|
"KL/rejected_KL_mean": -490.6697998046875,
|
|
"KL/std": 385.791748046875,
|
|
"epoch": 0.328928046989721,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 183.40045166015625,
|
|
"fcm_dpo/q_t": 0.49022960662841797,
|
|
"grad_norm": 3.7661983966827393,
|
|
"learning_rate": 4.2586691858633747e-07,
|
|
"logits/chosen": -0.1584322154521942,
|
|
"logits/rejected": -0.1633099913597107,
|
|
"logps/chosen": -355.8647155761719,
|
|
"logps/ref_chosen": -48.59541320800781,
|
|
"logps/ref_rejected": -77.11648559570312,
|
|
"logps/rejected": -567.7862548828125,
|
|
"loss": 1.3501,
|
|
"margin_dpo/margin_mean": 183.40045166015625,
|
|
"margin_dpo/margin_std": 472.4094543457031,
|
|
"step": 224
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -362.7328186035156,
|
|
"KL/mean": -477.8741760253906,
|
|
"KL/rejected_KL_mean": -593.0155639648438,
|
|
"KL/std": 375.5802001953125,
|
|
"epoch": 0.3303964757709251,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 230.28273010253906,
|
|
"fcm_dpo/q_t": 0.4877273440361023,
|
|
"grad_norm": 3.3898706436157227,
|
|
"learning_rate": 4.249525076191759e-07,
|
|
"logits/chosen": -0.14341585338115692,
|
|
"logits/rejected": -0.15620696544647217,
|
|
"logps/chosen": -420.7332763671875,
|
|
"logps/ref_chosen": -58.000465393066406,
|
|
"logps/ref_rejected": -99.90291595458984,
|
|
"logps/rejected": -692.91845703125,
|
|
"loss": 1.3406,
|
|
"margin_dpo/margin_mean": 230.28274536132812,
|
|
"margin_dpo/margin_std": 511.2721862792969,
|
|
"step": 225
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -322.0030517578125,
|
|
"KL/mean": -404.18328857421875,
|
|
"KL/rejected_KL_mean": -486.36346435546875,
|
|
"KL/std": 324.75518798828125,
|
|
"epoch": 0.33186490455212925,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 164.36044311523438,
|
|
"fcm_dpo/q_t": 0.49121221899986267,
|
|
"grad_norm": 3.274152994155884,
|
|
"learning_rate": 4.2403348649073167e-07,
|
|
"logits/chosen": -0.19513732194900513,
|
|
"logits/rejected": -0.19808658957481384,
|
|
"logps/chosen": -380.90185546875,
|
|
"logps/ref_chosen": -58.898799896240234,
|
|
"logps/ref_rejected": -78.68775939941406,
|
|
"logps/rejected": -565.05126953125,
|
|
"loss": 1.3535,
|
|
"margin_dpo/margin_mean": 164.36044311523438,
|
|
"margin_dpo/margin_std": 427.1483154296875,
|
|
"step": 226
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -357.85113525390625,
|
|
"KL/mean": -482.2805480957031,
|
|
"KL/rejected_KL_mean": -606.7099609375,
|
|
"KL/std": 433.9095458984375,
|
|
"epoch": 0.3333333333333333,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 248.85882568359375,
|
|
"fcm_dpo/q_t": 0.48674476146698,
|
|
"grad_norm": 3.5155231952667236,
|
|
"learning_rate": 4.2310987941806615e-07,
|
|
"logits/chosen": -0.20451249182224274,
|
|
"logits/rejected": -0.20552946627140045,
|
|
"logps/chosen": -416.92333984375,
|
|
"logps/ref_chosen": -59.072181701660156,
|
|
"logps/ref_rejected": -99.41236877441406,
|
|
"logps/rejected": -706.122314453125,
|
|
"loss": 1.3364,
|
|
"margin_dpo/margin_mean": 248.85882568359375,
|
|
"margin_dpo/margin_std": 477.48724365234375,
|
|
"step": 227
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -384.4470520019531,
|
|
"KL/mean": -456.94488525390625,
|
|
"KL/rejected_KL_mean": -529.4427490234375,
|
|
"KL/std": 377.869140625,
|
|
"epoch": 0.33480176211453744,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 144.99569702148438,
|
|
"fcm_dpo/q_t": 0.4922485947608948,
|
|
"grad_norm": 3.4961659908294678,
|
|
"learning_rate": 4.2218171073908463e-07,
|
|
"logits/chosen": -0.15710148215293884,
|
|
"logits/rejected": -0.1570408046245575,
|
|
"logps/chosen": -450.33831787109375,
|
|
"logps/ref_chosen": -65.89128875732422,
|
|
"logps/ref_rejected": -91.04875183105469,
|
|
"logps/rejected": -620.4915161132812,
|
|
"loss": 1.3584,
|
|
"margin_dpo/margin_mean": 144.99569702148438,
|
|
"margin_dpo/margin_std": 505.9360656738281,
|
|
"step": 228
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -427.20635986328125,
|
|
"KL/mean": -472.91815185546875,
|
|
"KL/rejected_KL_mean": -518.6299438476562,
|
|
"KL/std": 411.049560546875,
|
|
"epoch": 0.33627019089574156,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 91.4235610961914,
|
|
"fcm_dpo/q_t": 0.49511969089508057,
|
|
"grad_norm": 4.707752227783203,
|
|
"learning_rate": 4.212490049118951e-07,
|
|
"logits/chosen": -0.23124007880687714,
|
|
"logits/rejected": -0.23222056031227112,
|
|
"logps/chosen": -497.9127502441406,
|
|
"logps/ref_chosen": -70.70637512207031,
|
|
"logps/ref_rejected": -84.52741241455078,
|
|
"logps/rejected": -603.1573486328125,
|
|
"loss": 1.3692,
|
|
"margin_dpo/margin_mean": 91.4235610961914,
|
|
"margin_dpo/margin_std": 452.021240234375,
|
|
"step": 229
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -248.62451171875,
|
|
"KL/mean": -390.31243896484375,
|
|
"KL/rejected_KL_mean": -532.0003662109375,
|
|
"KL/std": 303.40869140625,
|
|
"epoch": 0.3377386196769457,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 283.3758544921875,
|
|
"fcm_dpo/q_t": 0.48483169078826904,
|
|
"grad_norm": 3.5236966609954834,
|
|
"learning_rate": 4.203117865141635e-07,
|
|
"logits/chosen": -0.10768608748912811,
|
|
"logits/rejected": -0.11796237528324127,
|
|
"logps/chosen": -287.9065246582031,
|
|
"logps/ref_chosen": -39.282005310058594,
|
|
"logps/ref_rejected": -85.62191009521484,
|
|
"logps/rejected": -617.622314453125,
|
|
"loss": 1.3278,
|
|
"margin_dpo/margin_mean": 283.3758544921875,
|
|
"margin_dpo/margin_std": 349.93524169921875,
|
|
"step": 230
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -329.960693359375,
|
|
"KL/mean": -397.3656921386719,
|
|
"KL/rejected_KL_mean": -464.77069091796875,
|
|
"KL/std": 273.9499816894531,
|
|
"epoch": 0.3392070484581498,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 134.8100128173828,
|
|
"fcm_dpo/q_t": 0.49278855323791504,
|
|
"grad_norm": 3.0640761852264404,
|
|
"learning_rate": 4.1937008024246625e-07,
|
|
"logits/chosen": -0.19377461075782776,
|
|
"logits/rejected": -0.1955992579460144,
|
|
"logps/chosen": -393.23712158203125,
|
|
"logps/ref_chosen": -63.27644348144531,
|
|
"logps/ref_rejected": -74.1239013671875,
|
|
"logps/rejected": -538.8945922851562,
|
|
"loss": 1.359,
|
|
"margin_dpo/margin_mean": 134.81002807617188,
|
|
"margin_dpo/margin_std": 346.1912536621094,
|
|
"step": 231
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -400.56787109375,
|
|
"KL/mean": -449.58966064453125,
|
|
"KL/rejected_KL_mean": -498.61151123046875,
|
|
"KL/std": 334.50140380859375,
|
|
"epoch": 0.3406754772393539,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 98.04367065429688,
|
|
"fcm_dpo/q_t": 0.4947741627693176,
|
|
"grad_norm": 5.453951358795166,
|
|
"learning_rate": 4.1842391091163933e-07,
|
|
"logits/chosen": -0.19259104132652283,
|
|
"logits/rejected": -0.20236793160438538,
|
|
"logps/chosen": -471.3166198730469,
|
|
"logps/ref_chosen": -70.74876403808594,
|
|
"logps/ref_rejected": -83.97706604003906,
|
|
"logps/rejected": -582.5885620117188,
|
|
"loss": 1.3671,
|
|
"margin_dpo/margin_mean": 98.04367065429688,
|
|
"margin_dpo/margin_std": 388.8453369140625,
|
|
"step": 232
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -313.58184814453125,
|
|
"KL/mean": -445.1939697265625,
|
|
"KL/rejected_KL_mean": -576.8060302734375,
|
|
"KL/std": 369.1995849609375,
|
|
"epoch": 0.342143906020558,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 263.2242431640625,
|
|
"fcm_dpo/q_t": 0.48596352338790894,
|
|
"grad_norm": 4.564657211303711,
|
|
"learning_rate": 4.174733034541245e-07,
|
|
"logits/chosen": -0.19956615567207336,
|
|
"logits/rejected": -0.21853141486644745,
|
|
"logps/chosen": -368.46478271484375,
|
|
"logps/ref_chosen": -54.8829345703125,
|
|
"logps/ref_rejected": -107.4800796508789,
|
|
"logps/rejected": -684.2861328125,
|
|
"loss": 1.334,
|
|
"margin_dpo/margin_mean": 263.2242431640625,
|
|
"margin_dpo/margin_std": 516.1271362304688,
|
|
"step": 233
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -285.1370849609375,
|
|
"KL/mean": -427.95343017578125,
|
|
"KL/rejected_KL_mean": -570.7697143554688,
|
|
"KL/std": 362.5662841796875,
|
|
"epoch": 0.3436123348017621,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 285.6326904296875,
|
|
"fcm_dpo/q_t": 0.4847436547279358,
|
|
"grad_norm": 3.6827473640441895,
|
|
"learning_rate": 4.165182829193126e-07,
|
|
"logits/chosen": -0.20543476939201355,
|
|
"logits/rejected": -0.2246108204126358,
|
|
"logps/chosen": -329.2315673828125,
|
|
"logps/ref_chosen": -44.094520568847656,
|
|
"logps/ref_rejected": -100.00663757324219,
|
|
"logps/rejected": -670.7763671875,
|
|
"loss": 1.3283,
|
|
"margin_dpo/margin_mean": 285.6326599121094,
|
|
"margin_dpo/margin_std": 430.40399169921875,
|
|
"step": 234
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -379.80419921875,
|
|
"KL/mean": -452.5826416015625,
|
|
"KL/rejected_KL_mean": -525.361083984375,
|
|
"KL/std": 368.2503662109375,
|
|
"epoch": 0.34508076358296624,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 145.556884765625,
|
|
"fcm_dpo/q_t": 0.4922545552253723,
|
|
"grad_norm": 4.7766242027282715,
|
|
"learning_rate": 4.1555887447288255e-07,
|
|
"logits/chosen": -0.1708293855190277,
|
|
"logits/rejected": -0.18167471885681152,
|
|
"logps/chosen": -442.0421142578125,
|
|
"logps/ref_chosen": -62.237911224365234,
|
|
"logps/ref_rejected": -90.39506530761719,
|
|
"logps/rejected": -615.756103515625,
|
|
"loss": 1.3582,
|
|
"margin_dpo/margin_mean": 145.556884765625,
|
|
"margin_dpo/margin_std": 496.21075439453125,
|
|
"step": 235
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -263.3704833984375,
|
|
"KL/mean": -413.715087890625,
|
|
"KL/rejected_KL_mean": -564.0596923828125,
|
|
"KL/std": 374.974609375,
|
|
"epoch": 0.3465491923641703,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 300.689208984375,
|
|
"fcm_dpo/q_t": 0.48393750190734863,
|
|
"grad_norm": 10.610316276550293,
|
|
"learning_rate": 4.1459510339613946e-07,
|
|
"logits/chosen": -0.1809891164302826,
|
|
"logits/rejected": -0.1887068748474121,
|
|
"logps/chosen": -312.71185302734375,
|
|
"logps/ref_chosen": -49.34136199951172,
|
|
"logps/ref_rejected": -103.51162719726562,
|
|
"logps/rejected": -667.5713500976562,
|
|
"loss": 1.325,
|
|
"margin_dpo/margin_mean": 300.689208984375,
|
|
"margin_dpo/margin_std": 420.67352294921875,
|
|
"step": 236
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -378.86138916015625,
|
|
"KL/mean": -480.03680419921875,
|
|
"KL/rejected_KL_mean": -581.2122802734375,
|
|
"KL/std": 396.18927001953125,
|
|
"epoch": 0.34801762114537443,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 202.35086059570312,
|
|
"fcm_dpo/q_t": 0.4892232120037079,
|
|
"grad_norm": 4.470569610595703,
|
|
"learning_rate": 4.136269950853473e-07,
|
|
"logits/chosen": -0.180775448679924,
|
|
"logits/rejected": -0.19322538375854492,
|
|
"logps/chosen": -433.0295104980469,
|
|
"logps/ref_chosen": -54.168121337890625,
|
|
"logps/ref_rejected": -94.78036499023438,
|
|
"logps/rejected": -675.9926147460938,
|
|
"loss": 1.3464,
|
|
"margin_dpo/margin_mean": 202.35086059570312,
|
|
"margin_dpo/margin_std": 515.0560913085938,
|
|
"step": 237
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -354.31298828125,
|
|
"KL/mean": -458.6822509765625,
|
|
"KL/rejected_KL_mean": -563.051513671875,
|
|
"KL/std": 386.575439453125,
|
|
"epoch": 0.34948604992657856,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 208.73849487304688,
|
|
"fcm_dpo/q_t": 0.48885422945022583,
|
|
"grad_norm": 4.142996311187744,
|
|
"learning_rate": 4.126545750510605e-07,
|
|
"logits/chosen": -0.18518191576004028,
|
|
"logits/rejected": -0.19046491384506226,
|
|
"logps/chosen": -408.2861328125,
|
|
"logps/ref_chosen": -53.973121643066406,
|
|
"logps/ref_rejected": -89.41795349121094,
|
|
"logps/rejected": -652.469482421875,
|
|
"loss": 1.3443,
|
|
"margin_dpo/margin_mean": 208.7384796142578,
|
|
"margin_dpo/margin_std": 444.2572021484375,
|
|
"step": 238
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -395.993408203125,
|
|
"KL/mean": -492.276123046875,
|
|
"KL/rejected_KL_mean": -588.558837890625,
|
|
"KL/std": 340.7003173828125,
|
|
"epoch": 0.3509544787077827,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 192.56541442871094,
|
|
"fcm_dpo/q_t": 0.4896959066390991,
|
|
"grad_norm": 3.825457811355591,
|
|
"learning_rate": 4.116778689174514e-07,
|
|
"logits/chosen": -0.16619744896888733,
|
|
"logits/rejected": -0.18394985795021057,
|
|
"logps/chosen": -454.09124755859375,
|
|
"logps/ref_chosen": -58.09782409667969,
|
|
"logps/ref_rejected": -93.59294128417969,
|
|
"logps/rejected": -682.1517333984375,
|
|
"loss": 1.3476,
|
|
"margin_dpo/margin_mean": 192.5654296875,
|
|
"margin_dpo/margin_std": 428.50225830078125,
|
|
"step": 239
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -454.6864013671875,
|
|
"KL/mean": -506.56591796875,
|
|
"KL/rejected_KL_mean": -558.4453735351562,
|
|
"KL/std": 395.06903076171875,
|
|
"epoch": 0.3524229074889868,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 103.75895690917969,
|
|
"fcm_dpo/q_t": 0.49443739652633667,
|
|
"grad_norm": 3.9248123168945312,
|
|
"learning_rate": 4.106969024216348e-07,
|
|
"logits/chosen": -0.19379109144210815,
|
|
"logits/rejected": -0.19150272011756897,
|
|
"logps/chosen": -515.3009033203125,
|
|
"logps/ref_chosen": -60.6144905090332,
|
|
"logps/ref_rejected": -74.1185302734375,
|
|
"logps/rejected": -632.5639038085938,
|
|
"loss": 1.367,
|
|
"margin_dpo/margin_mean": 103.75895690917969,
|
|
"margin_dpo/margin_std": 491.4389953613281,
|
|
"step": 240
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -433.6764221191406,
|
|
"KL/mean": -542.6292724609375,
|
|
"KL/rejected_KL_mean": -651.5821533203125,
|
|
"KL/std": 436.55352783203125,
|
|
"epoch": 0.35389133627019087,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 217.90567016601562,
|
|
"fcm_dpo/q_t": 0.4883774518966675,
|
|
"grad_norm": 4.787429332733154,
|
|
"learning_rate": 4.097117014129903e-07,
|
|
"logits/chosen": -0.22374919056892395,
|
|
"logits/rejected": -0.22779551148414612,
|
|
"logps/chosen": -499.76751708984375,
|
|
"logps/ref_chosen": -66.091064453125,
|
|
"logps/ref_rejected": -88.06088256835938,
|
|
"logps/rejected": -739.6430053710938,
|
|
"loss": 1.3435,
|
|
"margin_dpo/margin_mean": 217.9056854248047,
|
|
"margin_dpo/margin_std": 529.632568359375,
|
|
"step": 241
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -466.04278564453125,
|
|
"KL/mean": -527.4720458984375,
|
|
"KL/rejected_KL_mean": -588.9013671875,
|
|
"KL/std": 415.62457275390625,
|
|
"epoch": 0.355359765051395,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 122.85851287841797,
|
|
"fcm_dpo/q_t": 0.4934060573577881,
|
|
"grad_norm": 4.762578010559082,
|
|
"learning_rate": 4.087222918524807e-07,
|
|
"logits/chosen": -0.2294858694076538,
|
|
"logits/rejected": -0.23094022274017334,
|
|
"logps/chosen": -533.90673828125,
|
|
"logps/ref_chosen": -67.86392974853516,
|
|
"logps/ref_rejected": -83.36033630371094,
|
|
"logps/rejected": -672.2616577148438,
|
|
"loss": 1.364,
|
|
"margin_dpo/margin_mean": 122.85851287841797,
|
|
"margin_dpo/margin_std": 575.6241455078125,
|
|
"step": 242
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -402.603759765625,
|
|
"KL/mean": -486.40032958984375,
|
|
"KL/rejected_KL_mean": -570.1968994140625,
|
|
"KL/std": 361.3595886230469,
|
|
"epoch": 0.3568281938325991,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 167.5931396484375,
|
|
"fcm_dpo/q_t": 0.4910268187522888,
|
|
"grad_norm": 4.299717903137207,
|
|
"learning_rate": 4.07728699811968e-07,
|
|
"logits/chosen": -0.22352221608161926,
|
|
"logits/rejected": -0.23877659440040588,
|
|
"logps/chosen": -465.6880187988281,
|
|
"logps/ref_chosen": -63.0842399597168,
|
|
"logps/ref_rejected": -76.33563232421875,
|
|
"logps/rejected": -646.5325927734375,
|
|
"loss": 1.3528,
|
|
"margin_dpo/margin_mean": 167.59312438964844,
|
|
"margin_dpo/margin_std": 422.52716064453125,
|
|
"step": 243
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -334.23394775390625,
|
|
"KL/mean": -451.40582275390625,
|
|
"KL/rejected_KL_mean": -568.57763671875,
|
|
"KL/std": 372.09405517578125,
|
|
"epoch": 0.35829662261380324,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 234.34368896484375,
|
|
"fcm_dpo/q_t": 0.48746874928474426,
|
|
"grad_norm": 4.2224202156066895,
|
|
"learning_rate": 4.067309514735267e-07,
|
|
"logits/chosen": -0.2551266849040985,
|
|
"logits/rejected": -0.27021169662475586,
|
|
"logps/chosen": -395.3746643066406,
|
|
"logps/ref_chosen": -61.140689849853516,
|
|
"logps/ref_rejected": -94.89193725585938,
|
|
"logps/rejected": -663.4696044921875,
|
|
"loss": 1.3385,
|
|
"margin_dpo/margin_mean": 234.34368896484375,
|
|
"margin_dpo/margin_std": 386.5777893066406,
|
|
"step": 244
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -376.46820068359375,
|
|
"KL/mean": -438.8470153808594,
|
|
"KL/rejected_KL_mean": -501.2258605957031,
|
|
"KL/std": 329.3731689453125,
|
|
"epoch": 0.35976505139500736,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 124.75768280029297,
|
|
"fcm_dpo/q_t": 0.4933217763900757,
|
|
"grad_norm": 3.401407480239868,
|
|
"learning_rate": 4.057290731287531e-07,
|
|
"logits/chosen": -0.2517125606536865,
|
|
"logits/rejected": -0.24654418230056763,
|
|
"logps/chosen": -443.73046875,
|
|
"logps/ref_chosen": -67.26228332519531,
|
|
"logps/ref_rejected": -87.64010620117188,
|
|
"logps/rejected": -588.865966796875,
|
|
"loss": 1.3615,
|
|
"margin_dpo/margin_mean": 124.75768280029297,
|
|
"margin_dpo/margin_std": 384.70745849609375,
|
|
"step": 245
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -330.4073486328125,
|
|
"KL/mean": -413.43792724609375,
|
|
"KL/rejected_KL_mean": -496.468505859375,
|
|
"KL/std": 368.08837890625,
|
|
"epoch": 0.36123348017621143,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 166.0610809326172,
|
|
"fcm_dpo/q_t": 0.4911186397075653,
|
|
"grad_norm": 4.492984294891357,
|
|
"learning_rate": 4.047230911780736e-07,
|
|
"logits/chosen": -0.25868698954582214,
|
|
"logits/rejected": -0.2564406394958496,
|
|
"logps/chosen": -397.10430908203125,
|
|
"logps/ref_chosen": -66.69696807861328,
|
|
"logps/ref_rejected": -84.34634399414062,
|
|
"logps/rejected": -580.8148193359375,
|
|
"loss": 1.353,
|
|
"margin_dpo/margin_mean": 166.0610809326172,
|
|
"margin_dpo/margin_std": 415.17022705078125,
|
|
"step": 246
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -346.08819580078125,
|
|
"KL/mean": -501.11810302734375,
|
|
"KL/rejected_KL_mean": -656.1480712890625,
|
|
"KL/std": 380.09771728515625,
|
|
"epoch": 0.36270190895741555,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 310.059814453125,
|
|
"fcm_dpo/q_t": 0.48344653844833374,
|
|
"grad_norm": 6.050650596618652,
|
|
"learning_rate": 4.0371303213004814e-07,
|
|
"logits/chosen": -0.23237216472625732,
|
|
"logits/rejected": -0.24301677942276,
|
|
"logps/chosen": -402.69354248046875,
|
|
"logps/ref_chosen": -56.6053466796875,
|
|
"logps/ref_rejected": -106.29326629638672,
|
|
"logps/rejected": -762.4412841796875,
|
|
"loss": 1.3237,
|
|
"margin_dpo/margin_mean": 310.0598449707031,
|
|
"margin_dpo/margin_std": 467.6119689941406,
|
|
"step": 247
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -282.9842529296875,
|
|
"KL/mean": -411.3094482421875,
|
|
"KL/rejected_KL_mean": -539.6346435546875,
|
|
"KL/std": 325.2694396972656,
|
|
"epoch": 0.3641703377386197,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 256.65032958984375,
|
|
"fcm_dpo/q_t": 0.4862789809703827,
|
|
"grad_norm": 4.713438034057617,
|
|
"learning_rate": 4.0269892260067197e-07,
|
|
"logits/chosen": -0.18797361850738525,
|
|
"logits/rejected": -0.21102368831634521,
|
|
"logps/chosen": -327.0274963378906,
|
|
"logps/ref_chosen": -44.043216705322266,
|
|
"logps/ref_rejected": -91.85687255859375,
|
|
"logps/rejected": -631.4915161132812,
|
|
"loss": 1.3337,
|
|
"margin_dpo/margin_mean": 256.65032958984375,
|
|
"margin_dpo/margin_std": 363.392822265625,
|
|
"step": 248
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -423.87060546875,
|
|
"KL/mean": -470.3067626953125,
|
|
"KL/rejected_KL_mean": -516.742919921875,
|
|
"KL/std": 390.5693054199219,
|
|
"epoch": 0.3656387665198238,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 92.87237548828125,
|
|
"fcm_dpo/q_t": 0.49507373571395874,
|
|
"grad_norm": 3.928159236907959,
|
|
"learning_rate": 4.0168078931267426e-07,
|
|
"logits/chosen": -0.16814130544662476,
|
|
"logits/rejected": -0.17811042070388794,
|
|
"logps/chosen": -486.31292724609375,
|
|
"logps/ref_chosen": -62.442352294921875,
|
|
"logps/ref_rejected": -80.46806335449219,
|
|
"logps/rejected": -597.2110595703125,
|
|
"loss": 1.3692,
|
|
"margin_dpo/margin_mean": 92.87238311767578,
|
|
"margin_dpo/margin_std": 484.80364990234375,
|
|
"step": 249
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -410.2246398925781,
|
|
"KL/mean": -485.90521240234375,
|
|
"KL/rejected_KL_mean": -561.5857543945312,
|
|
"KL/std": 377.61981201171875,
|
|
"epoch": 0.3671071953010279,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 151.361083984375,
|
|
"fcm_dpo/q_t": 0.49189114570617676,
|
|
"grad_norm": 4.632651329040527,
|
|
"learning_rate": 4.006586590948141e-07,
|
|
"logits/chosen": -0.1685868501663208,
|
|
"logits/rejected": -0.17339754104614258,
|
|
"logps/chosen": -475.861328125,
|
|
"logps/ref_chosen": -65.63668823242188,
|
|
"logps/ref_rejected": -73.87184143066406,
|
|
"logps/rejected": -635.4575805664062,
|
|
"loss": 1.3563,
|
|
"margin_dpo/margin_mean": 151.361083984375,
|
|
"margin_dpo/margin_std": 436.9773254394531,
|
|
"step": 250
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -401.18408203125,
|
|
"KL/mean": -462.01068115234375,
|
|
"KL/rejected_KL_mean": -522.8372192382812,
|
|
"KL/std": 380.92681884765625,
|
|
"epoch": 0.368575624082232,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 121.65315246582031,
|
|
"fcm_dpo/q_t": 0.49349498748779297,
|
|
"grad_norm": 4.47353458404541,
|
|
"learning_rate": 3.9963255888117325e-07,
|
|
"logits/chosen": -0.1774073839187622,
|
|
"logits/rejected": -0.18519741296768188,
|
|
"logps/chosen": -458.3667907714844,
|
|
"logps/ref_chosen": -57.182716369628906,
|
|
"logps/ref_rejected": -77.66343688964844,
|
|
"logps/rejected": -600.5006713867188,
|
|
"loss": 1.3627,
|
|
"margin_dpo/margin_mean": 121.65316772460938,
|
|
"margin_dpo/margin_std": 439.44793701171875,
|
|
"step": 251
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -429.9615173339844,
|
|
"KL/mean": -518.6616821289062,
|
|
"KL/rejected_KL_mean": -607.36181640625,
|
|
"KL/std": 336.4705505371094,
|
|
"epoch": 0.3700440528634361,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 177.4003143310547,
|
|
"fcm_dpo/q_t": 0.4905024766921997,
|
|
"grad_norm": 4.228276252746582,
|
|
"learning_rate": 3.9860251571044666e-07,
|
|
"logits/chosen": -0.20860856771469116,
|
|
"logits/rejected": -0.19886748492717743,
|
|
"logps/chosen": -501.64715576171875,
|
|
"logps/ref_chosen": -71.68563842773438,
|
|
"logps/ref_rejected": -84.75799560546875,
|
|
"logps/rejected": -692.1198120117188,
|
|
"loss": 1.3501,
|
|
"margin_dpo/margin_mean": 177.4003143310547,
|
|
"margin_dpo/margin_std": 356.32275390625,
|
|
"step": 252
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -392.266357421875,
|
|
"KL/mean": -508.9656066894531,
|
|
"KL/rejected_KL_mean": -625.664794921875,
|
|
"KL/std": 355.4100341796875,
|
|
"epoch": 0.37151248164464024,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 233.3984832763672,
|
|
"fcm_dpo/q_t": 0.4875214695930481,
|
|
"grad_norm": 3.7770397663116455,
|
|
"learning_rate": 3.9756855672522986e-07,
|
|
"logits/chosen": -0.19992950558662415,
|
|
"logits/rejected": -0.19830244779586792,
|
|
"logps/chosen": -461.4002990722656,
|
|
"logps/ref_chosen": -69.1339340209961,
|
|
"logps/ref_rejected": -98.70252990722656,
|
|
"logps/rejected": -724.3673706054688,
|
|
"loss": 1.3391,
|
|
"margin_dpo/margin_mean": 233.39849853515625,
|
|
"margin_dpo/margin_std": 419.17742919921875,
|
|
"step": 253
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -392.25958251953125,
|
|
"KL/mean": -469.58319091796875,
|
|
"KL/rejected_KL_mean": -546.90673828125,
|
|
"KL/std": 388.672607421875,
|
|
"epoch": 0.37298091042584436,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 154.64715576171875,
|
|
"fcm_dpo/q_t": 0.4917369484901428,
|
|
"grad_norm": 4.755723476409912,
|
|
"learning_rate": 3.965307091713037e-07,
|
|
"logits/chosen": -0.14065057039260864,
|
|
"logits/rejected": -0.1489948332309723,
|
|
"logps/chosen": -446.41461181640625,
|
|
"logps/ref_chosen": -54.154998779296875,
|
|
"logps/ref_rejected": -90.30764770507812,
|
|
"logps/rejected": -637.21435546875,
|
|
"loss": 1.3558,
|
|
"margin_dpo/margin_mean": 154.64715576171875,
|
|
"margin_dpo/margin_std": 455.6612854003906,
|
|
"step": 254
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -382.1084899902344,
|
|
"KL/mean": -481.79656982421875,
|
|
"KL/rejected_KL_mean": -581.484619140625,
|
|
"KL/std": 380.31219482421875,
|
|
"epoch": 0.3744493392070485,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 199.37615966796875,
|
|
"fcm_dpo/q_t": 0.48935091495513916,
|
|
"grad_norm": 3.1735172271728516,
|
|
"learning_rate": 3.954890003969163e-07,
|
|
"logits/chosen": -0.12860265374183655,
|
|
"logits/rejected": -0.1277311146259308,
|
|
"logps/chosen": -439.2501525878906,
|
|
"logps/ref_chosen": -57.14167022705078,
|
|
"logps/ref_rejected": -90.2085952758789,
|
|
"logps/rejected": -671.6932373046875,
|
|
"loss": 1.3468,
|
|
"margin_dpo/margin_mean": 199.37615966796875,
|
|
"margin_dpo/margin_std": 478.7210388183594,
|
|
"step": 255
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -343.8599548339844,
|
|
"KL/mean": -461.6353454589844,
|
|
"KL/rejected_KL_mean": -579.4107666015625,
|
|
"KL/std": 413.0328674316406,
|
|
"epoch": 0.37591776798825255,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 235.55078125,
|
|
"fcm_dpo/q_t": 0.4874057471752167,
|
|
"grad_norm": 4.488049030303955,
|
|
"learning_rate": 3.944434578520628e-07,
|
|
"logits/chosen": -0.10436695069074631,
|
|
"logits/rejected": -0.12606412172317505,
|
|
"logps/chosen": -399.0234375,
|
|
"logps/ref_chosen": -55.163490295410156,
|
|
"logps/ref_rejected": -92.56291961669922,
|
|
"logps/rejected": -671.9736328125,
|
|
"loss": 1.3388,
|
|
"margin_dpo/margin_mean": 235.55078125,
|
|
"margin_dpo/margin_std": 458.3751220703125,
|
|
"step": 256
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -310.64581298828125,
|
|
"KL/mean": -431.5376281738281,
|
|
"KL/rejected_KL_mean": -552.429443359375,
|
|
"KL/std": 358.80474853515625,
|
|
"epoch": 0.37738619676945667,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 241.78359985351562,
|
|
"fcm_dpo/q_t": 0.4870920777320862,
|
|
"grad_norm": 4.349320888519287,
|
|
"learning_rate": 3.933941090877615e-07,
|
|
"logits/chosen": -0.13144733011722565,
|
|
"logits/rejected": -0.13504740595817566,
|
|
"logps/chosen": -360.06951904296875,
|
|
"logps/ref_chosen": -49.42369842529297,
|
|
"logps/ref_rejected": -79.53791809082031,
|
|
"logps/rejected": -631.9673461914062,
|
|
"loss": 1.3377,
|
|
"margin_dpo/margin_mean": 241.78358459472656,
|
|
"margin_dpo/margin_std": 457.3230895996094,
|
|
"step": 257
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -410.16473388671875,
|
|
"KL/mean": -527.0640869140625,
|
|
"KL/rejected_KL_mean": -643.96337890625,
|
|
"KL/std": 395.2886047363281,
|
|
"epoch": 0.3788546255506608,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 233.79864501953125,
|
|
"fcm_dpo/q_t": 0.4875029921531677,
|
|
"grad_norm": 3.8991451263427734,
|
|
"learning_rate": 3.923409817553284e-07,
|
|
"logits/chosen": -0.10106329619884491,
|
|
"logits/rejected": -0.10983704030513763,
|
|
"logps/chosen": -469.54888916015625,
|
|
"logps/ref_chosen": -59.384124755859375,
|
|
"logps/ref_rejected": -95.99010467529297,
|
|
"logps/rejected": -739.9534912109375,
|
|
"loss": 1.3397,
|
|
"margin_dpo/margin_mean": 233.79864501953125,
|
|
"margin_dpo/margin_std": 500.43988037109375,
|
|
"step": 258
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -380.03558349609375,
|
|
"KL/mean": -472.2444152832031,
|
|
"KL/rejected_KL_mean": -564.4532470703125,
|
|
"KL/std": 359.38470458984375,
|
|
"epoch": 0.3803230543318649,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 184.4176788330078,
|
|
"fcm_dpo/q_t": 0.49013984203338623,
|
|
"grad_norm": 3.6584014892578125,
|
|
"learning_rate": 3.9128410360564793e-07,
|
|
"logits/chosen": -0.19331489503383636,
|
|
"logits/rejected": -0.2091323733329773,
|
|
"logps/chosen": -432.8639221191406,
|
|
"logps/ref_chosen": -52.828346252441406,
|
|
"logps/ref_rejected": -89.191650390625,
|
|
"logps/rejected": -653.6448974609375,
|
|
"loss": 1.3494,
|
|
"margin_dpo/margin_mean": 184.41766357421875,
|
|
"margin_dpo/margin_std": 437.661865234375,
|
|
"step": 259
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -365.8872375488281,
|
|
"KL/mean": -518.8782958984375,
|
|
"KL/rejected_KL_mean": -671.8694458007812,
|
|
"KL/std": 458.7395324707031,
|
|
"epoch": 0.38179148311306904,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 305.98223876953125,
|
|
"fcm_dpo/q_t": 0.48385488986968994,
|
|
"grad_norm": 3.9910993576049805,
|
|
"learning_rate": 3.9022350248844246e-07,
|
|
"logits/chosen": -0.15303289890289307,
|
|
"logits/rejected": -0.16400694847106934,
|
|
"logps/chosen": -413.3049011230469,
|
|
"logps/ref_chosen": -47.41767501831055,
|
|
"logps/ref_rejected": -95.08978271484375,
|
|
"logps/rejected": -766.959228515625,
|
|
"loss": 1.3258,
|
|
"margin_dpo/margin_mean": 305.98223876953125,
|
|
"margin_dpo/margin_std": 550.6990966796875,
|
|
"step": 260
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -419.76605224609375,
|
|
"KL/mean": -529.2583618164062,
|
|
"KL/rejected_KL_mean": -638.7506713867188,
|
|
"KL/std": 460.34503173828125,
|
|
"epoch": 0.3832599118942731,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 218.98464965820312,
|
|
"fcm_dpo/q_t": 0.4883573055267334,
|
|
"grad_norm": 5.955752372741699,
|
|
"learning_rate": 3.891592063515376e-07,
|
|
"logits/chosen": -0.08432209491729736,
|
|
"logits/rejected": -0.09958672523498535,
|
|
"logps/chosen": -472.79742431640625,
|
|
"logps/ref_chosen": -53.03137969970703,
|
|
"logps/ref_rejected": -88.51494598388672,
|
|
"logps/rejected": -727.265625,
|
|
"loss": 1.3439,
|
|
"margin_dpo/margin_mean": 218.98464965820312,
|
|
"margin_dpo/margin_std": 580.8803100585938,
|
|
"step": 261
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -492.68560791015625,
|
|
"KL/mean": -573.60986328125,
|
|
"KL/rejected_KL_mean": -654.5340576171875,
|
|
"KL/std": 429.9180603027344,
|
|
"epoch": 0.38472834067547723,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 161.84848022460938,
|
|
"fcm_dpo/q_t": 0.49136343598365784,
|
|
"grad_norm": 5.33870792388916,
|
|
"learning_rate": 3.880912432401264e-07,
|
|
"logits/chosen": -0.10938498377799988,
|
|
"logits/rejected": -0.10725777596235275,
|
|
"logps/chosen": -552.3057861328125,
|
|
"logps/ref_chosen": -59.620140075683594,
|
|
"logps/ref_rejected": -86.41853332519531,
|
|
"logps/rejected": -740.95263671875,
|
|
"loss": 1.3542,
|
|
"margin_dpo/margin_mean": 161.84848022460938,
|
|
"margin_dpo/margin_std": 447.91815185546875,
|
|
"step": 262
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -442.79217529296875,
|
|
"KL/mean": -572.5969848632812,
|
|
"KL/rejected_KL_mean": -702.40185546875,
|
|
"KL/std": 522.2442016601562,
|
|
"epoch": 0.38619676945668135,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 259.609619140625,
|
|
"fcm_dpo/q_t": 0.4861585795879364,
|
|
"grad_norm": 3.8708550930023193,
|
|
"learning_rate": 3.870196412960302e-07,
|
|
"logits/chosen": -0.12460441887378693,
|
|
"logits/rejected": -0.13606280088424683,
|
|
"logps/chosen": -502.2131042480469,
|
|
"logps/ref_chosen": -59.42094421386719,
|
|
"logps/ref_rejected": -96.85720825195312,
|
|
"logps/rejected": -799.259033203125,
|
|
"loss": 1.3362,
|
|
"margin_dpo/margin_mean": 259.609619140625,
|
|
"margin_dpo/margin_std": 626.2482299804688,
|
|
"step": 263
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -469.1533508300781,
|
|
"KL/mean": -582.8370361328125,
|
|
"KL/rejected_KL_mean": -696.5206909179688,
|
|
"KL/std": 439.26593017578125,
|
|
"epoch": 0.3876651982378855,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 227.36734008789062,
|
|
"fcm_dpo/q_t": 0.4878886938095093,
|
|
"grad_norm": 3.9945685863494873,
|
|
"learning_rate": 3.8594442875695665e-07,
|
|
"logits/chosen": -0.15970063209533691,
|
|
"logits/rejected": -0.17009925842285156,
|
|
"logps/chosen": -531.8754272460938,
|
|
"logps/ref_chosen": -62.722084045410156,
|
|
"logps/ref_rejected": -93.85620880126953,
|
|
"logps/rejected": -790.376953125,
|
|
"loss": 1.342,
|
|
"margin_dpo/margin_mean": 227.36734008789062,
|
|
"margin_dpo/margin_std": 566.97900390625,
|
|
"step": 264
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -494.053466796875,
|
|
"KL/mean": -583.5369873046875,
|
|
"KL/rejected_KL_mean": -673.0205078125,
|
|
"KL/std": 433.59552001953125,
|
|
"epoch": 0.3891336270190896,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 178.96697998046875,
|
|
"fcm_dpo/q_t": 0.49045416712760925,
|
|
"grad_norm": 4.128632068634033,
|
|
"learning_rate": 3.848656339557562e-07,
|
|
"logits/chosen": -0.09494436532258987,
|
|
"logits/rejected": -0.10667555034160614,
|
|
"logps/chosen": -556.02490234375,
|
|
"logps/ref_chosen": -61.971466064453125,
|
|
"logps/ref_rejected": -88.02059936523438,
|
|
"logps/rejected": -761.0410766601562,
|
|
"loss": 1.3517,
|
|
"margin_dpo/margin_mean": 178.96697998046875,
|
|
"margin_dpo/margin_std": 544.5645751953125,
|
|
"step": 265
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -539.0609741210938,
|
|
"KL/mean": -561.758544921875,
|
|
"KL/rejected_KL_mean": -584.4561767578125,
|
|
"KL/std": 398.85784912109375,
|
|
"epoch": 0.39060205580029367,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 45.395179748535156,
|
|
"fcm_dpo/q_t": 0.49756669998168945,
|
|
"grad_norm": 6.132102966308594,
|
|
"learning_rate": 3.8378328531967507e-07,
|
|
"logits/chosen": -0.11285087466239929,
|
|
"logits/rejected": -0.12594473361968994,
|
|
"logps/chosen": -606.16064453125,
|
|
"logps/ref_chosen": -67.09967041015625,
|
|
"logps/ref_rejected": -67.97122192382812,
|
|
"logps/rejected": -652.4273681640625,
|
|
"loss": 1.3796,
|
|
"margin_dpo/margin_mean": 45.39517593383789,
|
|
"margin_dpo/margin_std": 510.51422119140625,
|
|
"step": 266
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -456.233642578125,
|
|
"KL/mean": -543.4124755859375,
|
|
"KL/rejected_KL_mean": -630.59130859375,
|
|
"KL/std": 447.64996337890625,
|
|
"epoch": 0.3920704845814978,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 174.35768127441406,
|
|
"fcm_dpo/q_t": 0.49071285128593445,
|
|
"grad_norm": 4.379497528076172,
|
|
"learning_rate": 3.8269741136960646e-07,
|
|
"logits/chosen": -0.13587993383407593,
|
|
"logits/rejected": -0.14027976989746094,
|
|
"logps/chosen": -525.204345703125,
|
|
"logps/ref_chosen": -68.97075653076172,
|
|
"logps/ref_rejected": -90.16844940185547,
|
|
"logps/rejected": -720.759765625,
|
|
"loss": 1.3531,
|
|
"margin_dpo/margin_mean": 174.357666015625,
|
|
"margin_dpo/margin_std": 578.6551513671875,
|
|
"step": 267
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -435.64410400390625,
|
|
"KL/mean": -557.5379638671875,
|
|
"KL/rejected_KL_mean": -679.4317626953125,
|
|
"KL/std": 507.63592529296875,
|
|
"epoch": 0.3935389133627019,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 243.78765869140625,
|
|
"fcm_dpo/q_t": 0.48697763681411743,
|
|
"grad_norm": 4.641546249389648,
|
|
"learning_rate": 3.8160804071933894e-07,
|
|
"logits/chosen": -0.17142120003700256,
|
|
"logits/rejected": -0.17281493544578552,
|
|
"logps/chosen": -491.54443359375,
|
|
"logps/ref_chosen": -55.90031051635742,
|
|
"logps/ref_rejected": -101.64763641357422,
|
|
"logps/rejected": -781.0794067382812,
|
|
"loss": 1.3387,
|
|
"margin_dpo/margin_mean": 243.78765869140625,
|
|
"margin_dpo/margin_std": 590.0145263671875,
|
|
"step": 268
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -501.7329406738281,
|
|
"KL/mean": -626.0015869140625,
|
|
"KL/rejected_KL_mean": -750.270263671875,
|
|
"KL/std": 443.25628662109375,
|
|
"epoch": 0.39500734214390604,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 248.5373077392578,
|
|
"fcm_dpo/q_t": 0.4867471754550934,
|
|
"grad_norm": 4.408675193786621,
|
|
"learning_rate": 3.8051520207480204e-07,
|
|
"logits/chosen": -0.17563828825950623,
|
|
"logits/rejected": -0.192486971616745,
|
|
"logps/chosen": -571.7724609375,
|
|
"logps/ref_chosen": -70.03955841064453,
|
|
"logps/ref_rejected": -107.34937286376953,
|
|
"logps/rejected": -857.61962890625,
|
|
"loss": 1.3379,
|
|
"margin_dpo/margin_mean": 248.53729248046875,
|
|
"margin_dpo/margin_std": 601.0906982421875,
|
|
"step": 269
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -464.98211669921875,
|
|
"KL/mean": -577.4337158203125,
|
|
"KL/rejected_KL_mean": -689.8851928710938,
|
|
"KL/std": 445.669189453125,
|
|
"epoch": 0.3964757709251101,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 224.90313720703125,
|
|
"fcm_dpo/q_t": 0.48800837993621826,
|
|
"grad_norm": 4.137222766876221,
|
|
"learning_rate": 3.794189242333106e-07,
|
|
"logits/chosen": -0.21441274881362915,
|
|
"logits/rejected": -0.22988632321357727,
|
|
"logps/chosen": -534.515625,
|
|
"logps/ref_chosen": -69.53347778320312,
|
|
"logps/ref_rejected": -109.92864990234375,
|
|
"logps/rejected": -799.8138427734375,
|
|
"loss": 1.3429,
|
|
"margin_dpo/margin_mean": 224.9031219482422,
|
|
"margin_dpo/margin_std": 603.002685546875,
|
|
"step": 270
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -383.33953857421875,
|
|
"KL/mean": -498.1918640136719,
|
|
"KL/rejected_KL_mean": -613.044189453125,
|
|
"KL/std": 389.174560546875,
|
|
"epoch": 0.39794419970631423,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 229.7046661376953,
|
|
"fcm_dpo/q_t": 0.48772376775741577,
|
|
"grad_norm": 3.387704849243164,
|
|
"learning_rate": 3.7831923608280514e-07,
|
|
"logits/chosen": -0.15242159366607666,
|
|
"logits/rejected": -0.17212322354316711,
|
|
"logps/chosen": -440.1041259765625,
|
|
"logps/ref_chosen": -56.76456832885742,
|
|
"logps/ref_rejected": -92.51383972167969,
|
|
"logps/rejected": -705.5580444335938,
|
|
"loss": 1.3403,
|
|
"margin_dpo/margin_mean": 229.70465087890625,
|
|
"margin_dpo/margin_std": 480.60894775390625,
|
|
"step": 271
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -352.65478515625,
|
|
"KL/mean": -527.4397583007812,
|
|
"KL/rejected_KL_mean": -702.2247314453125,
|
|
"KL/std": 438.55596923828125,
|
|
"epoch": 0.39941262848751835,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 349.5699768066406,
|
|
"fcm_dpo/q_t": 0.48131972551345825,
|
|
"grad_norm": 8.041435241699219,
|
|
"learning_rate": 3.772161666010912e-07,
|
|
"logits/chosen": -0.15468257665634155,
|
|
"logits/rejected": -0.16906066238880157,
|
|
"logps/chosen": -402.15191650390625,
|
|
"logps/ref_chosen": -49.497154235839844,
|
|
"logps/ref_rejected": -105.54279327392578,
|
|
"logps/rejected": -807.767578125,
|
|
"loss": 1.3154,
|
|
"margin_dpo/margin_mean": 349.5699462890625,
|
|
"margin_dpo/margin_std": 480.5194091796875,
|
|
"step": 272
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -454.3601989746094,
|
|
"KL/mean": -596.029541015625,
|
|
"KL/rejected_KL_mean": -737.698974609375,
|
|
"KL/std": 466.94976806640625,
|
|
"epoch": 0.4008810572687225,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 283.33868408203125,
|
|
"fcm_dpo/q_t": 0.48487192392349243,
|
|
"grad_norm": 4.082789421081543,
|
|
"learning_rate": 3.761097448550755e-07,
|
|
"logits/chosen": -0.16157250106334686,
|
|
"logits/rejected": -0.16911356151103973,
|
|
"logps/chosen": -517.3355712890625,
|
|
"logps/ref_chosen": -62.97539520263672,
|
|
"logps/ref_rejected": -92.49858093261719,
|
|
"logps/rejected": -830.197509765625,
|
|
"loss": 1.3297,
|
|
"margin_dpo/margin_mean": 283.33868408203125,
|
|
"margin_dpo/margin_std": 531.16064453125,
|
|
"step": 273
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -471.19451904296875,
|
|
"KL/mean": -561.68115234375,
|
|
"KL/rejected_KL_mean": -652.167724609375,
|
|
"KL/std": 369.1112365722656,
|
|
"epoch": 0.4023494860499266,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 180.9732208251953,
|
|
"fcm_dpo/q_t": 0.49033892154693604,
|
|
"grad_norm": 4.708371639251709,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": -0.10799916088581085,
|
|
"logits/rejected": -0.10551673173904419,
|
|
"logps/chosen": -526.8622436523438,
|
|
"logps/ref_chosen": -55.66770935058594,
|
|
"logps/ref_rejected": -77.33308410644531,
|
|
"logps/rejected": -729.5008544921875,
|
|
"loss": 1.3505,
|
|
"margin_dpo/margin_mean": 180.97323608398438,
|
|
"margin_dpo/margin_std": 471.4014587402344,
|
|
"step": 274
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -417.6435546875,
|
|
"KL/mean": -547.9447631835938,
|
|
"KL/rejected_KL_mean": -678.2459716796875,
|
|
"KL/std": 425.20245361328125,
|
|
"epoch": 0.40381791483113066,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 260.6024169921875,
|
|
"fcm_dpo/q_t": 0.48607248067855835,
|
|
"grad_norm": 5.847025394439697,
|
|
"learning_rate": 3.738869612786737e-07,
|
|
"logits/chosen": -0.15583306550979614,
|
|
"logits/rejected": -0.16265209019184113,
|
|
"logps/chosen": -466.2382507324219,
|
|
"logps/ref_chosen": -48.594703674316406,
|
|
"logps/ref_rejected": -93.30369567871094,
|
|
"logps/rejected": -771.5496826171875,
|
|
"loss": 1.3342,
|
|
"margin_dpo/margin_mean": 260.6024169921875,
|
|
"margin_dpo/margin_std": 507.73193359375,
|
|
"step": 275
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -472.4503173828125,
|
|
"KL/mean": -557.5875244140625,
|
|
"KL/rejected_KL_mean": -642.7247314453125,
|
|
"KL/std": 441.7587890625,
|
|
"epoch": 0.4052863436123348,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 170.2743377685547,
|
|
"fcm_dpo/q_t": 0.4909123182296753,
|
|
"grad_norm": 5.609163284301758,
|
|
"learning_rate": 3.7277065802070204e-07,
|
|
"logits/chosen": -0.14788061380386353,
|
|
"logits/rejected": -0.1665324866771698,
|
|
"logps/chosen": -529.0277099609375,
|
|
"logps/ref_chosen": -56.57740783691406,
|
|
"logps/ref_rejected": -70.36566925048828,
|
|
"logps/rejected": -713.09033203125,
|
|
"loss": 1.3544,
|
|
"margin_dpo/margin_mean": 170.2743377685547,
|
|
"margin_dpo/margin_std": 601.065185546875,
|
|
"step": 276
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -494.017578125,
|
|
"KL/mean": -627.9515380859375,
|
|
"KL/rejected_KL_mean": -761.8856201171875,
|
|
"KL/std": 439.302001953125,
|
|
"epoch": 0.4067547723935389,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 267.8680725097656,
|
|
"fcm_dpo/q_t": 0.4856921434402466,
|
|
"grad_norm": 4.117493629455566,
|
|
"learning_rate": 3.71651119641714e-07,
|
|
"logits/chosen": -0.14363861083984375,
|
|
"logits/rejected": -0.13916437327861786,
|
|
"logps/chosen": -550.2891235351562,
|
|
"logps/ref_chosen": -56.27156066894531,
|
|
"logps/ref_rejected": -92.88127136230469,
|
|
"logps/rejected": -854.766845703125,
|
|
"loss": 1.333,
|
|
"margin_dpo/margin_mean": 267.8680419921875,
|
|
"margin_dpo/margin_std": 535.217529296875,
|
|
"step": 277
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -475.8216552734375,
|
|
"KL/mean": -630.0145263671875,
|
|
"KL/rejected_KL_mean": -784.2073974609375,
|
|
"KL/std": 520.323486328125,
|
|
"epoch": 0.40822320117474303,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 308.3857116699219,
|
|
"fcm_dpo/q_t": 0.4835778772830963,
|
|
"grad_norm": 4.502967357635498,
|
|
"learning_rate": 3.705283756425872e-07,
|
|
"logits/chosen": -0.10104444622993469,
|
|
"logits/rejected": -0.11467007547616959,
|
|
"logps/chosen": -528.7635498046875,
|
|
"logps/ref_chosen": -52.94194030761719,
|
|
"logps/ref_rejected": -91.25357818603516,
|
|
"logps/rejected": -875.4609375,
|
|
"loss": 1.3266,
|
|
"margin_dpo/margin_mean": 308.3857116699219,
|
|
"margin_dpo/margin_std": 675.7715454101562,
|
|
"step": 278
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -527.8968505859375,
|
|
"KL/mean": -687.3350830078125,
|
|
"KL/rejected_KL_mean": -846.7733154296875,
|
|
"KL/std": 570.1357421875,
|
|
"epoch": 0.40969162995594716,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 318.8764953613281,
|
|
"fcm_dpo/q_t": 0.4832102060317993,
|
|
"grad_norm": 6.868563175201416,
|
|
"learning_rate": 3.6940245560867e-07,
|
|
"logits/chosen": -0.11306406557559967,
|
|
"logits/rejected": -0.1245322972536087,
|
|
"logps/chosen": -576.5381469726562,
|
|
"logps/ref_chosen": -48.641319274902344,
|
|
"logps/ref_rejected": -87.8514404296875,
|
|
"logps/rejected": -934.624755859375,
|
|
"loss": 1.3266,
|
|
"margin_dpo/margin_mean": 318.8764953613281,
|
|
"margin_dpo/margin_std": 792.3270874023438,
|
|
"step": 279
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -510.629150390625,
|
|
"KL/mean": -667.2784423828125,
|
|
"KL/rejected_KL_mean": -823.9278564453125,
|
|
"KL/std": 481.0506591796875,
|
|
"epoch": 0.4111600587371512,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 313.29864501953125,
|
|
"fcm_dpo/q_t": 0.48329728841781616,
|
|
"grad_norm": 4.903005123138428,
|
|
"learning_rate": 3.6827338920900253e-07,
|
|
"logits/chosen": -0.13975031673908234,
|
|
"logits/rejected": -0.1519356071949005,
|
|
"logps/chosen": -569.42626953125,
|
|
"logps/ref_chosen": -58.797122955322266,
|
|
"logps/ref_rejected": -98.61885070800781,
|
|
"logps/rejected": -922.546630859375,
|
|
"loss": 1.3249,
|
|
"margin_dpo/margin_mean": 313.29864501953125,
|
|
"margin_dpo/margin_std": 634.85693359375,
|
|
"step": 280
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -512.4186401367188,
|
|
"KL/mean": -614.4647216796875,
|
|
"KL/rejected_KL_mean": -716.5108032226562,
|
|
"KL/std": 448.61798095703125,
|
|
"epoch": 0.41262848751835535,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 204.09217834472656,
|
|
"fcm_dpo/q_t": 0.48906901478767395,
|
|
"grad_norm": 6.864736557006836,
|
|
"learning_rate": 3.6714120619553435e-07,
|
|
"logits/chosen": -0.14065885543823242,
|
|
"logits/rejected": -0.15409055352210999,
|
|
"logps/chosen": -567.9071655273438,
|
|
"logps/ref_chosen": -55.488521575927734,
|
|
"logps/ref_rejected": -80.88258361816406,
|
|
"logps/rejected": -797.3933715820312,
|
|
"loss": 1.3468,
|
|
"margin_dpo/margin_mean": 204.09219360351562,
|
|
"margin_dpo/margin_std": 557.4766845703125,
|
|
"step": 281
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -550.32373046875,
|
|
"KL/mean": -651.2105712890625,
|
|
"KL/rejected_KL_mean": -752.0973510742188,
|
|
"KL/std": 611.913818359375,
|
|
"epoch": 0.41409691629955947,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 201.77359008789062,
|
|
"fcm_dpo/q_t": 0.48951423168182373,
|
|
"grad_norm": 6.504598140716553,
|
|
"learning_rate": 3.660059364023408e-07,
|
|
"logits/chosen": -0.18802456557750702,
|
|
"logits/rejected": -0.1919810175895691,
|
|
"logps/chosen": -623.3939208984375,
|
|
"logps/ref_chosen": -73.07014465332031,
|
|
"logps/ref_rejected": -95.35098266601562,
|
|
"logps/rejected": -847.4483642578125,
|
|
"loss": 1.3497,
|
|
"margin_dpo/margin_mean": 201.77359008789062,
|
|
"margin_dpo/margin_std": 712.3954467773438,
|
|
"step": 282
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -534.77490234375,
|
|
"KL/mean": -697.13916015625,
|
|
"KL/rejected_KL_mean": -859.5034790039062,
|
|
"KL/std": 511.788818359375,
|
|
"epoch": 0.4155653450807636,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 324.72857666015625,
|
|
"fcm_dpo/q_t": 0.4826442003250122,
|
|
"grad_norm": 6.365213871002197,
|
|
"learning_rate": 3.6486760974483685e-07,
|
|
"logits/chosen": -0.13428352773189545,
|
|
"logits/rejected": -0.14583396911621094,
|
|
"logps/chosen": -596.67333984375,
|
|
"logps/ref_chosen": -61.89844512939453,
|
|
"logps/ref_rejected": -96.98655700683594,
|
|
"logps/rejected": -956.489990234375,
|
|
"loss": 1.3214,
|
|
"margin_dpo/margin_mean": 324.72857666015625,
|
|
"margin_dpo/margin_std": 536.8545532226562,
|
|
"step": 283
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -439.5884704589844,
|
|
"KL/mean": -578.688232421875,
|
|
"KL/rejected_KL_mean": -717.7880249023438,
|
|
"KL/std": 436.02374267578125,
|
|
"epoch": 0.4170337738619677,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 278.19952392578125,
|
|
"fcm_dpo/q_t": 0.48516762256622314,
|
|
"grad_norm": 5.291563987731934,
|
|
"learning_rate": 3.6372625621898863e-07,
|
|
"logits/chosen": -0.18925726413726807,
|
|
"logits/rejected": -0.18296435475349426,
|
|
"logps/chosen": -498.0240173339844,
|
|
"logps/ref_chosen": -58.4355354309082,
|
|
"logps/ref_rejected": -93.46926879882812,
|
|
"logps/rejected": -811.25732421875,
|
|
"loss": 1.3307,
|
|
"margin_dpo/margin_mean": 278.19952392578125,
|
|
"margin_dpo/margin_std": 519.5226440429688,
|
|
"step": 284
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -519.1543579101562,
|
|
"KL/mean": -606.4476928710938,
|
|
"KL/rejected_KL_mean": -693.740966796875,
|
|
"KL/std": 376.2451171875,
|
|
"epoch": 0.4185022026431718,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 174.586669921875,
|
|
"fcm_dpo/q_t": 0.4906350374221802,
|
|
"grad_norm": 4.204360485076904,
|
|
"learning_rate": 3.625819059005228e-07,
|
|
"logits/chosen": -0.14472293853759766,
|
|
"logits/rejected": -0.1445767879486084,
|
|
"logps/chosen": -585.3865966796875,
|
|
"logps/ref_chosen": -66.23219299316406,
|
|
"logps/ref_rejected": -99.1268310546875,
|
|
"logps/rejected": -792.8677978515625,
|
|
"loss": 1.3521,
|
|
"margin_dpo/margin_mean": 174.586669921875,
|
|
"margin_dpo/margin_std": 497.9844970703125,
|
|
"step": 285
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -518.2225341796875,
|
|
"KL/mean": -618.0490112304688,
|
|
"KL/rejected_KL_mean": -717.87548828125,
|
|
"KL/std": 502.3042297363281,
|
|
"epoch": 0.4199706314243759,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 199.6529541015625,
|
|
"fcm_dpo/q_t": 0.48932021856307983,
|
|
"grad_norm": 4.919650077819824,
|
|
"learning_rate": 3.614345889441346e-07,
|
|
"logits/chosen": -0.14968830347061157,
|
|
"logits/rejected": -0.1579841524362564,
|
|
"logps/chosen": -591.173583984375,
|
|
"logps/ref_chosen": -72.95100402832031,
|
|
"logps/ref_rejected": -88.58845520019531,
|
|
"logps/rejected": -806.4639282226562,
|
|
"loss": 1.3487,
|
|
"margin_dpo/margin_mean": 199.6529541015625,
|
|
"margin_dpo/margin_std": 646.603515625,
|
|
"step": 286
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -417.3997497558594,
|
|
"KL/mean": -486.1492919921875,
|
|
"KL/rejected_KL_mean": -554.8988647460938,
|
|
"KL/std": 410.86279296875,
|
|
"epoch": 0.42143906020558003,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 137.49908447265625,
|
|
"fcm_dpo/q_t": 0.49265867471694946,
|
|
"grad_norm": 3.5626771450042725,
|
|
"learning_rate": 3.6028433558269275e-07,
|
|
"logits/chosen": -0.14080211520195007,
|
|
"logits/rejected": -0.1525568813085556,
|
|
"logps/chosen": -478.94091796875,
|
|
"logps/ref_chosen": -61.54115295410156,
|
|
"logps/ref_rejected": -77.69607543945312,
|
|
"logps/rejected": -632.594970703125,
|
|
"loss": 1.36,
|
|
"margin_dpo/margin_mean": 137.49908447265625,
|
|
"margin_dpo/margin_std": 493.0873718261719,
|
|
"step": 287
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -323.7350769042969,
|
|
"KL/mean": -437.349365234375,
|
|
"KL/rejected_KL_mean": -550.963623046875,
|
|
"KL/std": 331.2504577636719,
|
|
"epoch": 0.42290748898678415,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 227.22857666015625,
|
|
"fcm_dpo/q_t": 0.4878368079662323,
|
|
"grad_norm": 3.0430126190185547,
|
|
"learning_rate": 3.5913117612644327e-07,
|
|
"logits/chosen": -0.1575808823108673,
|
|
"logits/rejected": -0.16179470717906952,
|
|
"logps/chosen": -380.39630126953125,
|
|
"logps/ref_chosen": -56.661224365234375,
|
|
"logps/ref_rejected": -87.33570098876953,
|
|
"logps/rejected": -638.29931640625,
|
|
"loss": 1.3402,
|
|
"margin_dpo/margin_mean": 227.22857666015625,
|
|
"margin_dpo/margin_std": 409.6346435546875,
|
|
"step": 288
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -281.5063171386719,
|
|
"KL/mean": -421.5150451660156,
|
|
"KL/rejected_KL_mean": -561.5237426757812,
|
|
"KL/std": 394.57177734375,
|
|
"epoch": 0.4243759177679883,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 280.01739501953125,
|
|
"fcm_dpo/q_t": 0.48507311940193176,
|
|
"grad_norm": 6.204428195953369,
|
|
"learning_rate": 3.5797514096221024e-07,
|
|
"logits/chosen": -0.11954732984304428,
|
|
"logits/rejected": -0.1284411996603012,
|
|
"logps/chosen": -326.7367248535156,
|
|
"logps/ref_chosen": -45.23039245605469,
|
|
"logps/ref_rejected": -87.64266967773438,
|
|
"logps/rejected": -649.1663818359375,
|
|
"loss": 1.3298,
|
|
"margin_dpo/margin_mean": 280.01739501953125,
|
|
"margin_dpo/margin_std": 468.9491882324219,
|
|
"step": 289
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -315.4320983886719,
|
|
"KL/mean": -489.9077453613281,
|
|
"KL/rejected_KL_mean": -664.3833618164062,
|
|
"KL/std": 450.1668701171875,
|
|
"epoch": 0.42584434654919234,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 348.9513244628906,
|
|
"fcm_dpo/q_t": 0.48141106963157654,
|
|
"grad_norm": 7.722722053527832,
|
|
"learning_rate": 3.568162605525952e-07,
|
|
"logits/chosen": -0.12268570065498352,
|
|
"logits/rejected": -0.14039406180381775,
|
|
"logps/chosen": -370.9035949707031,
|
|
"logps/ref_chosen": -55.47149658203125,
|
|
"logps/ref_rejected": -116.70857238769531,
|
|
"logps/rejected": -781.0919189453125,
|
|
"loss": 1.3162,
|
|
"margin_dpo/margin_mean": 348.9512939453125,
|
|
"margin_dpo/margin_std": 530.4649658203125,
|
|
"step": 290
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -322.58355712890625,
|
|
"KL/mean": -421.381103515625,
|
|
"KL/rejected_KL_mean": -520.1787109375,
|
|
"KL/std": 350.70855712890625,
|
|
"epoch": 0.42731277533039647,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 197.59518432617188,
|
|
"fcm_dpo/q_t": 0.4894236922264099,
|
|
"grad_norm": 4.782266139984131,
|
|
"learning_rate": 3.5565456543517485e-07,
|
|
"logits/chosen": -0.14178414642810822,
|
|
"logits/rejected": -0.1538257598876953,
|
|
"logps/chosen": -385.8439025878906,
|
|
"logps/ref_chosen": -63.26036834716797,
|
|
"logps/ref_rejected": -89.29708862304688,
|
|
"logps/rejected": -609.475830078125,
|
|
"loss": 1.3469,
|
|
"margin_dpo/margin_mean": 197.5951690673828,
|
|
"margin_dpo/margin_std": 465.94696044921875,
|
|
"step": 291
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -300.87261962890625,
|
|
"KL/mean": -426.175537109375,
|
|
"KL/rejected_KL_mean": -551.4783935546875,
|
|
"KL/std": 354.70635986328125,
|
|
"epoch": 0.4287812041116006,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 250.60580444335938,
|
|
"fcm_dpo/q_t": 0.48658984899520874,
|
|
"grad_norm": 4.581497669219971,
|
|
"learning_rate": 3.5449008622169583e-07,
|
|
"logits/chosen": -0.1559632122516632,
|
|
"logits/rejected": -0.16916170716285706,
|
|
"logps/chosen": -354.7911376953125,
|
|
"logps/ref_chosen": -53.91852951049805,
|
|
"logps/ref_rejected": -89.96138000488281,
|
|
"logps/rejected": -641.4398193359375,
|
|
"loss": 1.3355,
|
|
"margin_dpo/margin_mean": 250.60580444335938,
|
|
"margin_dpo/margin_std": 433.79608154296875,
|
|
"step": 292
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -365.57171630859375,
|
|
"KL/mean": -445.22955322265625,
|
|
"KL/rejected_KL_mean": -524.887451171875,
|
|
"KL/std": 410.1599426269531,
|
|
"epoch": 0.4302496328928047,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 159.31570434570312,
|
|
"fcm_dpo/q_t": 0.4914965033531189,
|
|
"grad_norm": 3.991530656814575,
|
|
"learning_rate": 3.5332285359726846e-07,
|
|
"logits/chosen": -0.15570124983787537,
|
|
"logits/rejected": -0.1721823364496231,
|
|
"logps/chosen": -425.94775390625,
|
|
"logps/ref_chosen": -60.376033782958984,
|
|
"logps/ref_rejected": -77.85244750976562,
|
|
"logps/rejected": -602.7398681640625,
|
|
"loss": 1.3547,
|
|
"margin_dpo/margin_mean": 159.315673828125,
|
|
"margin_dpo/margin_std": 447.989501953125,
|
|
"step": 293
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -289.04150390625,
|
|
"KL/mean": -404.94580078125,
|
|
"KL/rejected_KL_mean": -520.8502197265625,
|
|
"KL/std": 356.51409912109375,
|
|
"epoch": 0.43171806167400884,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 231.8087158203125,
|
|
"fcm_dpo/q_t": 0.4876137375831604,
|
|
"grad_norm": 5.801051616668701,
|
|
"learning_rate": 3.5215289831955786e-07,
|
|
"logits/chosen": -0.11081454157829285,
|
|
"logits/rejected": -0.12332391738891602,
|
|
"logps/chosen": -337.1290283203125,
|
|
"logps/ref_chosen": -48.0875358581543,
|
|
"logps/ref_rejected": -81.89698791503906,
|
|
"logps/rejected": -602.7471923828125,
|
|
"loss": 1.3401,
|
|
"margin_dpo/margin_mean": 231.80870056152344,
|
|
"margin_dpo/margin_std": 499.6016540527344,
|
|
"step": 294
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -377.87554931640625,
|
|
"KL/mean": -518.6566162109375,
|
|
"KL/rejected_KL_mean": -659.437744140625,
|
|
"KL/std": 447.6157531738281,
|
|
"epoch": 0.4331864904552129,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 281.56219482421875,
|
|
"fcm_dpo/q_t": 0.48502007126808167,
|
|
"grad_norm": 3.4757144451141357,
|
|
"learning_rate": 3.509802512179737e-07,
|
|
"logits/chosen": -0.13640262186527252,
|
|
"logits/rejected": -0.13862435519695282,
|
|
"logps/chosen": -427.80023193359375,
|
|
"logps/ref_chosen": -49.92467498779297,
|
|
"logps/ref_rejected": -87.45632934570312,
|
|
"logps/rejected": -746.89404296875,
|
|
"loss": 1.3308,
|
|
"margin_dpo/margin_mean": 281.56219482421875,
|
|
"margin_dpo/margin_std": 583.3878173828125,
|
|
"step": 295
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -489.06207275390625,
|
|
"KL/mean": -576.8760986328125,
|
|
"KL/rejected_KL_mean": -664.6901245117188,
|
|
"KL/std": 395.9642333984375,
|
|
"epoch": 0.434654919236417,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 175.62806701660156,
|
|
"fcm_dpo/q_t": 0.49063578248023987,
|
|
"grad_norm": 3.92090106010437,
|
|
"learning_rate": 3.498049431928577e-07,
|
|
"logits/chosen": -0.12152425199747086,
|
|
"logits/rejected": -0.12600529193878174,
|
|
"logps/chosen": -554.5533447265625,
|
|
"logps/ref_chosen": -65.49124145507812,
|
|
"logps/ref_rejected": -93.08908081054688,
|
|
"logps/rejected": -757.7791748046875,
|
|
"loss": 1.3523,
|
|
"margin_dpo/margin_mean": 175.62808227539062,
|
|
"margin_dpo/margin_std": 526.7503662109375,
|
|
"step": 296
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -468.5126037597656,
|
|
"KL/mean": -587.1865234375,
|
|
"KL/rejected_KL_mean": -705.8604736328125,
|
|
"KL/std": 483.5227966308594,
|
|
"epoch": 0.43612334801762115,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 237.34788513183594,
|
|
"fcm_dpo/q_t": 0.48738476634025574,
|
|
"grad_norm": 5.903518199920654,
|
|
"learning_rate": 3.486270052146694e-07,
|
|
"logits/chosen": -0.19832845032215118,
|
|
"logits/rejected": -0.213718980550766,
|
|
"logps/chosen": -524.9895629882812,
|
|
"logps/ref_chosen": -56.476951599121094,
|
|
"logps/ref_rejected": -95.1385498046875,
|
|
"logps/rejected": -800.9990234375,
|
|
"loss": 1.3395,
|
|
"margin_dpo/margin_mean": 237.34786987304688,
|
|
"margin_dpo/margin_std": 542.2998046875,
|
|
"step": 297
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -583.92919921875,
|
|
"KL/mean": -760.1380004882812,
|
|
"KL/rejected_KL_mean": -936.3467407226562,
|
|
"KL/std": 648.7681884765625,
|
|
"epoch": 0.43759177679882527,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 352.41748046875,
|
|
"fcm_dpo/q_t": 0.4814052879810333,
|
|
"grad_norm": 5.72074556350708,
|
|
"learning_rate": 3.474464683231698e-07,
|
|
"logits/chosen": -0.15087062120437622,
|
|
"logits/rejected": -0.1606747806072235,
|
|
"logps/chosen": -651.25439453125,
|
|
"logps/ref_chosen": -67.32516479492188,
|
|
"logps/ref_rejected": -116.66217041015625,
|
|
"logps/rejected": -1053.0089111328125,
|
|
"loss": 1.3195,
|
|
"margin_dpo/margin_mean": 352.41748046875,
|
|
"margin_dpo/margin_std": 788.0234375,
|
|
"step": 298
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -473.8212890625,
|
|
"KL/mean": -579.8349609375,
|
|
"KL/rejected_KL_mean": -685.8486328125,
|
|
"KL/std": 470.7117919921875,
|
|
"epoch": 0.4390602055800294,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 212.02735900878906,
|
|
"fcm_dpo/q_t": 0.4887089133262634,
|
|
"grad_norm": 7.583713054656982,
|
|
"learning_rate": 3.462633636266041e-07,
|
|
"logits/chosen": -0.23694965243339539,
|
|
"logits/rejected": -0.2537599205970764,
|
|
"logps/chosen": -522.7833862304688,
|
|
"logps/ref_chosen": -48.96209716796875,
|
|
"logps/ref_rejected": -84.32823944091797,
|
|
"logps/rejected": -770.1768798828125,
|
|
"loss": 1.3463,
|
|
"margin_dpo/margin_mean": 212.02735900878906,
|
|
"margin_dpo/margin_std": 633.6422119140625,
|
|
"step": 299
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -611.6695556640625,
|
|
"KL/mean": -765.8885498046875,
|
|
"KL/rejected_KL_mean": -920.107666015625,
|
|
"KL/std": 540.96484375,
|
|
"epoch": 0.44052863436123346,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 308.4380798339844,
|
|
"fcm_dpo/q_t": 0.48355600237846375,
|
|
"grad_norm": 7.862407207489014,
|
|
"learning_rate": 3.4507772230088147e-07,
|
|
"logits/chosen": -0.19781388342380524,
|
|
"logits/rejected": -0.21643571555614471,
|
|
"logps/chosen": -670.7432861328125,
|
|
"logps/ref_chosen": -59.07371139526367,
|
|
"logps/ref_rejected": -95.9664535522461,
|
|
"logps/rejected": -1016.0740966796875,
|
|
"loss": 1.327,
|
|
"margin_dpo/margin_mean": 308.4381103515625,
|
|
"margin_dpo/margin_std": 709.8028564453125,
|
|
"step": 300
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -573.643310546875,
|
|
"KL/mean": -723.9158935546875,
|
|
"KL/rejected_KL_mean": -874.1885375976562,
|
|
"KL/std": 545.066162109375,
|
|
"epoch": 0.4419970631424376,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 300.5451965332031,
|
|
"fcm_dpo/q_t": 0.48413360118865967,
|
|
"grad_norm": 6.303064346313477,
|
|
"learning_rate": 3.4388957558875316e-07,
|
|
"logits/chosen": -0.19899940490722656,
|
|
"logits/rejected": -0.21283400058746338,
|
|
"logps/chosen": -630.8927001953125,
|
|
"logps/ref_chosen": -57.249366760253906,
|
|
"logps/ref_rejected": -92.35354614257812,
|
|
"logps/rejected": -966.5421142578125,
|
|
"loss": 1.3293,
|
|
"margin_dpo/margin_mean": 300.5451965332031,
|
|
"margin_dpo/margin_std": 747.690185546875,
|
|
"step": 301
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -525.3902587890625,
|
|
"KL/mean": -676.2835693359375,
|
|
"KL/rejected_KL_mean": -827.1768798828125,
|
|
"KL/std": 550.9168701171875,
|
|
"epoch": 0.4434654919236417,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 301.78662109375,
|
|
"fcm_dpo/q_t": 0.4839833378791809,
|
|
"grad_norm": 6.866415500640869,
|
|
"learning_rate": 3.426989547989902e-07,
|
|
"logits/chosen": -0.2429056167602539,
|
|
"logits/rejected": -0.23922690749168396,
|
|
"logps/chosen": -576.5882568359375,
|
|
"logps/ref_chosen": -51.197994232177734,
|
|
"logps/ref_rejected": -97.22636413574219,
|
|
"logps/rejected": -924.4032592773438,
|
|
"loss": 1.3291,
|
|
"margin_dpo/margin_mean": 301.78662109375,
|
|
"margin_dpo/margin_std": 743.7776489257812,
|
|
"step": 302
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -630.103515625,
|
|
"KL/mean": -715.7218017578125,
|
|
"KL/rejected_KL_mean": -801.3401489257812,
|
|
"KL/std": 567.647705078125,
|
|
"epoch": 0.44493392070484583,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 171.23655700683594,
|
|
"fcm_dpo/q_t": 0.4908214807510376,
|
|
"grad_norm": 6.0209126472473145,
|
|
"learning_rate": 3.4150589130555773e-07,
|
|
"logits/chosen": -0.23265428841114044,
|
|
"logits/rejected": -0.24132990837097168,
|
|
"logps/chosen": -696.8175048828125,
|
|
"logps/ref_chosen": -66.71394348144531,
|
|
"logps/ref_rejected": -86.94542694091797,
|
|
"logps/rejected": -888.2855834960938,
|
|
"loss": 1.3558,
|
|
"margin_dpo/margin_mean": 171.23655700683594,
|
|
"margin_dpo/margin_std": 712.87548828125,
|
|
"step": 303
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -616.35107421875,
|
|
"KL/mean": -709.6608276367188,
|
|
"KL/rejected_KL_mean": -802.9705810546875,
|
|
"KL/std": 510.59539794921875,
|
|
"epoch": 0.44640234948604995,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 186.61947631835938,
|
|
"fcm_dpo/q_t": 0.49002233147621155,
|
|
"grad_norm": 5.741556167602539,
|
|
"learning_rate": 3.403104165467883e-07,
|
|
"logits/chosen": -0.2312694489955902,
|
|
"logits/rejected": -0.2368827909231186,
|
|
"logps/chosen": -688.3017578125,
|
|
"logps/ref_chosen": -71.95069885253906,
|
|
"logps/ref_rejected": -90.47203063964844,
|
|
"logps/rejected": -893.442626953125,
|
|
"loss": 1.3511,
|
|
"margin_dpo/margin_mean": 186.61947631835938,
|
|
"margin_dpo/margin_std": 618.4774169921875,
|
|
"step": 304
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -570.4090576171875,
|
|
"KL/mean": -677.1749267578125,
|
|
"KL/rejected_KL_mean": -783.9407958984375,
|
|
"KL/std": 503.7247009277344,
|
|
"epoch": 0.447870778267254,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 213.53167724609375,
|
|
"fcm_dpo/q_t": 0.4886142611503601,
|
|
"grad_norm": 5.134280681610107,
|
|
"learning_rate": 3.391125620245535e-07,
|
|
"logits/chosen": -0.27188488841056824,
|
|
"logits/rejected": -0.2840760350227356,
|
|
"logps/chosen": -637.204345703125,
|
|
"logps/ref_chosen": -66.79523468017578,
|
|
"logps/ref_rejected": -92.75459289550781,
|
|
"logps/rejected": -876.6953735351562,
|
|
"loss": 1.3456,
|
|
"margin_dpo/margin_mean": 213.53167724609375,
|
|
"margin_dpo/margin_std": 633.0828857421875,
|
|
"step": 305
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -537.7818603515625,
|
|
"KL/mean": -625.0843505859375,
|
|
"KL/rejected_KL_mean": -712.386962890625,
|
|
"KL/std": 452.66357421875,
|
|
"epoch": 0.44933920704845814,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 174.60516357421875,
|
|
"fcm_dpo/q_t": 0.4906894564628601,
|
|
"grad_norm": 5.521829128265381,
|
|
"learning_rate": 3.3791235930343417e-07,
|
|
"logits/chosen": -0.19213925302028656,
|
|
"logits/rejected": -0.19932372868061066,
|
|
"logps/chosen": -607.4656982421875,
|
|
"logps/ref_chosen": -69.68389892578125,
|
|
"logps/ref_rejected": -85.15919494628906,
|
|
"logps/rejected": -797.546142578125,
|
|
"loss": 1.3524,
|
|
"margin_dpo/margin_mean": 174.6051483154297,
|
|
"margin_dpo/margin_std": 521.4979858398438,
|
|
"step": 306
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -450.71343994140625,
|
|
"KL/mean": -559.7984619140625,
|
|
"KL/rejected_KL_mean": -668.8834228515625,
|
|
"KL/std": 426.73651123046875,
|
|
"epoch": 0.45080763582966227,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 218.17001342773438,
|
|
"fcm_dpo/q_t": 0.4883463978767395,
|
|
"grad_norm": 4.214844226837158,
|
|
"learning_rate": 3.367098400098881e-07,
|
|
"logits/chosen": -0.22033822536468506,
|
|
"logits/rejected": -0.224267840385437,
|
|
"logps/chosen": -520.8788452148438,
|
|
"logps/ref_chosen": -70.16542053222656,
|
|
"logps/ref_rejected": -86.97230529785156,
|
|
"logps/rejected": -755.855712890625,
|
|
"loss": 1.3434,
|
|
"margin_dpo/margin_mean": 218.17001342773438,
|
|
"margin_dpo/margin_std": 541.6478271484375,
|
|
"step": 307
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -401.5634765625,
|
|
"KL/mean": -537.181396484375,
|
|
"KL/rejected_KL_mean": -672.79931640625,
|
|
"KL/std": 440.7513122558594,
|
|
"epoch": 0.4522760646108664,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 271.23590087890625,
|
|
"fcm_dpo/q_t": 0.48551321029663086,
|
|
"grad_norm": 4.456334590911865,
|
|
"learning_rate": 3.355050358314172e-07,
|
|
"logits/chosen": -0.19973334670066833,
|
|
"logits/rejected": -0.2119927853345871,
|
|
"logps/chosen": -456.8084716796875,
|
|
"logps/ref_chosen": -55.2449951171875,
|
|
"logps/ref_rejected": -79.37226104736328,
|
|
"logps/rejected": -752.171630859375,
|
|
"loss": 1.3319,
|
|
"margin_dpo/margin_mean": 271.2358703613281,
|
|
"margin_dpo/margin_std": 503.54925537109375,
|
|
"step": 308
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -369.8251953125,
|
|
"KL/mean": -482.3707275390625,
|
|
"KL/rejected_KL_mean": -594.9161987304688,
|
|
"KL/std": 423.8313903808594,
|
|
"epoch": 0.45374449339207046,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 225.0910186767578,
|
|
"fcm_dpo/q_t": 0.48792463541030884,
|
|
"grad_norm": 4.421578407287598,
|
|
"learning_rate": 3.3429797851573183e-07,
|
|
"logits/chosen": -0.16784876585006714,
|
|
"logits/rejected": -0.18317045271396637,
|
|
"logps/chosen": -418.7843017578125,
|
|
"logps/ref_chosen": -48.959083557128906,
|
|
"logps/ref_rejected": -82.34072875976562,
|
|
"logps/rejected": -677.2569580078125,
|
|
"loss": 1.3417,
|
|
"margin_dpo/margin_mean": 225.0910186767578,
|
|
"margin_dpo/margin_std": 493.40032958984375,
|
|
"step": 309
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -440.13433837890625,
|
|
"KL/mean": -548.5415649414062,
|
|
"KL/rejected_KL_mean": -656.9487915039062,
|
|
"KL/std": 404.796875,
|
|
"epoch": 0.4552129221732746,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 216.81442260742188,
|
|
"fcm_dpo/q_t": 0.48840105533599854,
|
|
"grad_norm": 3.7958574295043945,
|
|
"learning_rate": 3.3308869986991487e-07,
|
|
"logits/chosen": -0.15935003757476807,
|
|
"logits/rejected": -0.17317405343055725,
|
|
"logps/chosen": -502.8761291503906,
|
|
"logps/ref_chosen": -62.74177932739258,
|
|
"logps/ref_rejected": -79.9302978515625,
|
|
"logps/rejected": -736.8790893554688,
|
|
"loss": 1.3427,
|
|
"margin_dpo/margin_mean": 216.81442260742188,
|
|
"margin_dpo/margin_std": 443.70294189453125,
|
|
"step": 310
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -415.8055419921875,
|
|
"KL/mean": -523.12060546875,
|
|
"KL/rejected_KL_mean": -630.4356689453125,
|
|
"KL/std": 363.2186279296875,
|
|
"epoch": 0.4566813509544787,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 214.63011169433594,
|
|
"fcm_dpo/q_t": 0.4885249733924866,
|
|
"grad_norm": 4.3125104904174805,
|
|
"learning_rate": 3.3187723175958346e-07,
|
|
"logits/chosen": -0.21918794512748718,
|
|
"logits/rejected": -0.22478339076042175,
|
|
"logps/chosen": -468.83349609375,
|
|
"logps/ref_chosen": -53.02798080444336,
|
|
"logps/ref_rejected": -77.43820190429688,
|
|
"logps/rejected": -707.8738403320312,
|
|
"loss": 1.3428,
|
|
"margin_dpo/margin_mean": 214.63011169433594,
|
|
"margin_dpo/margin_std": 417.1317443847656,
|
|
"step": 311
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -427.406005859375,
|
|
"KL/mean": -540.7687377929688,
|
|
"KL/rejected_KL_mean": -654.1314697265625,
|
|
"KL/std": 440.4998779296875,
|
|
"epoch": 0.4581497797356828,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 226.72544860839844,
|
|
"fcm_dpo/q_t": 0.4879283308982849,
|
|
"grad_norm": 4.058272361755371,
|
|
"learning_rate": 3.306636061080487e-07,
|
|
"logits/chosen": -0.17278644442558289,
|
|
"logits/rejected": -0.18266448378562927,
|
|
"logps/chosen": -476.7982177734375,
|
|
"logps/ref_chosen": -49.39221954345703,
|
|
"logps/ref_rejected": -75.79280853271484,
|
|
"logps/rejected": -729.92431640625,
|
|
"loss": 1.3417,
|
|
"margin_dpo/margin_mean": 226.7254638671875,
|
|
"margin_dpo/margin_std": 535.8837280273438,
|
|
"step": 312
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -382.3973388671875,
|
|
"KL/mean": -525.8050537109375,
|
|
"KL/rejected_KL_mean": -669.2127685546875,
|
|
"KL/std": 460.6397399902344,
|
|
"epoch": 0.45961820851688695,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 286.8153991699219,
|
|
"fcm_dpo/q_t": 0.48471421003341675,
|
|
"grad_norm": 4.11210298538208,
|
|
"learning_rate": 3.2944785489547537e-07,
|
|
"logits/chosen": -0.18957430124282837,
|
|
"logits/rejected": -0.2057763636112213,
|
|
"logps/chosen": -432.55010986328125,
|
|
"logps/ref_chosen": -50.152740478515625,
|
|
"logps/ref_rejected": -86.40620422363281,
|
|
"logps/rejected": -755.6190185546875,
|
|
"loss": 1.3291,
|
|
"margin_dpo/margin_mean": 286.8153991699219,
|
|
"margin_dpo/margin_std": 513.8978271484375,
|
|
"step": 313
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -473.88775634765625,
|
|
"KL/mean": -590.21435546875,
|
|
"KL/rejected_KL_mean": -706.541015625,
|
|
"KL/std": 460.99267578125,
|
|
"epoch": 0.461086637298091,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 232.65328979492188,
|
|
"fcm_dpo/q_t": 0.4876299202442169,
|
|
"grad_norm": 3.9764833450317383,
|
|
"learning_rate": 3.2823001015803857e-07,
|
|
"logits/chosen": -0.23278236389160156,
|
|
"logits/rejected": -0.2444659173488617,
|
|
"logps/chosen": -531.1253051757812,
|
|
"logps/ref_chosen": -57.237579345703125,
|
|
"logps/ref_rejected": -97.5965347290039,
|
|
"logps/rejected": -804.1375732421875,
|
|
"loss": 1.3414,
|
|
"margin_dpo/margin_mean": 232.65330505371094,
|
|
"margin_dpo/margin_std": 601.2679443359375,
|
|
"step": 314
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -396.1183166503906,
|
|
"KL/mean": -510.1334228515625,
|
|
"KL/rejected_KL_mean": -624.1485595703125,
|
|
"KL/std": 454.22100830078125,
|
|
"epoch": 0.46255506607929514,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 228.03021240234375,
|
|
"fcm_dpo/q_t": 0.4878466725349426,
|
|
"grad_norm": 4.002105236053467,
|
|
"learning_rate": 3.270101039870797e-07,
|
|
"logits/chosen": -0.1942490041255951,
|
|
"logits/rejected": -0.20377588272094727,
|
|
"logps/chosen": -445.18792724609375,
|
|
"logps/ref_chosen": -49.06958770751953,
|
|
"logps/ref_rejected": -85.68087768554688,
|
|
"logps/rejected": -709.8294067382812,
|
|
"loss": 1.3415,
|
|
"margin_dpo/margin_mean": 228.03021240234375,
|
|
"margin_dpo/margin_std": 553.0503540039062,
|
|
"step": 315
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -419.86102294921875,
|
|
"KL/mean": -615.5865478515625,
|
|
"KL/rejected_KL_mean": -811.3121948242188,
|
|
"KL/std": 490.57891845703125,
|
|
"epoch": 0.46402349486049926,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 391.451171875,
|
|
"fcm_dpo/q_t": 0.4791603088378906,
|
|
"grad_norm": 5.238584995269775,
|
|
"learning_rate": 3.2578816852826086e-07,
|
|
"logits/chosen": -0.2003282904624939,
|
|
"logits/rejected": -0.20365044474601746,
|
|
"logps/chosen": -474.12176513671875,
|
|
"logps/ref_chosen": -54.26074981689453,
|
|
"logps/ref_rejected": -101.2814712524414,
|
|
"logps/rejected": -912.5936279296875,
|
|
"loss": 1.3085,
|
|
"margin_dpo/margin_mean": 391.451171875,
|
|
"margin_dpo/margin_std": 604.1680908203125,
|
|
"step": 316
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -472.4468994140625,
|
|
"KL/mean": -679.812744140625,
|
|
"KL/rejected_KL_mean": -887.178466796875,
|
|
"KL/std": 529.8192138671875,
|
|
"epoch": 0.4654919236417034,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 414.73162841796875,
|
|
"fcm_dpo/q_t": 0.47804775834083557,
|
|
"grad_norm": 4.577226638793945,
|
|
"learning_rate": 3.2456423598071783e-07,
|
|
"logits/chosen": -0.2022346556186676,
|
|
"logits/rejected": -0.2135677933692932,
|
|
"logps/chosen": -528.5411376953125,
|
|
"logps/ref_chosen": -56.094207763671875,
|
|
"logps/ref_rejected": -100.69905090332031,
|
|
"logps/rejected": -987.8775634765625,
|
|
"loss": 1.3053,
|
|
"margin_dpo/margin_mean": 414.73162841796875,
|
|
"margin_dpo/margin_std": 667.3003540039062,
|
|
"step": 317
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -563.6049194335938,
|
|
"KL/mean": -657.4984130859375,
|
|
"KL/rejected_KL_mean": -751.3919067382812,
|
|
"KL/std": 476.6451721191406,
|
|
"epoch": 0.4669603524229075,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 187.7869873046875,
|
|
"fcm_dpo/q_t": 0.48997563123703003,
|
|
"grad_norm": 5.925684452056885,
|
|
"learning_rate": 3.233383385962115e-07,
|
|
"logits/chosen": -0.2424195110797882,
|
|
"logits/rejected": -0.2464376538991928,
|
|
"logps/chosen": -628.2506103515625,
|
|
"logps/ref_chosen": -64.64569854736328,
|
|
"logps/ref_rejected": -82.76425170898438,
|
|
"logps/rejected": -834.1561279296875,
|
|
"loss": 1.3496,
|
|
"margin_dpo/margin_mean": 187.78700256347656,
|
|
"margin_dpo/margin_std": 521.0235595703125,
|
|
"step": 318
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -436.5062255859375,
|
|
"KL/mean": -647.1476440429688,
|
|
"KL/rejected_KL_mean": -857.7891235351562,
|
|
"KL/std": 617.1673583984375,
|
|
"epoch": 0.4684287812041116,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 421.28289794921875,
|
|
"fcm_dpo/q_t": 0.47768670320510864,
|
|
"grad_norm": 7.061251163482666,
|
|
"learning_rate": 3.2211050867827805e-07,
|
|
"logits/chosen": -0.19704844057559967,
|
|
"logits/rejected": -0.20887039601802826,
|
|
"logps/chosen": -485.88995361328125,
|
|
"logps/ref_chosen": -49.383758544921875,
|
|
"logps/ref_rejected": -113.90650939941406,
|
|
"logps/rejected": -971.6956176757812,
|
|
"loss": 1.3037,
|
|
"margin_dpo/margin_mean": 421.28289794921875,
|
|
"margin_dpo/margin_std": 688.0316162109375,
|
|
"step": 319
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -505.7894592285156,
|
|
"KL/mean": -670.9154052734375,
|
|
"KL/rejected_KL_mean": -836.041259765625,
|
|
"KL/std": 497.88201904296875,
|
|
"epoch": 0.4698972099853157,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 330.25177001953125,
|
|
"fcm_dpo/q_t": 0.4823932647705078,
|
|
"grad_norm": 4.2582244873046875,
|
|
"learning_rate": 3.208807785813777e-07,
|
|
"logits/chosen": -0.15992069244384766,
|
|
"logits/rejected": -0.17337793111801147,
|
|
"logps/chosen": -565.2943725585938,
|
|
"logps/ref_chosen": -59.50489044189453,
|
|
"logps/ref_rejected": -97.66717529296875,
|
|
"logps/rejected": -933.70849609375,
|
|
"loss": 1.3212,
|
|
"margin_dpo/margin_mean": 330.25177001953125,
|
|
"margin_dpo/margin_std": 617.7380981445312,
|
|
"step": 320
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -604.0084228515625,
|
|
"KL/mean": -743.7211303710938,
|
|
"KL/rejected_KL_mean": -883.433837890625,
|
|
"KL/std": 562.9217529296875,
|
|
"epoch": 0.4713656387665198,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 279.4254455566406,
|
|
"fcm_dpo/q_t": 0.4851454198360443,
|
|
"grad_norm": 5.155450344085693,
|
|
"learning_rate": 3.1964918071004217e-07,
|
|
"logits/chosen": -0.14682038128376007,
|
|
"logits/rejected": -0.15771764516830444,
|
|
"logps/chosen": -665.5570678710938,
|
|
"logps/ref_chosen": -61.548683166503906,
|
|
"logps/ref_rejected": -91.64103698730469,
|
|
"logps/rejected": -975.074951171875,
|
|
"loss": 1.3322,
|
|
"margin_dpo/margin_mean": 279.42547607421875,
|
|
"margin_dpo/margin_std": 654.2274169921875,
|
|
"step": 321
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -511.4791259765625,
|
|
"KL/mean": -655.0626831054688,
|
|
"KL/rejected_KL_mean": -798.646240234375,
|
|
"KL/std": 467.83734130859375,
|
|
"epoch": 0.47283406754772395,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 287.16717529296875,
|
|
"fcm_dpo/q_t": 0.4846848249435425,
|
|
"grad_norm": 5.708774089813232,
|
|
"learning_rate": 3.184157475180207e-07,
|
|
"logits/chosen": -0.17217618227005005,
|
|
"logits/rejected": -0.18240973353385925,
|
|
"logps/chosen": -568.7691650390625,
|
|
"logps/ref_chosen": -57.29003143310547,
|
|
"logps/ref_rejected": -95.74992370605469,
|
|
"logps/rejected": -894.396240234375,
|
|
"loss": 1.3294,
|
|
"margin_dpo/margin_mean": 287.16717529296875,
|
|
"margin_dpo/margin_std": 554.1672973632812,
|
|
"step": 322
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -529.456787109375,
|
|
"KL/mean": -639.5374755859375,
|
|
"KL/rejected_KL_mean": -749.6182861328125,
|
|
"KL/std": 478.3594970703125,
|
|
"epoch": 0.47430249632892807,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 220.1614990234375,
|
|
"fcm_dpo/q_t": 0.4882357716560364,
|
|
"grad_norm": 5.594461441040039,
|
|
"learning_rate": 3.171805115074251e-07,
|
|
"logits/chosen": -0.1582409143447876,
|
|
"logits/rejected": -0.17803357541561127,
|
|
"logps/chosen": -580.690673828125,
|
|
"logps/ref_chosen": -51.23395919799805,
|
|
"logps/ref_rejected": -75.06192016601562,
|
|
"logps/rejected": -824.68017578125,
|
|
"loss": 1.3433,
|
|
"margin_dpo/margin_mean": 220.1614990234375,
|
|
"margin_dpo/margin_std": 536.7333984375,
|
|
"step": 323
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -634.5806884765625,
|
|
"KL/mean": -732.8265380859375,
|
|
"KL/rejected_KL_mean": -831.072509765625,
|
|
"KL/std": 546.5975952148438,
|
|
"epoch": 0.47577092511013214,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 196.49183654785156,
|
|
"fcm_dpo/q_t": 0.48958471417427063,
|
|
"grad_norm": 6.335326194763184,
|
|
"learning_rate": 3.1594350522787295e-07,
|
|
"logits/chosen": -0.18952664732933044,
|
|
"logits/rejected": -0.20132169127464294,
|
|
"logps/chosen": -699.7158203125,
|
|
"logps/ref_chosen": -65.13516998291016,
|
|
"logps/ref_rejected": -86.47750854492188,
|
|
"logps/rejected": -917.5499877929688,
|
|
"loss": 1.3508,
|
|
"margin_dpo/margin_mean": 196.49185180664062,
|
|
"margin_dpo/margin_std": 704.073974609375,
|
|
"step": 324
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -488.4103698730469,
|
|
"KL/mean": -557.0770263671875,
|
|
"KL/rejected_KL_mean": -625.7437133789062,
|
|
"KL/std": 416.86346435546875,
|
|
"epoch": 0.47723935389133626,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 137.33335876464844,
|
|
"fcm_dpo/q_t": 0.4926797151565552,
|
|
"grad_norm": 5.764030456542969,
|
|
"learning_rate": 3.147047612756302e-07,
|
|
"logits/chosen": -0.23148123919963837,
|
|
"logits/rejected": -0.2455836832523346,
|
|
"logps/chosen": -544.6259765625,
|
|
"logps/ref_chosen": -56.215599060058594,
|
|
"logps/ref_rejected": -70.08592987060547,
|
|
"logps/rejected": -695.8296508789062,
|
|
"loss": 1.3603,
|
|
"margin_dpo/margin_mean": 137.3333740234375,
|
|
"margin_dpo/margin_std": 519.9088745117188,
|
|
"step": 325
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -546.6886596679688,
|
|
"KL/mean": -624.8822021484375,
|
|
"KL/rejected_KL_mean": -703.0758666992188,
|
|
"KL/std": 415.1629333496094,
|
|
"epoch": 0.4787077826725404,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 156.38719177246094,
|
|
"fcm_dpo/q_t": 0.4916384220123291,
|
|
"grad_norm": 5.459661960601807,
|
|
"learning_rate": 3.134643122927519e-07,
|
|
"logits/chosen": -0.23426343500614166,
|
|
"logits/rejected": -0.24521896243095398,
|
|
"logps/chosen": -619.4136352539062,
|
|
"logps/ref_chosen": -72.72496032714844,
|
|
"logps/ref_rejected": -79.8467788696289,
|
|
"logps/rejected": -782.922607421875,
|
|
"loss": 1.3553,
|
|
"margin_dpo/margin_mean": 156.38717651367188,
|
|
"margin_dpo/margin_std": 438.69140625,
|
|
"step": 326
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -510.6233825683594,
|
|
"KL/mean": -662.1951904296875,
|
|
"KL/rejected_KL_mean": -813.7669677734375,
|
|
"KL/std": 460.990478515625,
|
|
"epoch": 0.4801762114537445,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 303.1435852050781,
|
|
"fcm_dpo/q_t": 0.48379653692245483,
|
|
"grad_norm": 6.2359209060668945,
|
|
"learning_rate": 3.1222219096622264e-07,
|
|
"logits/chosen": -0.19739487767219543,
|
|
"logits/rejected": -0.2052072286605835,
|
|
"logps/chosen": -579.7578125,
|
|
"logps/ref_chosen": -69.13441467285156,
|
|
"logps/ref_rejected": -111.93377685546875,
|
|
"logps/rejected": -925.7008056640625,
|
|
"loss": 1.3262,
|
|
"margin_dpo/margin_mean": 303.14361572265625,
|
|
"margin_dpo/margin_std": 575.9744873046875,
|
|
"step": 327
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -442.1131591796875,
|
|
"KL/mean": -588.37548828125,
|
|
"KL/rejected_KL_mean": -734.6378173828125,
|
|
"KL/std": 493.975830078125,
|
|
"epoch": 0.48164464023494863,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 292.524658203125,
|
|
"fcm_dpo/q_t": 0.4843920171260834,
|
|
"grad_norm": 5.148623943328857,
|
|
"learning_rate": 3.1097843002709427e-07,
|
|
"logits/chosen": -0.2125837206840515,
|
|
"logits/rejected": -0.2268255650997162,
|
|
"logps/chosen": -501.80035400390625,
|
|
"logps/ref_chosen": -59.68719482421875,
|
|
"logps/ref_rejected": -90.85499572753906,
|
|
"logps/rejected": -825.4927978515625,
|
|
"loss": 1.3284,
|
|
"margin_dpo/margin_mean": 292.524658203125,
|
|
"margin_dpo/margin_std": 573.4638671875,
|
|
"step": 328
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -491.8397216796875,
|
|
"KL/mean": -647.6664428710938,
|
|
"KL/rejected_KL_mean": -803.4930419921875,
|
|
"KL/std": 506.99774169921875,
|
|
"epoch": 0.4831130690161527,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 311.6534118652344,
|
|
"fcm_dpo/q_t": 0.48337361216545105,
|
|
"grad_norm": 4.15192985534668,
|
|
"learning_rate": 3.0973306224962437e-07,
|
|
"logits/chosen": -0.1763824224472046,
|
|
"logits/rejected": -0.19332997500896454,
|
|
"logps/chosen": -557.0859375,
|
|
"logps/ref_chosen": -65.2461929321289,
|
|
"logps/ref_rejected": -100.69770812988281,
|
|
"logps/rejected": -904.1907958984375,
|
|
"loss": 1.325,
|
|
"margin_dpo/margin_mean": 311.65338134765625,
|
|
"margin_dpo/margin_std": 609.5528564453125,
|
|
"step": 329
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -342.6683349609375,
|
|
"KL/mean": -501.531982421875,
|
|
"KL/rejected_KL_mean": -660.3956298828125,
|
|
"KL/std": 415.15380859375,
|
|
"epoch": 0.4845814977973568,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 317.727294921875,
|
|
"fcm_dpo/q_t": 0.483031302690506,
|
|
"grad_norm": 7.011267185211182,
|
|
"learning_rate": 3.084861204504122e-07,
|
|
"logits/chosen": -0.14472857117652893,
|
|
"logits/rejected": -0.1459788680076599,
|
|
"logps/chosen": -389.66668701171875,
|
|
"logps/ref_chosen": -46.998348236083984,
|
|
"logps/ref_rejected": -86.87684631347656,
|
|
"logps/rejected": -747.2724609375,
|
|
"loss": 1.3222,
|
|
"margin_dpo/margin_mean": 317.7272644042969,
|
|
"margin_dpo/margin_std": 499.0316162109375,
|
|
"step": 330
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -391.0220642089844,
|
|
"KL/mean": -550.6251220703125,
|
|
"KL/rejected_KL_mean": -710.228271484375,
|
|
"KL/std": 350.403076171875,
|
|
"epoch": 0.48604992657856094,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 319.2061462402344,
|
|
"fcm_dpo/q_t": 0.4829198718070984,
|
|
"grad_norm": 5.114991664886475,
|
|
"learning_rate": 3.072376374875335e-07,
|
|
"logits/chosen": -0.16779181361198425,
|
|
"logits/rejected": -0.1757754683494568,
|
|
"logps/chosen": -441.54632568359375,
|
|
"logps/ref_chosen": -50.52424621582031,
|
|
"logps/ref_rejected": -89.01544189453125,
|
|
"logps/rejected": -799.2437133789062,
|
|
"loss": 1.3205,
|
|
"margin_dpo/margin_mean": 319.2061462402344,
|
|
"margin_dpo/margin_std": 355.31463623046875,
|
|
"step": 331
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -429.01025390625,
|
|
"KL/mean": -513.4847412109375,
|
|
"KL/rejected_KL_mean": -597.9593505859375,
|
|
"KL/std": 391.617919921875,
|
|
"epoch": 0.48751835535976507,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 168.9490966796875,
|
|
"fcm_dpo/q_t": 0.4909921884536743,
|
|
"grad_norm": 4.994297981262207,
|
|
"learning_rate": 3.059876462596758e-07,
|
|
"logits/chosen": -0.14846037328243256,
|
|
"logits/rejected": -0.15179188549518585,
|
|
"logps/chosen": -478.1905212402344,
|
|
"logps/ref_chosen": -49.18028259277344,
|
|
"logps/ref_rejected": -76.48515319824219,
|
|
"logps/rejected": -674.4444580078125,
|
|
"loss": 1.3528,
|
|
"margin_dpo/margin_mean": 168.9490966796875,
|
|
"margin_dpo/margin_std": 455.59197998046875,
|
|
"step": 332
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -516.8207397460938,
|
|
"KL/mean": -647.05615234375,
|
|
"KL/rejected_KL_mean": -777.2916259765625,
|
|
"KL/std": 472.027099609375,
|
|
"epoch": 0.4889867841409692,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 260.4708251953125,
|
|
"fcm_dpo/q_t": 0.48608171939849854,
|
|
"grad_norm": 5.020219326019287,
|
|
"learning_rate": 3.0473617970527015e-07,
|
|
"logits/chosen": -0.22815987467765808,
|
|
"logits/rejected": -0.24586617946624756,
|
|
"logps/chosen": -580.5764770507812,
|
|
"logps/ref_chosen": -63.75574493408203,
|
|
"logps/ref_rejected": -95.04411315917969,
|
|
"logps/rejected": -872.335693359375,
|
|
"loss": 1.3361,
|
|
"margin_dpo/margin_mean": 260.4708251953125,
|
|
"margin_dpo/margin_std": 628.2245483398438,
|
|
"step": 333
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -535.3387451171875,
|
|
"KL/mean": -647.19287109375,
|
|
"KL/rejected_KL_mean": -759.047119140625,
|
|
"KL/std": 525.2220458984375,
|
|
"epoch": 0.49045521292217326,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 223.70834350585938,
|
|
"fcm_dpo/q_t": 0.48809105157852173,
|
|
"grad_norm": 5.767882823944092,
|
|
"learning_rate": 3.034832708016243e-07,
|
|
"logits/chosen": -0.2623814344406128,
|
|
"logits/rejected": -0.27332639694213867,
|
|
"logps/chosen": -602.3184814453125,
|
|
"logps/ref_chosen": -66.97975158691406,
|
|
"logps/ref_rejected": -95.31692504882812,
|
|
"logps/rejected": -854.364013671875,
|
|
"loss": 1.3445,
|
|
"margin_dpo/margin_mean": 223.70834350585938,
|
|
"margin_dpo/margin_std": 687.914794921875,
|
|
"step": 334
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -553.5545654296875,
|
|
"KL/mean": -645.1278076171875,
|
|
"KL/rejected_KL_mean": -736.701171875,
|
|
"KL/std": 531.672119140625,
|
|
"epoch": 0.4919236417033774,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 183.14663696289062,
|
|
"fcm_dpo/q_t": 0.4902423322200775,
|
|
"grad_norm": 5.44934606552124,
|
|
"learning_rate": 3.022289525640531e-07,
|
|
"logits/chosen": -0.2666965126991272,
|
|
"logits/rejected": -0.27081555128097534,
|
|
"logps/chosen": -616.0970458984375,
|
|
"logps/ref_chosen": -62.54248046875,
|
|
"logps/ref_rejected": -87.61770629882812,
|
|
"logps/rejected": -824.31884765625,
|
|
"loss": 1.351,
|
|
"margin_dpo/margin_mean": 183.14662170410156,
|
|
"margin_dpo/margin_std": 550.9996948242188,
|
|
"step": 335
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -554.318115234375,
|
|
"KL/mean": -750.877685546875,
|
|
"KL/rejected_KL_mean": -947.4371948242188,
|
|
"KL/std": 650.4267578125,
|
|
"epoch": 0.4933920704845815,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 393.1190490722656,
|
|
"fcm_dpo/q_t": 0.47911518812179565,
|
|
"grad_norm": 7.3909687995910645,
|
|
"learning_rate": 3.009732580450086e-07,
|
|
"logits/chosen": -0.21140223741531372,
|
|
"logits/rejected": -0.21927091479301453,
|
|
"logps/chosen": -608.8492431640625,
|
|
"logps/ref_chosen": -54.53115463256836,
|
|
"logps/ref_rejected": -104.40424346923828,
|
|
"logps/rejected": -1051.8414306640625,
|
|
"loss": 1.3116,
|
|
"margin_dpo/margin_mean": 393.11907958984375,
|
|
"margin_dpo/margin_std": 823.8309326171875,
|
|
"step": 336
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -571.74658203125,
|
|
"KL/mean": -740.125732421875,
|
|
"KL/rejected_KL_mean": -908.5048828125,
|
|
"KL/std": 476.75103759765625,
|
|
"epoch": 0.4948604992657856,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 336.75836181640625,
|
|
"fcm_dpo/q_t": 0.4820534288883209,
|
|
"grad_norm": 6.862320899963379,
|
|
"learning_rate": 2.9971622033320914e-07,
|
|
"logits/chosen": -0.25504040718078613,
|
|
"logits/rejected": -0.2622433006763458,
|
|
"logps/chosen": -636.875244140625,
|
|
"logps/ref_chosen": -65.12869262695312,
|
|
"logps/ref_rejected": -101.72701263427734,
|
|
"logps/rejected": -1010.23193359375,
|
|
"loss": 1.3197,
|
|
"margin_dpo/margin_mean": 336.75836181640625,
|
|
"margin_dpo/margin_std": 612.5994873046875,
|
|
"step": 337
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -500.8605651855469,
|
|
"KL/mean": -660.3416748046875,
|
|
"KL/rejected_KL_mean": -819.82275390625,
|
|
"KL/std": 447.0543212890625,
|
|
"epoch": 0.49632892804698975,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 318.9621887207031,
|
|
"fcm_dpo/q_t": 0.48294728994369507,
|
|
"grad_norm": 5.339162349700928,
|
|
"learning_rate": 2.984578725527675e-07,
|
|
"logits/chosen": -0.25964879989624023,
|
|
"logits/rejected": -0.27327626943588257,
|
|
"logps/chosen": -559.2832641601562,
|
|
"logps/ref_chosen": -58.422706604003906,
|
|
"logps/ref_rejected": -89.06854248046875,
|
|
"logps/rejected": -908.8912963867188,
|
|
"loss": 1.3215,
|
|
"margin_dpo/margin_mean": 318.9621887207031,
|
|
"margin_dpo/margin_std": 463.43841552734375,
|
|
"step": 338
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -594.2242431640625,
|
|
"KL/mean": -748.5675048828125,
|
|
"KL/rejected_KL_mean": -902.9107666015625,
|
|
"KL/std": 501.53179931640625,
|
|
"epoch": 0.4977973568281938,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 308.68658447265625,
|
|
"fcm_dpo/q_t": 0.48354700207710266,
|
|
"grad_norm": 6.215461730957031,
|
|
"learning_rate": 2.9719824786231796e-07,
|
|
"logits/chosen": -0.2669673264026642,
|
|
"logits/rejected": -0.27526944875717163,
|
|
"logps/chosen": -654.219482421875,
|
|
"logps/ref_chosen": -59.99531555175781,
|
|
"logps/ref_rejected": -103.9109115600586,
|
|
"logps/rejected": -1006.8216552734375,
|
|
"loss": 1.3258,
|
|
"margin_dpo/margin_mean": 308.6865539550781,
|
|
"margin_dpo/margin_std": 577.9491577148438,
|
|
"step": 339
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -631.038330078125,
|
|
"KL/mean": -757.1285400390625,
|
|
"KL/rejected_KL_mean": -883.21875,
|
|
"KL/std": 515.5421752929688,
|
|
"epoch": 0.49926578560939794,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 252.18031311035156,
|
|
"fcm_dpo/q_t": 0.48659059405326843,
|
|
"grad_norm": 7.41240930557251,
|
|
"learning_rate": 2.959373794541426e-07,
|
|
"logits/chosen": -0.2925180494785309,
|
|
"logits/rejected": -0.2974250912666321,
|
|
"logps/chosen": -683.8685913085938,
|
|
"logps/ref_chosen": -52.83022689819336,
|
|
"logps/ref_rejected": -73.10723114013672,
|
|
"logps/rejected": -956.325927734375,
|
|
"loss": 1.3383,
|
|
"margin_dpo/margin_mean": 252.18031311035156,
|
|
"margin_dpo/margin_std": 664.0989990234375,
|
|
"step": 340
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -659.0855712890625,
|
|
"KL/mean": -854.7269287109375,
|
|
"KL/rejected_KL_mean": -1050.3682861328125,
|
|
"KL/std": 603.756591796875,
|
|
"epoch": 0.5007342143906021,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 391.2827453613281,
|
|
"fcm_dpo/q_t": 0.47924739122390747,
|
|
"grad_norm": 8.219803810119629,
|
|
"learning_rate": 2.946753005532965e-07,
|
|
"logits/chosen": -0.30277353525161743,
|
|
"logits/rejected": -0.31048113107681274,
|
|
"logps/chosen": -706.9853515625,
|
|
"logps/ref_chosen": -47.899803161621094,
|
|
"logps/ref_rejected": -101.80987548828125,
|
|
"logps/rejected": -1152.17822265625,
|
|
"loss": 1.3112,
|
|
"margin_dpo/margin_mean": 391.28277587890625,
|
|
"margin_dpo/margin_std": 779.0953369140625,
|
|
"step": 341
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -684.173828125,
|
|
"KL/mean": -808.3017578125,
|
|
"KL/rejected_KL_mean": -932.4296875,
|
|
"KL/std": 543.6679077148438,
|
|
"epoch": 0.5022026431718062,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 248.255859375,
|
|
"fcm_dpo/q_t": 0.4867771863937378,
|
|
"grad_norm": 7.662797451019287,
|
|
"learning_rate": 2.934120444167326e-07,
|
|
"logits/chosen": -0.3431963324546814,
|
|
"logits/rejected": -0.34687745571136475,
|
|
"logps/chosen": -756.1705322265625,
|
|
"logps/ref_chosen": -71.99664306640625,
|
|
"logps/ref_rejected": -92.58959197998047,
|
|
"logps/rejected": -1025.019287109375,
|
|
"loss": 1.3395,
|
|
"margin_dpo/margin_mean": 248.255859375,
|
|
"margin_dpo/margin_std": 699.264892578125,
|
|
"step": 342
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -627.5256958007812,
|
|
"KL/mean": -823.489013671875,
|
|
"KL/rejected_KL_mean": -1019.4523315429688,
|
|
"KL/std": 555.4841918945312,
|
|
"epoch": 0.5036710719530103,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 391.92657470703125,
|
|
"fcm_dpo/q_t": 0.4790971279144287,
|
|
"grad_norm": 7.085535526275635,
|
|
"learning_rate": 2.9214764433242476e-07,
|
|
"logits/chosen": -0.31986576318740845,
|
|
"logits/rejected": -0.33300432562828064,
|
|
"logps/chosen": -681.9312744140625,
|
|
"logps/ref_chosen": -54.405616760253906,
|
|
"logps/ref_rejected": -111.04142761230469,
|
|
"logps/rejected": -1130.4937744140625,
|
|
"loss": 1.3076,
|
|
"margin_dpo/margin_mean": 391.92657470703125,
|
|
"margin_dpo/margin_std": 557.3361206054688,
|
|
"step": 343
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -689.4730224609375,
|
|
"KL/mean": -871.2525634765625,
|
|
"KL/rejected_KL_mean": -1053.0322265625,
|
|
"KL/std": 726.9136962890625,
|
|
"epoch": 0.5051395007342144,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 363.5591735839844,
|
|
"fcm_dpo/q_t": 0.48091691732406616,
|
|
"grad_norm": 9.887747764587402,
|
|
"learning_rate": 2.9088213361849126e-07,
|
|
"logits/chosen": -0.27034977078437805,
|
|
"logits/rejected": -0.2850009799003601,
|
|
"logps/chosen": -743.4376220703125,
|
|
"logps/ref_chosen": -53.96466827392578,
|
|
"logps/ref_rejected": -90.62336730957031,
|
|
"logps/rejected": -1143.655517578125,
|
|
"loss": 1.3198,
|
|
"margin_dpo/margin_mean": 363.55914306640625,
|
|
"margin_dpo/margin_std": 895.619140625,
|
|
"step": 344
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -792.3978881835938,
|
|
"KL/mean": -966.5239868164062,
|
|
"KL/rejected_KL_mean": -1140.650146484375,
|
|
"KL/std": 544.9487915039062,
|
|
"epoch": 0.5066079295154186,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 348.252197265625,
|
|
"fcm_dpo/q_t": 0.48146194219589233,
|
|
"grad_norm": 7.0328803062438965,
|
|
"learning_rate": 2.896155456223163e-07,
|
|
"logits/chosen": -0.279766321182251,
|
|
"logits/rejected": -0.2895045280456543,
|
|
"logps/chosen": -854.0836181640625,
|
|
"logps/ref_chosen": -61.685699462890625,
|
|
"logps/ref_rejected": -99.49041748046875,
|
|
"logps/rejected": -1240.1405029296875,
|
|
"loss": 1.3185,
|
|
"margin_dpo/margin_mean": 348.2522277832031,
|
|
"margin_dpo/margin_std": 693.2333984375,
|
|
"step": 345
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -717.4666748046875,
|
|
"KL/mean": -873.6412353515625,
|
|
"KL/rejected_KL_mean": -1029.8157958984375,
|
|
"KL/std": 560.0009155273438,
|
|
"epoch": 0.5080763582966226,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 312.34906005859375,
|
|
"fcm_dpo/q_t": 0.4833087921142578,
|
|
"grad_norm": 7.525673866271973,
|
|
"learning_rate": 2.883479137196714e-07,
|
|
"logits/chosen": -0.24226948618888855,
|
|
"logits/rejected": -0.2475300431251526,
|
|
"logps/chosen": -772.7229614257812,
|
|
"logps/ref_chosen": -55.256263732910156,
|
|
"logps/ref_rejected": -77.41532135009766,
|
|
"logps/rejected": -1107.2310791015625,
|
|
"loss": 1.3241,
|
|
"margin_dpo/margin_mean": 312.34906005859375,
|
|
"margin_dpo/margin_std": 564.9100952148438,
|
|
"step": 346
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -728.3167114257812,
|
|
"KL/mean": -851.8155517578125,
|
|
"KL/rejected_KL_mean": -975.3143310546875,
|
|
"KL/std": 575.36181640625,
|
|
"epoch": 0.5095447870778267,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 246.99755859375,
|
|
"fcm_dpo/q_t": 0.4868515729904175,
|
|
"grad_norm": 8.351125717163086,
|
|
"learning_rate": 2.8707927131383614e-07,
|
|
"logits/chosen": -0.20478571951389313,
|
|
"logits/rejected": -0.21141314506530762,
|
|
"logps/chosen": -785.8829345703125,
|
|
"logps/ref_chosen": -57.56623840332031,
|
|
"logps/ref_rejected": -92.35509490966797,
|
|
"logps/rejected": -1067.66943359375,
|
|
"loss": 1.3399,
|
|
"margin_dpo/margin_mean": 246.99752807617188,
|
|
"margin_dpo/margin_std": 715.6409912109375,
|
|
"step": 347
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -581.2276611328125,
|
|
"KL/mean": -719.3048095703125,
|
|
"KL/rejected_KL_mean": -857.382080078125,
|
|
"KL/std": 477.3961181640625,
|
|
"epoch": 0.5110132158590308,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 276.1544494628906,
|
|
"fcm_dpo/q_t": 0.485293447971344,
|
|
"grad_norm": 5.781659126281738,
|
|
"learning_rate": 2.858096518347179e-07,
|
|
"logits/chosen": -0.2297358363866806,
|
|
"logits/rejected": -0.2374425232410431,
|
|
"logps/chosen": -637.5453491210938,
|
|
"logps/ref_chosen": -56.31770324707031,
|
|
"logps/ref_rejected": -89.13836669921875,
|
|
"logps/rejected": -946.5204467773438,
|
|
"loss": 1.3329,
|
|
"margin_dpo/margin_mean": 276.1544494628906,
|
|
"margin_dpo/margin_std": 637.7725219726562,
|
|
"step": 348
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -541.278564453125,
|
|
"KL/mean": -737.878662109375,
|
|
"KL/rejected_KL_mean": -934.478759765625,
|
|
"KL/std": 600.9308471679688,
|
|
"epoch": 0.5124816446402349,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 393.2002258300781,
|
|
"fcm_dpo/q_t": 0.47914326190948486,
|
|
"grad_norm": 16.43769645690918,
|
|
"learning_rate": 2.845390887379706e-07,
|
|
"logits/chosen": -0.16476523876190186,
|
|
"logits/rejected": -0.1687617003917694,
|
|
"logps/chosen": -599.3040771484375,
|
|
"logps/ref_chosen": -58.025516510009766,
|
|
"logps/ref_rejected": -97.50515747070312,
|
|
"logps/rejected": -1031.98388671875,
|
|
"loss": 1.3102,
|
|
"margin_dpo/margin_mean": 393.2002258300781,
|
|
"margin_dpo/margin_std": 747.94921875,
|
|
"step": 349
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -617.7612915039062,
|
|
"KL/mean": -776.6915283203125,
|
|
"KL/rejected_KL_mean": -935.6217041015625,
|
|
"KL/std": 595.83447265625,
|
|
"epoch": 0.5139500734214391,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 317.8603820800781,
|
|
"fcm_dpo/q_t": 0.4831061363220215,
|
|
"grad_norm": 10.051895141601562,
|
|
"learning_rate": 2.8326761550411346e-07,
|
|
"logits/chosen": -0.21435394883155823,
|
|
"logits/rejected": -0.23840287327766418,
|
|
"logps/chosen": -682.091796875,
|
|
"logps/ref_chosen": -64.33049011230469,
|
|
"logps/ref_rejected": -89.87164306640625,
|
|
"logps/rejected": -1025.4932861328125,
|
|
"loss": 1.3262,
|
|
"margin_dpo/margin_mean": 317.8603820800781,
|
|
"margin_dpo/margin_std": 751.39208984375,
|
|
"step": 350
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -567.7418823242188,
|
|
"KL/mean": -760.047119140625,
|
|
"KL/rejected_KL_mean": -952.3523559570312,
|
|
"KL/std": 605.5358276367188,
|
|
"epoch": 0.5154185022026432,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 384.61053466796875,
|
|
"fcm_dpo/q_t": 0.4796157479286194,
|
|
"grad_norm": 6.818278789520264,
|
|
"learning_rate": 2.819952656376487e-07,
|
|
"logits/chosen": -0.23846173286437988,
|
|
"logits/rejected": -0.25154590606689453,
|
|
"logps/chosen": -628.4140014648438,
|
|
"logps/ref_chosen": -60.6721305847168,
|
|
"logps/ref_rejected": -101.5654296875,
|
|
"logps/rejected": -1053.9178466796875,
|
|
"loss": 1.3132,
|
|
"margin_dpo/margin_mean": 384.61053466796875,
|
|
"margin_dpo/margin_std": 796.4315795898438,
|
|
"step": 351
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -745.8148193359375,
|
|
"KL/mean": -805.4356689453125,
|
|
"KL/rejected_KL_mean": -865.0565185546875,
|
|
"KL/std": 641.70947265625,
|
|
"epoch": 0.5168869309838473,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 119.24172973632812,
|
|
"fcm_dpo/q_t": 0.49365556240081787,
|
|
"grad_norm": 10.332298278808594,
|
|
"learning_rate": 2.8072207266617854e-07,
|
|
"logits/chosen": -0.2152498960494995,
|
|
"logits/rejected": -0.20715099573135376,
|
|
"logps/chosen": -816.75830078125,
|
|
"logps/ref_chosen": -70.9434585571289,
|
|
"logps/ref_rejected": -76.6419677734375,
|
|
"logps/rejected": -941.698486328125,
|
|
"loss": 1.3666,
|
|
"margin_dpo/margin_mean": 119.2417221069336,
|
|
"margin_dpo/margin_std": 703.8898315429688,
|
|
"step": 352
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -665.606689453125,
|
|
"KL/mean": -830.3818359375,
|
|
"KL/rejected_KL_mean": -995.156982421875,
|
|
"KL/std": 639.630859375,
|
|
"epoch": 0.5183553597650514,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 329.55035400390625,
|
|
"fcm_dpo/q_t": 0.4826132357120514,
|
|
"grad_norm": 10.239563941955566,
|
|
"learning_rate": 2.794480701395219e-07,
|
|
"logits/chosen": -0.24213480949401855,
|
|
"logits/rejected": -0.2454732209444046,
|
|
"logps/chosen": -724.001953125,
|
|
"logps/ref_chosen": -58.39533996582031,
|
|
"logps/ref_rejected": -80.33553314208984,
|
|
"logps/rejected": -1075.4925537109375,
|
|
"loss": 1.3252,
|
|
"margin_dpo/margin_mean": 329.55035400390625,
|
|
"margin_dpo/margin_std": 818.1527099609375,
|
|
"step": 353
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -645.8775634765625,
|
|
"KL/mean": -835.401123046875,
|
|
"KL/rejected_KL_mean": -1024.924560546875,
|
|
"KL/std": 729.5835571289062,
|
|
"epoch": 0.5198237885462555,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 379.0469665527344,
|
|
"fcm_dpo/q_t": 0.48002883791923523,
|
|
"grad_norm": 9.386272430419922,
|
|
"learning_rate": 2.781732916288303e-07,
|
|
"logits/chosen": -0.27470749616622925,
|
|
"logits/rejected": -0.28340351581573486,
|
|
"logps/chosen": -705.6806030273438,
|
|
"logps/ref_chosen": -59.80299377441406,
|
|
"logps/ref_rejected": -88.75750732421875,
|
|
"logps/rejected": -1113.68212890625,
|
|
"loss": 1.3138,
|
|
"margin_dpo/margin_mean": 379.0469970703125,
|
|
"margin_dpo/margin_std": 765.1248779296875,
|
|
"step": 354
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -676.1866455078125,
|
|
"KL/mean": -879.7360229492188,
|
|
"KL/rejected_KL_mean": -1083.285400390625,
|
|
"KL/std": 590.9983520507812,
|
|
"epoch": 0.5212922173274597,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 407.0987243652344,
|
|
"fcm_dpo/q_t": 0.47829318046569824,
|
|
"grad_norm": 7.246441841125488,
|
|
"learning_rate": 2.7689777072570284e-07,
|
|
"logits/chosen": -0.3153689503669739,
|
|
"logits/rejected": -0.32817888259887695,
|
|
"logps/chosen": -730.315185546875,
|
|
"logps/ref_chosen": -54.12849807739258,
|
|
"logps/ref_rejected": -82.40606689453125,
|
|
"logps/rejected": -1165.69140625,
|
|
"loss": 1.3055,
|
|
"margin_dpo/margin_mean": 407.0987243652344,
|
|
"margin_dpo/margin_std": 628.2813720703125,
|
|
"step": 355
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -843.4148559570312,
|
|
"KL/mean": -896.946533203125,
|
|
"KL/rejected_KL_mean": -950.4781494140625,
|
|
"KL/std": 694.555908203125,
|
|
"epoch": 0.5227606461086637,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 107.06331634521484,
|
|
"fcm_dpo/q_t": 0.4944502115249634,
|
|
"grad_norm": 13.43985366821289,
|
|
"learning_rate": 2.7562154104130176e-07,
|
|
"logits/chosen": -0.3528267443180084,
|
|
"logits/rejected": -0.346576452255249,
|
|
"logps/chosen": -908.088623046875,
|
|
"logps/ref_chosen": -64.6738052368164,
|
|
"logps/ref_rejected": -75.89926147460938,
|
|
"logps/rejected": -1026.37744140625,
|
|
"loss": 1.3731,
|
|
"margin_dpo/margin_mean": 107.06331634521484,
|
|
"margin_dpo/margin_std": 916.18310546875,
|
|
"step": 356
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -742.629638671875,
|
|
"KL/mean": -851.8154907226562,
|
|
"KL/rejected_KL_mean": -961.0013427734375,
|
|
"KL/std": 635.5723876953125,
|
|
"epoch": 0.5242290748898678,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 218.3716583251953,
|
|
"fcm_dpo/q_t": 0.48831281065940857,
|
|
"grad_norm": 11.403450965881348,
|
|
"learning_rate": 2.7434463620546594e-07,
|
|
"logits/chosen": -0.34209227561950684,
|
|
"logits/rejected": -0.3437783718109131,
|
|
"logps/chosen": -795.35546875,
|
|
"logps/ref_chosen": -52.725799560546875,
|
|
"logps/ref_rejected": -86.84115600585938,
|
|
"logps/rejected": -1047.842529296875,
|
|
"loss": 1.3475,
|
|
"margin_dpo/margin_mean": 218.3716583251953,
|
|
"margin_dpo/margin_std": 797.1085205078125,
|
|
"step": 357
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -685.59619140625,
|
|
"KL/mean": -857.2205200195312,
|
|
"KL/rejected_KL_mean": -1028.844970703125,
|
|
"KL/std": 697.257568359375,
|
|
"epoch": 0.5256975036710719,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 343.24871826171875,
|
|
"fcm_dpo/q_t": 0.4819628894329071,
|
|
"grad_norm": 6.759915828704834,
|
|
"learning_rate": 2.730670898658255e-07,
|
|
"logits/chosen": -0.38823795318603516,
|
|
"logits/rejected": -0.39373278617858887,
|
|
"logps/chosen": -748.8016357421875,
|
|
"logps/ref_chosen": -63.20543670654297,
|
|
"logps/ref_rejected": -88.373291015625,
|
|
"logps/rejected": -1117.21826171875,
|
|
"loss": 1.3229,
|
|
"margin_dpo/margin_mean": 343.24871826171875,
|
|
"margin_dpo/margin_std": 876.3508911132812,
|
|
"step": 358
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -853.0894775390625,
|
|
"KL/mean": -1017.8627319335938,
|
|
"KL/rejected_KL_mean": -1182.635986328125,
|
|
"KL/std": 709.51904296875,
|
|
"epoch": 0.527165932452276,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 329.54638671875,
|
|
"fcm_dpo/q_t": 0.4825381338596344,
|
|
"grad_norm": 10.495577812194824,
|
|
"learning_rate": 2.717889356869146e-07,
|
|
"logits/chosen": -0.3527218997478485,
|
|
"logits/rejected": -0.3608393669128418,
|
|
"logps/chosen": -909.459716796875,
|
|
"logps/ref_chosen": -56.370216369628906,
|
|
"logps/ref_rejected": -82.17375183105469,
|
|
"logps/rejected": -1264.8096923828125,
|
|
"loss": 1.3279,
|
|
"margin_dpo/margin_mean": 329.54638671875,
|
|
"margin_dpo/margin_std": 976.3280639648438,
|
|
"step": 359
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -797.648193359375,
|
|
"KL/mean": -908.0338745117188,
|
|
"KL/rejected_KL_mean": -1018.4195556640625,
|
|
"KL/std": 799.6057739257812,
|
|
"epoch": 0.5286343612334802,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 220.7712860107422,
|
|
"fcm_dpo/q_t": 0.4882170855998993,
|
|
"grad_norm": 9.871207237243652,
|
|
"learning_rate": 2.7051020734928443e-07,
|
|
"logits/chosen": -0.30490997433662415,
|
|
"logits/rejected": -0.2990133762359619,
|
|
"logps/chosen": -849.108642578125,
|
|
"logps/ref_chosen": -51.460384368896484,
|
|
"logps/ref_rejected": -69.83892059326172,
|
|
"logps/rejected": -1088.2584228515625,
|
|
"loss": 1.3449,
|
|
"margin_dpo/margin_mean": 220.77130126953125,
|
|
"margin_dpo/margin_std": 689.1402587890625,
|
|
"step": 360
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -762.4368896484375,
|
|
"KL/mean": -887.9981689453125,
|
|
"KL/rejected_KL_mean": -1013.559326171875,
|
|
"KL/std": 713.6858520507812,
|
|
"epoch": 0.5301027900146843,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 251.12237548828125,
|
|
"fcm_dpo/q_t": 0.4866461157798767,
|
|
"grad_norm": 6.593760967254639,
|
|
"learning_rate": 2.6923093854861593e-07,
|
|
"logits/chosen": -0.2922874689102173,
|
|
"logits/rejected": -0.3055328130722046,
|
|
"logps/chosen": -816.3064575195312,
|
|
"logps/ref_chosen": -53.86951446533203,
|
|
"logps/ref_rejected": -90.7692642211914,
|
|
"logps/rejected": -1104.32861328125,
|
|
"loss": 1.3401,
|
|
"margin_dpo/margin_mean": 251.12237548828125,
|
|
"margin_dpo/margin_std": 772.4447631835938,
|
|
"step": 361
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -624.3651123046875,
|
|
"KL/mean": -883.3717041015625,
|
|
"KL/rejected_KL_mean": -1142.37841796875,
|
|
"KL/std": 716.97802734375,
|
|
"epoch": 0.5315712187958884,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 518.0132446289062,
|
|
"fcm_dpo/q_t": 0.47263094782829285,
|
|
"grad_norm": 13.076441764831543,
|
|
"learning_rate": 2.679511629948319e-07,
|
|
"logits/chosen": -0.30570244789123535,
|
|
"logits/rejected": -0.3104197382926941,
|
|
"logps/chosen": -683.004150390625,
|
|
"logps/ref_chosen": -58.639060974121094,
|
|
"logps/ref_rejected": -105.58195495605469,
|
|
"logps/rejected": -1247.9603271484375,
|
|
"loss": 1.2863,
|
|
"margin_dpo/margin_mean": 518.01318359375,
|
|
"margin_dpo/margin_std": 827.43408203125,
|
|
"step": 362
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -607.5775146484375,
|
|
"KL/mean": -829.8275146484375,
|
|
"KL/rejected_KL_mean": -1052.0775146484375,
|
|
"KL/std": 665.65380859375,
|
|
"epoch": 0.5330396475770925,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 444.5,
|
|
"fcm_dpo/q_t": 0.4764130711555481,
|
|
"grad_norm": 6.851832389831543,
|
|
"learning_rate": 2.6667091441120816e-07,
|
|
"logits/chosen": -0.29445457458496094,
|
|
"logits/rejected": -0.31152263283729553,
|
|
"logps/chosen": -652.1359252929688,
|
|
"logps/ref_chosen": -44.558380126953125,
|
|
"logps/ref_rejected": -74.69496154785156,
|
|
"logps/rejected": -1126.7724609375,
|
|
"loss": 1.2996,
|
|
"margin_dpo/margin_mean": 444.5,
|
|
"margin_dpo/margin_std": 740.0186767578125,
|
|
"step": 363
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -626.3096923828125,
|
|
"KL/mean": -794.18212890625,
|
|
"KL/rejected_KL_mean": -962.0546264648438,
|
|
"KL/std": 631.0501708984375,
|
|
"epoch": 0.5345080763582967,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 335.74493408203125,
|
|
"fcm_dpo/q_t": 0.48214179277420044,
|
|
"grad_norm": 7.443741321563721,
|
|
"learning_rate": 2.6539022653348575e-07,
|
|
"logits/chosen": -0.30771857500076294,
|
|
"logits/rejected": -0.32001155614852905,
|
|
"logps/chosen": -675.204345703125,
|
|
"logps/ref_chosen": -48.894622802734375,
|
|
"logps/ref_rejected": -91.395751953125,
|
|
"logps/rejected": -1053.4503173828125,
|
|
"loss": 1.3219,
|
|
"margin_dpo/margin_mean": 335.7449035644531,
|
|
"margin_dpo/margin_std": 725.6460571289062,
|
|
"step": 364
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -686.364013671875,
|
|
"KL/mean": -917.268798828125,
|
|
"KL/rejected_KL_mean": -1148.173583984375,
|
|
"KL/std": 698.85107421875,
|
|
"epoch": 0.5359765051395007,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 461.8096923828125,
|
|
"fcm_dpo/q_t": 0.4755679666996002,
|
|
"grad_norm": 10.261777877807617,
|
|
"learning_rate": 2.641091331089811e-07,
|
|
"logits/chosen": -0.31375348567962646,
|
|
"logits/rejected": -0.3348388671875,
|
|
"logps/chosen": -737.856689453125,
|
|
"logps/ref_chosen": -51.49274444580078,
|
|
"logps/ref_rejected": -92.70166778564453,
|
|
"logps/rejected": -1240.875244140625,
|
|
"loss": 1.2973,
|
|
"margin_dpo/margin_mean": 461.8096923828125,
|
|
"margin_dpo/margin_std": 814.092041015625,
|
|
"step": 365
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -645.4697265625,
|
|
"KL/mean": -870.0846557617188,
|
|
"KL/rejected_KL_mean": -1094.69970703125,
|
|
"KL/std": 782.6004028320312,
|
|
"epoch": 0.5374449339207048,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 449.22991943359375,
|
|
"fcm_dpo/q_t": 0.4762273132801056,
|
|
"grad_norm": 7.029806613922119,
|
|
"learning_rate": 2.6282766789569736e-07,
|
|
"logits/chosen": -0.32882604002952576,
|
|
"logits/rejected": -0.3432251214981079,
|
|
"logps/chosen": -690.1902465820312,
|
|
"logps/ref_chosen": -44.7205696105957,
|
|
"logps/ref_rejected": -83.31040954589844,
|
|
"logps/rejected": -1178.010009765625,
|
|
"loss": 1.3052,
|
|
"margin_dpo/margin_mean": 449.2299499511719,
|
|
"margin_dpo/margin_std": 1045.4915771484375,
|
|
"step": 366
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -684.59326171875,
|
|
"KL/mean": -842.3504638671875,
|
|
"KL/rejected_KL_mean": -1000.1076049804688,
|
|
"KL/std": 633.3516845703125,
|
|
"epoch": 0.5389133627019089,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 315.514404296875,
|
|
"fcm_dpo/q_t": 0.4832141101360321,
|
|
"grad_norm": 6.7098283767700195,
|
|
"learning_rate": 2.615458646614349e-07,
|
|
"logits/chosen": -0.3017085790634155,
|
|
"logits/rejected": -0.3029378056526184,
|
|
"logps/chosen": -742.9986572265625,
|
|
"logps/ref_chosen": -58.405418395996094,
|
|
"logps/ref_rejected": -76.75132751464844,
|
|
"logps/rejected": -1076.85888671875,
|
|
"loss": 1.326,
|
|
"margin_dpo/margin_mean": 315.514404296875,
|
|
"margin_dpo/margin_std": 734.252685546875,
|
|
"step": 367
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -611.4556884765625,
|
|
"KL/mean": -943.615966796875,
|
|
"KL/rejected_KL_mean": -1275.7762451171875,
|
|
"KL/std": 775.81884765625,
|
|
"epoch": 0.540381791483113,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 664.3204345703125,
|
|
"fcm_dpo/q_t": 0.46484458446502686,
|
|
"grad_norm": 25.336750030517578,
|
|
"learning_rate": 2.6026375718290083e-07,
|
|
"logits/chosen": -0.2958596348762512,
|
|
"logits/rejected": -0.3062882423400879,
|
|
"logps/chosen": -655.908203125,
|
|
"logps/ref_chosen": -44.452518463134766,
|
|
"logps/ref_rejected": -98.55526733398438,
|
|
"logps/rejected": -1374.33154296875,
|
|
"loss": 1.2562,
|
|
"margin_dpo/margin_mean": 664.3204345703125,
|
|
"margin_dpo/margin_std": 804.1547241210938,
|
|
"step": 368
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -814.8972778320312,
|
|
"KL/mean": -954.0897216796875,
|
|
"KL/rejected_KL_mean": -1093.2823486328125,
|
|
"KL/std": 686.5308227539062,
|
|
"epoch": 0.5418502202643172,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 278.38507080078125,
|
|
"fcm_dpo/q_t": 0.48527395725250244,
|
|
"grad_norm": 9.702912330627441,
|
|
"learning_rate": 2.589813792448196e-07,
|
|
"logits/chosen": -0.3168843388557434,
|
|
"logits/rejected": -0.32195502519607544,
|
|
"logps/chosen": -886.27880859375,
|
|
"logps/ref_chosen": -71.38150024414062,
|
|
"logps/ref_rejected": -91.29582214355469,
|
|
"logps/rejected": -1184.5782470703125,
|
|
"loss": 1.337,
|
|
"margin_dpo/margin_mean": 278.38507080078125,
|
|
"margin_dpo/margin_std": 897.73291015625,
|
|
"step": 369
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -883.7716064453125,
|
|
"KL/mean": -960.92041015625,
|
|
"KL/rejected_KL_mean": -1038.0693359375,
|
|
"KL/std": 785.7099609375,
|
|
"epoch": 0.5433186490455213,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 154.29771423339844,
|
|
"fcm_dpo/q_t": 0.491788387298584,
|
|
"grad_norm": 19.420272827148438,
|
|
"learning_rate": 2.5769876463904263e-07,
|
|
"logits/chosen": -0.3010944128036499,
|
|
"logits/rejected": -0.301439106464386,
|
|
"logps/chosen": -955.3790283203125,
|
|
"logps/ref_chosen": -71.60749816894531,
|
|
"logps/ref_rejected": -97.25978088378906,
|
|
"logps/rejected": -1135.3291015625,
|
|
"loss": 1.3638,
|
|
"margin_dpo/margin_mean": 154.2977294921875,
|
|
"margin_dpo/margin_std": 940.9684448242188,
|
|
"step": 370
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -807.43408203125,
|
|
"KL/mean": -988.1475219726562,
|
|
"KL/rejected_KL_mean": -1168.861083984375,
|
|
"KL/std": 789.1993408203125,
|
|
"epoch": 0.5447870778267254,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 361.4268798828125,
|
|
"fcm_dpo/q_t": 0.48083311319351196,
|
|
"grad_norm": 7.047267913818359,
|
|
"learning_rate": 2.5641594716365744e-07,
|
|
"logits/chosen": -0.3067021667957306,
|
|
"logits/rejected": -0.3089839816093445,
|
|
"logps/chosen": -876.8486328125,
|
|
"logps/ref_chosen": -69.41448974609375,
|
|
"logps/ref_rejected": -99.17217254638672,
|
|
"logps/rejected": -1268.033203125,
|
|
"loss": 1.3188,
|
|
"margin_dpo/margin_mean": 361.4268798828125,
|
|
"margin_dpo/margin_std": 860.834228515625,
|
|
"step": 371
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -753.4928588867188,
|
|
"KL/mean": -979.7862548828125,
|
|
"KL/rejected_KL_mean": -1206.079833984375,
|
|
"KL/std": 863.1746215820312,
|
|
"epoch": 0.5462555066079295,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 452.5868835449219,
|
|
"fcm_dpo/q_t": 0.47651100158691406,
|
|
"grad_norm": 18.109085083007812,
|
|
"learning_rate": 2.551329606220976e-07,
|
|
"logits/chosen": -0.28337526321411133,
|
|
"logits/rejected": -0.30206161737442017,
|
|
"logps/chosen": -815.3108520507812,
|
|
"logps/ref_chosen": -61.8179931640625,
|
|
"logps/ref_rejected": -78.53948974609375,
|
|
"logps/rejected": -1284.6192626953125,
|
|
"loss": 1.3037,
|
|
"margin_dpo/margin_mean": 452.5869140625,
|
|
"margin_dpo/margin_std": 1032.625732421875,
|
|
"step": 372
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -822.2640380859375,
|
|
"KL/mean": -992.8091430664062,
|
|
"KL/rejected_KL_mean": -1163.354248046875,
|
|
"KL/std": 807.1524047851562,
|
|
"epoch": 0.5477239353891337,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 341.0901794433594,
|
|
"fcm_dpo/q_t": 0.4819341003894806,
|
|
"grad_norm": 9.496335983276367,
|
|
"learning_rate": 2.538498388222517e-07,
|
|
"logits/chosen": -0.25716885924339294,
|
|
"logits/rejected": -0.26763084530830383,
|
|
"logps/chosen": -886.481201171875,
|
|
"logps/ref_chosen": -64.21713256835938,
|
|
"logps/ref_rejected": -85.95960998535156,
|
|
"logps/rejected": -1249.31396484375,
|
|
"loss": 1.324,
|
|
"margin_dpo/margin_mean": 341.0901794433594,
|
|
"margin_dpo/margin_std": 889.7625122070312,
|
|
"step": 373
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -697.943115234375,
|
|
"KL/mean": -933.2283325195312,
|
|
"KL/rejected_KL_mean": -1168.5135498046875,
|
|
"KL/std": 870.4952392578125,
|
|
"epoch": 0.5491923641703378,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 470.57049560546875,
|
|
"fcm_dpo/q_t": 0.47532325983047485,
|
|
"grad_norm": 11.988911628723145,
|
|
"learning_rate": 2.525666155755725e-07,
|
|
"logits/chosen": -0.3318430483341217,
|
|
"logits/rejected": -0.33818480372428894,
|
|
"logps/chosen": -768.59326171875,
|
|
"logps/ref_chosen": -70.65018463134766,
|
|
"logps/ref_rejected": -93.64016723632812,
|
|
"logps/rejected": -1262.1536865234375,
|
|
"loss": 1.3009,
|
|
"margin_dpo/margin_mean": 470.57049560546875,
|
|
"margin_dpo/margin_std": 1058.4764404296875,
|
|
"step": 374
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -752.4849243164062,
|
|
"KL/mean": -971.480712890625,
|
|
"KL/rejected_KL_mean": -1190.4764404296875,
|
|
"KL/std": 719.1031494140625,
|
|
"epoch": 0.5506607929515418,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 437.9915771484375,
|
|
"fcm_dpo/q_t": 0.47678330540657043,
|
|
"grad_norm": 14.679875373840332,
|
|
"learning_rate": 2.512833246961859e-07,
|
|
"logits/chosen": -0.33781394362449646,
|
|
"logits/rejected": -0.35372868180274963,
|
|
"logps/chosen": -812.565185546875,
|
|
"logps/ref_chosen": -60.080223083496094,
|
|
"logps/ref_rejected": -88.93830871582031,
|
|
"logps/rejected": -1279.414794921875,
|
|
"loss": 1.3037,
|
|
"margin_dpo/margin_mean": 437.9915771484375,
|
|
"margin_dpo/margin_std": 881.63671875,
|
|
"step": 375
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -957.6834716796875,
|
|
"KL/mean": -1178.202880859375,
|
|
"KL/rejected_KL_mean": -1398.72216796875,
|
|
"KL/std": 931.96435546875,
|
|
"epoch": 0.5521292217327459,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 441.0386962890625,
|
|
"fcm_dpo/q_t": 0.4765964150428772,
|
|
"grad_norm": 15.095070838928223,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": -0.3589029312133789,
|
|
"logits/rejected": -0.3605373501777649,
|
|
"logps/chosen": -1020.34375,
|
|
"logps/ref_chosen": -62.660308837890625,
|
|
"logps/ref_rejected": -105.52660369873047,
|
|
"logps/rejected": -1504.248779296875,
|
|
"loss": 1.3074,
|
|
"margin_dpo/margin_mean": 441.0386962890625,
|
|
"margin_dpo/margin_std": 1077.4156494140625,
|
|
"step": 376
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1065.7764892578125,
|
|
"KL/mean": -1337.8623046875,
|
|
"KL/rejected_KL_mean": -1609.947998046875,
|
|
"KL/std": 1192.957763671875,
|
|
"epoch": 0.55359765051395,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 544.1714477539062,
|
|
"fcm_dpo/q_t": 0.47193536162376404,
|
|
"grad_norm": 10.05336856842041,
|
|
"learning_rate": 2.487166753038141e-07,
|
|
"logits/chosen": -0.3872339129447937,
|
|
"logits/rejected": -0.40310975909233093,
|
|
"logps/chosen": -1120.2552490234375,
|
|
"logps/ref_chosen": -54.478736877441406,
|
|
"logps/ref_rejected": -98.70335388183594,
|
|
"logps/rejected": -1708.6513671875,
|
|
"loss": 1.2974,
|
|
"margin_dpo/margin_mean": 544.1714477539062,
|
|
"margin_dpo/margin_std": 1465.950439453125,
|
|
"step": 377
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -899.20458984375,
|
|
"KL/mean": -1161.714111328125,
|
|
"KL/rejected_KL_mean": -1424.2236328125,
|
|
"KL/std": 916.3400268554688,
|
|
"epoch": 0.5550660792951542,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 525.0191650390625,
|
|
"fcm_dpo/q_t": 0.4725556969642639,
|
|
"grad_norm": 14.26543140411377,
|
|
"learning_rate": 2.4743338442442754e-07,
|
|
"logits/chosen": -0.42005640268325806,
|
|
"logits/rejected": -0.44007664918899536,
|
|
"logps/chosen": -944.2251586914062,
|
|
"logps/ref_chosen": -45.02053451538086,
|
|
"logps/ref_rejected": -88.0469741821289,
|
|
"logps/rejected": -1512.270751953125,
|
|
"loss": 1.2936,
|
|
"margin_dpo/margin_mean": 525.0191650390625,
|
|
"margin_dpo/margin_std": 1212.896728515625,
|
|
"step": 378
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1210.5340576171875,
|
|
"KL/mean": -1521.940673828125,
|
|
"KL/rejected_KL_mean": -1833.34716796875,
|
|
"KL/std": 1398.197509765625,
|
|
"epoch": 0.5565345080763583,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 622.81298828125,
|
|
"fcm_dpo/q_t": 0.4689289629459381,
|
|
"grad_norm": 16.45003318786621,
|
|
"learning_rate": 2.461501611777483e-07,
|
|
"logits/chosen": -0.41934919357299805,
|
|
"logits/rejected": -0.44911909103393555,
|
|
"logps/chosen": -1263.7161865234375,
|
|
"logps/ref_chosen": -53.182098388671875,
|
|
"logps/ref_rejected": -114.3001708984375,
|
|
"logps/rejected": -1947.6473388671875,
|
|
"loss": 1.2913,
|
|
"margin_dpo/margin_mean": 622.81298828125,
|
|
"margin_dpo/margin_std": 1755.5107421875,
|
|
"step": 379
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1069.46337890625,
|
|
"KL/mean": -1406.3856201171875,
|
|
"KL/rejected_KL_mean": -1743.3077392578125,
|
|
"KL/std": 1310.0059814453125,
|
|
"epoch": 0.5580029368575624,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 673.8443603515625,
|
|
"fcm_dpo/q_t": 0.4651104509830475,
|
|
"grad_norm": 16.879432678222656,
|
|
"learning_rate": 2.4486703937790243e-07,
|
|
"logits/chosen": -0.43862009048461914,
|
|
"logits/rejected": -0.4662778377532959,
|
|
"logps/chosen": -1120.8165283203125,
|
|
"logps/ref_chosen": -51.3530387878418,
|
|
"logps/ref_rejected": -104.19169616699219,
|
|
"logps/rejected": -1847.49951171875,
|
|
"loss": 1.2767,
|
|
"margin_dpo/margin_mean": 673.8442993164062,
|
|
"margin_dpo/margin_std": 1641.3406982421875,
|
|
"step": 380
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1037.765380859375,
|
|
"KL/mean": -1180.1309814453125,
|
|
"KL/rejected_KL_mean": -1322.49658203125,
|
|
"KL/std": 1107.69970703125,
|
|
"epoch": 0.5594713656387665,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 284.7312316894531,
|
|
"fcm_dpo/q_t": 0.48559147119522095,
|
|
"grad_norm": 25.2132625579834,
|
|
"learning_rate": 2.435840528363426e-07,
|
|
"logits/chosen": -0.4325522482395172,
|
|
"logits/rejected": -0.4366803765296936,
|
|
"logps/chosen": -1095.568359375,
|
|
"logps/ref_chosen": -57.80306625366211,
|
|
"logps/ref_rejected": -79.21940612792969,
|
|
"logps/rejected": -1401.716064453125,
|
|
"loss": 1.3524,
|
|
"margin_dpo/margin_mean": 284.73126220703125,
|
|
"margin_dpo/margin_std": 1517.94580078125,
|
|
"step": 381
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -853.733642578125,
|
|
"KL/mean": -1091.995361328125,
|
|
"KL/rejected_KL_mean": -1330.25732421875,
|
|
"KL/std": 791.60595703125,
|
|
"epoch": 0.5609397944199707,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 476.52362060546875,
|
|
"fcm_dpo/q_t": 0.47479626536369324,
|
|
"grad_norm": 9.29355525970459,
|
|
"learning_rate": 2.4230123536095745e-07,
|
|
"logits/chosen": -0.4174577593803406,
|
|
"logits/rejected": -0.42780351638793945,
|
|
"logps/chosen": -919.75390625,
|
|
"logps/ref_chosen": -66.02030181884766,
|
|
"logps/ref_rejected": -110.71016693115234,
|
|
"logps/rejected": -1440.9674072265625,
|
|
"loss": 1.2955,
|
|
"margin_dpo/margin_mean": 476.52362060546875,
|
|
"margin_dpo/margin_std": 863.3676147460938,
|
|
"step": 382
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -779.2030029296875,
|
|
"KL/mean": -1037.3272705078125,
|
|
"KL/rejected_KL_mean": -1295.451416015625,
|
|
"KL/std": 926.0235595703125,
|
|
"epoch": 0.5624082232011748,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 516.2486572265625,
|
|
"fcm_dpo/q_t": 0.47278502583503723,
|
|
"grad_norm": 11.367318153381348,
|
|
"learning_rate": 2.4101862075518037e-07,
|
|
"logits/chosen": -0.39227527379989624,
|
|
"logits/rejected": -0.41163206100463867,
|
|
"logps/chosen": -829.594482421875,
|
|
"logps/ref_chosen": -50.39148712158203,
|
|
"logps/ref_rejected": -93.71589660644531,
|
|
"logps/rejected": -1389.16748046875,
|
|
"loss": 1.2977,
|
|
"margin_dpo/margin_mean": 516.2485961914062,
|
|
"margin_dpo/margin_std": 1285.308837890625,
|
|
"step": 383
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -682.0059814453125,
|
|
"KL/mean": -844.9951171875,
|
|
"KL/rejected_KL_mean": -1007.984375,
|
|
"KL/std": 720.017578125,
|
|
"epoch": 0.5638766519823789,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 325.9783935546875,
|
|
"fcm_dpo/q_t": 0.48265010118484497,
|
|
"grad_norm": 9.724451065063477,
|
|
"learning_rate": 2.397362428170992e-07,
|
|
"logits/chosen": -0.4297202527523041,
|
|
"logits/rejected": -0.4280009865760803,
|
|
"logps/chosen": -734.0520629882812,
|
|
"logps/ref_chosen": -52.046104431152344,
|
|
"logps/ref_rejected": -85.76089477539062,
|
|
"logps/rejected": -1093.7452392578125,
|
|
"loss": 1.3234,
|
|
"margin_dpo/margin_mean": 325.9783630371094,
|
|
"margin_dpo/margin_std": 708.7644653320312,
|
|
"step": 384
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -567.719970703125,
|
|
"KL/mean": -734.8634033203125,
|
|
"KL/rejected_KL_mean": -902.0068359375,
|
|
"KL/std": 536.7576904296875,
|
|
"epoch": 0.5653450807635829,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 334.28692626953125,
|
|
"fcm_dpo/q_t": 0.4821450114250183,
|
|
"grad_norm": 15.683626174926758,
|
|
"learning_rate": 2.3845413533856514e-07,
|
|
"logits/chosen": -0.36829620599746704,
|
|
"logits/rejected": -0.37683045864105225,
|
|
"logps/chosen": -633.2720947265625,
|
|
"logps/ref_chosen": -65.55215454101562,
|
|
"logps/ref_rejected": -77.82792663574219,
|
|
"logps/rejected": -979.834716796875,
|
|
"loss": 1.3211,
|
|
"margin_dpo/margin_mean": 334.28692626953125,
|
|
"margin_dpo/margin_std": 671.5125732421875,
|
|
"step": 385
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -539.3319091796875,
|
|
"KL/mean": -767.2230224609375,
|
|
"KL/rejected_KL_mean": -995.1141357421875,
|
|
"KL/std": 658.7130126953125,
|
|
"epoch": 0.566813509544787,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 455.7822265625,
|
|
"fcm_dpo/q_t": 0.4757852554321289,
|
|
"grad_norm": 15.568609237670898,
|
|
"learning_rate": 2.3717233210430254e-07,
|
|
"logits/chosen": -0.35743653774261475,
|
|
"logits/rejected": -0.36365118622779846,
|
|
"logps/chosen": -597.5537109375,
|
|
"logps/ref_chosen": -58.22185516357422,
|
|
"logps/ref_rejected": -92.32742309570312,
|
|
"logps/rejected": -1087.441650390625,
|
|
"loss": 1.2979,
|
|
"margin_dpo/margin_mean": 455.7822570800781,
|
|
"margin_dpo/margin_std": 782.3114013671875,
|
|
"step": 386
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -720.632080078125,
|
|
"KL/mean": -928.3619384765625,
|
|
"KL/rejected_KL_mean": -1136.091796875,
|
|
"KL/std": 814.0172119140625,
|
|
"epoch": 0.5682819383259912,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 415.4595947265625,
|
|
"fcm_dpo/q_t": 0.4782324433326721,
|
|
"grad_norm": 9.539994239807129,
|
|
"learning_rate": 2.3589086689101889e-07,
|
|
"logits/chosen": -0.4057348966598511,
|
|
"logits/rejected": -0.41669708490371704,
|
|
"logps/chosen": -787.051513671875,
|
|
"logps/ref_chosen": -66.41944885253906,
|
|
"logps/ref_rejected": -92.16915893554688,
|
|
"logps/rejected": -1228.2608642578125,
|
|
"loss": 1.3127,
|
|
"margin_dpo/margin_mean": 415.45965576171875,
|
|
"margin_dpo/margin_std": 1032.929443359375,
|
|
"step": 387
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -648.9365234375,
|
|
"KL/mean": -877.4620971679688,
|
|
"KL/rejected_KL_mean": -1105.9876708984375,
|
|
"KL/std": 719.4478759765625,
|
|
"epoch": 0.5697503671071953,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 457.05108642578125,
|
|
"fcm_dpo/q_t": 0.47609227895736694,
|
|
"grad_norm": 7.468353271484375,
|
|
"learning_rate": 2.3460977346651428e-07,
|
|
"logits/chosen": -0.3469131588935852,
|
|
"logits/rejected": -0.3659330904483795,
|
|
"logps/chosen": -699.0660400390625,
|
|
"logps/ref_chosen": -50.129459381103516,
|
|
"logps/ref_rejected": -104.43305969238281,
|
|
"logps/rejected": -1210.420654296875,
|
|
"loss": 1.3012,
|
|
"margin_dpo/margin_mean": 457.05108642578125,
|
|
"margin_dpo/margin_std": 926.29736328125,
|
|
"step": 388
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -743.910888671875,
|
|
"KL/mean": -964.4112548828125,
|
|
"KL/rejected_KL_mean": -1184.91162109375,
|
|
"KL/std": 831.6654052734375,
|
|
"epoch": 0.5712187958883994,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 441.00067138671875,
|
|
"fcm_dpo/q_t": 0.4765745997428894,
|
|
"grad_norm": 15.217109680175781,
|
|
"learning_rate": 2.3332908558879177e-07,
|
|
"logits/chosen": -0.394728422164917,
|
|
"logits/rejected": -0.4151400327682495,
|
|
"logps/chosen": -801.8175048828125,
|
|
"logps/ref_chosen": -57.906593322753906,
|
|
"logps/ref_rejected": -77.91454315185547,
|
|
"logps/rejected": -1262.826171875,
|
|
"loss": 1.3065,
|
|
"margin_dpo/margin_mean": 441.00067138671875,
|
|
"margin_dpo/margin_std": 1042.0787353515625,
|
|
"step": 389
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -730.2432861328125,
|
|
"KL/mean": -956.1160888671875,
|
|
"KL/rejected_KL_mean": -1181.9888916015625,
|
|
"KL/std": 893.8538208007812,
|
|
"epoch": 0.5726872246696035,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 451.74566650390625,
|
|
"fcm_dpo/q_t": 0.4772656559944153,
|
|
"grad_norm": 14.349326133728027,
|
|
"learning_rate": 2.320488370051681e-07,
|
|
"logits/chosen": -0.36197686195373535,
|
|
"logits/rejected": -0.3790530562400818,
|
|
"logps/chosen": -779.4691772460938,
|
|
"logps/ref_chosen": -49.22591781616211,
|
|
"logps/ref_rejected": -85.5281982421875,
|
|
"logps/rejected": -1267.51708984375,
|
|
"loss": 1.3101,
|
|
"margin_dpo/margin_mean": 451.7456970214844,
|
|
"margin_dpo/margin_std": 1213.085693359375,
|
|
"step": 390
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -934.3198852539062,
|
|
"KL/mean": -995.6199951171875,
|
|
"KL/rejected_KL_mean": -1056.920166015625,
|
|
"KL/std": 943.6309814453125,
|
|
"epoch": 0.5741556534508077,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 122.60029602050781,
|
|
"fcm_dpo/q_t": 0.49323803186416626,
|
|
"grad_norm": 19.008642196655273,
|
|
"learning_rate": 2.3076906145138405e-07,
|
|
"logits/chosen": -0.35622894763946533,
|
|
"logits/rejected": -0.3512190580368042,
|
|
"logps/chosen": -998.6495361328125,
|
|
"logps/ref_chosen": -64.32965087890625,
|
|
"logps/ref_rejected": -86.73820495605469,
|
|
"logps/rejected": -1143.6583251953125,
|
|
"loss": 1.3729,
|
|
"margin_dpo/margin_mean": 122.60028076171875,
|
|
"margin_dpo/margin_std": 1043.1915283203125,
|
|
"step": 391
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -610.44775390625,
|
|
"KL/mean": -939.5457763671875,
|
|
"KL/rejected_KL_mean": -1268.643798828125,
|
|
"KL/std": 832.3636474609375,
|
|
"epoch": 0.5756240822320118,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 658.1959838867188,
|
|
"fcm_dpo/q_t": 0.4652509093284607,
|
|
"grad_norm": 24.722393035888672,
|
|
"learning_rate": 2.294897926507156e-07,
|
|
"logits/chosen": -0.356148362159729,
|
|
"logits/rejected": -0.3655499219894409,
|
|
"logps/chosen": -663.9517822265625,
|
|
"logps/ref_chosen": -53.50397872924805,
|
|
"logps/ref_rejected": -102.34584045410156,
|
|
"logps/rejected": -1370.9896240234375,
|
|
"loss": 1.2592,
|
|
"margin_dpo/margin_mean": 658.1959228515625,
|
|
"margin_dpo/margin_std": 873.6790771484375,
|
|
"step": 392
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -772.148681640625,
|
|
"KL/mean": -1021.87158203125,
|
|
"KL/rejected_KL_mean": -1271.5946044921875,
|
|
"KL/std": 1009.1136474609375,
|
|
"epoch": 0.5770925110132159,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 499.4459228515625,
|
|
"fcm_dpo/q_t": 0.4741513729095459,
|
|
"grad_norm": 10.550104141235352,
|
|
"learning_rate": 2.2821106431308543e-07,
|
|
"logits/chosen": -0.36827749013900757,
|
|
"logits/rejected": -0.38175201416015625,
|
|
"logps/chosen": -818.62255859375,
|
|
"logps/ref_chosen": -46.473915100097656,
|
|
"logps/ref_rejected": -71.96885681152344,
|
|
"logps/rejected": -1343.5634765625,
|
|
"loss": 1.3002,
|
|
"margin_dpo/margin_mean": 499.4459228515625,
|
|
"margin_dpo/margin_std": 1268.5267333984375,
|
|
"step": 393
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -866.2552490234375,
|
|
"KL/mean": -1105.1195068359375,
|
|
"KL/rejected_KL_mean": -1343.98388671875,
|
|
"KL/std": 1064.1612548828125,
|
|
"epoch": 0.57856093979442,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 477.7286376953125,
|
|
"fcm_dpo/q_t": 0.4750671088695526,
|
|
"grad_norm": 10.232756614685059,
|
|
"learning_rate": 2.2693291013417452e-07,
|
|
"logits/chosen": -0.39185428619384766,
|
|
"logits/rejected": -0.4062337279319763,
|
|
"logps/chosen": -919.1668090820312,
|
|
"logps/ref_chosen": -52.91154861450195,
|
|
"logps/ref_rejected": -90.8226318359375,
|
|
"logps/rejected": -1434.8065185546875,
|
|
"loss": 1.2995,
|
|
"margin_dpo/margin_mean": 477.7286376953125,
|
|
"margin_dpo/margin_std": 1065.57275390625,
|
|
"step": 394
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -986.03759765625,
|
|
"KL/mean": -1240.0714111328125,
|
|
"KL/rejected_KL_mean": -1494.10498046875,
|
|
"KL/std": 1024.42822265625,
|
|
"epoch": 0.580029368575624,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 508.0675048828125,
|
|
"fcm_dpo/q_t": 0.473642498254776,
|
|
"grad_norm": 12.610128402709961,
|
|
"learning_rate": 2.2565536379453404e-07,
|
|
"logits/chosen": -0.4311944246292114,
|
|
"logits/rejected": -0.44871261715888977,
|
|
"logps/chosen": -1048.583740234375,
|
|
"logps/ref_chosen": -62.546112060546875,
|
|
"logps/ref_rejected": -83.78262329101562,
|
|
"logps/rejected": -1577.8876953125,
|
|
"loss": 1.3004,
|
|
"margin_dpo/margin_mean": 508.0675048828125,
|
|
"margin_dpo/margin_std": 1298.3829345703125,
|
|
"step": 395
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1033.08447265625,
|
|
"KL/mean": -1220.91015625,
|
|
"KL/rejected_KL_mean": -1408.7357177734375,
|
|
"KL/std": 1084.0780029296875,
|
|
"epoch": 0.5814977973568282,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 375.6512451171875,
|
|
"fcm_dpo/q_t": 0.4801104664802551,
|
|
"grad_norm": 10.486356735229492,
|
|
"learning_rate": 2.2437845895869825e-07,
|
|
"logits/chosen": -0.43420612812042236,
|
|
"logits/rejected": -0.4390130639076233,
|
|
"logps/chosen": -1102.0804443359375,
|
|
"logps/ref_chosen": -68.99594116210938,
|
|
"logps/ref_rejected": -88.64665985107422,
|
|
"logps/rejected": -1497.38232421875,
|
|
"loss": 1.3169,
|
|
"margin_dpo/margin_mean": 375.6512451171875,
|
|
"margin_dpo/margin_std": 911.8556518554688,
|
|
"step": 396
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1034.108154296875,
|
|
"KL/mean": -1459.783447265625,
|
|
"KL/rejected_KL_mean": -1885.45849609375,
|
|
"KL/std": 1350.3555908203125,
|
|
"epoch": 0.5829662261380323,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 851.3504028320312,
|
|
"fcm_dpo/q_t": 0.45677071809768677,
|
|
"grad_norm": 15.112154006958008,
|
|
"learning_rate": 2.2310222927429716e-07,
|
|
"logits/chosen": -0.4225345849990845,
|
|
"logits/rejected": -0.46220195293426514,
|
|
"logps/chosen": -1095.3853759765625,
|
|
"logps/ref_chosen": -61.27716827392578,
|
|
"logps/ref_rejected": -103.11612701416016,
|
|
"logps/rejected": -1988.57470703125,
|
|
"loss": 1.2402,
|
|
"margin_dpo/margin_mean": 851.3504028320312,
|
|
"margin_dpo/margin_std": 1585.188720703125,
|
|
"step": 397
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1233.375244140625,
|
|
"KL/mean": -1574.85107421875,
|
|
"KL/rejected_KL_mean": -1916.3270263671875,
|
|
"KL/std": 1434.3673095703125,
|
|
"epoch": 0.5844346549192364,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 682.951904296875,
|
|
"fcm_dpo/q_t": 0.46584078669548035,
|
|
"grad_norm": 23.73081398010254,
|
|
"learning_rate": 2.2182670837116972e-07,
|
|
"logits/chosen": -0.4878053069114685,
|
|
"logits/rejected": -0.5119198560714722,
|
|
"logps/chosen": -1301.5267333984375,
|
|
"logps/ref_chosen": -68.15155029296875,
|
|
"logps/ref_rejected": -108.52360534667969,
|
|
"logps/rejected": -2024.8505859375,
|
|
"loss": 1.2838,
|
|
"margin_dpo/margin_mean": 682.951904296875,
|
|
"margin_dpo/margin_std": 1882.985595703125,
|
|
"step": 398
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -991.1121826171875,
|
|
"KL/mean": -1204.9122314453125,
|
|
"KL/rejected_KL_mean": -1418.7122802734375,
|
|
"KL/std": 1063.78857421875,
|
|
"epoch": 0.5859030837004405,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 427.60003662109375,
|
|
"fcm_dpo/q_t": 0.4776034951210022,
|
|
"grad_norm": 33.797943115234375,
|
|
"learning_rate": 2.2055192986047804e-07,
|
|
"logits/chosen": -0.4256382882595062,
|
|
"logits/rejected": -0.4321112334728241,
|
|
"logps/chosen": -1052.001953125,
|
|
"logps/ref_chosen": -60.889801025390625,
|
|
"logps/ref_rejected": -77.965576171875,
|
|
"logps/rejected": -1496.6778564453125,
|
|
"loss": 1.3194,
|
|
"margin_dpo/margin_mean": 427.60003662109375,
|
|
"margin_dpo/margin_std": 1410.41748046875,
|
|
"step": 399
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -877.5531005859375,
|
|
"KL/mean": -1235.20166015625,
|
|
"KL/rejected_KL_mean": -1592.8502197265625,
|
|
"KL/std": 992.3692016601562,
|
|
"epoch": 0.5873715124816447,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 715.2971801757812,
|
|
"fcm_dpo/q_t": 0.46237778663635254,
|
|
"grad_norm": 10.352943420410156,
|
|
"learning_rate": 2.192779273338215e-07,
|
|
"logits/chosen": -0.426511287689209,
|
|
"logits/rejected": -0.44806522130966187,
|
|
"logps/chosen": -941.1966552734375,
|
|
"logps/ref_chosen": -63.64359664916992,
|
|
"logps/ref_rejected": -105.252685546875,
|
|
"logps/rejected": -1698.1029052734375,
|
|
"loss": 1.2569,
|
|
"margin_dpo/margin_mean": 715.2971801757812,
|
|
"margin_dpo/margin_std": 1255.512451171875,
|
|
"step": 400
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -964.7339477539062,
|
|
"KL/mean": -1108.6376953125,
|
|
"KL/rejected_KL_mean": -1252.54150390625,
|
|
"KL/std": 1027.1666259765625,
|
|
"epoch": 0.5888399412628488,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 287.8074951171875,
|
|
"fcm_dpo/q_t": 0.48464304208755493,
|
|
"grad_norm": 16.782976150512695,
|
|
"learning_rate": 2.1800473436235136e-07,
|
|
"logits/chosen": -0.3969235122203827,
|
|
"logits/rejected": -0.39872997999191284,
|
|
"logps/chosen": -1021.89697265625,
|
|
"logps/ref_chosen": -57.16303253173828,
|
|
"logps/ref_rejected": -83.79249572753906,
|
|
"logps/rejected": -1336.333984375,
|
|
"loss": 1.3476,
|
|
"margin_dpo/margin_mean": 287.8074951171875,
|
|
"margin_dpo/margin_std": 1398.988037109375,
|
|
"step": 401
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -710.2174072265625,
|
|
"KL/mean": -1041.47021484375,
|
|
"KL/rejected_KL_mean": -1372.72314453125,
|
|
"KL/std": 974.0499877929688,
|
|
"epoch": 0.5903083700440529,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 662.5057983398438,
|
|
"fcm_dpo/q_t": 0.4654858708381653,
|
|
"grad_norm": 8.760465621948242,
|
|
"learning_rate": 2.1673238449588665e-07,
|
|
"logits/chosen": -0.4055900573730469,
|
|
"logits/rejected": -0.4254668951034546,
|
|
"logps/chosen": -760.957763671875,
|
|
"logps/ref_chosen": -50.74037170410156,
|
|
"logps/ref_rejected": -81.0460433959961,
|
|
"logps/rejected": -1453.7691650390625,
|
|
"loss": 1.264,
|
|
"margin_dpo/margin_mean": 662.5057373046875,
|
|
"margin_dpo/margin_std": 1141.46630859375,
|
|
"step": 402
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -607.0709228515625,
|
|
"KL/mean": -857.822021484375,
|
|
"KL/rejected_KL_mean": -1108.5732421875,
|
|
"KL/std": 957.948486328125,
|
|
"epoch": 0.591776798825257,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 501.5023498535156,
|
|
"fcm_dpo/q_t": 0.47387418150901794,
|
|
"grad_norm": 21.134559631347656,
|
|
"learning_rate": 2.154609112620295e-07,
|
|
"logits/chosen": -0.3894062042236328,
|
|
"logits/rejected": -0.41035395860671997,
|
|
"logps/chosen": -654.2182006835938,
|
|
"logps/ref_chosen": -47.14731216430664,
|
|
"logps/ref_rejected": -77.2666015625,
|
|
"logps/rejected": -1185.83984375,
|
|
"loss": 1.2943,
|
|
"margin_dpo/margin_mean": 501.5023193359375,
|
|
"margin_dpo/margin_std": 1025.334228515625,
|
|
"step": 403
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -573.6932373046875,
|
|
"KL/mean": -833.4141845703125,
|
|
"KL/rejected_KL_mean": -1093.13525390625,
|
|
"KL/std": 773.28173828125,
|
|
"epoch": 0.593245227606461,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 519.4420776367188,
|
|
"fcm_dpo/q_t": 0.47276201844215393,
|
|
"grad_norm": 26.00446128845215,
|
|
"learning_rate": 2.1419034816528218e-07,
|
|
"logits/chosen": -0.3456383943557739,
|
|
"logits/rejected": -0.35928741097450256,
|
|
"logps/chosen": -621.5684814453125,
|
|
"logps/ref_chosen": -47.875274658203125,
|
|
"logps/ref_rejected": -77.15499877929688,
|
|
"logps/rejected": -1170.290283203125,
|
|
"loss": 1.2903,
|
|
"margin_dpo/margin_mean": 519.4420166015625,
|
|
"margin_dpo/margin_std": 1035.61083984375,
|
|
"step": 404
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -729.4765625,
|
|
"KL/mean": -907.5888671875,
|
|
"KL/rejected_KL_mean": -1085.7010498046875,
|
|
"KL/std": 880.6702880859375,
|
|
"epoch": 0.5947136563876652,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 356.22454833984375,
|
|
"fcm_dpo/q_t": 0.48145124316215515,
|
|
"grad_norm": 15.456494331359863,
|
|
"learning_rate": 2.129207286861638e-07,
|
|
"logits/chosen": -0.37179362773895264,
|
|
"logits/rejected": -0.37930575013160706,
|
|
"logps/chosen": -794.6394653320312,
|
|
"logps/ref_chosen": -65.16290283203125,
|
|
"logps/ref_rejected": -87.18678283691406,
|
|
"logps/rejected": -1172.887939453125,
|
|
"loss": 1.3278,
|
|
"margin_dpo/margin_mean": 356.22454833984375,
|
|
"margin_dpo/margin_std": 1170.162841796875,
|
|
"step": 405
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -589.8623046875,
|
|
"KL/mean": -890.648681640625,
|
|
"KL/rejected_KL_mean": -1191.43505859375,
|
|
"KL/std": 858.3298950195312,
|
|
"epoch": 0.5961820851688693,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 601.5728149414062,
|
|
"fcm_dpo/q_t": 0.4688172936439514,
|
|
"grad_norm": 22.13282585144043,
|
|
"learning_rate": 2.1165208628032861e-07,
|
|
"logits/chosen": -0.38036268949508667,
|
|
"logits/rejected": -0.4102054834365845,
|
|
"logps/chosen": -639.6031494140625,
|
|
"logps/ref_chosen": -49.740814208984375,
|
|
"logps/ref_rejected": -92.07862854003906,
|
|
"logps/rejected": -1283.513671875,
|
|
"loss": 1.2762,
|
|
"margin_dpo/margin_mean": 601.5728759765625,
|
|
"margin_dpo/margin_std": 1076.5947265625,
|
|
"step": 406
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -646.1621704101562,
|
|
"KL/mean": -756.427978515625,
|
|
"KL/rejected_KL_mean": -866.6937255859375,
|
|
"KL/std": 696.524169921875,
|
|
"epoch": 0.5976505139500734,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 220.5315399169922,
|
|
"fcm_dpo/q_t": 0.48832640051841736,
|
|
"grad_norm": 9.917305946350098,
|
|
"learning_rate": 2.1038445437768375e-07,
|
|
"logits/chosen": -0.3698161244392395,
|
|
"logits/rejected": -0.3660295903682709,
|
|
"logps/chosen": -702.4928588867188,
|
|
"logps/ref_chosen": -56.33069610595703,
|
|
"logps/ref_rejected": -77.51209259033203,
|
|
"logps/rejected": -944.205810546875,
|
|
"loss": 1.3508,
|
|
"margin_dpo/margin_mean": 220.5315399169922,
|
|
"margin_dpo/margin_std": 968.973876953125,
|
|
"step": 407
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -636.0607299804688,
|
|
"KL/mean": -787.4437866210938,
|
|
"KL/rejected_KL_mean": -938.826904296875,
|
|
"KL/std": 689.7813720703125,
|
|
"epoch": 0.5991189427312775,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 302.76611328125,
|
|
"fcm_dpo/q_t": 0.4842342138290405,
|
|
"grad_norm": 18.907068252563477,
|
|
"learning_rate": 2.0911786638150872e-07,
|
|
"logits/chosen": -0.36819028854370117,
|
|
"logits/rejected": -0.37358710169792175,
|
|
"logps/chosen": -705.8500366210938,
|
|
"logps/ref_chosen": -69.789306640625,
|
|
"logps/ref_rejected": -90.09693908691406,
|
|
"logps/rejected": -1028.923828125,
|
|
"loss": 1.3326,
|
|
"margin_dpo/margin_mean": 302.7661437988281,
|
|
"margin_dpo/margin_std": 921.2728881835938,
|
|
"step": 408
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -648.5272216796875,
|
|
"KL/mean": -830.0584716796875,
|
|
"KL/rejected_KL_mean": -1011.5897216796875,
|
|
"KL/std": 854.2369384765625,
|
|
"epoch": 0.6005873715124816,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 363.0625305175781,
|
|
"fcm_dpo/q_t": 0.4812033176422119,
|
|
"grad_norm": 13.033660888671875,
|
|
"learning_rate": 2.0785235566757517e-07,
|
|
"logits/chosen": -0.34932005405426025,
|
|
"logits/rejected": -0.3593984544277191,
|
|
"logps/chosen": -715.8446044921875,
|
|
"logps/ref_chosen": -67.31744384765625,
|
|
"logps/ref_rejected": -84.904296875,
|
|
"logps/rejected": -1096.4940185546875,
|
|
"loss": 1.3264,
|
|
"margin_dpo/margin_mean": 363.0625305175781,
|
|
"margin_dpo/margin_std": 1165.644775390625,
|
|
"step": 409
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -634.2596435546875,
|
|
"KL/mean": -884.68310546875,
|
|
"KL/rejected_KL_mean": -1135.1065673828125,
|
|
"KL/std": 723.4794311523438,
|
|
"epoch": 0.6020558002936858,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 500.8468322753906,
|
|
"fcm_dpo/q_t": 0.4734492003917694,
|
|
"grad_norm": 18.643348693847656,
|
|
"learning_rate": 2.065879555832674e-07,
|
|
"logits/chosen": -0.406690776348114,
|
|
"logits/rejected": -0.41948169469833374,
|
|
"logps/chosen": -685.7250366210938,
|
|
"logps/ref_chosen": -51.465354919433594,
|
|
"logps/ref_rejected": -83.198974609375,
|
|
"logps/rejected": -1218.3055419921875,
|
|
"loss": 1.2898,
|
|
"margin_dpo/margin_mean": 500.84686279296875,
|
|
"margin_dpo/margin_std": 840.258544921875,
|
|
"step": 410
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -756.8233642578125,
|
|
"KL/mean": -1003.681640625,
|
|
"KL/rejected_KL_mean": -1250.539794921875,
|
|
"KL/std": 1031.4246826171875,
|
|
"epoch": 0.6035242290748899,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 493.71636962890625,
|
|
"fcm_dpo/q_t": 0.4749707281589508,
|
|
"grad_norm": 13.850335121154785,
|
|
"learning_rate": 2.0532469944670343e-07,
|
|
"logits/chosen": -0.41555359959602356,
|
|
"logits/rejected": -0.43157142400741577,
|
|
"logps/chosen": -809.1306762695312,
|
|
"logps/ref_chosen": -52.30727005004883,
|
|
"logps/ref_rejected": -80.69495391845703,
|
|
"logps/rejected": -1331.2347412109375,
|
|
"loss": 1.3047,
|
|
"margin_dpo/margin_mean": 493.71636962890625,
|
|
"margin_dpo/margin_std": 1287.54248046875,
|
|
"step": 411
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -895.0568237304688,
|
|
"KL/mean": -1222.3822021484375,
|
|
"KL/rejected_KL_mean": -1549.7076416015625,
|
|
"KL/std": 1135.393310546875,
|
|
"epoch": 0.604992657856094,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 654.65087890625,
|
|
"fcm_dpo/q_t": 0.466141939163208,
|
|
"grad_norm": 13.520402908325195,
|
|
"learning_rate": 2.0406262054585738e-07,
|
|
"logits/chosen": -0.4506192207336426,
|
|
"logits/rejected": -0.4855607748031616,
|
|
"logps/chosen": -948.200927734375,
|
|
"logps/ref_chosen": -53.144126892089844,
|
|
"logps/ref_rejected": -100.0608139038086,
|
|
"logps/rejected": -1649.7684326171875,
|
|
"loss": 1.2784,
|
|
"margin_dpo/margin_mean": 654.65087890625,
|
|
"margin_dpo/margin_std": 1556.40234375,
|
|
"step": 412
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1110.75390625,
|
|
"KL/mean": -1391.29638671875,
|
|
"KL/rejected_KL_mean": -1671.839111328125,
|
|
"KL/std": 1254.1436767578125,
|
|
"epoch": 0.6064610866372981,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 561.085205078125,
|
|
"fcm_dpo/q_t": 0.47032126784324646,
|
|
"grad_norm": 16.13272476196289,
|
|
"learning_rate": 2.0280175213768205e-07,
|
|
"logits/chosen": -0.4394839406013489,
|
|
"logits/rejected": -0.455962598323822,
|
|
"logps/chosen": -1172.3358154296875,
|
|
"logps/ref_chosen": -61.58196258544922,
|
|
"logps/ref_rejected": -99.47340393066406,
|
|
"logps/rejected": -1771.3125,
|
|
"loss": 1.2944,
|
|
"margin_dpo/margin_mean": 561.0852661132812,
|
|
"margin_dpo/margin_std": 1475.930908203125,
|
|
"step": 413
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -960.60693359375,
|
|
"KL/mean": -1207.114013671875,
|
|
"KL/rejected_KL_mean": -1453.6209716796875,
|
|
"KL/std": 1134.6884765625,
|
|
"epoch": 0.6079295154185022,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 493.0139465332031,
|
|
"fcm_dpo/q_t": 0.47513312101364136,
|
|
"grad_norm": 23.284347534179688,
|
|
"learning_rate": 2.0154212744723247e-07,
|
|
"logits/chosen": -0.43949374556541443,
|
|
"logits/rejected": -0.45314937829971313,
|
|
"logps/chosen": -1007.2384643554688,
|
|
"logps/ref_chosen": -46.63148498535156,
|
|
"logps/ref_rejected": -87.64653015136719,
|
|
"logps/rejected": -1541.2674560546875,
|
|
"loss": 1.3087,
|
|
"margin_dpo/margin_mean": 493.013916015625,
|
|
"margin_dpo/margin_std": 1478.425048828125,
|
|
"step": 414
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1096.2464599609375,
|
|
"KL/mean": -1322.900634765625,
|
|
"KL/rejected_KL_mean": -1549.5546875,
|
|
"KL/std": 1193.9111328125,
|
|
"epoch": 0.6093979441997063,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 453.30828857421875,
|
|
"fcm_dpo/q_t": 0.476369708776474,
|
|
"grad_norm": 14.205986022949219,
|
|
"learning_rate": 2.002837796667909e-07,
|
|
"logits/chosen": -0.5072834491729736,
|
|
"logits/rejected": -0.5145879983901978,
|
|
"logps/chosen": -1174.86474609375,
|
|
"logps/ref_chosen": -78.6182861328125,
|
|
"logps/ref_rejected": -100.47752380371094,
|
|
"logps/rejected": -1650.0322265625,
|
|
"loss": 1.3179,
|
|
"margin_dpo/margin_mean": 453.3082580566406,
|
|
"margin_dpo/margin_std": 1521.6240234375,
|
|
"step": 415
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -895.443115234375,
|
|
"KL/mean": -1211.381103515625,
|
|
"KL/rejected_KL_mean": -1527.31884765625,
|
|
"KL/std": 1104.52685546875,
|
|
"epoch": 0.6108663729809104,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 631.8758544921875,
|
|
"fcm_dpo/q_t": 0.4671364426612854,
|
|
"grad_norm": 18.065832138061523,
|
|
"learning_rate": 1.990267419549914e-07,
|
|
"logits/chosen": -0.42411717772483826,
|
|
"logits/rejected": -0.4482443332672119,
|
|
"logps/chosen": -953.7222900390625,
|
|
"logps/ref_chosen": -58.27912521362305,
|
|
"logps/ref_rejected": -90.56871795654297,
|
|
"logps/rejected": -1617.8875732421875,
|
|
"loss": 1.2725,
|
|
"margin_dpo/margin_mean": 631.8757934570312,
|
|
"margin_dpo/margin_std": 1231.4012451171875,
|
|
"step": 416
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -706.0615234375,
|
|
"KL/mean": -973.61181640625,
|
|
"KL/rejected_KL_mean": -1241.162109375,
|
|
"KL/std": 916.9801025390625,
|
|
"epoch": 0.6123348017621145,
|
|
"fcm_dpo/beta": 0.00021450650820042938,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 535.1005859375,
|
|
"fcm_dpo/q_t": 0.4720717668533325,
|
|
"grad_norm": 10.751476287841797,
|
|
"learning_rate": 1.9777104743594686e-07,
|
|
"logits/chosen": -0.4009503126144409,
|
|
"logits/rejected": -0.4133146107196808,
|
|
"logps/chosen": -756.26025390625,
|
|
"logps/ref_chosen": -50.1987190246582,
|
|
"logps/ref_rejected": -68.15184020996094,
|
|
"logps/rejected": -1309.31396484375,
|
|
"loss": 1.2872,
|
|
"margin_dpo/margin_mean": 535.1005859375,
|
|
"margin_dpo/margin_std": 1025.5020751953125,
|
|
"step": 417
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -857.3790893554688,
|
|
"KL/mean": -1115.459716796875,
|
|
"KL/rejected_KL_mean": -1373.5404052734375,
|
|
"KL/std": 974.062255859375,
|
|
"epoch": 0.6138032305433186,
|
|
"fcm_dpo/beta": 0.00038203209987841547,
|
|
"fcm_dpo/delta": 0.09407737106084824,
|
|
"fcm_dpo/margin": 516.1611938476562,
|
|
"fcm_dpo/q_t": 0.47300809621810913,
|
|
"grad_norm": 26.765777587890625,
|
|
"learning_rate": 1.965167291983757e-07,
|
|
"logits/chosen": -0.45634764432907104,
|
|
"logits/rejected": -0.4652099609375,
|
|
"logps/chosen": -939.3576049804688,
|
|
"logps/ref_chosen": -81.97846984863281,
|
|
"logps/ref_rejected": -104.69148254394531,
|
|
"logps/rejected": -1478.23193359375,
|
|
"loss": 1.2179,
|
|
"margin_dpo/margin_mean": 516.1611938476562,
|
|
"margin_dpo/margin_std": 1353.4468994140625,
|
|
"step": 418
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -676.5275268554688,
|
|
"KL/mean": -1033.2412109375,
|
|
"KL/rejected_KL_mean": -1389.954833984375,
|
|
"KL/std": 921.36865234375,
|
|
"epoch": 0.6152716593245228,
|
|
"fcm_dpo/beta": 0.0005777660990133882,
|
|
"fcm_dpo/delta": 0.011424558237195015,
|
|
"fcm_dpo/margin": 713.4273071289062,
|
|
"fcm_dpo/q_t": 0.4135735332965851,
|
|
"grad_norm": 23.824268341064453,
|
|
"learning_rate": 1.9526382029472988e-07,
|
|
"logits/chosen": -0.406105101108551,
|
|
"logits/rejected": -0.43193238973617554,
|
|
"logps/chosen": -729.4761962890625,
|
|
"logps/ref_chosen": -52.948646545410156,
|
|
"logps/ref_rejected": -91.58309936523438,
|
|
"logps/rejected": -1481.5379638671875,
|
|
"loss": 1.0996,
|
|
"margin_dpo/margin_mean": 713.4273071289062,
|
|
"margin_dpo/margin_std": 1061.15673828125,
|
|
"step": 419
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1244.012939453125,
|
|
"KL/mean": -1279.085693359375,
|
|
"KL/rejected_KL_mean": -1314.1585693359375,
|
|
"KL/std": 1466.0330810546875,
|
|
"epoch": 0.6167400881057269,
|
|
"fcm_dpo/beta": 0.000616069242823869,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 70.14578247070312,
|
|
"fcm_dpo/q_t": 0.47235944867134094,
|
|
"grad_norm": 252.84213256835938,
|
|
"learning_rate": 1.9401235374032425e-07,
|
|
"logits/chosen": -0.4455627501010895,
|
|
"logits/rejected": -0.4392239451408386,
|
|
"logps/chosen": -1321.78271484375,
|
|
"logps/ref_chosen": -77.7699203491211,
|
|
"logps/ref_rejected": -69.31985473632812,
|
|
"logps/rejected": -1383.478515625,
|
|
"loss": 1.5994,
|
|
"margin_dpo/margin_mean": 70.14576721191406,
|
|
"margin_dpo/margin_std": 2004.8504638671875,
|
|
"step": 420
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -741.7689208984375,
|
|
"KL/mean": -928.7274780273438,
|
|
"KL/rejected_KL_mean": -1115.68603515625,
|
|
"KL/std": 884.5521240234375,
|
|
"epoch": 0.618208516886931,
|
|
"fcm_dpo/beta": 0.000668178778141737,
|
|
"fcm_dpo/delta": 0.015629228204488754,
|
|
"fcm_dpo/margin": 373.91705322265625,
|
|
"fcm_dpo/q_t": 0.4435460865497589,
|
|
"grad_norm": 44.24656677246094,
|
|
"learning_rate": 1.9276236251246653e-07,
|
|
"logits/chosen": -0.45810630917549133,
|
|
"logits/rejected": -0.4658089280128479,
|
|
"logps/chosen": -795.5347900390625,
|
|
"logps/ref_chosen": -53.765865325927734,
|
|
"logps/ref_rejected": -89.28144836425781,
|
|
"logps/rejected": -1204.967529296875,
|
|
"loss": 1.2457,
|
|
"margin_dpo/margin_mean": 373.91705322265625,
|
|
"margin_dpo/margin_std": 1011.509765625,
|
|
"step": 421
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -949.32373046875,
|
|
"KL/mean": -1192.359130859375,
|
|
"KL/rejected_KL_mean": -1435.394775390625,
|
|
"KL/std": 1171.419677734375,
|
|
"epoch": 0.6196769456681351,
|
|
"fcm_dpo/beta": 0.0010301879374310374,
|
|
"fcm_dpo/delta": 0.06974077969789505,
|
|
"fcm_dpo/margin": 486.0709533691406,
|
|
"fcm_dpo/q_t": 0.4236348569393158,
|
|
"grad_norm": 89.19212341308594,
|
|
"learning_rate": 1.9151387954958792e-07,
|
|
"logits/chosen": -0.4476343095302582,
|
|
"logits/rejected": -0.464691162109375,
|
|
"logps/chosen": -1017.95751953125,
|
|
"logps/ref_chosen": -68.6337661743164,
|
|
"logps/ref_rejected": -87.86351013183594,
|
|
"logps/rejected": -1523.2581787109375,
|
|
"loss": 1.3495,
|
|
"margin_dpo/margin_mean": 486.0709228515625,
|
|
"margin_dpo/margin_std": 1334.6302490234375,
|
|
"step": 422
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -751.098876953125,
|
|
"KL/mean": -995.6851806640625,
|
|
"KL/rejected_KL_mean": -1240.2713623046875,
|
|
"KL/std": 885.8568115234375,
|
|
"epoch": 0.6211453744493393,
|
|
"fcm_dpo/beta": 0.00025757343973964453,
|
|
"fcm_dpo/delta": -0.1725754588842392,
|
|
"fcm_dpo/margin": 489.1725158691406,
|
|
"fcm_dpo/q_t": 0.4245114326477051,
|
|
"grad_norm": 22.50880241394043,
|
|
"learning_rate": 1.902669377503756e-07,
|
|
"logits/chosen": -0.45179301500320435,
|
|
"logits/rejected": -0.46642887592315674,
|
|
"logps/chosen": -806.0891723632812,
|
|
"logps/ref_chosen": -54.99030303955078,
|
|
"logps/ref_rejected": -86.30654907226562,
|
|
"logps/rejected": -1326.577880859375,
|
|
"loss": 1.2827,
|
|
"margin_dpo/margin_mean": 489.1724853515625,
|
|
"margin_dpo/margin_std": 1063.1331787109375,
|
|
"step": 423
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -590.9034423828125,
|
|
"KL/mean": -812.5848388671875,
|
|
"KL/rejected_KL_mean": -1034.266357421875,
|
|
"KL/std": 783.5313720703125,
|
|
"epoch": 0.6226138032305433,
|
|
"fcm_dpo/beta": 0.00025757343973964453,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 443.36285400390625,
|
|
"fcm_dpo/q_t": 0.47206780314445496,
|
|
"grad_norm": 10.314723014831543,
|
|
"learning_rate": 1.890215699729057e-07,
|
|
"logits/chosen": -0.38947632908821106,
|
|
"logits/rejected": -0.3932424783706665,
|
|
"logps/chosen": -646.9153442382812,
|
|
"logps/ref_chosen": -56.01192092895508,
|
|
"logps/ref_rejected": -66.47896575927734,
|
|
"logps/rejected": -1100.745361328125,
|
|
"loss": 1.2904,
|
|
"margin_dpo/margin_mean": 443.36285400390625,
|
|
"margin_dpo/margin_std": 945.1494750976562,
|
|
"step": 424
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -649.426513671875,
|
|
"KL/mean": -852.2593994140625,
|
|
"KL/rejected_KL_mean": -1055.0921630859375,
|
|
"KL/std": 737.5302124023438,
|
|
"epoch": 0.6240822320117474,
|
|
"fcm_dpo/beta": 0.00025757343973964453,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 405.6656188964844,
|
|
"fcm_dpo/q_t": 0.4741717576980591,
|
|
"grad_norm": 16.524267196655273,
|
|
"learning_rate": 1.8777780903377732e-07,
|
|
"logits/chosen": -0.36731043457984924,
|
|
"logits/rejected": -0.38339000940322876,
|
|
"logps/chosen": -696.2955322265625,
|
|
"logps/ref_chosen": -46.86899948120117,
|
|
"logps/ref_rejected": -95.92545318603516,
|
|
"logps/rejected": -1151.017578125,
|
|
"loss": 1.3006,
|
|
"margin_dpo/margin_mean": 405.6656188964844,
|
|
"margin_dpo/margin_std": 980.6445922851562,
|
|
"step": 425
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -496.1842041015625,
|
|
"KL/mean": -661.1705322265625,
|
|
"KL/rejected_KL_mean": -826.15673828125,
|
|
"KL/std": 525.2192993164062,
|
|
"epoch": 0.6255506607929515,
|
|
"fcm_dpo/beta": 0.00025757343973964453,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 329.9725341796875,
|
|
"fcm_dpo/q_t": 0.4789263606071472,
|
|
"grad_norm": 17.888946533203125,
|
|
"learning_rate": 1.8653568770724803e-07,
|
|
"logits/chosen": -0.3716368079185486,
|
|
"logits/rejected": -0.363339364528656,
|
|
"logps/chosen": -572.7677612304688,
|
|
"logps/ref_chosen": -76.58354187011719,
|
|
"logps/ref_rejected": -81.26658630371094,
|
|
"logps/rejected": -907.42333984375,
|
|
"loss": 1.3107,
|
|
"margin_dpo/margin_mean": 329.9725646972656,
|
|
"margin_dpo/margin_std": 637.1539916992188,
|
|
"step": 426
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -430.772216796875,
|
|
"KL/mean": -575.3130493164062,
|
|
"KL/rejected_KL_mean": -719.8538818359375,
|
|
"KL/std": 486.26800537109375,
|
|
"epoch": 0.6270190895741556,
|
|
"fcm_dpo/beta": 0.00025757343973964453,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 289.08160400390625,
|
|
"fcm_dpo/q_t": 0.481527715921402,
|
|
"grad_norm": 23.913623809814453,
|
|
"learning_rate": 1.8529523872436977e-07,
|
|
"logits/chosen": -0.3612695634365082,
|
|
"logits/rejected": -0.35891193151474,
|
|
"logps/chosen": -495.6260986328125,
|
|
"logps/ref_chosen": -64.8538818359375,
|
|
"logps/ref_rejected": -78.5660171508789,
|
|
"logps/rejected": -798.4198608398438,
|
|
"loss": 1.3182,
|
|
"margin_dpo/margin_mean": 289.08160400390625,
|
|
"margin_dpo/margin_std": 542.7352294921875,
|
|
"step": 427
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -599.3250732421875,
|
|
"KL/mean": -861.975341796875,
|
|
"KL/rejected_KL_mean": -1124.62548828125,
|
|
"KL/std": 748.1722412109375,
|
|
"epoch": 0.6284875183553598,
|
|
"fcm_dpo/beta": 0.00025757343973964453,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 525.3004760742188,
|
|
"fcm_dpo/q_t": 0.46697381138801575,
|
|
"grad_norm": 43.41276168823242,
|
|
"learning_rate": 1.8405649477212697e-07,
|
|
"logits/chosen": -0.41682755947113037,
|
|
"logits/rejected": -0.4279855489730835,
|
|
"logps/chosen": -661.9617919921875,
|
|
"logps/ref_chosen": -62.63666534423828,
|
|
"logps/ref_rejected": -103.28181457519531,
|
|
"logps/rejected": -1227.9073486328125,
|
|
"loss": 1.2697,
|
|
"margin_dpo/margin_mean": 525.3004760742188,
|
|
"margin_dpo/margin_std": 932.6864013671875,
|
|
"step": 428
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -824.616943359375,
|
|
"KL/mean": -973.2047119140625,
|
|
"KL/rejected_KL_mean": -1121.79248046875,
|
|
"KL/std": 866.7525024414062,
|
|
"epoch": 0.6299559471365639,
|
|
"fcm_dpo/beta": 0.00025757343973964453,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 297.17559814453125,
|
|
"fcm_dpo/q_t": 0.48108816146850586,
|
|
"grad_norm": 22.956989288330078,
|
|
"learning_rate": 1.828194884925749e-07,
|
|
"logits/chosen": -0.42566531896591187,
|
|
"logits/rejected": -0.4174090027809143,
|
|
"logps/chosen": -905.8509521484375,
|
|
"logps/ref_chosen": -81.23401641845703,
|
|
"logps/ref_rejected": -91.79493713378906,
|
|
"logps/rejected": -1213.58740234375,
|
|
"loss": 1.3307,
|
|
"margin_dpo/margin_mean": 297.1755676269531,
|
|
"margin_dpo/margin_std": 1031.1573486328125,
|
|
"step": 429
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -615.6317138671875,
|
|
"KL/mean": -830.307861328125,
|
|
"KL/rejected_KL_mean": -1044.98388671875,
|
|
"KL/std": 806.1045532226562,
|
|
"epoch": 0.631424375917768,
|
|
"fcm_dpo/beta": 0.00025757343973964453,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 429.3522033691406,
|
|
"fcm_dpo/q_t": 0.473649799823761,
|
|
"grad_norm": 22.414127349853516,
|
|
"learning_rate": 1.8158425248197928e-07,
|
|
"logits/chosen": -0.42373788356781006,
|
|
"logits/rejected": -0.4414384365081787,
|
|
"logps/chosen": -676.5520629882812,
|
|
"logps/ref_chosen": -60.920326232910156,
|
|
"logps/ref_rejected": -104.42280578613281,
|
|
"logps/rejected": -1149.40673828125,
|
|
"loss": 1.2974,
|
|
"margin_dpo/margin_mean": 429.35223388671875,
|
|
"margin_dpo/margin_std": 1028.1934814453125,
|
|
"step": 430
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -606.7068481445312,
|
|
"KL/mean": -882.7003173828125,
|
|
"KL/rejected_KL_mean": -1158.69384765625,
|
|
"KL/std": 827.6256103515625,
|
|
"epoch": 0.6328928046989721,
|
|
"fcm_dpo/beta": 0.00025757343973964453,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 551.987060546875,
|
|
"fcm_dpo/q_t": 0.46554499864578247,
|
|
"grad_norm": 22.636369705200195,
|
|
"learning_rate": 1.8035081928995788e-07,
|
|
"logits/chosen": -0.41880902647972107,
|
|
"logits/rejected": -0.43652844429016113,
|
|
"logps/chosen": -664.0556030273438,
|
|
"logps/ref_chosen": -57.34874725341797,
|
|
"logps/ref_rejected": -92.84022521972656,
|
|
"logps/rejected": -1251.5340576171875,
|
|
"loss": 1.269,
|
|
"margin_dpo/margin_mean": 551.987060546875,
|
|
"margin_dpo/margin_std": 1050.994384765625,
|
|
"step": 431
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -738.18017578125,
|
|
"KL/mean": -1104.6925048828125,
|
|
"KL/rejected_KL_mean": -1471.204833984375,
|
|
"KL/std": 1035.17333984375,
|
|
"epoch": 0.6343612334801763,
|
|
"fcm_dpo/beta": 0.00038527188007719815,
|
|
"fcm_dpo/delta": 0.06889130920171738,
|
|
"fcm_dpo/margin": 733.0245971679688,
|
|
"fcm_dpo/q_t": 0.4547889232635498,
|
|
"grad_norm": 28.267343521118164,
|
|
"learning_rate": 1.791192214186223e-07,
|
|
"logits/chosen": -0.43129995465278625,
|
|
"logits/rejected": -0.4455578923225403,
|
|
"logps/chosen": -809.2550048828125,
|
|
"logps/ref_chosen": -71.07479095458984,
|
|
"logps/ref_rejected": -98.57952880859375,
|
|
"logps/rejected": -1569.7843017578125,
|
|
"loss": 1.1671,
|
|
"margin_dpo/margin_mean": 733.0245971679688,
|
|
"margin_dpo/margin_std": 1155.4307861328125,
|
|
"step": 432
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1069.6123046875,
|
|
"KL/mean": -1359.947265625,
|
|
"KL/rejected_KL_mean": -1650.2822265625,
|
|
"KL/std": 1036.75927734375,
|
|
"epoch": 0.6358296622613803,
|
|
"fcm_dpo/beta": 0.0006475095869973302,
|
|
"fcm_dpo/delta": -0.06112983077764511,
|
|
"fcm_dpo/margin": 580.669921875,
|
|
"fcm_dpo/q_t": 0.4136975407600403,
|
|
"grad_norm": 42.71298599243164,
|
|
"learning_rate": 1.7788949132172193e-07,
|
|
"logits/chosen": -0.5129560232162476,
|
|
"logits/rejected": -0.5314432382583618,
|
|
"logps/chosen": -1127.885498046875,
|
|
"logps/ref_chosen": -58.273193359375,
|
|
"logps/ref_rejected": -95.95089721679688,
|
|
"logps/rejected": -1746.233154296875,
|
|
"loss": 1.2069,
|
|
"margin_dpo/margin_mean": 580.669921875,
|
|
"margin_dpo/margin_std": 1201.7972412109375,
|
|
"step": 433
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1070.666748046875,
|
|
"KL/mean": -1364.0224609375,
|
|
"KL/rejected_KL_mean": -1657.3782958984375,
|
|
"KL/std": 1538.32275390625,
|
|
"epoch": 0.6372980910425844,
|
|
"fcm_dpo/beta": 0.0002783612289931625,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 586.7115478515625,
|
|
"fcm_dpo/q_t": 0.4694068133831024,
|
|
"grad_norm": 28.23455810546875,
|
|
"learning_rate": 1.7666166140378853e-07,
|
|
"logits/chosen": -0.5182013511657715,
|
|
"logits/rejected": -0.5297871232032776,
|
|
"logps/chosen": -1132.640380859375,
|
|
"logps/ref_chosen": -61.97370147705078,
|
|
"logps/ref_rejected": -78.49861145019531,
|
|
"logps/rejected": -1735.876953125,
|
|
"loss": 1.2991,
|
|
"margin_dpo/margin_mean": 586.7114868164062,
|
|
"margin_dpo/margin_std": 2062.91650390625,
|
|
"step": 434
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1081.22021484375,
|
|
"KL/mean": -1402.046875,
|
|
"KL/rejected_KL_mean": -1722.873779296875,
|
|
"KL/std": 1234.45849609375,
|
|
"epoch": 0.6387665198237885,
|
|
"fcm_dpo/beta": 0.0002783612289931625,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 641.6536865234375,
|
|
"fcm_dpo/q_t": 0.45899978280067444,
|
|
"grad_norm": 16.346473693847656,
|
|
"learning_rate": 1.7543576401928218e-07,
|
|
"logits/chosen": -0.5801471471786499,
|
|
"logits/rejected": -0.5891480445861816,
|
|
"logps/chosen": -1132.72216796875,
|
|
"logps/ref_chosen": -51.502052307128906,
|
|
"logps/ref_rejected": -87.56689453125,
|
|
"logps/rejected": -1810.440673828125,
|
|
"loss": 1.2605,
|
|
"margin_dpo/margin_mean": 641.6536865234375,
|
|
"margin_dpo/margin_std": 1534.156494140625,
|
|
"step": 435
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1159.755126953125,
|
|
"KL/mean": -1270.0465087890625,
|
|
"KL/rejected_KL_mean": -1380.3380126953125,
|
|
"KL/std": 917.111328125,
|
|
"epoch": 0.6402349486049926,
|
|
"fcm_dpo/beta": 0.0002783612289931625,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 220.5829315185547,
|
|
"fcm_dpo/q_t": 0.4855788052082062,
|
|
"grad_norm": 49.0058708190918,
|
|
"learning_rate": 1.742118314717391e-07,
|
|
"logits/chosen": -0.5959898233413696,
|
|
"logits/rejected": -0.5875068306922913,
|
|
"logps/chosen": -1231.15869140625,
|
|
"logps/ref_chosen": -71.40371704101562,
|
|
"logps/ref_rejected": -82.72775268554688,
|
|
"logps/rejected": -1463.065673828125,
|
|
"loss": 1.3546,
|
|
"margin_dpo/margin_mean": 220.58294677734375,
|
|
"margin_dpo/margin_std": 1219.72509765625,
|
|
"step": 436
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1182.003173828125,
|
|
"KL/mean": -1367.249755859375,
|
|
"KL/rejected_KL_mean": -1552.496337890625,
|
|
"KL/std": 1103.9197998046875,
|
|
"epoch": 0.6417033773861968,
|
|
"fcm_dpo/beta": 0.0002783612289931625,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 370.4931640625,
|
|
"fcm_dpo/q_t": 0.4748278260231018,
|
|
"grad_norm": 45.79405212402344,
|
|
"learning_rate": 1.7298989601292036e-07,
|
|
"logits/chosen": -0.5938007235527039,
|
|
"logits/rejected": -0.5952749252319336,
|
|
"logps/chosen": -1246.747314453125,
|
|
"logps/ref_chosen": -64.7442626953125,
|
|
"logps/ref_rejected": -82.04356384277344,
|
|
"logps/rejected": -1634.539794921875,
|
|
"loss": 1.3213,
|
|
"margin_dpo/margin_mean": 370.4931640625,
|
|
"margin_dpo/margin_std": 1342.3785400390625,
|
|
"step": 437
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1072.06591796875,
|
|
"KL/mean": -1324.57470703125,
|
|
"KL/rejected_KL_mean": -1577.08349609375,
|
|
"KL/std": 1039.245849609375,
|
|
"epoch": 0.6431718061674009,
|
|
"fcm_dpo/beta": 0.0002783612289931625,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 505.0177001953125,
|
|
"fcm_dpo/q_t": 0.46574634313583374,
|
|
"grad_norm": 17.366430282592773,
|
|
"learning_rate": 1.7176998984196144e-07,
|
|
"logits/chosen": -0.5239421129226685,
|
|
"logits/rejected": -0.5232424736022949,
|
|
"logps/chosen": -1131.08447265625,
|
|
"logps/ref_chosen": -59.0186653137207,
|
|
"logps/ref_rejected": -83.07682800292969,
|
|
"logps/rejected": -1660.160400390625,
|
|
"loss": 1.2731,
|
|
"margin_dpo/margin_mean": 505.0177307128906,
|
|
"margin_dpo/margin_std": 1082.669677734375,
|
|
"step": 438
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -940.6976318359375,
|
|
"KL/mean": -1246.2998046875,
|
|
"KL/rejected_KL_mean": -1551.90185546875,
|
|
"KL/std": 1239.5167236328125,
|
|
"epoch": 0.644640234948605,
|
|
"fcm_dpo/beta": 0.00039391464088112116,
|
|
"fcm_dpo/delta": 0.034721456468105316,
|
|
"fcm_dpo/margin": 611.2044067382812,
|
|
"fcm_dpo/q_t": 0.4621936082839966,
|
|
"grad_norm": 33.72039794921875,
|
|
"learning_rate": 1.7055214510452458e-07,
|
|
"logits/chosen": -0.5249619483947754,
|
|
"logits/rejected": -0.5584954023361206,
|
|
"logps/chosen": -994.481689453125,
|
|
"logps/ref_chosen": -53.78407669067383,
|
|
"logps/ref_rejected": -83.98545837402344,
|
|
"logps/rejected": -1635.887451171875,
|
|
"loss": 1.2525,
|
|
"margin_dpo/margin_mean": 611.2044067382812,
|
|
"margin_dpo/margin_std": 1597.325439453125,
|
|
"step": 439
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -993.7286376953125,
|
|
"KL/mean": -1251.96337890625,
|
|
"KL/rejected_KL_mean": -1510.1981201171875,
|
|
"KL/std": 1111.6761474609375,
|
|
"epoch": 0.6461086637298091,
|
|
"fcm_dpo/beta": 0.0006941439351066947,
|
|
"fcm_dpo/delta": 0.0925978422164917,
|
|
"fcm_dpo/margin": 516.469482421875,
|
|
"fcm_dpo/q_t": 0.4560081958770752,
|
|
"grad_norm": 44.43145751953125,
|
|
"learning_rate": 1.6933639389195134e-07,
|
|
"logits/chosen": -0.5849356651306152,
|
|
"logits/rejected": -0.5931211709976196,
|
|
"logps/chosen": -1072.29541015625,
|
|
"logps/ref_chosen": -78.56671905517578,
|
|
"logps/ref_rejected": -96.49775695800781,
|
|
"logps/rejected": -1606.6959228515625,
|
|
"loss": 1.2332,
|
|
"margin_dpo/margin_mean": 516.469482421875,
|
|
"margin_dpo/margin_std": 1347.8345947265625,
|
|
"step": 440
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1073.631103515625,
|
|
"KL/mean": -1258.8494873046875,
|
|
"KL/rejected_KL_mean": -1444.06787109375,
|
|
"KL/std": 1060.948486328125,
|
|
"epoch": 0.6475770925110133,
|
|
"fcm_dpo/beta": 0.0010014603612944484,
|
|
"fcm_dpo/delta": -0.04774302989244461,
|
|
"fcm_dpo/margin": 370.4366455078125,
|
|
"fcm_dpo/q_t": 0.40342986583709717,
|
|
"grad_norm": 167.0828399658203,
|
|
"learning_rate": 1.681227682404166e-07,
|
|
"logits/chosen": -0.5610836744308472,
|
|
"logits/rejected": -0.566421389579773,
|
|
"logps/chosen": -1134.45556640625,
|
|
"logps/ref_chosen": -60.824440002441406,
|
|
"logps/ref_rejected": -96.47080993652344,
|
|
"logps/rejected": -1540.5386962890625,
|
|
"loss": 1.3984,
|
|
"margin_dpo/margin_mean": 370.4366455078125,
|
|
"margin_dpo/margin_std": 1346.23828125,
|
|
"step": 441
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -677.8002319335938,
|
|
"KL/mean": -963.8506469726562,
|
|
"KL/rejected_KL_mean": -1249.901123046875,
|
|
"KL/std": 910.0428466796875,
|
|
"epoch": 0.6490455212922174,
|
|
"fcm_dpo/beta": 0.0006988396635279059,
|
|
"fcm_dpo/delta": -0.21735386550426483,
|
|
"fcm_dpo/margin": 572.1007080078125,
|
|
"fcm_dpo/q_t": 0.39990121126174927,
|
|
"grad_norm": 58.9281120300293,
|
|
"learning_rate": 1.669113001300851e-07,
|
|
"logits/chosen": -0.5104144215583801,
|
|
"logits/rejected": -0.5223550796508789,
|
|
"logps/chosen": -724.8114624023438,
|
|
"logps/ref_chosen": -47.01121520996094,
|
|
"logps/ref_rejected": -76.53926086425781,
|
|
"logps/rejected": -1326.4403076171875,
|
|
"loss": 1.3103,
|
|
"margin_dpo/margin_mean": 572.1007080078125,
|
|
"margin_dpo/margin_std": 1180.846923828125,
|
|
"step": 442
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -682.045654296875,
|
|
"KL/mean": -812.193359375,
|
|
"KL/rejected_KL_mean": -942.3411254882812,
|
|
"KL/std": 811.3766479492188,
|
|
"epoch": 0.6505139500734214,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 260.2955627441406,
|
|
"fcm_dpo/q_t": 0.4954499900341034,
|
|
"grad_norm": 5.396309852600098,
|
|
"learning_rate": 1.6570202148426815e-07,
|
|
"logits/chosen": -0.477186918258667,
|
|
"logits/rejected": -0.4765312373638153,
|
|
"logps/chosen": -753.318603515625,
|
|
"logps/ref_chosen": -71.27301788330078,
|
|
"logps/ref_rejected": -86.679931640625,
|
|
"logps/rejected": -1029.0211181640625,
|
|
"loss": 1.3694,
|
|
"margin_dpo/margin_mean": 260.2955322265625,
|
|
"margin_dpo/margin_std": 990.7225341796875,
|
|
"step": 443
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -610.6414184570312,
|
|
"KL/mean": -837.4876708984375,
|
|
"KL/rejected_KL_mean": -1064.3338623046875,
|
|
"KL/std": 821.0623779296875,
|
|
"epoch": 0.6519823788546255,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 453.6924743652344,
|
|
"fcm_dpo/q_t": 0.4920549988746643,
|
|
"grad_norm": 5.598912239074707,
|
|
"learning_rate": 1.6449496416858282e-07,
|
|
"logits/chosen": -0.42270809412002563,
|
|
"logits/rejected": -0.42965829372406006,
|
|
"logps/chosen": -667.8551025390625,
|
|
"logps/ref_chosen": -57.213706970214844,
|
|
"logps/ref_rejected": -97.25489807128906,
|
|
"logps/rejected": -1161.5887451171875,
|
|
"loss": 1.3558,
|
|
"margin_dpo/margin_mean": 453.69244384765625,
|
|
"margin_dpo/margin_std": 947.6956787109375,
|
|
"step": 444
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -393.3489685058594,
|
|
"KL/mean": -593.373291015625,
|
|
"KL/rejected_KL_mean": -793.3975830078125,
|
|
"KL/std": 562.3740234375,
|
|
"epoch": 0.6534508076358296,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 400.0485534667969,
|
|
"fcm_dpo/q_t": 0.4929931163787842,
|
|
"grad_norm": 7.666253089904785,
|
|
"learning_rate": 1.6329015999011182e-07,
|
|
"logits/chosen": -0.38615715503692627,
|
|
"logits/rejected": -0.38465434312820435,
|
|
"logps/chosen": -460.6487731933594,
|
|
"logps/ref_chosen": -67.29979705810547,
|
|
"logps/ref_rejected": -92.68267059326172,
|
|
"logps/rejected": -886.0802001953125,
|
|
"loss": 1.3592,
|
|
"margin_dpo/margin_mean": 400.048583984375,
|
|
"margin_dpo/margin_std": 728.7763061523438,
|
|
"step": 445
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -392.439208984375,
|
|
"KL/mean": -607.7997436523438,
|
|
"KL/rejected_KL_mean": -823.1602783203125,
|
|
"KL/std": 526.53857421875,
|
|
"epoch": 0.6549192364170338,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 430.7210693359375,
|
|
"fcm_dpo/q_t": 0.4924466013908386,
|
|
"grad_norm": 7.6744914054870605,
|
|
"learning_rate": 1.6208764069656578e-07,
|
|
"logits/chosen": -0.3997993469238281,
|
|
"logits/rejected": -0.41034257411956787,
|
|
"logps/chosen": -451.5377197265625,
|
|
"logps/ref_chosen": -59.098487854003906,
|
|
"logps/ref_rejected": -101.26419067382812,
|
|
"logps/rejected": -924.4244384765625,
|
|
"loss": 1.3568,
|
|
"margin_dpo/margin_mean": 430.7210693359375,
|
|
"margin_dpo/margin_std": 591.9511108398438,
|
|
"step": 446
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -391.9793701171875,
|
|
"KL/mean": -633.8006591796875,
|
|
"KL/rejected_KL_mean": -875.6220703125,
|
|
"KL/std": 740.5856323242188,
|
|
"epoch": 0.6563876651982379,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 483.64263916015625,
|
|
"fcm_dpo/q_t": 0.49153846502304077,
|
|
"grad_norm": 7.994978427886963,
|
|
"learning_rate": 1.608874379754465e-07,
|
|
"logits/chosen": -0.434841513633728,
|
|
"logits/rejected": -0.4521018862724304,
|
|
"logps/chosen": -448.0547180175781,
|
|
"logps/ref_chosen": -56.07533264160156,
|
|
"logps/ref_rejected": -98.69475555419922,
|
|
"logps/rejected": -974.3168334960938,
|
|
"loss": 1.3536,
|
|
"margin_dpo/margin_mean": 483.64263916015625,
|
|
"margin_dpo/margin_std": 850.4795532226562,
|
|
"step": 447
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -371.1406555175781,
|
|
"KL/mean": -603.8892822265625,
|
|
"KL/rejected_KL_mean": -836.6378784179688,
|
|
"KL/std": 571.3126220703125,
|
|
"epoch": 0.657856093979442,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 465.4972229003906,
|
|
"fcm_dpo/q_t": 0.49184244871139526,
|
|
"grad_norm": 7.214312553405762,
|
|
"learning_rate": 1.5968958345321177e-07,
|
|
"logits/chosen": -0.4187026917934418,
|
|
"logits/rejected": -0.4283428192138672,
|
|
"logps/chosen": -431.1445007324219,
|
|
"logps/ref_chosen": -60.00384521484375,
|
|
"logps/ref_rejected": -102.26465606689453,
|
|
"logps/rejected": -938.9025268554688,
|
|
"loss": 1.3545,
|
|
"margin_dpo/margin_mean": 465.4972229003906,
|
|
"margin_dpo/margin_std": 671.6864013671875,
|
|
"step": 448
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -494.8532409667969,
|
|
"KL/mean": -674.86083984375,
|
|
"KL/rejected_KL_mean": -854.8682861328125,
|
|
"KL/std": 626.908203125,
|
|
"epoch": 0.6593245227606461,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 360.01507568359375,
|
|
"fcm_dpo/q_t": 0.4936937391757965,
|
|
"grad_norm": 8.744072914123535,
|
|
"learning_rate": 1.584941086944423e-07,
|
|
"logits/chosen": -0.43162280321121216,
|
|
"logits/rejected": -0.4255381226539612,
|
|
"logps/chosen": -562.3798828125,
|
|
"logps/ref_chosen": -67.52661895751953,
|
|
"logps/ref_rejected": -88.59690856933594,
|
|
"logps/rejected": -943.4652099609375,
|
|
"loss": 1.3619,
|
|
"margin_dpo/margin_mean": 360.01507568359375,
|
|
"margin_dpo/margin_std": 755.4654541015625,
|
|
"step": 449
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -329.07965087890625,
|
|
"KL/mean": -557.316650390625,
|
|
"KL/rejected_KL_mean": -785.5537109375,
|
|
"KL/std": 504.0699462890625,
|
|
"epoch": 0.6607929515418502,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 456.47406005859375,
|
|
"fcm_dpo/q_t": 0.49199458956718445,
|
|
"grad_norm": 10.857216835021973,
|
|
"learning_rate": 1.573010452010098e-07,
|
|
"logits/chosen": -0.38689732551574707,
|
|
"logits/rejected": -0.40724536776542664,
|
|
"logps/chosen": -386.1877746582031,
|
|
"logps/ref_chosen": -57.10811996459961,
|
|
"logps/ref_rejected": -102.75494384765625,
|
|
"logps/rejected": -888.3086547851562,
|
|
"loss": 1.3549,
|
|
"margin_dpo/margin_mean": 456.47406005859375,
|
|
"margin_dpo/margin_std": 548.2633666992188,
|
|
"step": 450
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -549.2373046875,
|
|
"KL/mean": -656.2725830078125,
|
|
"KL/rejected_KL_mean": -763.307861328125,
|
|
"KL/std": 700.226318359375,
|
|
"epoch": 0.6622613803230544,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 214.07054138183594,
|
|
"fcm_dpo/q_t": 0.4962322413921356,
|
|
"grad_norm": 6.819939613342285,
|
|
"learning_rate": 1.5611042441124687e-07,
|
|
"logits/chosen": -0.44771769642829895,
|
|
"logits/rejected": -0.43964436650276184,
|
|
"logps/chosen": -607.7061767578125,
|
|
"logps/ref_chosen": -58.46883010864258,
|
|
"logps/ref_rejected": -72.92941284179688,
|
|
"logps/rejected": -836.2373046875,
|
|
"loss": 1.3725,
|
|
"margin_dpo/margin_mean": 214.07054138183594,
|
|
"margin_dpo/margin_std": 986.6764526367188,
|
|
"step": 451
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -501.6789855957031,
|
|
"KL/mean": -689.10693359375,
|
|
"KL/rejected_KL_mean": -876.5348510742188,
|
|
"KL/std": 670.5494384765625,
|
|
"epoch": 0.6637298091042585,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 374.8558654785156,
|
|
"fcm_dpo/q_t": 0.4934268891811371,
|
|
"grad_norm": 3.6413915157318115,
|
|
"learning_rate": 1.549222776991186e-07,
|
|
"logits/chosen": -0.4632149636745453,
|
|
"logits/rejected": -0.47757190465927124,
|
|
"logps/chosen": -552.069580078125,
|
|
"logps/ref_chosen": -50.39055252075195,
|
|
"logps/ref_rejected": -97.77142333984375,
|
|
"logps/rejected": -974.3062744140625,
|
|
"loss": 1.3608,
|
|
"margin_dpo/margin_mean": 374.85589599609375,
|
|
"margin_dpo/margin_std": 725.793701171875,
|
|
"step": 452
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -527.853515625,
|
|
"KL/mean": -772.795654296875,
|
|
"KL/rejected_KL_mean": -1017.73779296875,
|
|
"KL/std": 713.8123779296875,
|
|
"epoch": 0.6651982378854625,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 489.8843078613281,
|
|
"fcm_dpo/q_t": 0.4914173185825348,
|
|
"grad_norm": 6.951712608337402,
|
|
"learning_rate": 1.5373663637339584e-07,
|
|
"logits/chosen": -0.4904746413230896,
|
|
"logits/rejected": -0.4950370788574219,
|
|
"logps/chosen": -585.568359375,
|
|
"logps/ref_chosen": -57.71485137939453,
|
|
"logps/ref_rejected": -82.20741271972656,
|
|
"logps/rejected": -1099.9451904296875,
|
|
"loss": 1.3532,
|
|
"margin_dpo/margin_mean": 489.88427734375,
|
|
"margin_dpo/margin_std": 879.7352294921875,
|
|
"step": 453
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -756.3292236328125,
|
|
"KL/mean": -986.6734619140625,
|
|
"KL/rejected_KL_mean": -1217.017578125,
|
|
"KL/std": 858.3633422851562,
|
|
"epoch": 0.6666666666666666,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 460.6884765625,
|
|
"fcm_dpo/q_t": 0.4919373393058777,
|
|
"grad_norm": 4.276784896850586,
|
|
"learning_rate": 1.5255353167683017e-07,
|
|
"logits/chosen": -0.48667120933532715,
|
|
"logits/rejected": -0.4960416555404663,
|
|
"logps/chosen": -817.27490234375,
|
|
"logps/ref_chosen": -60.945648193359375,
|
|
"logps/ref_rejected": -84.95079040527344,
|
|
"logps/rejected": -1301.968505859375,
|
|
"loss": 1.3556,
|
|
"margin_dpo/margin_mean": 460.6884765625,
|
|
"margin_dpo/margin_std": 1064.3143310546875,
|
|
"step": 454
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -746.62548828125,
|
|
"KL/mean": -1129.1025390625,
|
|
"KL/rejected_KL_mean": -1511.579345703125,
|
|
"KL/std": 1107.7918701171875,
|
|
"epoch": 0.6681350954478708,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 764.9539794921875,
|
|
"fcm_dpo/q_t": 0.4866509437561035,
|
|
"grad_norm": 7.09210729598999,
|
|
"learning_rate": 1.5137299478533064e-07,
|
|
"logits/chosen": -0.5352227687835693,
|
|
"logits/rejected": -0.5812809467315674,
|
|
"logps/chosen": -791.51220703125,
|
|
"logps/ref_chosen": -44.88671112060547,
|
|
"logps/ref_rejected": -115.30147552490234,
|
|
"logps/rejected": -1626.880859375,
|
|
"loss": 1.3358,
|
|
"margin_dpo/margin_mean": 764.953857421875,
|
|
"margin_dpo/margin_std": 1381.763671875,
|
|
"step": 455
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1079.772216796875,
|
|
"KL/mean": -1386.8994140625,
|
|
"KL/rejected_KL_mean": -1694.026611328125,
|
|
"KL/std": 1328.23046875,
|
|
"epoch": 0.6696035242290749,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 614.2544555664062,
|
|
"fcm_dpo/q_t": 0.48927462100982666,
|
|
"grad_norm": 4.1105146408081055,
|
|
"learning_rate": 1.5019505680714232e-07,
|
|
"logits/chosen": -0.5510225296020508,
|
|
"logits/rejected": -0.5779255628585815,
|
|
"logps/chosen": -1136.8089599609375,
|
|
"logps/ref_chosen": -57.036781311035156,
|
|
"logps/ref_rejected": -105.21784210205078,
|
|
"logps/rejected": -1799.244384765625,
|
|
"loss": 1.3461,
|
|
"margin_dpo/margin_mean": 614.25439453125,
|
|
"margin_dpo/margin_std": 1356.8857421875,
|
|
"step": 456
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -928.1408081054688,
|
|
"KL/mean": -1333.076416015625,
|
|
"KL/rejected_KL_mean": -1738.0120849609375,
|
|
"KL/std": 1123.67529296875,
|
|
"epoch": 0.671071953010279,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 809.871337890625,
|
|
"fcm_dpo/q_t": 0.4858279526233673,
|
|
"grad_norm": 6.433835506439209,
|
|
"learning_rate": 1.4901974878202627e-07,
|
|
"logits/chosen": -0.6030972003936768,
|
|
"logits/rejected": -0.6290802359580994,
|
|
"logps/chosen": -982.38330078125,
|
|
"logps/ref_chosen": -54.24253845214844,
|
|
"logps/ref_rejected": -85.10956573486328,
|
|
"logps/rejected": -1823.12158203125,
|
|
"loss": 1.3325,
|
|
"margin_dpo/margin_mean": 809.871337890625,
|
|
"margin_dpo/margin_std": 1329.330810546875,
|
|
"step": 457
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1218.3466796875,
|
|
"KL/mean": -1525.91845703125,
|
|
"KL/rejected_KL_mean": -1833.4901123046875,
|
|
"KL/std": 1092.089599609375,
|
|
"epoch": 0.6725403817914831,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 615.1434326171875,
|
|
"fcm_dpo/q_t": 0.4892372488975525,
|
|
"grad_norm": 5.473250865936279,
|
|
"learning_rate": 1.4784710168044212e-07,
|
|
"logits/chosen": -0.6052824854850769,
|
|
"logits/rejected": -0.6187411546707153,
|
|
"logps/chosen": -1273.755615234375,
|
|
"logps/ref_chosen": -55.40888214111328,
|
|
"logps/ref_rejected": -97.68325805664062,
|
|
"logps/rejected": -1931.17333984375,
|
|
"loss": 1.3456,
|
|
"margin_dpo/margin_mean": 615.1434326171875,
|
|
"margin_dpo/margin_std": 1242.5,
|
|
"step": 458
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1342.32373046875,
|
|
"KL/mean": -1710.5882568359375,
|
|
"KL/rejected_KL_mean": -2078.8525390625,
|
|
"KL/std": 1349.3956298828125,
|
|
"epoch": 0.6740088105726872,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 736.52880859375,
|
|
"fcm_dpo/q_t": 0.48712393641471863,
|
|
"grad_norm": 6.021612644195557,
|
|
"learning_rate": 1.466771464027316e-07,
|
|
"logits/chosen": -0.6226590871810913,
|
|
"logits/rejected": -0.6531392335891724,
|
|
"logps/chosen": -1388.88134765625,
|
|
"logps/ref_chosen": -46.55748748779297,
|
|
"logps/ref_rejected": -86.16854095458984,
|
|
"logps/rejected": -2165.02099609375,
|
|
"loss": 1.339,
|
|
"margin_dpo/margin_mean": 736.52880859375,
|
|
"margin_dpo/margin_std": 1749.3363037109375,
|
|
"step": 459
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1793.916259765625,
|
|
"KL/mean": -2120.57568359375,
|
|
"KL/rejected_KL_mean": -2447.23486328125,
|
|
"KL/std": 1669.0478515625,
|
|
"epoch": 0.6754772393538914,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 653.318603515625,
|
|
"fcm_dpo/q_t": 0.4886649549007416,
|
|
"grad_norm": 8.884930610656738,
|
|
"learning_rate": 1.4550991377830423e-07,
|
|
"logits/chosen": -0.6757256984710693,
|
|
"logits/rejected": -0.7148680686950684,
|
|
"logps/chosen": -1845.55126953125,
|
|
"logps/ref_chosen": -51.63489532470703,
|
|
"logps/ref_rejected": -104.11935424804688,
|
|
"logps/rejected": -2551.3544921875,
|
|
"loss": 1.3459,
|
|
"margin_dpo/margin_mean": 653.3186645507812,
|
|
"margin_dpo/margin_std": 1995.814697265625,
|
|
"step": 460
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1595.17822265625,
|
|
"KL/mean": -1813.5162353515625,
|
|
"KL/rejected_KL_mean": -2031.854248046875,
|
|
"KL/std": 1255.39697265625,
|
|
"epoch": 0.6769456681350955,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 436.6760559082031,
|
|
"fcm_dpo/q_t": 0.49233686923980713,
|
|
"grad_norm": 9.014615058898926,
|
|
"learning_rate": 1.4434543456482518e-07,
|
|
"logits/chosen": -0.7007662057876587,
|
|
"logits/rejected": -0.7139623761177063,
|
|
"logps/chosen": -1650.360107421875,
|
|
"logps/ref_chosen": -55.18195724487305,
|
|
"logps/ref_rejected": -86.47689819335938,
|
|
"logps/rejected": -2118.3310546875,
|
|
"loss": 1.359,
|
|
"margin_dpo/margin_mean": 436.67608642578125,
|
|
"margin_dpo/margin_std": 1588.939453125,
|
|
"step": 461
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1918.244384765625,
|
|
"KL/mean": -2012.81298828125,
|
|
"KL/rejected_KL_mean": -2107.3818359375,
|
|
"KL/std": 1555.3056640625,
|
|
"epoch": 0.6784140969162996,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 189.1370086669922,
|
|
"fcm_dpo/q_t": 0.4966925382614136,
|
|
"grad_norm": 25.653871536254883,
|
|
"learning_rate": 1.4318373944740484e-07,
|
|
"logits/chosen": -0.6912171244621277,
|
|
"logits/rejected": -0.6754558086395264,
|
|
"logps/chosen": -1988.172607421875,
|
|
"logps/ref_chosen": -69.92803192138672,
|
|
"logps/ref_rejected": -78.84111022949219,
|
|
"logps/rejected": -2186.22265625,
|
|
"loss": 1.3773,
|
|
"margin_dpo/margin_mean": 189.13702392578125,
|
|
"margin_dpo/margin_std": 1842.75927734375,
|
|
"step": 462
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1629.407958984375,
|
|
"KL/mean": -1926.924072265625,
|
|
"KL/rejected_KL_mean": -2224.440185546875,
|
|
"KL/std": 1563.47412109375,
|
|
"epoch": 0.6798825256975036,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 595.0322265625,
|
|
"fcm_dpo/q_t": 0.48969152569770813,
|
|
"grad_norm": 10.694584846496582,
|
|
"learning_rate": 1.4202485903778976e-07,
|
|
"logits/chosen": -0.6257902383804321,
|
|
"logits/rejected": -0.6413998603820801,
|
|
"logps/chosen": -1684.682373046875,
|
|
"logps/ref_chosen": -55.27437210083008,
|
|
"logps/ref_rejected": -89.02497863769531,
|
|
"logps/rejected": -2313.46533203125,
|
|
"loss": 1.3502,
|
|
"margin_dpo/margin_mean": 595.0322875976562,
|
|
"margin_dpo/margin_std": 2058.953125,
|
|
"step": 463
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1547.507080078125,
|
|
"KL/mean": -2104.552001953125,
|
|
"KL/rejected_KL_mean": -2661.59716796875,
|
|
"KL/std": 1696.195068359375,
|
|
"epoch": 0.6813509544787077,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1114.0902099609375,
|
|
"fcm_dpo/q_t": 0.48058074712753296,
|
|
"grad_norm": 6.717080593109131,
|
|
"learning_rate": 1.4086882387355658e-07,
|
|
"logits/chosen": -0.6039457321166992,
|
|
"logits/rejected": -0.6586399078369141,
|
|
"logps/chosen": -1598.4193115234375,
|
|
"logps/ref_chosen": -50.91230010986328,
|
|
"logps/ref_rejected": -102.4893798828125,
|
|
"logps/rejected": -2764.08642578125,
|
|
"loss": 1.3144,
|
|
"margin_dpo/margin_mean": 1114.0902099609375,
|
|
"margin_dpo/margin_std": 1911.744140625,
|
|
"step": 464
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1389.27978515625,
|
|
"KL/mean": -1775.259765625,
|
|
"KL/rejected_KL_mean": -2161.23974609375,
|
|
"KL/std": 1636.043212890625,
|
|
"epoch": 0.6828193832599119,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 771.9598388671875,
|
|
"fcm_dpo/q_t": 0.4866272807121277,
|
|
"grad_norm": 5.775012016296387,
|
|
"learning_rate": 1.3971566441730714e-07,
|
|
"logits/chosen": -0.5333804488182068,
|
|
"logits/rejected": -0.556584358215332,
|
|
"logps/chosen": -1449.396728515625,
|
|
"logps/ref_chosen": -60.116851806640625,
|
|
"logps/ref_rejected": -113.94602966308594,
|
|
"logps/rejected": -2275.185791015625,
|
|
"loss": 1.3381,
|
|
"margin_dpo/margin_mean": 771.9598388671875,
|
|
"margin_dpo/margin_std": 1992.7213134765625,
|
|
"step": 465
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1304.01220703125,
|
|
"KL/mean": -1642.657470703125,
|
|
"KL/rejected_KL_mean": -1981.302734375,
|
|
"KL/std": 1344.09765625,
|
|
"epoch": 0.684287812041116,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 677.29052734375,
|
|
"fcm_dpo/q_t": 0.4881631135940552,
|
|
"grad_norm": 4.701225757598877,
|
|
"learning_rate": 1.3856541105586545e-07,
|
|
"logits/chosen": -0.5309747457504272,
|
|
"logits/rejected": -0.5521206855773926,
|
|
"logps/chosen": -1356.9332275390625,
|
|
"logps/ref_chosen": -52.920921325683594,
|
|
"logps/ref_rejected": -90.3154296875,
|
|
"logps/rejected": -2071.6181640625,
|
|
"loss": 1.343,
|
|
"margin_dpo/margin_mean": 677.29052734375,
|
|
"margin_dpo/margin_std": 1733.6058349609375,
|
|
"step": 466
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -2066.13720703125,
|
|
"KL/mean": -2241.6923828125,
|
|
"KL/rejected_KL_mean": -2417.24755859375,
|
|
"KL/std": 1884.09375,
|
|
"epoch": 0.6857562408223201,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 351.11004638671875,
|
|
"fcm_dpo/q_t": 0.4938202500343323,
|
|
"grad_norm": 13.84892463684082,
|
|
"learning_rate": 1.3741809409947729e-07,
|
|
"logits/chosen": -0.5912429690361023,
|
|
"logits/rejected": -0.5880587100982666,
|
|
"logps/chosen": -2144.85302734375,
|
|
"logps/ref_chosen": -78.7158203125,
|
|
"logps/ref_rejected": -102.86019897460938,
|
|
"logps/rejected": -2520.10791015625,
|
|
"loss": 1.3708,
|
|
"margin_dpo/margin_mean": 351.1100158691406,
|
|
"margin_dpo/margin_std": 2587.14794921875,
|
|
"step": 467
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1504.03369140625,
|
|
"KL/mean": -1898.8018798828125,
|
|
"KL/rejected_KL_mean": -2293.570068359375,
|
|
"KL/std": 1620.427001953125,
|
|
"epoch": 0.6872246696035242,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 789.5361328125,
|
|
"fcm_dpo/q_t": 0.48619183897972107,
|
|
"grad_norm": 10.929081916809082,
|
|
"learning_rate": 1.362737437810114e-07,
|
|
"logits/chosen": -0.5442489385604858,
|
|
"logits/rejected": -0.5691794753074646,
|
|
"logps/chosen": -1573.9691162109375,
|
|
"logps/ref_chosen": -69.93536376953125,
|
|
"logps/ref_rejected": -101.02880859375,
|
|
"logps/rejected": -2394.598876953125,
|
|
"loss": 1.3377,
|
|
"margin_dpo/margin_mean": 789.5361938476562,
|
|
"margin_dpo/margin_std": 2186.982666015625,
|
|
"step": 468
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1296.493896484375,
|
|
"KL/mean": -1764.9542236328125,
|
|
"KL/rejected_KL_mean": -2233.41455078125,
|
|
"KL/std": 1456.62353515625,
|
|
"epoch": 0.6886930983847284,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 936.920654296875,
|
|
"fcm_dpo/q_t": 0.4836467504501343,
|
|
"grad_norm": 10.054937362670898,
|
|
"learning_rate": 1.351323902551631e-07,
|
|
"logits/chosen": -0.5154561996459961,
|
|
"logits/rejected": -0.545080840587616,
|
|
"logps/chosen": -1364.61865234375,
|
|
"logps/ref_chosen": -68.12469482421875,
|
|
"logps/ref_rejected": -104.78640747070312,
|
|
"logps/rejected": -2338.201171875,
|
|
"loss": 1.3254,
|
|
"margin_dpo/margin_mean": 936.920654296875,
|
|
"margin_dpo/margin_std": 1700.6353759765625,
|
|
"step": 469
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1085.716552734375,
|
|
"KL/mean": -1395.3118896484375,
|
|
"KL/rejected_KL_mean": -1704.9072265625,
|
|
"KL/std": 1203.486328125,
|
|
"epoch": 0.6901615271659325,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 619.190673828125,
|
|
"fcm_dpo/q_t": 0.4891727566719055,
|
|
"grad_norm": 4.784751892089844,
|
|
"learning_rate": 1.339940635976592e-07,
|
|
"logits/chosen": -0.4637961983680725,
|
|
"logits/rejected": -0.4888746738433838,
|
|
"logps/chosen": -1129.508544921875,
|
|
"logps/ref_chosen": -43.791927337646484,
|
|
"logps/ref_rejected": -82.70285034179688,
|
|
"logps/rejected": -1787.610107421875,
|
|
"loss": 1.3458,
|
|
"margin_dpo/margin_mean": 619.190673828125,
|
|
"margin_dpo/margin_std": 1384.6240234375,
|
|
"step": 470
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1458.683837890625,
|
|
"KL/mean": -1615.782958984375,
|
|
"KL/rejected_KL_mean": -1772.8822021484375,
|
|
"KL/std": 1343.892578125,
|
|
"epoch": 0.6916299559471366,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 314.1982727050781,
|
|
"fcm_dpo/q_t": 0.4945109486579895,
|
|
"grad_norm": 14.473358154296875,
|
|
"learning_rate": 1.3285879380446563e-07,
|
|
"logits/chosen": -0.5264901518821716,
|
|
"logits/rejected": -0.5227891206741333,
|
|
"logps/chosen": -1522.0234375,
|
|
"logps/ref_chosen": -63.33952331542969,
|
|
"logps/ref_rejected": -83.61048126220703,
|
|
"logps/rejected": -1856.49267578125,
|
|
"loss": 1.3675,
|
|
"margin_dpo/margin_mean": 314.19830322265625,
|
|
"margin_dpo/margin_std": 1578.263671875,
|
|
"step": 471
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1396.530029296875,
|
|
"KL/mean": -1693.50439453125,
|
|
"KL/rejected_KL_mean": -1990.4786376953125,
|
|
"KL/std": 1501.013671875,
|
|
"epoch": 0.6930983847283406,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 593.9485473632812,
|
|
"fcm_dpo/q_t": 0.4896436929702759,
|
|
"grad_norm": 5.583034992218018,
|
|
"learning_rate": 1.317266107909975e-07,
|
|
"logits/chosen": -0.5575324296951294,
|
|
"logits/rejected": -0.5637389421463013,
|
|
"logps/chosen": -1480.1962890625,
|
|
"logps/ref_chosen": -83.66610717773438,
|
|
"logps/ref_rejected": -117.20919799804688,
|
|
"logps/rejected": -2107.68798828125,
|
|
"loss": 1.3496,
|
|
"margin_dpo/margin_mean": 593.9485473632812,
|
|
"margin_dpo/margin_std": 1845.80224609375,
|
|
"step": 472
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1678.685791015625,
|
|
"KL/mean": -1702.64501953125,
|
|
"KL/rejected_KL_mean": -1726.6043701171875,
|
|
"KL/std": 1605.0902099609375,
|
|
"epoch": 0.6945668135095447,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 47.91863250732422,
|
|
"fcm_dpo/q_t": 0.49903565645217896,
|
|
"grad_norm": 10.380570411682129,
|
|
"learning_rate": 1.3059754439133002e-07,
|
|
"logits/chosen": -0.5147888660430908,
|
|
"logits/rejected": -0.5062965154647827,
|
|
"logps/chosen": -1742.1827392578125,
|
|
"logps/ref_chosen": -63.49696731567383,
|
|
"logps/ref_rejected": -81.14657592773438,
|
|
"logps/rejected": -1807.7509765625,
|
|
"loss": 1.389,
|
|
"margin_dpo/margin_mean": 47.918617248535156,
|
|
"margin_dpo/margin_std": 2152.85302734375,
|
|
"step": 473
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1045.2838134765625,
|
|
"KL/mean": -1396.827880859375,
|
|
"KL/rejected_KL_mean": -1748.3720703125,
|
|
"KL/std": 1296.06787109375,
|
|
"epoch": 0.6960352422907489,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 703.088134765625,
|
|
"fcm_dpo/q_t": 0.4877893924713135,
|
|
"grad_norm": 12.54921817779541,
|
|
"learning_rate": 1.2947162435741277e-07,
|
|
"logits/chosen": -0.44941064715385437,
|
|
"logits/rejected": -0.465454638004303,
|
|
"logps/chosen": -1097.895751953125,
|
|
"logps/ref_chosen": -52.6119384765625,
|
|
"logps/ref_rejected": -90.08041381835938,
|
|
"logps/rejected": -1838.452392578125,
|
|
"loss": 1.3413,
|
|
"margin_dpo/margin_mean": 703.088134765625,
|
|
"margin_dpo/margin_std": 1566.402099609375,
|
|
"step": 474
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -696.1770629882812,
|
|
"KL/mean": -1096.912109375,
|
|
"KL/rejected_KL_mean": -1497.64697265625,
|
|
"KL/std": 1150.5213623046875,
|
|
"epoch": 0.697503671071953,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 801.469970703125,
|
|
"fcm_dpo/q_t": 0.48600390553474426,
|
|
"grad_norm": 16.383455276489258,
|
|
"learning_rate": 1.2834888035828596e-07,
|
|
"logits/chosen": -0.43204575777053833,
|
|
"logits/rejected": -0.46623319387435913,
|
|
"logps/chosen": -738.6722412109375,
|
|
"logps/ref_chosen": -42.49519348144531,
|
|
"logps/ref_rejected": -90.06294250488281,
|
|
"logps/rejected": -1587.7099609375,
|
|
"loss": 1.3334,
|
|
"margin_dpo/margin_mean": 801.469970703125,
|
|
"margin_dpo/margin_std": 1452.920654296875,
|
|
"step": 475
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -922.133056640625,
|
|
"KL/mean": -1211.9774169921875,
|
|
"KL/rejected_KL_mean": -1501.82177734375,
|
|
"KL/std": 1047.379150390625,
|
|
"epoch": 0.6989720998531571,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 579.6885986328125,
|
|
"fcm_dpo/q_t": 0.4898582696914673,
|
|
"grad_norm": 7.450405597686768,
|
|
"learning_rate": 1.2722934197929802e-07,
|
|
"logits/chosen": -0.4594970941543579,
|
|
"logits/rejected": -0.4757981300354004,
|
|
"logps/chosen": -965.0824584960938,
|
|
"logps/ref_chosen": -42.94938278198242,
|
|
"logps/ref_rejected": -73.71023559570312,
|
|
"logps/rejected": -1575.531982421875,
|
|
"loss": 1.3477,
|
|
"margin_dpo/margin_mean": 579.6885986328125,
|
|
"margin_dpo/margin_std": 1162.8359375,
|
|
"step": 476
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1014.1900634765625,
|
|
"KL/mean": -1202.794189453125,
|
|
"KL/rejected_KL_mean": -1391.398193359375,
|
|
"KL/std": 987.1451416015625,
|
|
"epoch": 0.7004405286343612,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 377.20819091796875,
|
|
"fcm_dpo/q_t": 0.4933883547782898,
|
|
"grad_norm": 4.9775261878967285,
|
|
"learning_rate": 1.2611303872132631e-07,
|
|
"logits/chosen": -0.48688119649887085,
|
|
"logits/rejected": -0.47269320487976074,
|
|
"logps/chosen": -1084.962646484375,
|
|
"logps/ref_chosen": -70.77261352539062,
|
|
"logps/ref_rejected": -76.13737487792969,
|
|
"logps/rejected": -1467.53564453125,
|
|
"loss": 1.3615,
|
|
"margin_dpo/margin_mean": 377.20819091796875,
|
|
"margin_dpo/margin_std": 1102.049560546875,
|
|
"step": 477
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -741.496337890625,
|
|
"KL/mean": -1104.615966796875,
|
|
"KL/rejected_KL_mean": -1467.7357177734375,
|
|
"KL/std": 1056.3428955078125,
|
|
"epoch": 0.7019089574155654,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 726.2393798828125,
|
|
"fcm_dpo/q_t": 0.48729372024536133,
|
|
"grad_norm": 13.8693208694458,
|
|
"learning_rate": 1.2500000000000005e-07,
|
|
"logits/chosen": -0.4414135813713074,
|
|
"logits/rejected": -0.4705617129802704,
|
|
"logps/chosen": -782.9368286132812,
|
|
"logps/ref_chosen": -41.440513610839844,
|
|
"logps/ref_rejected": -85.36196899414062,
|
|
"logps/rejected": -1553.09765625,
|
|
"loss": 1.3379,
|
|
"margin_dpo/margin_mean": 726.2393798828125,
|
|
"margin_dpo/margin_std": 1235.259033203125,
|
|
"step": 478
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1152.8778076171875,
|
|
"KL/mean": -1485.199951171875,
|
|
"KL/rejected_KL_mean": -1817.52197265625,
|
|
"KL/std": 1490.449951171875,
|
|
"epoch": 0.7033773861967695,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 664.6441650390625,
|
|
"fcm_dpo/q_t": 0.4884074628353119,
|
|
"grad_norm": 4.815835952758789,
|
|
"learning_rate": 1.2389025514492456e-07,
|
|
"logits/chosen": -0.4350011944770813,
|
|
"logits/rejected": -0.4591215252876282,
|
|
"logps/chosen": -1206.7857666015625,
|
|
"logps/ref_chosen": -53.907920837402344,
|
|
"logps/ref_rejected": -95.1163330078125,
|
|
"logps/rejected": -1912.638427734375,
|
|
"loss": 1.343,
|
|
"margin_dpo/margin_mean": 664.6441650390625,
|
|
"margin_dpo/margin_std": 1489.7821044921875,
|
|
"step": 479
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1410.0196533203125,
|
|
"KL/mean": -1525.3935546875,
|
|
"KL/rejected_KL_mean": -1640.767578125,
|
|
"KL/std": 1323.5804443359375,
|
|
"epoch": 0.7048458149779736,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 230.74789428710938,
|
|
"fcm_dpo/q_t": 0.4959403872489929,
|
|
"grad_norm": 8.32140064239502,
|
|
"learning_rate": 1.227838333989088e-07,
|
|
"logits/chosen": -0.5214420557022095,
|
|
"logits/rejected": -0.5171930193901062,
|
|
"logps/chosen": -1468.702392578125,
|
|
"logps/ref_chosen": -58.682701110839844,
|
|
"logps/ref_rejected": -82.93248748779297,
|
|
"logps/rejected": -1723.7000732421875,
|
|
"loss": 1.3738,
|
|
"margin_dpo/margin_mean": 230.7479248046875,
|
|
"margin_dpo/margin_std": 1702.75048828125,
|
|
"step": 480
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1059.987548828125,
|
|
"KL/mean": -1491.16796875,
|
|
"KL/rejected_KL_mean": -1922.3486328125,
|
|
"KL/std": 1393.0323486328125,
|
|
"epoch": 0.7063142437591777,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 862.3609619140625,
|
|
"fcm_dpo/q_t": 0.484965980052948,
|
|
"grad_norm": 4.985651969909668,
|
|
"learning_rate": 1.2168076391719489e-07,
|
|
"logits/chosen": -0.5206190347671509,
|
|
"logits/rejected": -0.562711238861084,
|
|
"logps/chosen": -1114.9517822265625,
|
|
"logps/ref_chosen": -54.964271545410156,
|
|
"logps/ref_rejected": -92.42044067382812,
|
|
"logps/rejected": -2014.76904296875,
|
|
"loss": 1.33,
|
|
"margin_dpo/margin_mean": 862.3610229492188,
|
|
"margin_dpo/margin_std": 1640.7061767578125,
|
|
"step": 481
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1543.61083984375,
|
|
"KL/mean": -1666.1243896484375,
|
|
"KL/rejected_KL_mean": -1788.637939453125,
|
|
"KL/std": 1566.256591796875,
|
|
"epoch": 0.7077826725403817,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 245.027099609375,
|
|
"fcm_dpo/q_t": 0.49571603536605835,
|
|
"grad_norm": 9.695770263671875,
|
|
"learning_rate": 1.2058107576668938e-07,
|
|
"logits/chosen": -0.5083905458450317,
|
|
"logits/rejected": -0.5077981948852539,
|
|
"logps/chosen": -1611.164306640625,
|
|
"logps/ref_chosen": -67.553466796875,
|
|
"logps/ref_rejected": -87.58953857421875,
|
|
"logps/rejected": -1876.2275390625,
|
|
"loss": 1.374,
|
|
"margin_dpo/margin_mean": 245.02706909179688,
|
|
"margin_dpo/margin_std": 1968.4217529296875,
|
|
"step": 482
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1194.04248046875,
|
|
"KL/mean": -1479.513671875,
|
|
"KL/rejected_KL_mean": -1764.9847412109375,
|
|
"KL/std": 1215.059326171875,
|
|
"epoch": 0.7092511013215859,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 570.9423217773438,
|
|
"fcm_dpo/q_t": 0.4899991750717163,
|
|
"grad_norm": 7.4929304122924805,
|
|
"learning_rate": 1.194847979251979e-07,
|
|
"logits/chosen": -0.5671132206916809,
|
|
"logits/rejected": -0.5771450996398926,
|
|
"logps/chosen": -1257.372314453125,
|
|
"logps/ref_chosen": -63.32981872558594,
|
|
"logps/ref_rejected": -95.78697204589844,
|
|
"logps/rejected": -1860.771728515625,
|
|
"loss": 1.3497,
|
|
"margin_dpo/margin_mean": 570.9423217773438,
|
|
"margin_dpo/margin_std": 1546.977783203125,
|
|
"step": 483
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1078.635498046875,
|
|
"KL/mean": -1429.60009765625,
|
|
"KL/rejected_KL_mean": -1780.564697265625,
|
|
"KL/std": 1515.4169921875,
|
|
"epoch": 0.71071953010279,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 701.9291381835938,
|
|
"fcm_dpo/q_t": 0.48787161707878113,
|
|
"grad_norm": 6.547210216522217,
|
|
"learning_rate": 1.1839195928066101e-07,
|
|
"logits/chosen": -0.4985977113246918,
|
|
"logits/rejected": -0.5153236389160156,
|
|
"logps/chosen": -1137.773681640625,
|
|
"logps/ref_chosen": -59.13812255859375,
|
|
"logps/ref_rejected": -84.37144470214844,
|
|
"logps/rejected": -1864.93603515625,
|
|
"loss": 1.3425,
|
|
"margin_dpo/margin_mean": 701.9291381835938,
|
|
"margin_dpo/margin_std": 1918.406494140625,
|
|
"step": 484
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -998.8741455078125,
|
|
"KL/mean": -1385.538818359375,
|
|
"KL/rejected_KL_mean": -1772.2037353515625,
|
|
"KL/std": 1393.5875244140625,
|
|
"epoch": 0.7121879588839941,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 773.32958984375,
|
|
"fcm_dpo/q_t": 0.4865483045578003,
|
|
"grad_norm": 6.182162761688232,
|
|
"learning_rate": 1.1730258863039347e-07,
|
|
"logits/chosen": -0.5134660005569458,
|
|
"logits/rejected": -0.5451745390892029,
|
|
"logps/chosen": -1057.7236328125,
|
|
"logps/ref_chosen": -58.849571228027344,
|
|
"logps/ref_rejected": -103.36408233642578,
|
|
"logps/rejected": -1875.56787109375,
|
|
"loss": 1.3371,
|
|
"margin_dpo/margin_mean": 773.32958984375,
|
|
"margin_dpo/margin_std": 1857.324462890625,
|
|
"step": 485
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1066.927734375,
|
|
"KL/mean": -1443.706787109375,
|
|
"KL/rejected_KL_mean": -1820.4857177734375,
|
|
"KL/std": 1382.1572265625,
|
|
"epoch": 0.7136563876651982,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 753.5579833984375,
|
|
"fcm_dpo/q_t": 0.48684683442115784,
|
|
"grad_norm": 4.5740437507629395,
|
|
"learning_rate": 1.1621671468032493e-07,
|
|
"logits/chosen": -0.46131497621536255,
|
|
"logits/rejected": -0.49075794219970703,
|
|
"logps/chosen": -1122.1875,
|
|
"logps/ref_chosen": -55.25966262817383,
|
|
"logps/ref_rejected": -92.13936614990234,
|
|
"logps/rejected": -1912.625,
|
|
"loss": 1.3381,
|
|
"margin_dpo/margin_mean": 753.5579833984375,
|
|
"margin_dpo/margin_std": 1785.576171875,
|
|
"step": 486
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1072.845458984375,
|
|
"KL/mean": -1368.38525390625,
|
|
"KL/rejected_KL_mean": -1663.9249267578125,
|
|
"KL/std": 1040.571533203125,
|
|
"epoch": 0.7151248164464024,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 591.0794677734375,
|
|
"fcm_dpo/q_t": 0.4896491765975952,
|
|
"grad_norm": 9.592281341552734,
|
|
"learning_rate": 1.1513436604424378e-07,
|
|
"logits/chosen": -0.47907352447509766,
|
|
"logits/rejected": -0.4934772253036499,
|
|
"logps/chosen": -1125.90869140625,
|
|
"logps/ref_chosen": -53.06330871582031,
|
|
"logps/ref_rejected": -92.41883087158203,
|
|
"logps/rejected": -1756.34375,
|
|
"loss": 1.3469,
|
|
"margin_dpo/margin_mean": 591.0795288085938,
|
|
"margin_dpo/margin_std": 1158.6181640625,
|
|
"step": 487
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -761.5409545898438,
|
|
"KL/mean": -1126.8492431640625,
|
|
"KL/rejected_KL_mean": -1492.157470703125,
|
|
"KL/std": 1100.26611328125,
|
|
"epoch": 0.7165932452276065,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 730.6165161132812,
|
|
"fcm_dpo/q_t": 0.487249493598938,
|
|
"grad_norm": 11.395562171936035,
|
|
"learning_rate": 1.1405557124304335e-07,
|
|
"logits/chosen": -0.4758809208869934,
|
|
"logits/rejected": -0.49622082710266113,
|
|
"logps/chosen": -813.7691040039062,
|
|
"logps/ref_chosen": -52.22815704345703,
|
|
"logps/ref_rejected": -84.00656127929688,
|
|
"logps/rejected": -1576.1640625,
|
|
"loss": 1.3382,
|
|
"margin_dpo/margin_mean": 730.6165771484375,
|
|
"margin_dpo/margin_std": 1362.75,
|
|
"step": 488
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -914.7315673828125,
|
|
"KL/mean": -1159.464599609375,
|
|
"KL/rejected_KL_mean": -1404.197509765625,
|
|
"KL/std": 1258.12744140625,
|
|
"epoch": 0.7180616740088106,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 489.4659423828125,
|
|
"fcm_dpo/q_t": 0.4914560914039612,
|
|
"grad_norm": 5.10060977935791,
|
|
"learning_rate": 1.1298035870396985e-07,
|
|
"logits/chosen": -0.553819477558136,
|
|
"logits/rejected": -0.5615823268890381,
|
|
"logps/chosen": -970.72119140625,
|
|
"logps/ref_chosen": -55.989627838134766,
|
|
"logps/ref_rejected": -79.39812469482422,
|
|
"logps/rejected": -1483.595703125,
|
|
"loss": 1.3549,
|
|
"margin_dpo/margin_mean": 489.4659118652344,
|
|
"margin_dpo/margin_std": 1443.1240234375,
|
|
"step": 489
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -992.151123046875,
|
|
"KL/mean": -1396.121337890625,
|
|
"KL/rejected_KL_mean": -1800.091552734375,
|
|
"KL/std": 1577.330078125,
|
|
"epoch": 0.7195301027900147,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 807.9404296875,
|
|
"fcm_dpo/q_t": 0.4860500693321228,
|
|
"grad_norm": 6.243213653564453,
|
|
"learning_rate": 1.1190875675987355e-07,
|
|
"logits/chosen": -0.5025284290313721,
|
|
"logits/rejected": -0.5427592992782593,
|
|
"logps/chosen": -1044.517578125,
|
|
"logps/ref_chosen": -52.36639404296875,
|
|
"logps/ref_rejected": -110.4090576171875,
|
|
"logps/rejected": -1910.5006103515625,
|
|
"loss": 1.3358,
|
|
"margin_dpo/margin_mean": 807.9404296875,
|
|
"margin_dpo/margin_std": 1997.775146484375,
|
|
"step": 490
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -961.5281982421875,
|
|
"KL/mean": -1136.3218994140625,
|
|
"KL/rejected_KL_mean": -1311.1156005859375,
|
|
"KL/std": 1014.553955078125,
|
|
"epoch": 0.7209985315712188,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 349.58746337890625,
|
|
"fcm_dpo/q_t": 0.4938778281211853,
|
|
"grad_norm": 3.7116713523864746,
|
|
"learning_rate": 1.1084079364846241e-07,
|
|
"logits/chosen": -0.5231382846832275,
|
|
"logits/rejected": -0.5342763662338257,
|
|
"logps/chosen": -1021.6444702148438,
|
|
"logps/ref_chosen": -60.11626434326172,
|
|
"logps/ref_rejected": -73.27278900146484,
|
|
"logps/rejected": -1384.388427734375,
|
|
"loss": 1.3634,
|
|
"margin_dpo/margin_mean": 349.58746337890625,
|
|
"margin_dpo/margin_std": 1089.209228515625,
|
|
"step": 491
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1040.841796875,
|
|
"KL/mean": -1144.0247802734375,
|
|
"KL/rejected_KL_mean": -1247.20751953125,
|
|
"KL/std": 1073.8876953125,
|
|
"epoch": 0.7224669603524229,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 206.36590576171875,
|
|
"fcm_dpo/q_t": 0.4963826835155487,
|
|
"grad_norm": 5.653712272644043,
|
|
"learning_rate": 1.097764975115576e-07,
|
|
"logits/chosen": -0.5174509286880493,
|
|
"logits/rejected": -0.5069364905357361,
|
|
"logps/chosen": -1094.8359375,
|
|
"logps/ref_chosen": -53.994178771972656,
|
|
"logps/ref_rejected": -72.65962219238281,
|
|
"logps/rejected": -1319.8671875,
|
|
"loss": 1.3731,
|
|
"margin_dpo/margin_mean": 206.36590576171875,
|
|
"margin_dpo/margin_std": 1005.8743896484375,
|
|
"step": 492
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1245.180908203125,
|
|
"KL/mean": -1378.21533203125,
|
|
"KL/rejected_KL_mean": -1511.2498779296875,
|
|
"KL/std": 1198.19384765625,
|
|
"epoch": 0.723935389133627,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 266.0688781738281,
|
|
"fcm_dpo/q_t": 0.49536770582199097,
|
|
"grad_norm": 9.439188003540039,
|
|
"learning_rate": 1.0871589639435203e-07,
|
|
"logits/chosen": -0.5633972883224487,
|
|
"logits/rejected": -0.5487905740737915,
|
|
"logps/chosen": -1320.67822265625,
|
|
"logps/ref_chosen": -75.49723815917969,
|
|
"logps/ref_rejected": -87.32301330566406,
|
|
"logps/rejected": -1598.5728759765625,
|
|
"loss": 1.3702,
|
|
"margin_dpo/margin_mean": 266.06890869140625,
|
|
"margin_dpo/margin_std": 1407.188232421875,
|
|
"step": 493
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -743.1343994140625,
|
|
"KL/mean": -1110.82080078125,
|
|
"KL/rejected_KL_mean": -1478.5072021484375,
|
|
"KL/std": 1011.7152709960938,
|
|
"epoch": 0.7254038179148311,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 735.372802734375,
|
|
"fcm_dpo/q_t": 0.48713040351867676,
|
|
"grad_norm": 10.48877239227295,
|
|
"learning_rate": 1.0765901824467166e-07,
|
|
"logits/chosen": -0.4758908748626709,
|
|
"logits/rejected": -0.5146316289901733,
|
|
"logps/chosen": -784.49365234375,
|
|
"logps/ref_chosen": -41.35926818847656,
|
|
"logps/ref_rejected": -86.09136962890625,
|
|
"logps/rejected": -1564.5986328125,
|
|
"loss": 1.3374,
|
|
"margin_dpo/margin_mean": 735.372802734375,
|
|
"margin_dpo/margin_std": 1294.782470703125,
|
|
"step": 494
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1080.8409423828125,
|
|
"KL/mean": -1387.46923828125,
|
|
"KL/rejected_KL_mean": -1694.097412109375,
|
|
"KL/std": 1105.668212890625,
|
|
"epoch": 0.7268722466960352,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 613.2564086914062,
|
|
"fcm_dpo/q_t": 0.4892851710319519,
|
|
"grad_norm": 5.530251502990723,
|
|
"learning_rate": 1.0660589091223854e-07,
|
|
"logits/chosen": -0.6073266267776489,
|
|
"logits/rejected": -0.6195170879364014,
|
|
"logps/chosen": -1144.3759765625,
|
|
"logps/ref_chosen": -63.53507995605469,
|
|
"logps/ref_rejected": -91.42443084716797,
|
|
"logps/rejected": -1785.52197265625,
|
|
"loss": 1.347,
|
|
"margin_dpo/margin_mean": 613.25634765625,
|
|
"margin_dpo/margin_std": 1621.5955810546875,
|
|
"step": 495
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1282.886962890625,
|
|
"KL/mean": -1321.6571044921875,
|
|
"KL/rejected_KL_mean": -1360.427001953125,
|
|
"KL/std": 1002.799560546875,
|
|
"epoch": 0.7283406754772394,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 77.54009246826172,
|
|
"fcm_dpo/q_t": 0.49863314628601074,
|
|
"grad_norm": 8.960994720458984,
|
|
"learning_rate": 1.0555654214793722e-07,
|
|
"logits/chosen": -0.5475114583969116,
|
|
"logits/rejected": -0.5427240133285522,
|
|
"logps/chosen": -1355.4788818359375,
|
|
"logps/ref_chosen": -72.5919189453125,
|
|
"logps/ref_rejected": -84.32933807373047,
|
|
"logps/rejected": -1444.75634765625,
|
|
"loss": 1.3831,
|
|
"margin_dpo/margin_mean": 77.54010009765625,
|
|
"margin_dpo/margin_std": 1289.9385986328125,
|
|
"step": 496
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -949.1077270507812,
|
|
"KL/mean": -1054.76904296875,
|
|
"KL/rejected_KL_mean": -1160.430419921875,
|
|
"KL/std": 835.3856201171875,
|
|
"epoch": 0.7298091042584435,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 211.32281494140625,
|
|
"fcm_dpo/q_t": 0.4962995648384094,
|
|
"grad_norm": 3.6305837631225586,
|
|
"learning_rate": 1.0451099960308374e-07,
|
|
"logits/chosen": -0.5167309045791626,
|
|
"logits/rejected": -0.5133849382400513,
|
|
"logps/chosen": -1007.70166015625,
|
|
"logps/ref_chosen": -58.59397506713867,
|
|
"logps/ref_rejected": -76.28836822509766,
|
|
"logps/rejected": -1236.71875,
|
|
"loss": 1.3724,
|
|
"margin_dpo/margin_mean": 211.32281494140625,
|
|
"margin_dpo/margin_std": 839.1533813476562,
|
|
"step": 497
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1179.71044921875,
|
|
"KL/mean": -1404.1376953125,
|
|
"KL/rejected_KL_mean": -1628.5648193359375,
|
|
"KL/std": 1239.938232421875,
|
|
"epoch": 0.7312775330396476,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 448.8544006347656,
|
|
"fcm_dpo/q_t": 0.4921446144580841,
|
|
"grad_norm": 4.796504974365234,
|
|
"learning_rate": 1.0346929082869641e-07,
|
|
"logits/chosen": -0.5920599102973938,
|
|
"logits/rejected": -0.5912704467773438,
|
|
"logps/chosen": -1250.916015625,
|
|
"logps/ref_chosen": -71.20565795898438,
|
|
"logps/ref_rejected": -83.95803833007812,
|
|
"logps/rejected": -1712.52294921875,
|
|
"loss": 1.3579,
|
|
"margin_dpo/margin_mean": 448.8544006347656,
|
|
"margin_dpo/margin_std": 1510.208740234375,
|
|
"step": 498
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -847.0122680664062,
|
|
"KL/mean": -1311.192138671875,
|
|
"KL/rejected_KL_mean": -1775.3720703125,
|
|
"KL/std": 1260.3809814453125,
|
|
"epoch": 0.7327459618208517,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 928.3598022460938,
|
|
"fcm_dpo/q_t": 0.4837894141674042,
|
|
"grad_norm": 6.413288593292236,
|
|
"learning_rate": 1.0243144327477013e-07,
|
|
"logits/chosen": -0.5122471451759338,
|
|
"logits/rejected": -0.5531229972839355,
|
|
"logps/chosen": -898.2674560546875,
|
|
"logps/ref_chosen": -51.25519561767578,
|
|
"logps/ref_rejected": -101.07870483398438,
|
|
"logps/rejected": -1876.45068359375,
|
|
"loss": 1.3257,
|
|
"margin_dpo/margin_mean": 928.3598022460938,
|
|
"margin_dpo/margin_std": 1673.77978515625,
|
|
"step": 499
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1049.207763671875,
|
|
"KL/mean": -1407.1966552734375,
|
|
"KL/rejected_KL_mean": -1765.185546875,
|
|
"KL/std": 1214.736328125,
|
|
"epoch": 0.7342143906020558,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 715.9777221679688,
|
|
"fcm_dpo/q_t": 0.4875142276287079,
|
|
"grad_norm": 4.70263671875,
|
|
"learning_rate": 1.0139748428955333e-07,
|
|
"logits/chosen": -0.5440601706504822,
|
|
"logits/rejected": -0.5752770900726318,
|
|
"logps/chosen": -1106.2352294921875,
|
|
"logps/ref_chosen": -57.027442932128906,
|
|
"logps/ref_rejected": -93.93421173095703,
|
|
"logps/rejected": -1859.1197509765625,
|
|
"loss": 1.34,
|
|
"margin_dpo/margin_mean": 715.977783203125,
|
|
"margin_dpo/margin_std": 1645.666259765625,
|
|
"step": 500
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -943.0932006835938,
|
|
"KL/mean": -1164.473876953125,
|
|
"KL/rejected_KL_mean": -1385.8544921875,
|
|
"KL/std": 1028.2890625,
|
|
"epoch": 0.73568281938326,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 442.7614440917969,
|
|
"fcm_dpo/q_t": 0.4922564625740051,
|
|
"grad_norm": 5.623405933380127,
|
|
"learning_rate": 1.0036744111882672e-07,
|
|
"logits/chosen": -0.5240232944488525,
|
|
"logits/rejected": -0.5279651880264282,
|
|
"logps/chosen": -997.4527587890625,
|
|
"logps/ref_chosen": -54.359527587890625,
|
|
"logps/ref_rejected": -80.15670013427734,
|
|
"logps/rejected": -1466.01123046875,
|
|
"loss": 1.3582,
|
|
"margin_dpo/margin_mean": 442.76141357421875,
|
|
"margin_dpo/margin_std": 1491.604248046875,
|
|
"step": 501
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -986.6317749023438,
|
|
"KL/mean": -1168.455078125,
|
|
"KL/rejected_KL_mean": -1350.2783203125,
|
|
"KL/std": 1039.948486328125,
|
|
"epoch": 0.737151248164464,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 363.64642333984375,
|
|
"fcm_dpo/q_t": 0.4936283826828003,
|
|
"grad_norm": 8.514199256896973,
|
|
"learning_rate": 9.934134090518592e-08,
|
|
"logits/chosen": -0.5406728982925415,
|
|
"logits/rejected": -0.5402701497077942,
|
|
"logps/chosen": -1054.2322998046875,
|
|
"logps/ref_chosen": -67.60050964355469,
|
|
"logps/ref_rejected": -82.94876098632812,
|
|
"logps/rejected": -1433.22705078125,
|
|
"loss": 1.3624,
|
|
"margin_dpo/margin_mean": 363.64642333984375,
|
|
"margin_dpo/margin_std": 1048.6695556640625,
|
|
"step": 502
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -824.9329833984375,
|
|
"KL/mean": -1109.140869140625,
|
|
"KL/rejected_KL_mean": -1393.3486328125,
|
|
"KL/std": 1051.9873046875,
|
|
"epoch": 0.7386196769456681,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 568.41552734375,
|
|
"fcm_dpo/q_t": 0.4900738596916199,
|
|
"grad_norm": 6.4620280265808105,
|
|
"learning_rate": 9.831921068732571e-08,
|
|
"logits/chosen": -0.516451358795166,
|
|
"logits/rejected": -0.528926432132721,
|
|
"logps/chosen": -880.0114135742188,
|
|
"logps/ref_chosen": -55.078407287597656,
|
|
"logps/ref_rejected": -82.50544738769531,
|
|
"logps/rejected": -1475.85400390625,
|
|
"loss": 1.3488,
|
|
"margin_dpo/margin_mean": 568.41552734375,
|
|
"margin_dpo/margin_std": 1233.030029296875,
|
|
"step": 503
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -967.4698486328125,
|
|
"KL/mean": -1384.71337890625,
|
|
"KL/rejected_KL_mean": -1801.9569091796875,
|
|
"KL/std": 1324.2451171875,
|
|
"epoch": 0.7400881057268722,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 834.4871826171875,
|
|
"fcm_dpo/q_t": 0.485454797744751,
|
|
"grad_norm": 8.587855339050293,
|
|
"learning_rate": 9.730107739932805e-08,
|
|
"logits/chosen": -0.5549697279930115,
|
|
"logits/rejected": -0.5928115844726562,
|
|
"logps/chosen": -1027.435546875,
|
|
"logps/ref_chosen": -59.96575164794922,
|
|
"logps/ref_rejected": -103.76212310791016,
|
|
"logps/rejected": -1905.718994140625,
|
|
"loss": 1.332,
|
|
"margin_dpo/margin_mean": 834.4871826171875,
|
|
"margin_dpo/margin_std": 1630.097900390625,
|
|
"step": 504
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1171.11376953125,
|
|
"KL/mean": -1224.62890625,
|
|
"KL/rejected_KL_mean": -1278.1439208984375,
|
|
"KL/std": 1174.8758544921875,
|
|
"epoch": 0.7415565345080763,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 107.03019714355469,
|
|
"fcm_dpo/q_t": 0.4981127381324768,
|
|
"grad_norm": 7.053234577178955,
|
|
"learning_rate": 9.628696786995188e-08,
|
|
"logits/chosen": -0.5928906202316284,
|
|
"logits/rejected": -0.5843832492828369,
|
|
"logps/chosen": -1247.2685546875,
|
|
"logps/ref_chosen": -76.1549072265625,
|
|
"logps/ref_rejected": -88.58537292480469,
|
|
"logps/rejected": -1366.729248046875,
|
|
"loss": 1.3813,
|
|
"margin_dpo/margin_mean": 107.03020477294922,
|
|
"margin_dpo/margin_std": 1291.50732421875,
|
|
"step": 505
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -677.691650390625,
|
|
"KL/mean": -1025.074462890625,
|
|
"KL/rejected_KL_mean": -1372.457275390625,
|
|
"KL/std": 931.436767578125,
|
|
"epoch": 0.7430249632892805,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 694.765625,
|
|
"fcm_dpo/q_t": 0.48783206939697266,
|
|
"grad_norm": 6.811020851135254,
|
|
"learning_rate": 9.527690882192635e-08,
|
|
"logits/chosen": -0.561811089515686,
|
|
"logits/rejected": -0.5848267078399658,
|
|
"logps/chosen": -726.6522216796875,
|
|
"logps/ref_chosen": -48.96050262451172,
|
|
"logps/ref_rejected": -78.41505432128906,
|
|
"logps/rejected": -1450.872314453125,
|
|
"loss": 1.3395,
|
|
"margin_dpo/margin_mean": 694.765625,
|
|
"margin_dpo/margin_std": 1041.9476318359375,
|
|
"step": 506
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -971.807373046875,
|
|
"KL/mean": -1220.935302734375,
|
|
"KL/rejected_KL_mean": -1470.0631103515625,
|
|
"KL/std": 1240.0423583984375,
|
|
"epoch": 0.7444933920704846,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 498.2557067871094,
|
|
"fcm_dpo/q_t": 0.49129223823547363,
|
|
"grad_norm": 5.5018086433410645,
|
|
"learning_rate": 9.427092687124691e-08,
|
|
"logits/chosen": -0.5913007259368896,
|
|
"logits/rejected": -0.6078677177429199,
|
|
"logps/chosen": -1038.60888671875,
|
|
"logps/ref_chosen": -66.80149841308594,
|
|
"logps/ref_rejected": -95.37289428710938,
|
|
"logps/rejected": -1565.43603515625,
|
|
"loss": 1.354,
|
|
"margin_dpo/margin_mean": 498.2557373046875,
|
|
"margin_dpo/margin_std": 1392.36083984375,
|
|
"step": 507
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -939.2349853515625,
|
|
"KL/mean": -1200.439697265625,
|
|
"KL/rejected_KL_mean": -1461.644287109375,
|
|
"KL/std": 1131.05126953125,
|
|
"epoch": 0.7459618208516887,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 522.409423828125,
|
|
"fcm_dpo/q_t": 0.4908856451511383,
|
|
"grad_norm": 3.9309823513031006,
|
|
"learning_rate": 9.326904852647344e-08,
|
|
"logits/chosen": -0.5682249665260315,
|
|
"logits/rejected": -0.5859044194221497,
|
|
"logps/chosen": -1010.5384521484375,
|
|
"logps/ref_chosen": -71.303466796875,
|
|
"logps/ref_rejected": -95.6275405883789,
|
|
"logps/rejected": -1557.2718505859375,
|
|
"loss": 1.3528,
|
|
"margin_dpo/margin_mean": 522.409423828125,
|
|
"margin_dpo/margin_std": 1477.3380126953125,
|
|
"step": 508
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -770.8653564453125,
|
|
"KL/mean": -939.4214477539062,
|
|
"KL/rejected_KL_mean": -1107.9775390625,
|
|
"KL/std": 911.1104736328125,
|
|
"epoch": 0.7474302496328928,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 337.11224365234375,
|
|
"fcm_dpo/q_t": 0.49410343170166016,
|
|
"grad_norm": 5.916812419891357,
|
|
"learning_rate": 9.227130018803195e-08,
|
|
"logits/chosen": -0.5424858927726746,
|
|
"logits/rejected": -0.5421060919761658,
|
|
"logps/chosen": -834.684326171875,
|
|
"logps/ref_chosen": -63.81895065307617,
|
|
"logps/ref_rejected": -83.25643920898438,
|
|
"logps/rejected": -1191.2340087890625,
|
|
"loss": 1.3649,
|
|
"margin_dpo/margin_mean": 337.1122131347656,
|
|
"margin_dpo/margin_std": 1224.0833740234375,
|
|
"step": 509
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -931.734130859375,
|
|
"KL/mean": -1244.052978515625,
|
|
"KL/rejected_KL_mean": -1556.371826171875,
|
|
"KL/std": 978.7109375,
|
|
"epoch": 0.748898678414097,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 624.6377563476562,
|
|
"fcm_dpo/q_t": 0.4890550374984741,
|
|
"grad_norm": 4.738297462463379,
|
|
"learning_rate": 9.127770814751932e-08,
|
|
"logits/chosen": -0.5328891277313232,
|
|
"logits/rejected": -0.5605103969573975,
|
|
"logps/chosen": -983.612548828125,
|
|
"logps/ref_chosen": -51.878448486328125,
|
|
"logps/ref_rejected": -102.7651596069336,
|
|
"logps/rejected": -1659.1370849609375,
|
|
"loss": 1.344,
|
|
"margin_dpo/margin_mean": 624.6378173828125,
|
|
"margin_dpo/margin_std": 936.5046997070312,
|
|
"step": 510
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -913.4978637695312,
|
|
"KL/mean": -1169.09619140625,
|
|
"KL/rejected_KL_mean": -1424.6943359375,
|
|
"KL/std": 1147.4625244140625,
|
|
"epoch": 0.750367107195301,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 511.196533203125,
|
|
"fcm_dpo/q_t": 0.4910575747489929,
|
|
"grad_norm": 6.8262619972229,
|
|
"learning_rate": 9.028829858700973e-08,
|
|
"logits/chosen": -0.5740972757339478,
|
|
"logits/rejected": -0.5858516693115234,
|
|
"logps/chosen": -973.7359619140625,
|
|
"logps/ref_chosen": -60.23811721801758,
|
|
"logps/ref_rejected": -92.85676574707031,
|
|
"logps/rejected": -1517.5511474609375,
|
|
"loss": 1.3535,
|
|
"margin_dpo/margin_mean": 511.1965026855469,
|
|
"margin_dpo/margin_std": 1480.135009765625,
|
|
"step": 511
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -768.2742919921875,
|
|
"KL/mean": -1137.4937744140625,
|
|
"KL/rejected_KL_mean": -1506.713134765625,
|
|
"KL/std": 1220.0850830078125,
|
|
"epoch": 0.7518355359765051,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 738.43896484375,
|
|
"fcm_dpo/q_t": 0.4870929718017578,
|
|
"grad_norm": 6.630341529846191,
|
|
"learning_rate": 8.930309757836516e-08,
|
|
"logits/chosen": -0.5527737140655518,
|
|
"logits/rejected": -0.5937384366989136,
|
|
"logps/chosen": -823.1798095703125,
|
|
"logps/ref_chosen": -54.905494689941406,
|
|
"logps/ref_rejected": -81.87586975097656,
|
|
"logps/rejected": -1588.589111328125,
|
|
"loss": 1.3376,
|
|
"margin_dpo/margin_mean": 738.43896484375,
|
|
"margin_dpo/margin_std": 1411.79248046875,
|
|
"step": 512
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1048.791015625,
|
|
"KL/mean": -1195.49755859375,
|
|
"KL/rejected_KL_mean": -1342.2042236328125,
|
|
"KL/std": 1093.606689453125,
|
|
"epoch": 0.7533039647577092,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 293.4132080078125,
|
|
"fcm_dpo/q_t": 0.49488991498947144,
|
|
"grad_norm": 9.842702865600586,
|
|
"learning_rate": 8.832213108254863e-08,
|
|
"logits/chosen": -0.567176342010498,
|
|
"logits/rejected": -0.552276611328125,
|
|
"logps/chosen": -1113.70751953125,
|
|
"logps/ref_chosen": -64.91644287109375,
|
|
"logps/ref_rejected": -76.06245422363281,
|
|
"logps/rejected": -1418.2666015625,
|
|
"loss": 1.3686,
|
|
"margin_dpo/margin_mean": 293.4131774902344,
|
|
"margin_dpo/margin_std": 1416.5491943359375,
|
|
"step": 513
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1270.703369140625,
|
|
"KL/mean": -1452.780517578125,
|
|
"KL/rejected_KL_mean": -1634.857666015625,
|
|
"KL/std": 1571.225341796875,
|
|
"epoch": 0.7547723935389133,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 364.1544189453125,
|
|
"fcm_dpo/q_t": 0.4936813712120056,
|
|
"grad_norm": 8.145087242126465,
|
|
"learning_rate": 8.734542494893954e-08,
|
|
"logits/chosen": -0.6005032658576965,
|
|
"logits/rejected": -0.6026064157485962,
|
|
"logps/chosen": -1344.932861328125,
|
|
"logps/ref_chosen": -74.22957611083984,
|
|
"logps/ref_rejected": -78.945556640625,
|
|
"logps/rejected": -1713.80322265625,
|
|
"loss": 1.3656,
|
|
"margin_dpo/margin_mean": 364.1544494628906,
|
|
"margin_dpo/margin_std": 1899.677001953125,
|
|
"step": 514
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -822.61181640625,
|
|
"KL/mean": -1065.546142578125,
|
|
"KL/rejected_KL_mean": -1308.4805908203125,
|
|
"KL/std": 936.01123046875,
|
|
"epoch": 0.7562408223201175,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 485.86871337890625,
|
|
"fcm_dpo/q_t": 0.4914933145046234,
|
|
"grad_norm": 14.613423347473145,
|
|
"learning_rate": 8.637300491465272e-08,
|
|
"logits/chosen": -0.5557729601860046,
|
|
"logits/rejected": -0.5711244940757751,
|
|
"logps/chosen": -873.013427734375,
|
|
"logps/ref_chosen": -50.40156555175781,
|
|
"logps/ref_rejected": -87.09774780273438,
|
|
"logps/rejected": -1395.578369140625,
|
|
"loss": 1.3542,
|
|
"margin_dpo/margin_mean": 485.8686828613281,
|
|
"margin_dpo/margin_std": 1161.409423828125,
|
|
"step": 515
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1072.080810546875,
|
|
"KL/mean": -1407.988037109375,
|
|
"KL/rejected_KL_mean": -1743.8955078125,
|
|
"KL/std": 1599.38037109375,
|
|
"epoch": 0.7577092511013216,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 671.8146362304688,
|
|
"fcm_dpo/q_t": 0.4883235692977905,
|
|
"grad_norm": 7.461482048034668,
|
|
"learning_rate": 8.540489660386064e-08,
|
|
"logits/chosen": -0.5883182287216187,
|
|
"logits/rejected": -0.621609091758728,
|
|
"logps/chosen": -1136.7303466796875,
|
|
"logps/ref_chosen": -64.64956665039062,
|
|
"logps/ref_rejected": -111.72237396240234,
|
|
"logps/rejected": -1855.6177978515625,
|
|
"loss": 1.3438,
|
|
"margin_dpo/margin_mean": 671.8145751953125,
|
|
"margin_dpo/margin_std": 1814.15576171875,
|
|
"step": 516
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1061.081787109375,
|
|
"KL/mean": -1346.005859375,
|
|
"KL/rejected_KL_mean": -1630.9300537109375,
|
|
"KL/std": 1233.93896484375,
|
|
"epoch": 0.7591776798825257,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 569.8483276367188,
|
|
"fcm_dpo/q_t": 0.4900466501712799,
|
|
"grad_norm": 7.388690948486328,
|
|
"learning_rate": 8.444112552711752e-08,
|
|
"logits/chosen": -0.5805338621139526,
|
|
"logits/rejected": -0.5963911414146423,
|
|
"logps/chosen": -1121.995361328125,
|
|
"logps/ref_chosen": -60.913551330566406,
|
|
"logps/ref_rejected": -89.08308410644531,
|
|
"logps/rejected": -1720.01318359375,
|
|
"loss": 1.3499,
|
|
"margin_dpo/margin_mean": 569.8483276367188,
|
|
"margin_dpo/margin_std": 1582.470947265625,
|
|
"step": 517
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1056.78125,
|
|
"KL/mean": -1327.392578125,
|
|
"KL/rejected_KL_mean": -1598.0040283203125,
|
|
"KL/std": 1047.9013671875,
|
|
"epoch": 0.7606461086637298,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 541.22265625,
|
|
"fcm_dpo/q_t": 0.49052339792251587,
|
|
"grad_norm": 4.970123291015625,
|
|
"learning_rate": 8.348171708068747e-08,
|
|
"logits/chosen": -0.6027194261550903,
|
|
"logits/rejected": -0.6282007694244385,
|
|
"logps/chosen": -1114.2371826171875,
|
|
"logps/ref_chosen": -57.45589065551758,
|
|
"logps/ref_rejected": -85.31269836425781,
|
|
"logps/rejected": -1683.316650390625,
|
|
"loss": 1.3508,
|
|
"margin_dpo/margin_mean": 541.22265625,
|
|
"margin_dpo/margin_std": 1296.53173828125,
|
|
"step": 518
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1016.2708740234375,
|
|
"KL/mean": -1329.120849609375,
|
|
"KL/rejected_KL_mean": -1641.970947265625,
|
|
"KL/std": 1342.2520751953125,
|
|
"epoch": 0.762114537444934,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 625.7000732421875,
|
|
"fcm_dpo/q_t": 0.4891040325164795,
|
|
"grad_norm": 7.316686630249023,
|
|
"learning_rate": 8.25266965458755e-08,
|
|
"logits/chosen": -0.5834439992904663,
|
|
"logits/rejected": -0.5853796005249023,
|
|
"logps/chosen": -1090.334228515625,
|
|
"logps/ref_chosen": -74.06331634521484,
|
|
"logps/ref_rejected": -104.44416809082031,
|
|
"logps/rejected": -1746.415283203125,
|
|
"loss": 1.347,
|
|
"margin_dpo/margin_mean": 625.7000732421875,
|
|
"margin_dpo/margin_std": 1800.982421875,
|
|
"step": 519
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1117.6549072265625,
|
|
"KL/mean": -1392.603759765625,
|
|
"KL/rejected_KL_mean": -1667.552490234375,
|
|
"KL/std": 1104.5682373046875,
|
|
"epoch": 0.7635829662261381,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 549.8974609375,
|
|
"fcm_dpo/q_t": 0.49038708209991455,
|
|
"grad_norm": 6.708014965057373,
|
|
"learning_rate": 8.15760890883607e-08,
|
|
"logits/chosen": -0.5891440510749817,
|
|
"logits/rejected": -0.6028249859809875,
|
|
"logps/chosen": -1187.9547119140625,
|
|
"logps/ref_chosen": -70.2998275756836,
|
|
"logps/ref_rejected": -99.98133850097656,
|
|
"logps/rejected": -1767.533935546875,
|
|
"loss": 1.3504,
|
|
"margin_dpo/margin_mean": 549.8974609375,
|
|
"margin_dpo/margin_std": 1362.424560546875,
|
|
"step": 520
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -902.4603271484375,
|
|
"KL/mean": -1152.8935546875,
|
|
"KL/rejected_KL_mean": -1403.32666015625,
|
|
"KL/std": 1028.9302978515625,
|
|
"epoch": 0.7650513950073421,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 500.8663330078125,
|
|
"fcm_dpo/q_t": 0.4912322163581848,
|
|
"grad_norm": 4.412991523742676,
|
|
"learning_rate": 8.062991975753378e-08,
|
|
"logits/chosen": -0.5892634391784668,
|
|
"logits/rejected": -0.5929391980171204,
|
|
"logps/chosen": -960.603271484375,
|
|
"logps/ref_chosen": -58.14292526245117,
|
|
"logps/ref_rejected": -83.28060913085938,
|
|
"logps/rejected": -1486.6072998046875,
|
|
"loss": 1.3533,
|
|
"margin_dpo/margin_mean": 500.8663330078125,
|
|
"margin_dpo/margin_std": 1217.1341552734375,
|
|
"step": 521
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1055.103271484375,
|
|
"KL/mean": -1362.3404541015625,
|
|
"KL/rejected_KL_mean": -1669.57763671875,
|
|
"KL/std": 1621.5078125,
|
|
"epoch": 0.7665198237885462,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 614.4742431640625,
|
|
"fcm_dpo/q_t": 0.4893651604652405,
|
|
"grad_norm": 4.380655288696289,
|
|
"learning_rate": 7.968821348583643e-08,
|
|
"logits/chosen": -0.5628671646118164,
|
|
"logits/rejected": -0.5865719318389893,
|
|
"logps/chosen": -1101.6510009765625,
|
|
"logps/ref_chosen": -46.54766845703125,
|
|
"logps/ref_rejected": -66.01388549804688,
|
|
"logps/rejected": -1735.591552734375,
|
|
"loss": 1.3489,
|
|
"margin_dpo/margin_mean": 614.4742431640625,
|
|
"margin_dpo/margin_std": 2025.41748046875,
|
|
"step": 522
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1028.6043701171875,
|
|
"KL/mean": -1341.892578125,
|
|
"KL/rejected_KL_mean": -1655.180908203125,
|
|
"KL/std": 1353.0596923828125,
|
|
"epoch": 0.7679882525697503,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 626.5765380859375,
|
|
"fcm_dpo/q_t": 0.48908570408821106,
|
|
"grad_norm": 7.474643707275391,
|
|
"learning_rate": 7.875099508810484e-08,
|
|
"logits/chosen": -0.6015632152557373,
|
|
"logits/rejected": -0.6170424222946167,
|
|
"logps/chosen": -1090.3740234375,
|
|
"logps/ref_chosen": -61.76960372924805,
|
|
"logps/ref_rejected": -83.76141357421875,
|
|
"logps/rejected": -1738.9423828125,
|
|
"loss": 1.346,
|
|
"margin_dpo/margin_mean": 626.5764770507812,
|
|
"margin_dpo/margin_std": 1597.18505859375,
|
|
"step": 523
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1175.5543212890625,
|
|
"KL/mean": -1509.3448486328125,
|
|
"KL/rejected_KL_mean": -1843.1353759765625,
|
|
"KL/std": 1454.0643310546875,
|
|
"epoch": 0.7694566813509545,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 667.5810546875,
|
|
"fcm_dpo/q_t": 0.48826104402542114,
|
|
"grad_norm": 5.199390411376953,
|
|
"learning_rate": 7.781828926091535e-08,
|
|
"logits/chosen": -0.6230287551879883,
|
|
"logits/rejected": -0.6203910112380981,
|
|
"logps/chosen": -1253.6263427734375,
|
|
"logps/ref_chosen": -78.0720443725586,
|
|
"logps/ref_rejected": -81.30198669433594,
|
|
"logps/rejected": -1924.4373779296875,
|
|
"loss": 1.3434,
|
|
"margin_dpo/margin_mean": 667.5810546875,
|
|
"margin_dpo/margin_std": 1561.3837890625,
|
|
"step": 524
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -979.4048461914062,
|
|
"KL/mean": -1361.4544677734375,
|
|
"KL/rejected_KL_mean": -1743.50390625,
|
|
"KL/std": 1232.875732421875,
|
|
"epoch": 0.7709251101321586,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 764.09912109375,
|
|
"fcm_dpo/q_t": 0.4866584539413452,
|
|
"grad_norm": 7.510035037994385,
|
|
"learning_rate": 7.689012058193384e-08,
|
|
"logits/chosen": -0.5819898843765259,
|
|
"logits/rejected": -0.6207159757614136,
|
|
"logps/chosen": -1030.232666015625,
|
|
"logps/ref_chosen": -50.827857971191406,
|
|
"logps/ref_rejected": -100.05294036865234,
|
|
"logps/rejected": -1843.556884765625,
|
|
"loss": 1.3363,
|
|
"margin_dpo/margin_mean": 764.09912109375,
|
|
"margin_dpo/margin_std": 1525.293212890625,
|
|
"step": 525
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1006.8646240234375,
|
|
"KL/mean": -1335.72021484375,
|
|
"KL/rejected_KL_mean": -1664.5760498046875,
|
|
"KL/std": 1145.358642578125,
|
|
"epoch": 0.7723935389133627,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 657.71142578125,
|
|
"fcm_dpo/q_t": 0.4884929656982422,
|
|
"grad_norm": 4.733247756958008,
|
|
"learning_rate": 7.596651350926836e-08,
|
|
"logits/chosen": -0.6167298555374146,
|
|
"logits/rejected": -0.6341676712036133,
|
|
"logps/chosen": -1070.0318603515625,
|
|
"logps/ref_chosen": -63.167236328125,
|
|
"logps/ref_rejected": -86.30934143066406,
|
|
"logps/rejected": -1750.8853759765625,
|
|
"loss": 1.3427,
|
|
"margin_dpo/margin_mean": 657.71142578125,
|
|
"margin_dpo/margin_std": 1287.364990234375,
|
|
"step": 526
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1146.7838134765625,
|
|
"KL/mean": -1294.6519775390625,
|
|
"KL/rejected_KL_mean": -1442.520263671875,
|
|
"KL/std": 1127.4002685546875,
|
|
"epoch": 0.7738619676945668,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 295.73638916015625,
|
|
"fcm_dpo/q_t": 0.49481672048568726,
|
|
"grad_norm": 8.143030166625977,
|
|
"learning_rate": 7.504749238082414e-08,
|
|
"logits/chosen": -0.6452538967132568,
|
|
"logits/rejected": -0.6307415962219238,
|
|
"logps/chosen": -1217.91259765625,
|
|
"logps/ref_chosen": -71.12867736816406,
|
|
"logps/ref_rejected": -78.3425521850586,
|
|
"logps/rejected": -1520.86279296875,
|
|
"loss": 1.3671,
|
|
"margin_dpo/margin_mean": 295.73638916015625,
|
|
"margin_dpo/margin_std": 1076.11376953125,
|
|
"step": 527
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -924.047607421875,
|
|
"KL/mean": -1256.195556640625,
|
|
"KL/rejected_KL_mean": -1588.343505859375,
|
|
"KL/std": 1196.41650390625,
|
|
"epoch": 0.775330396475771,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 664.2958374023438,
|
|
"fcm_dpo/q_t": 0.4884028434753418,
|
|
"grad_norm": 4.170558929443359,
|
|
"learning_rate": 7.413308141366254e-08,
|
|
"logits/chosen": -0.6017875671386719,
|
|
"logits/rejected": -0.6084692478179932,
|
|
"logps/chosen": -992.1370849609375,
|
|
"logps/ref_chosen": -68.0894546508789,
|
|
"logps/ref_rejected": -93.91006469726562,
|
|
"logps/rejected": -1682.2535400390625,
|
|
"loss": 1.3432,
|
|
"margin_dpo/margin_mean": 664.2958374023438,
|
|
"margin_dpo/margin_std": 1553.2027587890625,
|
|
"step": 528
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1159.14599609375,
|
|
"KL/mean": -1451.514404296875,
|
|
"KL/rejected_KL_mean": -1743.8826904296875,
|
|
"KL/std": 1228.73193359375,
|
|
"epoch": 0.7767988252569751,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 584.7366333007812,
|
|
"fcm_dpo/q_t": 0.48979973793029785,
|
|
"grad_norm": 4.807751655578613,
|
|
"learning_rate": 7.322330470336313e-08,
|
|
"logits/chosen": -0.5786510109901428,
|
|
"logits/rejected": -0.5997955799102783,
|
|
"logps/chosen": -1214.720947265625,
|
|
"logps/ref_chosen": -55.57495880126953,
|
|
"logps/ref_rejected": -89.20909118652344,
|
|
"logps/rejected": -1833.091796875,
|
|
"loss": 1.3491,
|
|
"margin_dpo/margin_mean": 584.7366333007812,
|
|
"margin_dpo/margin_std": 1670.16748046875,
|
|
"step": 529
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -882.88427734375,
|
|
"KL/mean": -1237.404296875,
|
|
"KL/rejected_KL_mean": -1591.9244384765625,
|
|
"KL/std": 1404.204345703125,
|
|
"epoch": 0.7782672540381792,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 709.0401000976562,
|
|
"fcm_dpo/q_t": 0.4876710772514343,
|
|
"grad_norm": 8.733548164367676,
|
|
"learning_rate": 7.231818622338822e-08,
|
|
"logits/chosen": -0.5622015595436096,
|
|
"logits/rejected": -0.5846884250640869,
|
|
"logps/chosen": -930.4857177734375,
|
|
"logps/ref_chosen": -47.601417541503906,
|
|
"logps/ref_rejected": -87.2845230102539,
|
|
"logps/rejected": -1679.208984375,
|
|
"loss": 1.3413,
|
|
"margin_dpo/margin_mean": 709.0401611328125,
|
|
"margin_dpo/margin_std": 1824.845458984375,
|
|
"step": 530
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -859.3704833984375,
|
|
"KL/mean": -1154.43994140625,
|
|
"KL/rejected_KL_mean": -1449.509521484375,
|
|
"KL/std": 1112.5634765625,
|
|
"epoch": 0.7797356828193832,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 590.1390380859375,
|
|
"fcm_dpo/q_t": 0.48967117071151733,
|
|
"grad_norm": 5.344662189483643,
|
|
"learning_rate": 7.141774982445147e-08,
|
|
"logits/chosen": -0.5867226123809814,
|
|
"logits/rejected": -0.6062394380569458,
|
|
"logps/chosen": -914.6165161132812,
|
|
"logps/ref_chosen": -55.246063232421875,
|
|
"logps/ref_rejected": -70.60598754882812,
|
|
"logps/rejected": -1520.115478515625,
|
|
"loss": 1.3468,
|
|
"margin_dpo/margin_mean": 590.1390380859375,
|
|
"margin_dpo/margin_std": 1108.8994140625,
|
|
"step": 531
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1094.8494873046875,
|
|
"KL/mean": -1391.7200927734375,
|
|
"KL/rejected_KL_mean": -1688.590576171875,
|
|
"KL/std": 1260.90576171875,
|
|
"epoch": 0.7812041116005873,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 593.7410278320312,
|
|
"fcm_dpo/q_t": 0.4896160364151001,
|
|
"grad_norm": 7.872839450836182,
|
|
"learning_rate": 7.052201923388953e-08,
|
|
"logits/chosen": -0.5879281759262085,
|
|
"logits/rejected": -0.5886451005935669,
|
|
"logps/chosen": -1165.135498046875,
|
|
"logps/ref_chosen": -70.28601837158203,
|
|
"logps/ref_rejected": -86.5913314819336,
|
|
"logps/rejected": -1775.181884765625,
|
|
"loss": 1.3489,
|
|
"margin_dpo/margin_mean": 593.740966796875,
|
|
"margin_dpo/margin_std": 1750.036376953125,
|
|
"step": 532
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -933.3600463867188,
|
|
"KL/mean": -1131.0,
|
|
"KL/rejected_KL_mean": -1328.639892578125,
|
|
"KL/std": 966.4365844726562,
|
|
"epoch": 0.7826725403817915,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 395.27978515625,
|
|
"fcm_dpo/q_t": 0.4930812120437622,
|
|
"grad_norm": 7.988112449645996,
|
|
"learning_rate": 6.963101805503646e-08,
|
|
"logits/chosen": -0.5576300024986267,
|
|
"logits/rejected": -0.5628528594970703,
|
|
"logps/chosen": -998.2151489257812,
|
|
"logps/ref_chosen": -64.8551025390625,
|
|
"logps/ref_rejected": -76.58805847167969,
|
|
"logps/rejected": -1405.2279052734375,
|
|
"loss": 1.3604,
|
|
"margin_dpo/margin_mean": 395.27978515625,
|
|
"margin_dpo/margin_std": 1123.464599609375,
|
|
"step": 533
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -992.2314453125,
|
|
"KL/mean": -1203.13671875,
|
|
"KL/rejected_KL_mean": -1414.042236328125,
|
|
"KL/std": 950.9990234375,
|
|
"epoch": 0.7841409691629956,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 421.810791015625,
|
|
"fcm_dpo/q_t": 0.49261125922203064,
|
|
"grad_norm": 3.841683864593506,
|
|
"learning_rate": 6.874476976660184e-08,
|
|
"logits/chosen": -0.5702451467514038,
|
|
"logits/rejected": -0.5771702527999878,
|
|
"logps/chosen": -1052.350830078125,
|
|
"logps/ref_chosen": -60.119388580322266,
|
|
"logps/ref_rejected": -78.54347229003906,
|
|
"logps/rejected": -1492.585693359375,
|
|
"loss": 1.3584,
|
|
"margin_dpo/margin_mean": 421.810791015625,
|
|
"margin_dpo/margin_std": 1089.174560546875,
|
|
"step": 534
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -888.7866821289062,
|
|
"KL/mean": -1202.163330078125,
|
|
"KL/rejected_KL_mean": -1515.5400390625,
|
|
"KL/std": 1040.59130859375,
|
|
"epoch": 0.7856093979441997,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 626.75341796875,
|
|
"fcm_dpo/q_t": 0.48904871940612793,
|
|
"grad_norm": 8.149191856384277,
|
|
"learning_rate": 6.786329772205246e-08,
|
|
"logits/chosen": -0.552294135093689,
|
|
"logits/rejected": -0.5709767937660217,
|
|
"logps/chosen": -943.1168823242188,
|
|
"logps/ref_chosen": -54.330238342285156,
|
|
"logps/ref_rejected": -96.30763244628906,
|
|
"logps/rejected": -1611.84765625,
|
|
"loss": 1.345,
|
|
"margin_dpo/margin_mean": 626.75341796875,
|
|
"margin_dpo/margin_std": 1275.3328857421875,
|
|
"step": 535
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -841.0042724609375,
|
|
"KL/mean": -1279.9410400390625,
|
|
"KL/rejected_KL_mean": -1718.8778076171875,
|
|
"KL/std": 1255.70849609375,
|
|
"epoch": 0.7870778267254038,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 877.8734130859375,
|
|
"fcm_dpo/q_t": 0.4847047030925751,
|
|
"grad_norm": 5.253050327301025,
|
|
"learning_rate": 6.698662514899638e-08,
|
|
"logits/chosen": -0.5551550388336182,
|
|
"logits/rejected": -0.5932704210281372,
|
|
"logps/chosen": -888.0848388671875,
|
|
"logps/ref_chosen": -47.08053207397461,
|
|
"logps/ref_rejected": -89.09783935546875,
|
|
"logps/rejected": -1807.9755859375,
|
|
"loss": 1.3289,
|
|
"margin_dpo/margin_mean": 877.8734741210938,
|
|
"margin_dpo/margin_std": 1561.20703125,
|
|
"step": 536
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -840.8751831054688,
|
|
"KL/mean": -1096.5126953125,
|
|
"KL/rejected_KL_mean": -1352.150146484375,
|
|
"KL/std": 1151.702880859375,
|
|
"epoch": 0.788546255506608,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 511.2747802734375,
|
|
"fcm_dpo/q_t": 0.49105796217918396,
|
|
"grad_norm": 7.912595748901367,
|
|
"learning_rate": 6.611477514857114e-08,
|
|
"logits/chosen": -0.5522580742835999,
|
|
"logits/rejected": -0.5589759945869446,
|
|
"logps/chosen": -898.6226806640625,
|
|
"logps/ref_chosen": -57.747467041015625,
|
|
"logps/ref_rejected": -70.43838500976562,
|
|
"logps/rejected": -1422.58837890625,
|
|
"loss": 1.3537,
|
|
"margin_dpo/margin_mean": 511.2747802734375,
|
|
"margin_dpo/margin_std": 1513.6673583984375,
|
|
"step": 537
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1396.2117919921875,
|
|
"KL/mean": -1632.014404296875,
|
|
"KL/rejected_KL_mean": -1867.817138671875,
|
|
"KL/std": 1259.4403076171875,
|
|
"epoch": 0.7900146842878121,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 471.6053466796875,
|
|
"fcm_dpo/q_t": 0.49175089597702026,
|
|
"grad_norm": 9.711124420166016,
|
|
"learning_rate": 6.524777069483525e-08,
|
|
"logits/chosen": -0.5915812253952026,
|
|
"logits/rejected": -0.5872704982757568,
|
|
"logps/chosen": -1462.627685546875,
|
|
"logps/ref_chosen": -66.41594696044922,
|
|
"logps/ref_rejected": -84.22808837890625,
|
|
"logps/rejected": -1952.045166015625,
|
|
"loss": 1.3565,
|
|
"margin_dpo/margin_mean": 471.6053466796875,
|
|
"margin_dpo/margin_std": 1497.396484375,
|
|
"step": 538
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1097.3935546875,
|
|
"KL/mean": -1476.907958984375,
|
|
"KL/rejected_KL_mean": -1856.42236328125,
|
|
"KL/std": 1343.3634033203125,
|
|
"epoch": 0.7914831130690162,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 759.0288696289062,
|
|
"fcm_dpo/q_t": 0.48674440383911133,
|
|
"grad_norm": 5.35037899017334,
|
|
"learning_rate": 6.438563463416221e-08,
|
|
"logits/chosen": -0.6155024766921997,
|
|
"logits/rejected": -0.6297394633293152,
|
|
"logps/chosen": -1155.886474609375,
|
|
"logps/ref_chosen": -58.492855072021484,
|
|
"logps/ref_rejected": -91.85395050048828,
|
|
"logps/rejected": -1948.2763671875,
|
|
"loss": 1.3363,
|
|
"margin_dpo/margin_mean": 759.0289306640625,
|
|
"margin_dpo/margin_std": 1419.5430908203125,
|
|
"step": 539
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1084.130615234375,
|
|
"KL/mean": -1591.833740234375,
|
|
"KL/rejected_KL_mean": -2099.53662109375,
|
|
"KL/std": 1494.089111328125,
|
|
"epoch": 0.7929515418502202,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1015.4059448242188,
|
|
"fcm_dpo/q_t": 0.48234492540359497,
|
|
"grad_norm": 17.94775390625,
|
|
"learning_rate": 6.352838968463919e-08,
|
|
"logits/chosen": -0.5842588543891907,
|
|
"logits/rejected": -0.6213306188583374,
|
|
"logps/chosen": -1147.61328125,
|
|
"logps/ref_chosen": -63.482513427734375,
|
|
"logps/ref_rejected": -116.42999267578125,
|
|
"logps/rejected": -2215.966796875,
|
|
"loss": 1.3212,
|
|
"margin_dpo/margin_mean": 1015.4058837890625,
|
|
"margin_dpo/margin_std": 1876.0716552734375,
|
|
"step": 540
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1515.512939453125,
|
|
"KL/mean": -1609.35546875,
|
|
"KL/rejected_KL_mean": -1703.19775390625,
|
|
"KL/std": 1322.71044921875,
|
|
"epoch": 0.7944199706314243,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 187.684814453125,
|
|
"fcm_dpo/q_t": 0.49672752618789673,
|
|
"grad_norm": 9.675416946411133,
|
|
"learning_rate": 6.267605843546767e-08,
|
|
"logits/chosen": -0.6445415019989014,
|
|
"logits/rejected": -0.6379104852676392,
|
|
"logps/chosen": -1593.7933349609375,
|
|
"logps/ref_chosen": -78.28036499023438,
|
|
"logps/ref_rejected": -103.273681640625,
|
|
"logps/rejected": -1806.471435546875,
|
|
"loss": 1.3765,
|
|
"margin_dpo/margin_mean": 187.68482971191406,
|
|
"margin_dpo/margin_std": 1610.0963134765625,
|
|
"step": 541
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1154.2373046875,
|
|
"KL/mean": -1602.501220703125,
|
|
"KL/rejected_KL_mean": -2050.76513671875,
|
|
"KL/std": 1615.48046875,
|
|
"epoch": 0.7958883994126285,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 896.5280151367188,
|
|
"fcm_dpo/q_t": 0.48452144861221313,
|
|
"grad_norm": 4.6800994873046875,
|
|
"learning_rate": 6.182866334636888e-08,
|
|
"logits/chosen": -0.6557344198226929,
|
|
"logits/rejected": -0.6881058216094971,
|
|
"logps/chosen": -1211.72216796875,
|
|
"logps/ref_chosen": -57.48497009277344,
|
|
"logps/ref_rejected": -96.47506713867188,
|
|
"logps/rejected": -2147.240234375,
|
|
"loss": 1.33,
|
|
"margin_dpo/margin_mean": 896.5280151367188,
|
|
"margin_dpo/margin_std": 2071.083740234375,
|
|
"step": 542
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1310.2457275390625,
|
|
"KL/mean": -1533.172119140625,
|
|
"KL/rejected_KL_mean": -1756.0986328125,
|
|
"KL/std": 1376.15087890625,
|
|
"epoch": 0.7973568281938326,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 445.85296630859375,
|
|
"fcm_dpo/q_t": 0.4922771453857422,
|
|
"grad_norm": 4.305510520935059,
|
|
"learning_rate": 6.098622674699147e-08,
|
|
"logits/chosen": -0.664671778678894,
|
|
"logits/rejected": -0.6874226331710815,
|
|
"logps/chosen": -1370.86328125,
|
|
"logps/ref_chosen": -60.61750793457031,
|
|
"logps/ref_rejected": -105.59896850585938,
|
|
"logps/rejected": -1861.6976318359375,
|
|
"loss": 1.36,
|
|
"margin_dpo/margin_mean": 445.8529357910156,
|
|
"margin_dpo/margin_std": 1953.89453125,
|
|
"step": 543
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1286.40478515625,
|
|
"KL/mean": -1731.963134765625,
|
|
"KL/rejected_KL_mean": -2177.521484375,
|
|
"KL/std": 1765.681396484375,
|
|
"epoch": 0.7988252569750367,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 891.1168212890625,
|
|
"fcm_dpo/q_t": 0.48461323976516724,
|
|
"grad_norm": 8.807916641235352,
|
|
"learning_rate": 6.01487708363232e-08,
|
|
"logits/chosen": -0.6092058420181274,
|
|
"logits/rejected": -0.6435132026672363,
|
|
"logps/chosen": -1346.047119140625,
|
|
"logps/ref_chosen": -59.642303466796875,
|
|
"logps/ref_rejected": -100.95469665527344,
|
|
"logps/rejected": -2278.476318359375,
|
|
"loss": 1.3311,
|
|
"margin_dpo/margin_mean": 891.1168212890625,
|
|
"margin_dpo/margin_std": 2284.972900390625,
|
|
"step": 544
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1260.89306640625,
|
|
"KL/mean": -1646.8466796875,
|
|
"KL/rejected_KL_mean": -2032.800537109375,
|
|
"KL/std": 1534.07568359375,
|
|
"epoch": 0.8002936857562408,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 771.9075927734375,
|
|
"fcm_dpo/q_t": 0.48656225204467773,
|
|
"grad_norm": 6.247243881225586,
|
|
"learning_rate": 5.9316317682106294e-08,
|
|
"logits/chosen": -0.5921393632888794,
|
|
"logits/rejected": -0.6117866635322571,
|
|
"logps/chosen": -1328.54150390625,
|
|
"logps/ref_chosen": -67.64859771728516,
|
|
"logps/ref_rejected": -95.90800476074219,
|
|
"logps/rejected": -2128.70849609375,
|
|
"loss": 1.337,
|
|
"margin_dpo/margin_mean": 771.9075317382812,
|
|
"margin_dpo/margin_std": 1806.5426025390625,
|
|
"step": 545
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1186.041748046875,
|
|
"KL/mean": -1404.183837890625,
|
|
"KL/rejected_KL_mean": -1622.325927734375,
|
|
"KL/std": 1201.185546875,
|
|
"epoch": 0.801762114537445,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 436.28411865234375,
|
|
"fcm_dpo/q_t": 0.4923797845840454,
|
|
"grad_norm": 12.886591911315918,
|
|
"learning_rate": 5.848888922025552e-08,
|
|
"logits/chosen": -0.6253443360328674,
|
|
"logits/rejected": -0.6243206262588501,
|
|
"logps/chosen": -1236.785888671875,
|
|
"logps/ref_chosen": -50.744232177734375,
|
|
"logps/ref_rejected": -81.86622619628906,
|
|
"logps/rejected": -1704.192138671875,
|
|
"loss": 1.3586,
|
|
"margin_dpo/margin_mean": 436.28411865234375,
|
|
"margin_dpo/margin_std": 1439.834716796875,
|
|
"step": 546
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1329.843505859375,
|
|
"KL/mean": -1714.03662109375,
|
|
"KL/rejected_KL_mean": -2098.2294921875,
|
|
"KL/std": 1930.3846435546875,
|
|
"epoch": 0.8032305433186491,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 768.38623046875,
|
|
"fcm_dpo/q_t": 0.4867956340312958,
|
|
"grad_norm": 9.838577270507812,
|
|
"learning_rate": 5.7666507254280265e-08,
|
|
"logits/chosen": -0.5816187858581543,
|
|
"logits/rejected": -0.6018866300582886,
|
|
"logps/chosen": -1403.53125,
|
|
"logps/ref_chosen": -73.6877212524414,
|
|
"logps/ref_rejected": -90.76136779785156,
|
|
"logps/rejected": -2188.990966796875,
|
|
"loss": 1.3413,
|
|
"margin_dpo/margin_mean": 768.38623046875,
|
|
"margin_dpo/margin_std": 2552.30029296875,
|
|
"step": 547
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1230.2508544921875,
|
|
"KL/mean": -1500.3779296875,
|
|
"KL/rejected_KL_mean": -1770.5048828125,
|
|
"KL/std": 1448.751708984375,
|
|
"epoch": 0.8046989720998532,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 540.2540283203125,
|
|
"fcm_dpo/q_t": 0.49062401056289673,
|
|
"grad_norm": 4.971753120422363,
|
|
"learning_rate": 5.684919345471029e-08,
|
|
"logits/chosen": -0.6555124521255493,
|
|
"logits/rejected": -0.6675156950950623,
|
|
"logps/chosen": -1295.4971923828125,
|
|
"logps/ref_chosen": -65.24634552001953,
|
|
"logps/ref_rejected": -94.11807250976562,
|
|
"logps/rejected": -1864.6229248046875,
|
|
"loss": 1.3527,
|
|
"margin_dpo/margin_mean": 540.2540283203125,
|
|
"margin_dpo/margin_std": 1749.078125,
|
|
"step": 548
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1231.84814453125,
|
|
"KL/mean": -1339.867431640625,
|
|
"KL/rejected_KL_mean": -1447.8865966796875,
|
|
"KL/std": 1256.0994873046875,
|
|
"epoch": 0.8061674008810573,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 216.03839111328125,
|
|
"fcm_dpo/q_t": 0.4962120056152344,
|
|
"grad_norm": 6.037284851074219,
|
|
"learning_rate": 5.603696935852426e-08,
|
|
"logits/chosen": -0.5892428159713745,
|
|
"logits/rejected": -0.5814424753189087,
|
|
"logps/chosen": -1281.060546875,
|
|
"logps/ref_chosen": -49.21235656738281,
|
|
"logps/ref_rejected": -73.91031646728516,
|
|
"logps/rejected": -1521.796875,
|
|
"loss": 1.3742,
|
|
"margin_dpo/margin_mean": 216.0384063720703,
|
|
"margin_dpo/margin_std": 1532.143310546875,
|
|
"step": 549
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1138.1700439453125,
|
|
"KL/mean": -1449.984375,
|
|
"KL/rejected_KL_mean": -1761.798828125,
|
|
"KL/std": 1296.947998046875,
|
|
"epoch": 0.8076358296622613,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 623.6287841796875,
|
|
"fcm_dpo/q_t": 0.489163339138031,
|
|
"grad_norm": 4.7602033615112305,
|
|
"learning_rate": 5.5229856368582376e-08,
|
|
"logits/chosen": -0.6239445209503174,
|
|
"logits/rejected": -0.6451071500778198,
|
|
"logps/chosen": -1194.97705078125,
|
|
"logps/ref_chosen": -56.80695343017578,
|
|
"logps/ref_rejected": -95.12580871582031,
|
|
"logps/rejected": -1856.924560546875,
|
|
"loss": 1.3463,
|
|
"margin_dpo/margin_mean": 623.6287841796875,
|
|
"margin_dpo/margin_std": 1583.240478515625,
|
|
"step": 550
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -903.1275634765625,
|
|
"KL/mean": -1493.087158203125,
|
|
"KL/rejected_KL_mean": -2083.04638671875,
|
|
"KL/std": 1501.976806640625,
|
|
"epoch": 0.8091042584434655,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1179.9189453125,
|
|
"fcm_dpo/q_t": 0.4793858528137207,
|
|
"grad_norm": 11.45347785949707,
|
|
"learning_rate": 5.4427875753062734e-08,
|
|
"logits/chosen": -0.5236402750015259,
|
|
"logits/rejected": -0.5718011260032654,
|
|
"logps/chosen": -962.23388671875,
|
|
"logps/ref_chosen": -59.10633087158203,
|
|
"logps/ref_rejected": -111.67280578613281,
|
|
"logps/rejected": -2194.71923828125,
|
|
"loss": 1.3087,
|
|
"margin_dpo/margin_mean": 1179.9189453125,
|
|
"margin_dpo/margin_std": 1684.580322265625,
|
|
"step": 551
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -930.559326171875,
|
|
"KL/mean": -1511.77001953125,
|
|
"KL/rejected_KL_mean": -2092.98046875,
|
|
"KL/std": 1509.347412109375,
|
|
"epoch": 0.8105726872246696,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1162.4212646484375,
|
|
"fcm_dpo/q_t": 0.4797826111316681,
|
|
"grad_norm": 5.244424343109131,
|
|
"learning_rate": 5.363104864490034e-08,
|
|
"logits/chosen": -0.5785896182060242,
|
|
"logits/rejected": -0.6230120658874512,
|
|
"logps/chosen": -992.9139404296875,
|
|
"logps/ref_chosen": -62.35459899902344,
|
|
"logps/ref_rejected": -104.56210327148438,
|
|
"logps/rejected": -2197.54248046875,
|
|
"loss": 1.3114,
|
|
"margin_dpo/margin_mean": 1162.421142578125,
|
|
"margin_dpo/margin_std": 1976.2950439453125,
|
|
"step": 552
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1078.3341064453125,
|
|
"KL/mean": -1394.103515625,
|
|
"KL/rejected_KL_mean": -1709.872802734375,
|
|
"KL/std": 1449.83984375,
|
|
"epoch": 0.8120411160058737,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 631.5387573242188,
|
|
"fcm_dpo/q_t": 0.4890480637550354,
|
|
"grad_norm": 3.988219976425171,
|
|
"learning_rate": 5.2839396041230415e-08,
|
|
"logits/chosen": -0.62043297290802,
|
|
"logits/rejected": -0.6310804486274719,
|
|
"logps/chosen": -1146.5928955078125,
|
|
"logps/ref_chosen": -68.25881958007812,
|
|
"logps/ref_rejected": -98.0971450805664,
|
|
"logps/rejected": -1807.969970703125,
|
|
"loss": 1.3467,
|
|
"margin_dpo/margin_mean": 631.5386962890625,
|
|
"margin_dpo/margin_std": 1802.6300048828125,
|
|
"step": 553
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1069.6845703125,
|
|
"KL/mean": -1364.756103515625,
|
|
"KL/rejected_KL_mean": -1659.8275146484375,
|
|
"KL/std": 1380.947998046875,
|
|
"epoch": 0.8135095447870778,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 590.1428833007812,
|
|
"fcm_dpo/q_t": 0.4896678328514099,
|
|
"grad_norm": 8.758122444152832,
|
|
"learning_rate": 5.205293880283551e-08,
|
|
"logits/chosen": -0.5634171962738037,
|
|
"logits/rejected": -0.5632882714271545,
|
|
"logps/chosen": -1137.63232421875,
|
|
"logps/ref_chosen": -67.94767761230469,
|
|
"logps/ref_rejected": -89.78272247314453,
|
|
"logps/rejected": -1749.6102294921875,
|
|
"loss": 1.35,
|
|
"margin_dpo/margin_mean": 590.1428833007812,
|
|
"margin_dpo/margin_std": 1906.44287109375,
|
|
"step": 554
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1021.588623046875,
|
|
"KL/mean": -1599.221923828125,
|
|
"KL/rejected_KL_mean": -2176.855224609375,
|
|
"KL/std": 1767.3104248046875,
|
|
"epoch": 0.8149779735682819,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1155.2666015625,
|
|
"fcm_dpo/q_t": 0.48013854026794434,
|
|
"grad_norm": 10.302849769592285,
|
|
"learning_rate": 5.127169765359515e-08,
|
|
"logits/chosen": -0.6012399792671204,
|
|
"logits/rejected": -0.6592134237289429,
|
|
"logps/chosen": -1074.919189453125,
|
|
"logps/ref_chosen": -53.33049011230469,
|
|
"logps/ref_rejected": -108.47937774658203,
|
|
"logps/rejected": -2285.33447265625,
|
|
"loss": 1.3141,
|
|
"margin_dpo/margin_mean": 1155.2666015625,
|
|
"margin_dpo/margin_std": 2393.09033203125,
|
|
"step": 555
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -982.6744384765625,
|
|
"KL/mean": -1277.244384765625,
|
|
"KL/rejected_KL_mean": -1571.814208984375,
|
|
"KL/std": 1051.68212890625,
|
|
"epoch": 0.8164464023494861,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 589.1397705078125,
|
|
"fcm_dpo/q_t": 0.4896823763847351,
|
|
"grad_norm": 13.509299278259277,
|
|
"learning_rate": 5.049569317994012e-08,
|
|
"logits/chosen": -0.615922212600708,
|
|
"logits/rejected": -0.6318573355674744,
|
|
"logps/chosen": -1041.31884765625,
|
|
"logps/ref_chosen": -58.64447021484375,
|
|
"logps/ref_rejected": -101.34040832519531,
|
|
"logps/rejected": -1673.1546630859375,
|
|
"loss": 1.3468,
|
|
"margin_dpo/margin_mean": 589.1397705078125,
|
|
"margin_dpo/margin_std": 1059.956787109375,
|
|
"step": 556
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1247.9884033203125,
|
|
"KL/mean": -1629.21337890625,
|
|
"KL/rejected_KL_mean": -2010.438232421875,
|
|
"KL/std": 1569.9039306640625,
|
|
"epoch": 0.8179148311306902,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 762.44970703125,
|
|
"fcm_dpo/q_t": 0.4867032766342163,
|
|
"grad_norm": 9.238645553588867,
|
|
"learning_rate": 4.9724945830310144e-08,
|
|
"logits/chosen": -0.6624920964241028,
|
|
"logits/rejected": -0.6939408779144287,
|
|
"logps/chosen": -1315.8291015625,
|
|
"logps/ref_chosen": -67.84066009521484,
|
|
"logps/ref_rejected": -109.93965911865234,
|
|
"logps/rejected": -2120.3779296875,
|
|
"loss": 1.3377,
|
|
"margin_dpo/margin_mean": 762.44970703125,
|
|
"margin_dpo/margin_std": 1803.6630859375,
|
|
"step": 557
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1106.59033203125,
|
|
"KL/mean": -1585.306396484375,
|
|
"KL/rejected_KL_mean": -2064.022216796875,
|
|
"KL/std": 1658.90380859375,
|
|
"epoch": 0.8193832599118943,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 957.43212890625,
|
|
"fcm_dpo/q_t": 0.48335278034210205,
|
|
"grad_norm": 7.449563980102539,
|
|
"learning_rate": 4.8959475914614554e-08,
|
|
"logits/chosen": -0.6731094717979431,
|
|
"logits/rejected": -0.7122669219970703,
|
|
"logps/chosen": -1168.9586181640625,
|
|
"logps/ref_chosen": -62.36824035644531,
|
|
"logps/ref_rejected": -102.16102600097656,
|
|
"logps/rejected": -2166.183349609375,
|
|
"loss": 1.3256,
|
|
"margin_dpo/margin_mean": 957.4320678710938,
|
|
"margin_dpo/margin_std": 2066.8369140625,
|
|
"step": 558
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1256.0625,
|
|
"KL/mean": -1648.341064453125,
|
|
"KL/rejected_KL_mean": -2040.619873046875,
|
|
"KL/std": 1527.5263671875,
|
|
"epoch": 0.8208516886930984,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 784.557373046875,
|
|
"fcm_dpo/q_t": 0.4863905608654022,
|
|
"grad_norm": 9.646005630493164,
|
|
"learning_rate": 4.8199303603697614e-08,
|
|
"logits/chosen": -0.70490962266922,
|
|
"logits/rejected": -0.7145035266876221,
|
|
"logps/chosen": -1316.814697265625,
|
|
"logps/ref_chosen": -60.752323150634766,
|
|
"logps/ref_rejected": -93.44229125976562,
|
|
"logps/rejected": -2134.06201171875,
|
|
"loss": 1.3373,
|
|
"margin_dpo/margin_mean": 784.557373046875,
|
|
"margin_dpo/margin_std": 2029.985595703125,
|
|
"step": 559
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1072.104736328125,
|
|
"KL/mean": -1330.7318115234375,
|
|
"KL/rejected_KL_mean": -1589.35888671875,
|
|
"KL/std": 1190.2357177734375,
|
|
"epoch": 0.8223201174743024,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 517.25390625,
|
|
"fcm_dpo/q_t": 0.4909706115722656,
|
|
"grad_norm": 8.734450340270996,
|
|
"learning_rate": 4.7444448928806615e-08,
|
|
"logits/chosen": -0.5731639862060547,
|
|
"logits/rejected": -0.5752190351486206,
|
|
"logps/chosen": -1130.2086181640625,
|
|
"logps/ref_chosen": -58.10382080078125,
|
|
"logps/ref_rejected": -79.99122619628906,
|
|
"logps/rejected": -1669.35009765625,
|
|
"loss": 1.3529,
|
|
"margin_dpo/margin_mean": 517.2539672851562,
|
|
"margin_dpo/margin_std": 1448.2685546875,
|
|
"step": 560
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1208.382568359375,
|
|
"KL/mean": -1487.0732421875,
|
|
"KL/rejected_KL_mean": -1765.763916015625,
|
|
"KL/std": 1353.732421875,
|
|
"epoch": 0.8237885462555066,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 557.3814697265625,
|
|
"fcm_dpo/q_t": 0.49026691913604736,
|
|
"grad_norm": 5.121155738830566,
|
|
"learning_rate": 4.669493178106432e-08,
|
|
"logits/chosen": -0.6740288734436035,
|
|
"logits/rejected": -0.6940807104110718,
|
|
"logps/chosen": -1259.29541015625,
|
|
"logps/ref_chosen": -50.912879943847656,
|
|
"logps/ref_rejected": -99.06856536865234,
|
|
"logps/rejected": -1864.83251953125,
|
|
"loss": 1.3516,
|
|
"margin_dpo/margin_mean": 557.3814697265625,
|
|
"margin_dpo/margin_std": 1794.720458984375,
|
|
"step": 561
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -977.68701171875,
|
|
"KL/mean": -1403.8623046875,
|
|
"KL/rejected_KL_mean": -1830.037841796875,
|
|
"KL/std": 1357.09423828125,
|
|
"epoch": 0.8252569750367107,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 852.3507080078125,
|
|
"fcm_dpo/q_t": 0.48517775535583496,
|
|
"grad_norm": 6.560747146606445,
|
|
"learning_rate": 4.5950771910944596e-08,
|
|
"logits/chosen": -0.6164703965187073,
|
|
"logits/rejected": -0.6432167291641235,
|
|
"logps/chosen": -1037.1513671875,
|
|
"logps/ref_chosen": -59.46440124511719,
|
|
"logps/ref_rejected": -96.54266357421875,
|
|
"logps/rejected": -1926.580322265625,
|
|
"loss": 1.331,
|
|
"margin_dpo/margin_mean": 852.3507690429688,
|
|
"margin_dpo/margin_std": 1607.605712890625,
|
|
"step": 562
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1360.0205078125,
|
|
"KL/mean": -1654.7742919921875,
|
|
"KL/rejected_KL_mean": -1949.5281982421875,
|
|
"KL/std": 1659.429931640625,
|
|
"epoch": 0.8267254038179148,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 589.5076904296875,
|
|
"fcm_dpo/q_t": 0.4897218942642212,
|
|
"grad_norm": 8.3922700881958,
|
|
"learning_rate": 4.521198892775202e-08,
|
|
"logits/chosen": -0.6155869960784912,
|
|
"logits/rejected": -0.6347240209579468,
|
|
"logps/chosen": -1420.628662109375,
|
|
"logps/ref_chosen": -60.60819625854492,
|
|
"logps/ref_rejected": -94.56770324707031,
|
|
"logps/rejected": -2044.095947265625,
|
|
"loss": 1.3502,
|
|
"margin_dpo/margin_mean": 589.5076904296875,
|
|
"margin_dpo/margin_std": 1928.743408203125,
|
|
"step": 563
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1113.524658203125,
|
|
"KL/mean": -1359.48486328125,
|
|
"KL/rejected_KL_mean": -1605.445068359375,
|
|
"KL/std": 1332.8668212890625,
|
|
"epoch": 0.8281938325991189,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 491.92047119140625,
|
|
"fcm_dpo/q_t": 0.49136167764663696,
|
|
"grad_norm": 7.608941555023193,
|
|
"learning_rate": 4.447860229910544e-08,
|
|
"logits/chosen": -0.6324988603591919,
|
|
"logits/rejected": -0.6346863508224487,
|
|
"logps/chosen": -1187.79296875,
|
|
"logps/ref_chosen": -74.26837921142578,
|
|
"logps/ref_rejected": -93.23818969726562,
|
|
"logps/rejected": -1698.6832275390625,
|
|
"loss": 1.3556,
|
|
"margin_dpo/margin_mean": 491.9204406738281,
|
|
"margin_dpo/margin_std": 1603.78125,
|
|
"step": 564
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1407.1187744140625,
|
|
"KL/mean": -1665.321044921875,
|
|
"KL/rejected_KL_mean": -1923.5233154296875,
|
|
"KL/std": 1837.506103515625,
|
|
"epoch": 0.8296622613803231,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 516.404541015625,
|
|
"fcm_dpo/q_t": 0.4908890128135681,
|
|
"grad_norm": 13.449864387512207,
|
|
"learning_rate": 4.375063135042445e-08,
|
|
"logits/chosen": -0.663166880607605,
|
|
"logits/rejected": -0.6663168668746948,
|
|
"logps/chosen": -1476.138671875,
|
|
"logps/ref_chosen": -69.0199203491211,
|
|
"logps/ref_rejected": -85.7789306640625,
|
|
"logps/rejected": -2009.30224609375,
|
|
"loss": 1.3573,
|
|
"margin_dpo/margin_mean": 516.404541015625,
|
|
"margin_dpo/margin_std": 2329.1201171875,
|
|
"step": 565
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1349.890869140625,
|
|
"KL/mean": -1728.78564453125,
|
|
"KL/rejected_KL_mean": -2107.68017578125,
|
|
"KL/std": 1578.119873046875,
|
|
"epoch": 0.8311306901615272,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 757.7894287109375,
|
|
"fcm_dpo/q_t": 0.48675039410591125,
|
|
"grad_norm": 5.23419713973999,
|
|
"learning_rate": 4.3028095264420525e-08,
|
|
"logits/chosen": -0.6609238386154175,
|
|
"logits/rejected": -0.6840362548828125,
|
|
"logps/chosen": -1416.436279296875,
|
|
"logps/ref_chosen": -66.5453109741211,
|
|
"logps/ref_rejected": -103.86932373046875,
|
|
"logps/rejected": -2211.5498046875,
|
|
"loss": 1.3386,
|
|
"margin_dpo/margin_mean": 757.7894287109375,
|
|
"margin_dpo/margin_std": 1948.74609375,
|
|
"step": 566
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1095.9215087890625,
|
|
"KL/mean": -1309.902099609375,
|
|
"KL/rejected_KL_mean": -1523.882568359375,
|
|
"KL/std": 1136.513427734375,
|
|
"epoch": 0.8325991189427313,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 427.9610595703125,
|
|
"fcm_dpo/q_t": 0.4924955368041992,
|
|
"grad_norm": 6.573452472686768,
|
|
"learning_rate": 4.231101308059165e-08,
|
|
"logits/chosen": -0.6667437553405762,
|
|
"logits/rejected": -0.676659107208252,
|
|
"logps/chosen": -1148.77978515625,
|
|
"logps/ref_chosen": -52.85829544067383,
|
|
"logps/ref_rejected": -85.37095642089844,
|
|
"logps/rejected": -1609.25341796875,
|
|
"loss": 1.3586,
|
|
"margin_dpo/margin_mean": 427.9610595703125,
|
|
"margin_dpo/margin_std": 1303.414306640625,
|
|
"step": 567
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -958.6199951171875,
|
|
"KL/mean": -1419.5540771484375,
|
|
"KL/rejected_KL_mean": -1880.4881591796875,
|
|
"KL/std": 1383.7803955078125,
|
|
"epoch": 0.8340675477239354,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 921.8681030273438,
|
|
"fcm_dpo/q_t": 0.4839458167552948,
|
|
"grad_norm": 5.986705780029297,
|
|
"learning_rate": 4.1599403694720145e-08,
|
|
"logits/chosen": -0.6187951564788818,
|
|
"logits/rejected": -0.6642179489135742,
|
|
"logps/chosen": -1003.8123779296875,
|
|
"logps/ref_chosen": -45.1923828125,
|
|
"logps/ref_rejected": -89.09236907958984,
|
|
"logps/rejected": -1969.58056640625,
|
|
"loss": 1.3265,
|
|
"margin_dpo/margin_mean": 921.8680419921875,
|
|
"margin_dpo/margin_std": 1768.663818359375,
|
|
"step": 568
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1272.758056640625,
|
|
"KL/mean": -1551.21484375,
|
|
"KL/rejected_KL_mean": -1829.671630859375,
|
|
"KL/std": 1590.348876953125,
|
|
"epoch": 0.8355359765051396,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 556.9136962890625,
|
|
"fcm_dpo/q_t": 0.4902852773666382,
|
|
"grad_norm": 6.188879013061523,
|
|
"learning_rate": 4.089328585837512e-08,
|
|
"logits/chosen": -0.6311490535736084,
|
|
"logits/rejected": -0.6400599479675293,
|
|
"logps/chosen": -1336.478515625,
|
|
"logps/ref_chosen": -63.72056198120117,
|
|
"logps/ref_rejected": -79.10325622558594,
|
|
"logps/rejected": -1908.77490234375,
|
|
"loss": 1.3522,
|
|
"margin_dpo/margin_mean": 556.9136962890625,
|
|
"margin_dpo/margin_std": 1879.6124267578125,
|
|
"step": 569
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1346.568603515625,
|
|
"KL/mean": -1499.462890625,
|
|
"KL/rejected_KL_mean": -1652.357177734375,
|
|
"KL/std": 1435.96875,
|
|
"epoch": 0.8370044052863436,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 305.7886962890625,
|
|
"fcm_dpo/q_t": 0.4945737421512604,
|
|
"grad_norm": 8.600349426269531,
|
|
"learning_rate": 4.019267817841834e-08,
|
|
"logits/chosen": -0.6977522373199463,
|
|
"logits/rejected": -0.6893934011459351,
|
|
"logps/chosen": -1408.18310546875,
|
|
"logps/ref_chosen": -61.61454391479492,
|
|
"logps/ref_rejected": -82.14186096191406,
|
|
"logps/rejected": -1734.4991455078125,
|
|
"loss": 1.3696,
|
|
"margin_dpo/margin_mean": 305.7887268066406,
|
|
"margin_dpo/margin_std": 1899.91796875,
|
|
"step": 570
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1334.66357421875,
|
|
"KL/mean": -1602.7347412109375,
|
|
"KL/rejected_KL_mean": -1870.8057861328125,
|
|
"KL/std": 1609.7418212890625,
|
|
"epoch": 0.8384728340675477,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 536.1421508789062,
|
|
"fcm_dpo/q_t": 0.49062836170196533,
|
|
"grad_norm": 9.382889747619629,
|
|
"learning_rate": 3.9497599116513705e-08,
|
|
"logits/chosen": -0.6136207580566406,
|
|
"logits/rejected": -0.6315397024154663,
|
|
"logps/chosen": -1387.7177734375,
|
|
"logps/ref_chosen": -53.05406188964844,
|
|
"logps/ref_rejected": -91.33682250976562,
|
|
"logps/rejected": -1962.142578125,
|
|
"loss": 1.3535,
|
|
"margin_dpo/margin_mean": 536.1421508789062,
|
|
"margin_dpo/margin_std": 1897.035400390625,
|
|
"step": 571
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1041.9559326171875,
|
|
"KL/mean": -1387.749267578125,
|
|
"KL/rejected_KL_mean": -1733.54248046875,
|
|
"KL/std": 1231.0006103515625,
|
|
"epoch": 0.8399412628487518,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 691.5866088867188,
|
|
"fcm_dpo/q_t": 0.48791706562042236,
|
|
"grad_norm": 5.244479179382324,
|
|
"learning_rate": 3.880806698864086e-08,
|
|
"logits/chosen": -0.6141128540039062,
|
|
"logits/rejected": -0.6465499401092529,
|
|
"logps/chosen": -1090.415283203125,
|
|
"logps/ref_chosen": -48.45928955078125,
|
|
"logps/ref_rejected": -83.55703735351562,
|
|
"logps/rejected": -1817.099609375,
|
|
"loss": 1.3413,
|
|
"margin_dpo/margin_mean": 691.5865478515625,
|
|
"margin_dpo/margin_std": 1555.69189453125,
|
|
"step": 572
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1082.886474609375,
|
|
"KL/mean": -1360.33203125,
|
|
"KL/rejected_KL_mean": -1637.777587890625,
|
|
"KL/std": 1212.4599609375,
|
|
"epoch": 0.8414096916299559,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 554.8909912109375,
|
|
"fcm_dpo/q_t": 0.49030232429504395,
|
|
"grad_norm": 9.327940940856934,
|
|
"learning_rate": 3.812409996461275e-08,
|
|
"logits/chosen": -0.606751561164856,
|
|
"logits/rejected": -0.6213085651397705,
|
|
"logps/chosen": -1134.5091552734375,
|
|
"logps/ref_chosen": -51.62262725830078,
|
|
"logps/ref_rejected": -85.32499694824219,
|
|
"logps/rejected": -1723.1025390625,
|
|
"loss": 1.3498,
|
|
"margin_dpo/margin_mean": 554.8909912109375,
|
|
"margin_dpo/margin_std": 1286.421142578125,
|
|
"step": 573
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1081.611328125,
|
|
"KL/mean": -1515.553466796875,
|
|
"KL/rejected_KL_mean": -1949.4954833984375,
|
|
"KL/std": 1357.4376220703125,
|
|
"epoch": 0.8428781204111601,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 867.8842163085938,
|
|
"fcm_dpo/q_t": 0.4848443865776062,
|
|
"grad_norm": 5.942359447479248,
|
|
"learning_rate": 3.74457160675965e-08,
|
|
"logits/chosen": -0.6815818548202515,
|
|
"logits/rejected": -0.7281221151351929,
|
|
"logps/chosen": -1132.65576171875,
|
|
"logps/ref_chosen": -51.04446029663086,
|
|
"logps/ref_rejected": -92.80640411376953,
|
|
"logps/rejected": -2042.302001953125,
|
|
"loss": 1.3294,
|
|
"margin_dpo/margin_mean": 867.8841552734375,
|
|
"margin_dpo/margin_std": 1571.0694580078125,
|
|
"step": 574
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1305.66552734375,
|
|
"KL/mean": -1599.9190673828125,
|
|
"KL/rejected_KL_mean": -1894.1724853515625,
|
|
"KL/std": 1470.6298828125,
|
|
"epoch": 0.8443465491923642,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 588.5068359375,
|
|
"fcm_dpo/q_t": 0.48974353075027466,
|
|
"grad_norm": 7.068485736846924,
|
|
"learning_rate": 3.677293317363864e-08,
|
|
"logits/chosen": -0.6297034025192261,
|
|
"logits/rejected": -0.6442345380783081,
|
|
"logps/chosen": -1377.455810546875,
|
|
"logps/ref_chosen": -71.7901382446289,
|
|
"logps/ref_rejected": -95.38619995117188,
|
|
"logps/rejected": -1989.55859375,
|
|
"loss": 1.3495,
|
|
"margin_dpo/margin_mean": 588.5068359375,
|
|
"margin_dpo/margin_std": 1797.72509765625,
|
|
"step": 575
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1039.58740234375,
|
|
"KL/mean": -1412.387451171875,
|
|
"KL/rejected_KL_mean": -1785.1873779296875,
|
|
"KL/std": 1391.4742431640625,
|
|
"epoch": 0.8458149779735683,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 745.5999755859375,
|
|
"fcm_dpo/q_t": 0.48703137040138245,
|
|
"grad_norm": 12.236254692077637,
|
|
"learning_rate": 3.6105769011194224e-08,
|
|
"logits/chosen": -0.6339189410209656,
|
|
"logits/rejected": -0.6675552129745483,
|
|
"logps/chosen": -1093.850341796875,
|
|
"logps/ref_chosen": -54.262962341308594,
|
|
"logps/ref_rejected": -100.75428009033203,
|
|
"logps/rejected": -1885.941650390625,
|
|
"loss": 1.3381,
|
|
"margin_dpo/margin_mean": 745.5999755859375,
|
|
"margin_dpo/margin_std": 1672.704345703125,
|
|
"step": 576
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1099.926513671875,
|
|
"KL/mean": -1301.64794921875,
|
|
"KL/rejected_KL_mean": -1503.369384765625,
|
|
"KL/std": 1184.22705078125,
|
|
"epoch": 0.8472834067547724,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 403.4429626464844,
|
|
"fcm_dpo/q_t": 0.49293655157089233,
|
|
"grad_norm": 6.366522312164307,
|
|
"learning_rate": 3.5444241160659304e-08,
|
|
"logits/chosen": -0.6079691648483276,
|
|
"logits/rejected": -0.6053318381309509,
|
|
"logps/chosen": -1161.836181640625,
|
|
"logps/ref_chosen": -61.909706115722656,
|
|
"logps/ref_rejected": -84.07069396972656,
|
|
"logps/rejected": -1587.440185546875,
|
|
"loss": 1.3608,
|
|
"margin_dpo/margin_mean": 403.4429626464844,
|
|
"margin_dpo/margin_std": 1407.972412109375,
|
|
"step": 577
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -874.846923828125,
|
|
"KL/mean": -1265.8436279296875,
|
|
"KL/rejected_KL_mean": -1656.84033203125,
|
|
"KL/std": 1145.15771484375,
|
|
"epoch": 0.8487518355359766,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 781.993408203125,
|
|
"fcm_dpo/q_t": 0.48631131649017334,
|
|
"grad_norm": 4.333878517150879,
|
|
"learning_rate": 3.478836705390808e-08,
|
|
"logits/chosen": -0.5466019511222839,
|
|
"logits/rejected": -0.5783486366271973,
|
|
"logps/chosen": -924.110595703125,
|
|
"logps/ref_chosen": -49.26368713378906,
|
|
"logps/ref_rejected": -83.4362564086914,
|
|
"logps/rejected": -1740.276611328125,
|
|
"loss": 1.3339,
|
|
"margin_dpo/margin_mean": 781.993408203125,
|
|
"margin_dpo/margin_std": 1137.68408203125,
|
|
"step": 578
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1144.806396484375,
|
|
"KL/mean": -1350.88134765625,
|
|
"KL/rejected_KL_mean": -1556.956298828125,
|
|
"KL/std": 1468.2509765625,
|
|
"epoch": 0.8502202643171806,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 412.1499938964844,
|
|
"fcm_dpo/q_t": 0.49288052320480347,
|
|
"grad_norm": 6.493415355682373,
|
|
"learning_rate": 3.41381639738331e-08,
|
|
"logits/chosen": -0.609187662601471,
|
|
"logits/rejected": -0.6248157024383545,
|
|
"logps/chosen": -1203.6922607421875,
|
|
"logps/ref_chosen": -58.88581848144531,
|
|
"logps/ref_rejected": -94.78762817382812,
|
|
"logps/rejected": -1651.743896484375,
|
|
"loss": 1.362,
|
|
"margin_dpo/margin_mean": 412.14996337890625,
|
|
"margin_dpo/margin_std": 1850.548095703125,
|
|
"step": 579
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -775.547119140625,
|
|
"KL/mean": -1131.787109375,
|
|
"KL/rejected_KL_mean": -1488.02685546875,
|
|
"KL/std": 1250.650390625,
|
|
"epoch": 0.8516886930983847,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 712.479736328125,
|
|
"fcm_dpo/q_t": 0.48757392168045044,
|
|
"grad_norm": 5.055511951446533,
|
|
"learning_rate": 3.349364905389032e-08,
|
|
"logits/chosen": -0.5566902160644531,
|
|
"logits/rejected": -0.5791162848472595,
|
|
"logps/chosen": -824.2539672851562,
|
|
"logps/ref_chosen": -48.70683670043945,
|
|
"logps/ref_rejected": -81.7583999633789,
|
|
"logps/rejected": -1569.7852783203125,
|
|
"loss": 1.3396,
|
|
"margin_dpo/margin_mean": 712.479736328125,
|
|
"margin_dpo/margin_std": 1466.5977783203125,
|
|
"step": 580
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1139.18310546875,
|
|
"KL/mean": -1376.47412109375,
|
|
"KL/rejected_KL_mean": -1613.764892578125,
|
|
"KL/std": 1217.5745849609375,
|
|
"epoch": 0.8531571218795888,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 474.5818786621094,
|
|
"fcm_dpo/q_t": 0.4917064607143402,
|
|
"grad_norm": 6.296034812927246,
|
|
"learning_rate": 3.285483927764726e-08,
|
|
"logits/chosen": -0.6595457196235657,
|
|
"logits/rejected": -0.6783765554428101,
|
|
"logps/chosen": -1201.4053955078125,
|
|
"logps/ref_chosen": -62.22235107421875,
|
|
"logps/ref_rejected": -91.73568725585938,
|
|
"logps/rejected": -1705.500732421875,
|
|
"loss": 1.3559,
|
|
"margin_dpo/margin_mean": 474.5818786621094,
|
|
"margin_dpo/margin_std": 1438.438232421875,
|
|
"step": 581
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1091.2376708984375,
|
|
"KL/mean": -1487.45654296875,
|
|
"KL/rejected_KL_mean": -1883.6754150390625,
|
|
"KL/std": 1237.882080078125,
|
|
"epoch": 0.8546255506607929,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 792.4378051757812,
|
|
"fcm_dpo/q_t": 0.4861498475074768,
|
|
"grad_norm": 11.559684753417969,
|
|
"learning_rate": 3.222175147833556e-08,
|
|
"logits/chosen": -0.6552462577819824,
|
|
"logits/rejected": -0.6873466968536377,
|
|
"logps/chosen": -1149.46630859375,
|
|
"logps/ref_chosen": -58.228660583496094,
|
|
"logps/ref_rejected": -110.06959533691406,
|
|
"logps/rejected": -1993.7449951171875,
|
|
"loss": 1.3342,
|
|
"margin_dpo/margin_mean": 792.437744140625,
|
|
"margin_dpo/margin_std": 1396.3656005859375,
|
|
"step": 582
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1086.141357421875,
|
|
"KL/mean": -1331.1463623046875,
|
|
"KL/rejected_KL_mean": -1576.1513671875,
|
|
"KL/std": 1458.034423828125,
|
|
"epoch": 0.856093979441997,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 490.01007080078125,
|
|
"fcm_dpo/q_t": 0.4915131628513336,
|
|
"grad_norm": 12.745737075805664,
|
|
"learning_rate": 3.159440233840763e-08,
|
|
"logits/chosen": -0.6324259042739868,
|
|
"logits/rejected": -0.6466102600097656,
|
|
"logps/chosen": -1143.004150390625,
|
|
"logps/ref_chosen": -56.86286163330078,
|
|
"logps/ref_rejected": -88.4039306640625,
|
|
"logps/rejected": -1664.5552978515625,
|
|
"loss": 1.3569,
|
|
"margin_dpo/margin_mean": 490.0100402832031,
|
|
"margin_dpo/margin_std": 1838.203125,
|
|
"step": 583
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1118.363037109375,
|
|
"KL/mean": -1506.4857177734375,
|
|
"KL/rejected_KL_mean": -1894.6083984375,
|
|
"KL/std": 1478.693359375,
|
|
"epoch": 0.8575624082232012,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 776.2452392578125,
|
|
"fcm_dpo/q_t": 0.4864642024040222,
|
|
"grad_norm": 16.514495849609375,
|
|
"learning_rate": 3.0972808389096635e-08,
|
|
"logits/chosen": -0.6495020389556885,
|
|
"logits/rejected": -0.6715532541275024,
|
|
"logps/chosen": -1175.2637939453125,
|
|
"logps/ref_chosen": -56.90068054199219,
|
|
"logps/ref_rejected": -97.63606262207031,
|
|
"logps/rejected": -1992.244384765625,
|
|
"loss": 1.3358,
|
|
"margin_dpo/margin_mean": 776.2452392578125,
|
|
"margin_dpo/margin_std": 1611.7119140625,
|
|
"step": 584
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1159.185546875,
|
|
"KL/mean": -1560.73681640625,
|
|
"KL/rejected_KL_mean": -1962.2880859375,
|
|
"KL/std": 1544.73486328125,
|
|
"epoch": 0.8590308370044053,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 803.1026611328125,
|
|
"fcm_dpo/q_t": 0.4860289692878723,
|
|
"grad_norm": 4.359808444976807,
|
|
"learning_rate": 3.035698600998121e-08,
|
|
"logits/chosen": -0.6542025804519653,
|
|
"logits/rejected": -0.6867420077323914,
|
|
"logps/chosen": -1220.159423828125,
|
|
"logps/ref_chosen": -60.973968505859375,
|
|
"logps/ref_rejected": -84.16952514648438,
|
|
"logps/rejected": -2046.4576416015625,
|
|
"loss": 1.3362,
|
|
"margin_dpo/margin_mean": 803.1026611328125,
|
|
"margin_dpo/margin_std": 2070.01416015625,
|
|
"step": 585
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1311.316162109375,
|
|
"KL/mean": -1604.206787109375,
|
|
"KL/rejected_KL_mean": -1897.097412109375,
|
|
"KL/std": 1670.98046875,
|
|
"epoch": 0.8604992657856094,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 585.7813720703125,
|
|
"fcm_dpo/q_t": 0.4897732436656952,
|
|
"grad_norm": 6.482140064239502,
|
|
"learning_rate": 2.974695142855388e-08,
|
|
"logits/chosen": -0.6228867173194885,
|
|
"logits/rejected": -0.6541084051132202,
|
|
"logps/chosen": -1368.171630859375,
|
|
"logps/ref_chosen": -56.85559844970703,
|
|
"logps/ref_rejected": -91.80261993408203,
|
|
"logps/rejected": -1988.9000244140625,
|
|
"loss": 1.3486,
|
|
"margin_dpo/margin_mean": 585.7813720703125,
|
|
"margin_dpo/margin_std": 1524.811767578125,
|
|
"step": 586
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -844.4599609375,
|
|
"KL/mean": -1170.924560546875,
|
|
"KL/rejected_KL_mean": -1497.38916015625,
|
|
"KL/std": 1240.437255859375,
|
|
"epoch": 0.8619676945668135,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 652.92919921875,
|
|
"fcm_dpo/q_t": 0.48856592178344727,
|
|
"grad_norm": 9.642579078674316,
|
|
"learning_rate": 2.9142720719793122e-08,
|
|
"logits/chosen": -0.6443250775337219,
|
|
"logits/rejected": -0.6658428907394409,
|
|
"logps/chosen": -889.151611328125,
|
|
"logps/ref_chosen": -44.69159698486328,
|
|
"logps/ref_rejected": -82.62385559082031,
|
|
"logps/rejected": -1580.0130615234375,
|
|
"loss": 1.3437,
|
|
"margin_dpo/margin_mean": 652.92919921875,
|
|
"margin_dpo/margin_std": 1477.093994140625,
|
|
"step": 587
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1141.727294921875,
|
|
"KL/mean": -1513.7247314453125,
|
|
"KL/rejected_KL_mean": -1885.722412109375,
|
|
"KL/std": 1343.573486328125,
|
|
"epoch": 0.8634361233480177,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 743.9951171875,
|
|
"fcm_dpo/q_t": 0.48701196908950806,
|
|
"grad_norm": 7.112242221832275,
|
|
"learning_rate": 2.8544309805740018e-08,
|
|
"logits/chosen": -0.6173974275588989,
|
|
"logits/rejected": -0.6490628719329834,
|
|
"logps/chosen": -1192.022216796875,
|
|
"logps/ref_chosen": -50.29494857788086,
|
|
"logps/ref_rejected": -107.36988067626953,
|
|
"logps/rejected": -1993.0921630859375,
|
|
"loss": 1.3384,
|
|
"margin_dpo/margin_mean": 743.9951171875,
|
|
"margin_dpo/margin_std": 1684.069091796875,
|
|
"step": 588
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1334.051513671875,
|
|
"KL/mean": -1767.8013916015625,
|
|
"KL/rejected_KL_mean": -2201.55126953125,
|
|
"KL/std": 1667.269287109375,
|
|
"epoch": 0.8649045521292217,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 867.4995727539062,
|
|
"fcm_dpo/q_t": 0.48490506410598755,
|
|
"grad_norm": 7.0310516357421875,
|
|
"learning_rate": 2.7951734455078786e-08,
|
|
"logits/chosen": -0.6013877391815186,
|
|
"logits/rejected": -0.6371986269950867,
|
|
"logps/chosen": -1393.9814453125,
|
|
"logps/ref_chosen": -59.929908752441406,
|
|
"logps/ref_rejected": -111.65534973144531,
|
|
"logps/rejected": -2313.20654296875,
|
|
"loss": 1.3308,
|
|
"margin_dpo/margin_mean": 867.4996337890625,
|
|
"margin_dpo/margin_std": 1884.604736328125,
|
|
"step": 589
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1027.572998046875,
|
|
"KL/mean": -1449.776611328125,
|
|
"KL/rejected_KL_mean": -1871.980224609375,
|
|
"KL/std": 1585.0914306640625,
|
|
"epoch": 0.8663729809104258,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 844.4072875976562,
|
|
"fcm_dpo/q_t": 0.4853343367576599,
|
|
"grad_norm": 6.130049228668213,
|
|
"learning_rate": 2.736501028272095e-08,
|
|
"logits/chosen": -0.6168742775917053,
|
|
"logits/rejected": -0.6527448892593384,
|
|
"logps/chosen": -1083.3828125,
|
|
"logps/ref_chosen": -55.80979537963867,
|
|
"logps/ref_rejected": -106.06282043457031,
|
|
"logps/rejected": -1978.0430908203125,
|
|
"loss": 1.3328,
|
|
"margin_dpo/margin_mean": 844.4072265625,
|
|
"margin_dpo/margin_std": 1984.2763671875,
|
|
"step": 590
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1019.3551025390625,
|
|
"KL/mean": -1367.507568359375,
|
|
"KL/rejected_KL_mean": -1715.6600341796875,
|
|
"KL/std": 1360.0946044921875,
|
|
"epoch": 0.8678414096916299,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 696.304931640625,
|
|
"fcm_dpo/q_t": 0.4878562390804291,
|
|
"grad_norm": 6.305633068084717,
|
|
"learning_rate": 2.678415274939408e-08,
|
|
"logits/chosen": -0.6568745970726013,
|
|
"logits/rejected": -0.6684634685516357,
|
|
"logps/chosen": -1075.595703125,
|
|
"logps/ref_chosen": -56.24061965942383,
|
|
"logps/ref_rejected": -83.78629302978516,
|
|
"logps/rejected": -1799.4462890625,
|
|
"loss": 1.342,
|
|
"margin_dpo/margin_mean": 696.304931640625,
|
|
"margin_dpo/margin_std": 1778.0966796875,
|
|
"step": 591
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1184.225830078125,
|
|
"KL/mean": -1476.70849609375,
|
|
"KL/rejected_KL_mean": -1769.1912841796875,
|
|
"KL/std": 1288.901123046875,
|
|
"epoch": 0.869309838472834,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 584.965576171875,
|
|
"fcm_dpo/q_t": 0.4897754192352295,
|
|
"grad_norm": 8.049051284790039,
|
|
"learning_rate": 2.6209177161234442e-08,
|
|
"logits/chosen": -0.6074360609054565,
|
|
"logits/rejected": -0.6240702271461487,
|
|
"logps/chosen": -1232.166015625,
|
|
"logps/ref_chosen": -47.94025421142578,
|
|
"logps/ref_rejected": -75.73287963867188,
|
|
"logps/rejected": -1844.9241943359375,
|
|
"loss": 1.3498,
|
|
"margin_dpo/margin_mean": 584.965576171875,
|
|
"margin_dpo/margin_std": 1812.1029052734375,
|
|
"step": 592
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1095.839599609375,
|
|
"KL/mean": -1293.16259765625,
|
|
"KL/rejected_KL_mean": -1490.4853515625,
|
|
"KL/std": 1231.2266845703125,
|
|
"epoch": 0.8707782672540382,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 394.645751953125,
|
|
"fcm_dpo/q_t": 0.493096262216568,
|
|
"grad_norm": 4.078222751617432,
|
|
"learning_rate": 2.564009866938349e-08,
|
|
"logits/chosen": -0.6004587411880493,
|
|
"logits/rejected": -0.6096137166023254,
|
|
"logps/chosen": -1144.5303955078125,
|
|
"logps/ref_chosen": -48.690757751464844,
|
|
"logps/ref_rejected": -60.90800094604492,
|
|
"logps/rejected": -1551.3934326171875,
|
|
"loss": 1.3614,
|
|
"margin_dpo/margin_mean": 394.645751953125,
|
|
"margin_dpo/margin_std": 1439.93701171875,
|
|
"step": 593
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1057.113037109375,
|
|
"KL/mean": -1357.0494384765625,
|
|
"KL/rejected_KL_mean": -1656.9857177734375,
|
|
"KL/std": 1238.75146484375,
|
|
"epoch": 0.8722466960352423,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 599.8726806640625,
|
|
"fcm_dpo/q_t": 0.4895210862159729,
|
|
"grad_norm": 7.381805419921875,
|
|
"learning_rate": 2.5076932269588708e-08,
|
|
"logits/chosen": -0.6215956211090088,
|
|
"logits/rejected": -0.6254661083221436,
|
|
"logps/chosen": -1112.0478515625,
|
|
"logps/ref_chosen": -54.93488693237305,
|
|
"logps/ref_rejected": -86.09967803955078,
|
|
"logps/rejected": -1743.08544921875,
|
|
"loss": 1.3476,
|
|
"margin_dpo/margin_mean": 599.8727416992188,
|
|
"margin_dpo/margin_std": 1491.129150390625,
|
|
"step": 594
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -986.426513671875,
|
|
"KL/mean": -1438.9229736328125,
|
|
"KL/rejected_KL_mean": -1891.41943359375,
|
|
"KL/std": 1537.6827392578125,
|
|
"epoch": 0.8737151248164464,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 904.9927978515625,
|
|
"fcm_dpo/q_t": 0.4843044877052307,
|
|
"grad_norm": 8.513747215270996,
|
|
"learning_rate": 2.451969280180849e-08,
|
|
"logits/chosen": -0.6116786003112793,
|
|
"logits/rejected": -0.6459616422653198,
|
|
"logps/chosen": -1035.846923828125,
|
|
"logps/ref_chosen": -49.4204216003418,
|
|
"logps/ref_rejected": -80.62731170654297,
|
|
"logps/rejected": -1972.046630859375,
|
|
"loss": 1.3286,
|
|
"margin_dpo/margin_mean": 904.992919921875,
|
|
"margin_dpo/margin_std": 1967.465087890625,
|
|
"step": 595
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1317.722900390625,
|
|
"KL/mean": -1460.748779296875,
|
|
"KL/rejected_KL_mean": -1603.77490234375,
|
|
"KL/std": 1665.4937744140625,
|
|
"epoch": 0.8751835535976505,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 286.0518798828125,
|
|
"fcm_dpo/q_t": 0.49503082036972046,
|
|
"grad_norm": 5.618963241577148,
|
|
"learning_rate": 2.396839494982103e-08,
|
|
"logits/chosen": -0.6746952533721924,
|
|
"logits/rejected": -0.6740427017211914,
|
|
"logps/chosen": -1377.5145263671875,
|
|
"logps/ref_chosen": -59.791683197021484,
|
|
"logps/ref_rejected": -80.09111785888672,
|
|
"logps/rejected": -1683.865966796875,
|
|
"loss": 1.3718,
|
|
"margin_dpo/margin_mean": 286.0519104003906,
|
|
"margin_dpo/margin_std": 2089.884521484375,
|
|
"step": 596
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1239.4058837890625,
|
|
"KL/mean": -1711.820556640625,
|
|
"KL/rejected_KL_mean": -2184.235107421875,
|
|
"KL/std": 1657.8519287109375,
|
|
"epoch": 0.8766519823788547,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 944.8291625976562,
|
|
"fcm_dpo/q_t": 0.4835811257362366,
|
|
"grad_norm": 6.500996112823486,
|
|
"learning_rate": 2.3423053240837514e-08,
|
|
"logits/chosen": -0.6523314118385315,
|
|
"logits/rejected": -0.6965519785881042,
|
|
"logps/chosen": -1296.666748046875,
|
|
"logps/ref_chosen": -57.26078796386719,
|
|
"logps/ref_rejected": -100.6937255859375,
|
|
"logps/rejected": -2284.9287109375,
|
|
"loss": 1.3265,
|
|
"margin_dpo/margin_mean": 944.8292236328125,
|
|
"margin_dpo/margin_std": 1901.935791015625,
|
|
"step": 597
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1068.144287109375,
|
|
"KL/mean": -1389.80712890625,
|
|
"KL/rejected_KL_mean": -1711.4698486328125,
|
|
"KL/std": 1323.55322265625,
|
|
"epoch": 0.8781204111600588,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 643.3255615234375,
|
|
"fcm_dpo/q_t": 0.4887608289718628,
|
|
"grad_norm": 4.988048076629639,
|
|
"learning_rate": 2.2883682045119062e-08,
|
|
"logits/chosen": -0.602400541305542,
|
|
"logits/rejected": -0.6199566721916199,
|
|
"logps/chosen": -1120.662841796875,
|
|
"logps/ref_chosen": -52.51850509643555,
|
|
"logps/ref_rejected": -89.44385528564453,
|
|
"logps/rejected": -1800.913818359375,
|
|
"loss": 1.3452,
|
|
"margin_dpo/margin_mean": 643.3255615234375,
|
|
"margin_dpo/margin_std": 1611.5604248046875,
|
|
"step": 598
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1212.7427978515625,
|
|
"KL/mean": -1437.69384765625,
|
|
"KL/rejected_KL_mean": -1662.644775390625,
|
|
"KL/std": 1404.40869140625,
|
|
"epoch": 0.8795888399412628,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 449.90191650390625,
|
|
"fcm_dpo/q_t": 0.4921151101589203,
|
|
"grad_norm": 5.333652973175049,
|
|
"learning_rate": 2.2350295575598367e-08,
|
|
"logits/chosen": -0.6382207274436951,
|
|
"logits/rejected": -0.6507253050804138,
|
|
"logps/chosen": -1262.5455322265625,
|
|
"logps/ref_chosen": -49.802677154541016,
|
|
"logps/ref_rejected": -82.978515625,
|
|
"logps/rejected": -1745.623291015625,
|
|
"loss": 1.3583,
|
|
"margin_dpo/margin_mean": 449.90191650390625,
|
|
"margin_dpo/margin_std": 1570.1396484375,
|
|
"step": 599
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1502.628173828125,
|
|
"KL/mean": -1721.712646484375,
|
|
"KL/rejected_KL_mean": -1940.797119140625,
|
|
"KL/std": 1682.96484375,
|
|
"epoch": 0.8810572687224669,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 438.1690673828125,
|
|
"fcm_dpo/q_t": 0.49244120717048645,
|
|
"grad_norm": 6.994654655456543,
|
|
"learning_rate": 2.1822907887504932e-08,
|
|
"logits/chosen": -0.6980600357055664,
|
|
"logits/rejected": -0.7049777507781982,
|
|
"logps/chosen": -1569.06298828125,
|
|
"logps/ref_chosen": -66.43487548828125,
|
|
"logps/ref_rejected": -85.45649719238281,
|
|
"logps/rejected": -2026.253662109375,
|
|
"loss": 1.3628,
|
|
"margin_dpo/margin_mean": 438.1690368652344,
|
|
"margin_dpo/margin_std": 2316.57080078125,
|
|
"step": 600
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1398.1495361328125,
|
|
"KL/mean": -1735.676513671875,
|
|
"KL/rejected_KL_mean": -2073.203369140625,
|
|
"KL/std": 1529.931640625,
|
|
"epoch": 0.882525697503671,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 675.0538330078125,
|
|
"fcm_dpo/q_t": 0.4882049858570099,
|
|
"grad_norm": 6.039072513580322,
|
|
"learning_rate": 2.1301532877994742e-08,
|
|
"logits/chosen": -0.6777108907699585,
|
|
"logits/rejected": -0.6996618509292603,
|
|
"logps/chosen": -1457.283203125,
|
|
"logps/ref_chosen": -59.13361358642578,
|
|
"logps/ref_rejected": -94.69093322753906,
|
|
"logps/rejected": -2167.894287109375,
|
|
"loss": 1.344,
|
|
"margin_dpo/margin_mean": 675.0538330078125,
|
|
"margin_dpo/margin_std": 1895.3690185546875,
|
|
"step": 601
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -859.4080810546875,
|
|
"KL/mean": -1380.189697265625,
|
|
"KL/rejected_KL_mean": -1900.97119140625,
|
|
"KL/std": 1388.6561279296875,
|
|
"epoch": 0.8839941262848752,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1041.5631103515625,
|
|
"fcm_dpo/q_t": 0.4818532466888428,
|
|
"grad_norm": 14.612542152404785,
|
|
"learning_rate": 2.0786184285784298e-08,
|
|
"logits/chosen": -0.6874485015869141,
|
|
"logits/rejected": -0.7312799692153931,
|
|
"logps/chosen": -908.0015869140625,
|
|
"logps/ref_chosen": -48.59352111816406,
|
|
"logps/ref_rejected": -87.6685562133789,
|
|
"logps/rejected": -1988.6396484375,
|
|
"loss": 1.3185,
|
|
"margin_dpo/margin_mean": 1041.5631103515625,
|
|
"margin_dpo/margin_std": 1663.5296630859375,
|
|
"step": 602
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1425.88330078125,
|
|
"KL/mean": -1779.10498046875,
|
|
"KL/rejected_KL_mean": -2132.32666015625,
|
|
"KL/std": 1935.75244140625,
|
|
"epoch": 0.8854625550660793,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 706.4432983398438,
|
|
"fcm_dpo/q_t": 0.48793095350265503,
|
|
"grad_norm": 14.034811019897461,
|
|
"learning_rate": 2.0276875690788204e-08,
|
|
"logits/chosen": -0.7051316499710083,
|
|
"logits/rejected": -0.7167890071868896,
|
|
"logps/chosen": -1496.2979736328125,
|
|
"logps/ref_chosen": -70.41461944580078,
|
|
"logps/ref_rejected": -100.32559967041016,
|
|
"logps/rejected": -2232.65234375,
|
|
"loss": 1.3446,
|
|
"margin_dpo/margin_mean": 706.4432983398438,
|
|
"margin_dpo/margin_std": 2401.55517578125,
|
|
"step": 603
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1154.302001953125,
|
|
"KL/mean": -1677.122802734375,
|
|
"KL/rejected_KL_mean": -2199.943603515625,
|
|
"KL/std": 1617.4171142578125,
|
|
"epoch": 0.8869309838472834,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1045.6416015625,
|
|
"fcm_dpo/q_t": 0.4818729758262634,
|
|
"grad_norm": 6.103328704833984,
|
|
"learning_rate": 1.977362051376158e-08,
|
|
"logits/chosen": -0.6412931680679321,
|
|
"logits/rejected": -0.6906075477600098,
|
|
"logps/chosen": -1200.760009765625,
|
|
"logps/ref_chosen": -46.45808029174805,
|
|
"logps/ref_rejected": -91.8544921875,
|
|
"logps/rejected": -2291.798095703125,
|
|
"loss": 1.3206,
|
|
"margin_dpo/margin_mean": 1045.6414794921875,
|
|
"margin_dpo/margin_std": 2262.8173828125,
|
|
"step": 604
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1212.1982421875,
|
|
"KL/mean": -1576.411865234375,
|
|
"KL/rejected_KL_mean": -1940.62548828125,
|
|
"KL/std": 1592.7261962890625,
|
|
"epoch": 0.8883994126284875,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 728.4273681640625,
|
|
"fcm_dpo/q_t": 0.48738065361976624,
|
|
"grad_norm": 7.565073013305664,
|
|
"learning_rate": 1.9276432015946446e-08,
|
|
"logits/chosen": -0.6622729301452637,
|
|
"logits/rejected": -0.6903505325317383,
|
|
"logps/chosen": -1278.447509765625,
|
|
"logps/ref_chosen": -66.24933624267578,
|
|
"logps/ref_rejected": -102.30496978759766,
|
|
"logps/rejected": -2042.930419921875,
|
|
"loss": 1.3417,
|
|
"margin_dpo/margin_mean": 728.4273681640625,
|
|
"margin_dpo/margin_std": 2157.499267578125,
|
|
"step": 605
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1284.903564453125,
|
|
"KL/mean": -1668.8912353515625,
|
|
"KL/rejected_KL_mean": -2052.87890625,
|
|
"KL/std": 2009.388916015625,
|
|
"epoch": 0.8898678414096917,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 767.9752807617188,
|
|
"fcm_dpo/q_t": 0.48712000250816345,
|
|
"grad_norm": 10.69570255279541,
|
|
"learning_rate": 1.8785323298722093e-08,
|
|
"logits/chosen": -0.6595591306686401,
|
|
"logits/rejected": -0.695212721824646,
|
|
"logps/chosen": -1339.72265625,
|
|
"logps/ref_chosen": -54.819122314453125,
|
|
"logps/ref_rejected": -98.37146759033203,
|
|
"logps/rejected": -2151.250244140625,
|
|
"loss": 1.3417,
|
|
"margin_dpo/margin_mean": 767.9752807617188,
|
|
"margin_dpo/margin_std": 2621.770263671875,
|
|
"step": 606
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1284.68359375,
|
|
"KL/mean": -1566.37744140625,
|
|
"KL/rejected_KL_mean": -1848.0712890625,
|
|
"KL/std": 1655.87255859375,
|
|
"epoch": 0.8913362701908958,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 563.3876342773438,
|
|
"fcm_dpo/q_t": 0.4901534914970398,
|
|
"grad_norm": 10.138040542602539,
|
|
"learning_rate": 1.8300307303259904e-08,
|
|
"logits/chosen": -0.6720389127731323,
|
|
"logits/rejected": -0.6777141690254211,
|
|
"logps/chosen": -1342.767578125,
|
|
"logps/ref_chosen": -58.08403778076172,
|
|
"logps/ref_rejected": -79.777099609375,
|
|
"logps/rejected": -1927.848388671875,
|
|
"loss": 1.3511,
|
|
"margin_dpo/margin_mean": 563.3876953125,
|
|
"margin_dpo/margin_std": 1770.9501953125,
|
|
"step": 607
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1165.683837890625,
|
|
"KL/mean": -1508.021240234375,
|
|
"KL/rejected_KL_mean": -1850.35888671875,
|
|
"KL/std": 1325.984130859375,
|
|
"epoch": 0.8928046989720999,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 684.6749877929688,
|
|
"fcm_dpo/q_t": 0.48802483081817627,
|
|
"grad_norm": 5.801738262176514,
|
|
"learning_rate": 1.7821396810182437e-08,
|
|
"logits/chosen": -0.6812508702278137,
|
|
"logits/rejected": -0.7010186910629272,
|
|
"logps/chosen": -1223.1346435546875,
|
|
"logps/ref_chosen": -57.450836181640625,
|
|
"logps/ref_rejected": -94.77339172363281,
|
|
"logps/rejected": -1945.1322021484375,
|
|
"loss": 1.3419,
|
|
"margin_dpo/margin_mean": 684.675048828125,
|
|
"margin_dpo/margin_std": 1577.7777099609375,
|
|
"step": 608
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1125.0963134765625,
|
|
"KL/mean": -1520.38818359375,
|
|
"KL/rejected_KL_mean": -1915.679931640625,
|
|
"KL/std": 1332.13232421875,
|
|
"epoch": 0.8942731277533039,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 790.583740234375,
|
|
"fcm_dpo/q_t": 0.48619094491004944,
|
|
"grad_norm": 10.305415153503418,
|
|
"learning_rate": 1.7348604439226617e-08,
|
|
"logits/chosen": -0.7046061754226685,
|
|
"logits/rejected": -0.7308906316757202,
|
|
"logps/chosen": -1183.901611328125,
|
|
"logps/ref_chosen": -58.805355072021484,
|
|
"logps/ref_rejected": -88.81600952148438,
|
|
"logps/rejected": -2004.4959716796875,
|
|
"loss": 1.3355,
|
|
"margin_dpo/margin_mean": 790.583740234375,
|
|
"margin_dpo/margin_std": 1792.01611328125,
|
|
"step": 609
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1282.007568359375,
|
|
"KL/mean": -1432.61962890625,
|
|
"KL/rejected_KL_mean": -1583.231689453125,
|
|
"KL/std": 1448.6640625,
|
|
"epoch": 0.895741556534508,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 301.22406005859375,
|
|
"fcm_dpo/q_t": 0.49473780393600464,
|
|
"grad_norm": 11.932384490966797,
|
|
"learning_rate": 1.6881942648911074e-08,
|
|
"logits/chosen": -0.6814596652984619,
|
|
"logits/rejected": -0.6760205030441284,
|
|
"logps/chosen": -1347.7025146484375,
|
|
"logps/ref_chosen": -65.69503784179688,
|
|
"logps/ref_rejected": -83.40538787841797,
|
|
"logps/rejected": -1666.636962890625,
|
|
"loss": 1.3701,
|
|
"margin_dpo/margin_mean": 301.22406005859375,
|
|
"margin_dpo/margin_std": 1961.03515625,
|
|
"step": 610
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1148.4498291015625,
|
|
"KL/mean": -1598.172607421875,
|
|
"KL/rejected_KL_mean": -2047.8955078125,
|
|
"KL/std": 1594.612060546875,
|
|
"epoch": 0.8972099853157122,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 899.4454956054688,
|
|
"fcm_dpo/q_t": 0.48433127999305725,
|
|
"grad_norm": 6.665002822875977,
|
|
"learning_rate": 1.6421423736208e-08,
|
|
"logits/chosen": -0.6560553312301636,
|
|
"logits/rejected": -0.6957667469978333,
|
|
"logps/chosen": -1201.04931640625,
|
|
"logps/ref_chosen": -52.59946823120117,
|
|
"logps/ref_rejected": -86.33099365234375,
|
|
"logps/rejected": -2134.226318359375,
|
|
"loss": 1.3283,
|
|
"margin_dpo/margin_mean": 899.4454345703125,
|
|
"margin_dpo/margin_std": 1819.7413330078125,
|
|
"step": 611
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1225.57861328125,
|
|
"KL/mean": -1524.23291015625,
|
|
"KL/rejected_KL_mean": -1822.88720703125,
|
|
"KL/std": 1422.014404296875,
|
|
"epoch": 0.8986784140969163,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 597.30859375,
|
|
"fcm_dpo/q_t": 0.4895847737789154,
|
|
"grad_norm": 8.166829109191895,
|
|
"learning_rate": 1.5967059836219042e-08,
|
|
"logits/chosen": -0.6424583792686462,
|
|
"logits/rejected": -0.671108067035675,
|
|
"logps/chosen": -1284.90234375,
|
|
"logps/ref_chosen": -59.32372283935547,
|
|
"logps/ref_rejected": -88.31239318847656,
|
|
"logps/rejected": -1911.19970703125,
|
|
"loss": 1.3491,
|
|
"margin_dpo/margin_mean": 597.30859375,
|
|
"margin_dpo/margin_std": 1860.5745849609375,
|
|
"step": 612
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1104.078857421875,
|
|
"KL/mean": -1501.256591796875,
|
|
"KL/rejected_KL_mean": -1898.4345703125,
|
|
"KL/std": 1444.134521484375,
|
|
"epoch": 0.9001468428781204,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 794.355712890625,
|
|
"fcm_dpo/q_t": 0.4861546754837036,
|
|
"grad_norm": 6.351512432098389,
|
|
"learning_rate": 1.551886292185553e-08,
|
|
"logits/chosen": -0.6626429557800293,
|
|
"logits/rejected": -0.7097995281219482,
|
|
"logps/chosen": -1163.808837890625,
|
|
"logps/ref_chosen": -59.72996520996094,
|
|
"logps/ref_rejected": -105.10752868652344,
|
|
"logps/rejected": -2003.5419921875,
|
|
"loss": 1.3355,
|
|
"margin_dpo/margin_mean": 794.3555908203125,
|
|
"margin_dpo/margin_std": 1837.294189453125,
|
|
"step": 613
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1072.1090087890625,
|
|
"KL/mean": -1519.0906982421875,
|
|
"KL/rejected_KL_mean": -1966.072509765625,
|
|
"KL/std": 1425.783447265625,
|
|
"epoch": 0.9016152716593245,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 893.96337890625,
|
|
"fcm_dpo/q_t": 0.4844154119491577,
|
|
"grad_norm": 8.47018051147461,
|
|
"learning_rate": 1.507684480352292e-08,
|
|
"logits/chosen": -0.6103712320327759,
|
|
"logits/rejected": -0.6593648195266724,
|
|
"logps/chosen": -1125.048095703125,
|
|
"logps/ref_chosen": -52.93898010253906,
|
|
"logps/ref_rejected": -104.67938232421875,
|
|
"logps/rejected": -2070.751953125,
|
|
"loss": 1.3294,
|
|
"margin_dpo/margin_mean": 893.96337890625,
|
|
"margin_dpo/margin_std": 1988.4140625,
|
|
"step": 614
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1103.2506103515625,
|
|
"KL/mean": -1468.0848388671875,
|
|
"KL/rejected_KL_mean": -1832.919189453125,
|
|
"KL/std": 1459.701171875,
|
|
"epoch": 0.9030837004405287,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 729.66845703125,
|
|
"fcm_dpo/q_t": 0.4873241186141968,
|
|
"grad_norm": 5.5929274559021,
|
|
"learning_rate": 1.4641017128809801e-08,
|
|
"logits/chosen": -0.6631003022193909,
|
|
"logits/rejected": -0.687023401260376,
|
|
"logps/chosen": -1169.06787109375,
|
|
"logps/ref_chosen": -65.81727600097656,
|
|
"logps/ref_rejected": -95.17749786376953,
|
|
"logps/rejected": -1928.0966796875,
|
|
"loss": 1.3401,
|
|
"margin_dpo/margin_mean": 729.66845703125,
|
|
"margin_dpo/margin_std": 1842.6663818359375,
|
|
"step": 615
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1339.16796875,
|
|
"KL/mean": -1633.54150390625,
|
|
"KL/rejected_KL_mean": -1927.914794921875,
|
|
"KL/std": 1515.873779296875,
|
|
"epoch": 0.9045521292217328,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 588.7468872070312,
|
|
"fcm_dpo/q_t": 0.489709734916687,
|
|
"grad_norm": 6.771057605743408,
|
|
"learning_rate": 1.4211391382180637e-08,
|
|
"logits/chosen": -0.6890289783477783,
|
|
"logits/rejected": -0.6939765214920044,
|
|
"logps/chosen": -1404.3009033203125,
|
|
"logps/ref_chosen": -65.13285827636719,
|
|
"logps/ref_rejected": -74.70050048828125,
|
|
"logps/rejected": -2002.615234375,
|
|
"loss": 1.3497,
|
|
"margin_dpo/margin_mean": 588.746826171875,
|
|
"margin_dpo/margin_std": 1862.400390625,
|
|
"step": 616
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1498.676513671875,
|
|
"KL/mean": -1554.958251953125,
|
|
"KL/rejected_KL_mean": -1611.240234375,
|
|
"KL/std": 1531.433837890625,
|
|
"epoch": 0.9060205580029369,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 112.56379699707031,
|
|
"fcm_dpo/q_t": 0.4979705214500427,
|
|
"grad_norm": 20.132686614990234,
|
|
"learning_rate": 1.378797888467345e-08,
|
|
"logits/chosen": -0.6434112787246704,
|
|
"logits/rejected": -0.6232435703277588,
|
|
"logps/chosen": -1561.68212890625,
|
|
"logps/ref_chosen": -63.005550384521484,
|
|
"logps/ref_rejected": -64.234130859375,
|
|
"logps/rejected": -1675.474365234375,
|
|
"loss": 1.3831,
|
|
"margin_dpo/margin_mean": 112.56379699707031,
|
|
"margin_dpo/margin_std": 1948.6531982421875,
|
|
"step": 617
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1434.4833984375,
|
|
"KL/mean": -1825.2366943359375,
|
|
"KL/rejected_KL_mean": -2215.989990234375,
|
|
"KL/std": 1607.75,
|
|
"epoch": 0.9074889867841409,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 781.506591796875,
|
|
"fcm_dpo/q_t": 0.4863564670085907,
|
|
"grad_norm": 9.600302696228027,
|
|
"learning_rate": 1.3370790793601371e-08,
|
|
"logits/chosen": -0.6751698851585388,
|
|
"logits/rejected": -0.7124015092849731,
|
|
"logps/chosen": -1501.584716796875,
|
|
"logps/ref_chosen": -67.10134887695312,
|
|
"logps/ref_rejected": -92.15340423583984,
|
|
"logps/rejected": -2308.1435546875,
|
|
"loss": 1.3373,
|
|
"margin_dpo/margin_mean": 781.506591796875,
|
|
"margin_dpo/margin_std": 1978.0693359375,
|
|
"step": 618
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1300.69775390625,
|
|
"KL/mean": -1552.975830078125,
|
|
"KL/rejected_KL_mean": -1805.2540283203125,
|
|
"KL/std": 1608.731689453125,
|
|
"epoch": 0.908957415565345,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 504.5563659667969,
|
|
"fcm_dpo/q_t": 0.4911313056945801,
|
|
"grad_norm": 7.555780410766602,
|
|
"learning_rate": 1.2959838102258535e-08,
|
|
"logits/chosen": -0.6084803342819214,
|
|
"logits/rejected": -0.6177768707275391,
|
|
"logps/chosen": -1356.6759033203125,
|
|
"logps/ref_chosen": -55.978233337402344,
|
|
"logps/ref_rejected": -93.1854019165039,
|
|
"logps/rejected": -1898.439453125,
|
|
"loss": 1.3564,
|
|
"margin_dpo/margin_mean": 504.55633544921875,
|
|
"margin_dpo/margin_std": 2043.724609375,
|
|
"step": 619
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1157.4849853515625,
|
|
"KL/mean": -1404.56494140625,
|
|
"KL/rejected_KL_mean": -1651.644775390625,
|
|
"KL/std": 1310.7586669921875,
|
|
"epoch": 0.9104258443465492,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 494.15985107421875,
|
|
"fcm_dpo/q_t": 0.49135249853134155,
|
|
"grad_norm": 6.826314449310303,
|
|
"learning_rate": 1.2555131639630567e-08,
|
|
"logits/chosen": -0.6301568746566772,
|
|
"logits/rejected": -0.6424782276153564,
|
|
"logps/chosen": -1217.282470703125,
|
|
"logps/ref_chosen": -59.79750061035156,
|
|
"logps/ref_rejected": -78.41075134277344,
|
|
"logps/rejected": -1730.0556640625,
|
|
"loss": 1.3558,
|
|
"margin_dpo/margin_mean": 494.15985107421875,
|
|
"margin_dpo/margin_std": 1771.6112060546875,
|
|
"step": 620
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1000.7633056640625,
|
|
"KL/mean": -1483.132568359375,
|
|
"KL/rejected_KL_mean": -1965.501708984375,
|
|
"KL/std": 1498.1484375,
|
|
"epoch": 0.9118942731277533,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 964.7385864257812,
|
|
"fcm_dpo/q_t": 0.48318374156951904,
|
|
"grad_norm": 6.330345630645752,
|
|
"learning_rate": 1.2156682070109086e-08,
|
|
"logits/chosen": -0.6472010612487793,
|
|
"logits/rejected": -0.6926671266555786,
|
|
"logps/chosen": -1054.697021484375,
|
|
"logps/ref_chosen": -53.93375778198242,
|
|
"logps/ref_rejected": -88.36951446533203,
|
|
"logps/rejected": -2053.871337890625,
|
|
"loss": 1.3245,
|
|
"margin_dpo/margin_mean": 964.7385864257812,
|
|
"margin_dpo/margin_std": 1884.843017578125,
|
|
"step": 621
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1014.5686645507812,
|
|
"KL/mean": -1319.3843994140625,
|
|
"KL/rejected_KL_mean": -1624.2001953125,
|
|
"KL/std": 1281.564697265625,
|
|
"epoch": 0.9133627019089574,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 609.6315307617188,
|
|
"fcm_dpo/q_t": 0.48939478397369385,
|
|
"grad_norm": 6.59990930557251,
|
|
"learning_rate": 1.1764499893210878e-08,
|
|
"logits/chosen": -0.6089707612991333,
|
|
"logits/rejected": -0.6166027784347534,
|
|
"logps/chosen": -1074.8544921875,
|
|
"logps/ref_chosen": -60.28582000732422,
|
|
"logps/ref_rejected": -85.51873779296875,
|
|
"logps/rejected": -1709.718994140625,
|
|
"loss": 1.3473,
|
|
"margin_dpo/margin_mean": 609.6315307617188,
|
|
"margin_dpo/margin_std": 1645.434814453125,
|
|
"step": 622
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1287.5997314453125,
|
|
"KL/mean": -1454.433349609375,
|
|
"KL/rejected_KL_mean": -1621.266845703125,
|
|
"KL/std": 1537.554443359375,
|
|
"epoch": 0.9148311306901615,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 333.66717529296875,
|
|
"fcm_dpo/q_t": 0.4941401183605194,
|
|
"grad_norm": 15.589156150817871,
|
|
"learning_rate": 1.1378595443300998e-08,
|
|
"logits/chosen": -0.6384043097496033,
|
|
"logits/rejected": -0.6354462504386902,
|
|
"logps/chosen": -1351.756591796875,
|
|
"logps/ref_chosen": -64.1569595336914,
|
|
"logps/ref_rejected": -85.08304595947266,
|
|
"logps/rejected": -1706.349853515625,
|
|
"loss": 1.3675,
|
|
"margin_dpo/margin_mean": 333.6671142578125,
|
|
"margin_dpo/margin_std": 1856.07421875,
|
|
"step": 623
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1173.687744140625,
|
|
"KL/mean": -1618.4342041015625,
|
|
"KL/rejected_KL_mean": -2063.1806640625,
|
|
"KL/std": 1536.404296875,
|
|
"epoch": 0.9162995594713657,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 889.4931640625,
|
|
"fcm_dpo/q_t": 0.4844810962677002,
|
|
"grad_norm": 5.250894546508789,
|
|
"learning_rate": 1.0998978889320582e-08,
|
|
"logits/chosen": -0.6779031753540039,
|
|
"logits/rejected": -0.6975363492965698,
|
|
"logps/chosen": -1245.6063232421875,
|
|
"logps/ref_chosen": -71.91862487792969,
|
|
"logps/ref_rejected": -97.13203430175781,
|
|
"logps/rejected": -2160.312744140625,
|
|
"loss": 1.3297,
|
|
"margin_dpo/margin_mean": 889.4930419921875,
|
|
"margin_dpo/margin_std": 1989.96533203125,
|
|
"step": 624
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1133.847412109375,
|
|
"KL/mean": -1514.25537109375,
|
|
"KL/rejected_KL_mean": -1894.663330078125,
|
|
"KL/std": 1352.28173828125,
|
|
"epoch": 0.9177679882525698,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 760.81591796875,
|
|
"fcm_dpo/q_t": 0.4867083430290222,
|
|
"grad_norm": 6.905755996704102,
|
|
"learning_rate": 1.0625660234518913e-08,
|
|
"logits/chosen": -0.5888922214508057,
|
|
"logits/rejected": -0.605700671672821,
|
|
"logps/chosen": -1192.189453125,
|
|
"logps/ref_chosen": -58.342071533203125,
|
|
"logps/ref_rejected": -86.09038543701172,
|
|
"logps/rejected": -1980.753662109375,
|
|
"loss": 1.3361,
|
|
"margin_dpo/margin_mean": 760.81591796875,
|
|
"margin_dpo/margin_std": 1390.97021484375,
|
|
"step": 625
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1193.1416015625,
|
|
"KL/mean": -1446.146728515625,
|
|
"KL/rejected_KL_mean": -1699.152099609375,
|
|
"KL/std": 1531.291259765625,
|
|
"epoch": 0.9192364170337739,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 506.01043701171875,
|
|
"fcm_dpo/q_t": 0.491168349981308,
|
|
"grad_norm": 6.740545749664307,
|
|
"learning_rate": 1.0258649316189721e-08,
|
|
"logits/chosen": -0.6106196045875549,
|
|
"logits/rejected": -0.6149002909660339,
|
|
"logps/chosen": -1268.254150390625,
|
|
"logps/ref_chosen": -75.11260986328125,
|
|
"logps/ref_rejected": -99.188720703125,
|
|
"logps/rejected": -1798.3408203125,
|
|
"loss": 1.3553,
|
|
"margin_dpo/margin_mean": 506.01043701171875,
|
|
"margin_dpo/margin_std": 1811.4095458984375,
|
|
"step": 626
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -773.64111328125,
|
|
"KL/mean": -1223.371337890625,
|
|
"KL/rejected_KL_mean": -1673.101318359375,
|
|
"KL/std": 1513.056884765625,
|
|
"epoch": 0.920704845814978,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 899.4602661132812,
|
|
"fcm_dpo/q_t": 0.4843396246433258,
|
|
"grad_norm": 10.459403991699219,
|
|
"learning_rate": 9.897955805412e-09,
|
|
"logits/chosen": -0.6010231971740723,
|
|
"logits/rejected": -0.6437910795211792,
|
|
"logps/chosen": -821.38427734375,
|
|
"logps/ref_chosen": -47.74314880371094,
|
|
"logps/ref_rejected": -106.75448608398438,
|
|
"logps/rejected": -1779.85595703125,
|
|
"loss": 1.3293,
|
|
"margin_dpo/margin_mean": 899.460205078125,
|
|
"margin_dpo/margin_std": 2023.463623046875,
|
|
"step": 627
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1010.72412109375,
|
|
"KL/mean": -1478.8046875,
|
|
"KL/rejected_KL_mean": -1946.8851318359375,
|
|
"KL/std": 1570.787109375,
|
|
"epoch": 0.922173274596182,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 936.1610107421875,
|
|
"fcm_dpo/q_t": 0.48373740911483765,
|
|
"grad_norm": 8.426224708557129,
|
|
"learning_rate": 9.543589206795238e-09,
|
|
"logits/chosen": -0.6274340152740479,
|
|
"logits/rejected": -0.6652277708053589,
|
|
"logps/chosen": -1070.906982421875,
|
|
"logps/ref_chosen": -60.182945251464844,
|
|
"logps/ref_rejected": -101.55467224121094,
|
|
"logps/rejected": -2048.439697265625,
|
|
"loss": 1.3262,
|
|
"margin_dpo/margin_mean": 936.1610717773438,
|
|
"margin_dpo/margin_std": 1925.196044921875,
|
|
"step": 628
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1270.9169921875,
|
|
"KL/mean": -1610.190673828125,
|
|
"KL/rejected_KL_mean": -1949.46435546875,
|
|
"KL/std": 1889.252197265625,
|
|
"epoch": 0.9236417033773862,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 678.5474243164062,
|
|
"fcm_dpo/q_t": 0.48802652955055237,
|
|
"grad_norm": 6.176880359649658,
|
|
"learning_rate": 9.19555885822887e-09,
|
|
"logits/chosen": -0.6713994741439819,
|
|
"logits/rejected": -0.6894298791885376,
|
|
"logps/chosen": -1335.13037109375,
|
|
"logps/ref_chosen": -64.21354675292969,
|
|
"logps/ref_rejected": -91.65367126464844,
|
|
"logps/rejected": -2041.1180419921875,
|
|
"loss": 1.3461,
|
|
"margin_dpo/margin_mean": 678.5474853515625,
|
|
"margin_dpo/margin_std": 2344.418701171875,
|
|
"step": 629
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1025.02001953125,
|
|
"KL/mean": -1086.37158203125,
|
|
"KL/rejected_KL_mean": -1147.72314453125,
|
|
"KL/std": 1131.6832275390625,
|
|
"epoch": 0.9251101321585903,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 122.7031021118164,
|
|
"fcm_dpo/q_t": 0.4978800415992737,
|
|
"grad_norm": 10.462772369384766,
|
|
"learning_rate": 8.85387393063622e-09,
|
|
"logits/chosen": -0.6387168169021606,
|
|
"logits/rejected": -0.6161863803863525,
|
|
"logps/chosen": -1084.31103515625,
|
|
"logps/ref_chosen": -59.29100036621094,
|
|
"logps/ref_rejected": -83.59829711914062,
|
|
"logps/rejected": -1231.3214111328125,
|
|
"loss": 1.3809,
|
|
"margin_dpo/margin_mean": 122.70311737060547,
|
|
"margin_dpo/margin_std": 1551.34130859375,
|
|
"step": 630
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1031.8355712890625,
|
|
"KL/mean": -1342.90185546875,
|
|
"KL/rejected_KL_mean": -1653.9681396484375,
|
|
"KL/std": 1206.7095947265625,
|
|
"epoch": 0.9265785609397944,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 622.132568359375,
|
|
"fcm_dpo/q_t": 0.48913437128067017,
|
|
"grad_norm": 6.256035327911377,
|
|
"learning_rate": 8.518543427732949e-09,
|
|
"logits/chosen": -0.643370509147644,
|
|
"logits/rejected": -0.6664153337478638,
|
|
"logps/chosen": -1091.2891845703125,
|
|
"logps/ref_chosen": -59.45360565185547,
|
|
"logps/ref_rejected": -80.95156860351562,
|
|
"logps/rejected": -1734.919677734375,
|
|
"loss": 1.3464,
|
|
"margin_dpo/margin_mean": 622.1326904296875,
|
|
"margin_dpo/margin_std": 1622.26513671875,
|
|
"step": 631
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1001.5509643554688,
|
|
"KL/mean": -1220.533935546875,
|
|
"KL/rejected_KL_mean": -1439.5169677734375,
|
|
"KL/std": 1116.1923828125,
|
|
"epoch": 0.9280469897209985,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 437.966064453125,
|
|
"fcm_dpo/q_t": 0.4923382103443146,
|
|
"grad_norm": 4.83076810836792,
|
|
"learning_rate": 8.189576185789637e-09,
|
|
"logits/chosen": -0.6454243659973145,
|
|
"logits/rejected": -0.6561754941940308,
|
|
"logps/chosen": -1062.902587890625,
|
|
"logps/ref_chosen": -61.35155487060547,
|
|
"logps/ref_rejected": -86.16017150878906,
|
|
"logps/rejected": -1525.6771240234375,
|
|
"loss": 1.359,
|
|
"margin_dpo/margin_mean": 437.966064453125,
|
|
"margin_dpo/margin_std": 1603.9329833984375,
|
|
"step": 632
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1161.1319580078125,
|
|
"KL/mean": -1383.982666015625,
|
|
"KL/rejected_KL_mean": -1606.83349609375,
|
|
"KL/std": 1076.3099365234375,
|
|
"epoch": 0.9295154185022027,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 445.701416015625,
|
|
"fcm_dpo/q_t": 0.4922114312648773,
|
|
"grad_norm": 5.985357284545898,
|
|
"learning_rate": 7.866980873399015e-09,
|
|
"logits/chosen": -0.6461182832717896,
|
|
"logits/rejected": -0.6646542549133301,
|
|
"logps/chosen": -1218.41015625,
|
|
"logps/ref_chosen": -57.27816390991211,
|
|
"logps/ref_rejected": -91.58395385742188,
|
|
"logps/rejected": -1698.4173583984375,
|
|
"loss": 1.3575,
|
|
"margin_dpo/margin_mean": 445.701416015625,
|
|
"margin_dpo/margin_std": 1343.09716796875,
|
|
"step": 633
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1291.4398193359375,
|
|
"KL/mean": -1477.098876953125,
|
|
"KL/rejected_KL_mean": -1662.758056640625,
|
|
"KL/std": 1391.55859375,
|
|
"epoch": 0.9309838472834068,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 371.3182373046875,
|
|
"fcm_dpo/q_t": 0.493512898683548,
|
|
"grad_norm": 6.06099796295166,
|
|
"learning_rate": 7.550765991247654e-09,
|
|
"logits/chosen": -0.6100882291793823,
|
|
"logits/rejected": -0.6161783337593079,
|
|
"logps/chosen": -1358.058837890625,
|
|
"logps/ref_chosen": -66.61896514892578,
|
|
"logps/ref_rejected": -107.12564849853516,
|
|
"logps/rejected": -1769.8836669921875,
|
|
"loss": 1.3639,
|
|
"margin_dpo/margin_mean": 371.3182373046875,
|
|
"margin_dpo/margin_std": 1650.0980224609375,
|
|
"step": 634
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1296.58935546875,
|
|
"KL/mean": -1485.6541748046875,
|
|
"KL/rejected_KL_mean": -1674.71875,
|
|
"KL/std": 1795.396728515625,
|
|
"epoch": 0.9324522760646109,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 378.1294860839844,
|
|
"fcm_dpo/q_t": 0.4934682250022888,
|
|
"grad_norm": 10.481958389282227,
|
|
"learning_rate": 7.240939871891699e-09,
|
|
"logits/chosen": -0.6858574151992798,
|
|
"logits/rejected": -0.6715748310089111,
|
|
"logps/chosen": -1370.544921875,
|
|
"logps/ref_chosen": -73.95551300048828,
|
|
"logps/ref_rejected": -82.50045776367188,
|
|
"logps/rejected": -1757.21923828125,
|
|
"loss": 1.3643,
|
|
"margin_dpo/margin_mean": 378.1295166015625,
|
|
"margin_dpo/margin_std": 1888.038818359375,
|
|
"step": 635
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1168.95947265625,
|
|
"KL/mean": -1345.31640625,
|
|
"KL/rejected_KL_mean": -1521.6732177734375,
|
|
"KL/std": 1588.704345703125,
|
|
"epoch": 0.933920704845815,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 352.7136535644531,
|
|
"fcm_dpo/q_t": 0.49370384216308594,
|
|
"grad_norm": 7.528316020965576,
|
|
"learning_rate": 6.937510679537628e-09,
|
|
"logits/chosen": -0.6131812334060669,
|
|
"logits/rejected": -0.6081274747848511,
|
|
"logps/chosen": -1228.58837890625,
|
|
"logps/ref_chosen": -59.628910064697266,
|
|
"logps/ref_rejected": -81.97883605957031,
|
|
"logps/rejected": -1603.652099609375,
|
|
"loss": 1.3669,
|
|
"margin_dpo/margin_mean": 352.713623046875,
|
|
"margin_dpo/margin_std": 2045.9951171875,
|
|
"step": 636
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -982.5504150390625,
|
|
"KL/mean": -1417.7578125,
|
|
"KL/rejected_KL_mean": -1852.9652099609375,
|
|
"KL/std": 1381.57568359375,
|
|
"epoch": 0.9353891336270191,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 870.4149169921875,
|
|
"fcm_dpo/q_t": 0.48480871319770813,
|
|
"grad_norm": 6.873878479003906,
|
|
"learning_rate": 6.640486409826785e-09,
|
|
"logits/chosen": -0.642035961151123,
|
|
"logits/rejected": -0.6937611103057861,
|
|
"logps/chosen": -1032.203125,
|
|
"logps/ref_chosen": -49.652687072753906,
|
|
"logps/ref_rejected": -98.40513610839844,
|
|
"logps/rejected": -1951.370361328125,
|
|
"loss": 1.3292,
|
|
"margin_dpo/margin_mean": 870.4149169921875,
|
|
"margin_dpo/margin_std": 1557.965087890625,
|
|
"step": 637
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -980.171142578125,
|
|
"KL/mean": -1293.06494140625,
|
|
"KL/rejected_KL_mean": -1605.9588623046875,
|
|
"KL/std": 1329.313720703125,
|
|
"epoch": 0.9368575624082232,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 625.7876586914062,
|
|
"fcm_dpo/q_t": 0.4890254735946655,
|
|
"grad_norm": 4.65677547454834,
|
|
"learning_rate": 6.349874889624962e-09,
|
|
"logits/chosen": -0.6139806509017944,
|
|
"logits/rejected": -0.6274735331535339,
|
|
"logps/chosen": -1038.327880859375,
|
|
"logps/ref_chosen": -58.156639099121094,
|
|
"logps/ref_rejected": -79.3014907836914,
|
|
"logps/rejected": -1685.26025390625,
|
|
"loss": 1.3471,
|
|
"margin_dpo/margin_mean": 625.78759765625,
|
|
"margin_dpo/margin_std": 1827.26708984375,
|
|
"step": 638
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1459.7200927734375,
|
|
"KL/mean": -1391.7880859375,
|
|
"KL/rejected_KL_mean": -1323.856201171875,
|
|
"KL/std": 1219.503662109375,
|
|
"epoch": 0.9383259911894273,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -135.86386108398438,
|
|
"fcm_dpo/q_t": 0.5023590326309204,
|
|
"grad_norm": 26.913433074951172,
|
|
"learning_rate": 6.065683776815933e-09,
|
|
"logits/chosen": -0.6304311752319336,
|
|
"logits/rejected": -0.5896961092948914,
|
|
"logps/chosen": -1532.0433349609375,
|
|
"logps/ref_chosen": -72.32319641113281,
|
|
"logps/ref_rejected": -74.2749252319336,
|
|
"logps/rejected": -1398.131103515625,
|
|
"loss": 1.3987,
|
|
"margin_dpo/margin_mean": -135.86386108398438,
|
|
"margin_dpo/margin_std": 1491.50634765625,
|
|
"step": 639
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -954.1146850585938,
|
|
"KL/mean": -1476.706298828125,
|
|
"KL/rejected_KL_mean": -1999.2978515625,
|
|
"KL/std": 1782.7164306640625,
|
|
"epoch": 0.9397944199706314,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1045.18310546875,
|
|
"fcm_dpo/q_t": 0.4820125997066498,
|
|
"grad_norm": 6.035217761993408,
|
|
"learning_rate": 5.7879205600998296e-09,
|
|
"logits/chosen": -0.591851532459259,
|
|
"logits/rejected": -0.6358239650726318,
|
|
"logps/chosen": -1010.2490234375,
|
|
"logps/ref_chosen": -56.13436508178711,
|
|
"logps/ref_rejected": -108.60014343261719,
|
|
"logps/rejected": -2107.89794921875,
|
|
"loss": 1.3206,
|
|
"margin_dpo/margin_mean": 1045.18310546875,
|
|
"margin_dpo/margin_std": 2260.031005859375,
|
|
"step": 640
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1025.90771484375,
|
|
"KL/mean": -1481.028076171875,
|
|
"KL/rejected_KL_mean": -1936.1484375,
|
|
"KL/std": 1427.94873046875,
|
|
"epoch": 0.9412628487518355,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 910.2408447265625,
|
|
"fcm_dpo/q_t": 0.4841909408569336,
|
|
"grad_norm": 10.455007553100586,
|
|
"learning_rate": 5.516592558795746e-09,
|
|
"logits/chosen": -0.5904731750488281,
|
|
"logits/rejected": -0.6200425624847412,
|
|
"logps/chosen": -1090.904541015625,
|
|
"logps/ref_chosen": -64.99689483642578,
|
|
"logps/ref_rejected": -86.99232482910156,
|
|
"logps/rejected": -2023.140869140625,
|
|
"loss": 1.3278,
|
|
"margin_dpo/margin_mean": 910.2408447265625,
|
|
"margin_dpo/margin_std": 1894.365966796875,
|
|
"step": 641
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1221.719970703125,
|
|
"KL/mean": -1628.12744140625,
|
|
"KL/rejected_KL_mean": -2034.5347900390625,
|
|
"KL/std": 1844.57666015625,
|
|
"epoch": 0.9427312775330396,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 812.81494140625,
|
|
"fcm_dpo/q_t": 0.4858717918395996,
|
|
"grad_norm": 5.243577003479004,
|
|
"learning_rate": 5.251706922648868e-09,
|
|
"logits/chosen": -0.5993083715438843,
|
|
"logits/rejected": -0.6353590488433838,
|
|
"logps/chosen": -1287.4091796875,
|
|
"logps/ref_chosen": -65.68924713134766,
|
|
"logps/ref_rejected": -110.24205017089844,
|
|
"logps/rejected": -2144.77685546875,
|
|
"loss": 1.3379,
|
|
"margin_dpo/margin_mean": 812.8150024414062,
|
|
"margin_dpo/margin_std": 2420.636474609375,
|
|
"step": 642
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1205.4664306640625,
|
|
"KL/mean": -1394.8968505859375,
|
|
"KL/rejected_KL_mean": -1584.3272705078125,
|
|
"KL/std": 1325.056640625,
|
|
"epoch": 0.9441997063142438,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 378.8608703613281,
|
|
"fcm_dpo/q_t": 0.4933786988258362,
|
|
"grad_norm": 5.956773281097412,
|
|
"learning_rate": 4.993270631642038e-09,
|
|
"logits/chosen": -0.6313992738723755,
|
|
"logits/rejected": -0.6435078382492065,
|
|
"logps/chosen": -1257.41650390625,
|
|
"logps/ref_chosen": -51.94999694824219,
|
|
"logps/ref_rejected": -87.46833801269531,
|
|
"logps/rejected": -1671.795654296875,
|
|
"loss": 1.3637,
|
|
"margin_dpo/margin_mean": 378.86090087890625,
|
|
"margin_dpo/margin_std": 1685.135986328125,
|
|
"step": 643
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1009.8892822265625,
|
|
"KL/mean": -1339.213623046875,
|
|
"KL/rejected_KL_mean": -1668.5380859375,
|
|
"KL/std": 1681.36669921875,
|
|
"epoch": 0.9456681350954479,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 658.6486206054688,
|
|
"fcm_dpo/q_t": 0.48869824409484863,
|
|
"grad_norm": 5.554941654205322,
|
|
"learning_rate": 4.741290495811873e-09,
|
|
"logits/chosen": -0.6091523170471191,
|
|
"logits/rejected": -0.6272720098495483,
|
|
"logps/chosen": -1068.906982421875,
|
|
"logps/ref_chosen": -59.017662048339844,
|
|
"logps/ref_rejected": -87.13668823242188,
|
|
"logps/rejected": -1755.6746826171875,
|
|
"loss": 1.3465,
|
|
"margin_dpo/margin_mean": 658.6486206054688,
|
|
"margin_dpo/margin_std": 1983.645751953125,
|
|
"step": 644
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1136.2744140625,
|
|
"KL/mean": -1262.766845703125,
|
|
"KL/rejected_KL_mean": -1389.25927734375,
|
|
"KL/std": 1186.4443359375,
|
|
"epoch": 0.947136563876652,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 252.98484802246094,
|
|
"fcm_dpo/q_t": 0.49558210372924805,
|
|
"grad_norm": 8.011646270751953,
|
|
"learning_rate": 4.495773155069299e-09,
|
|
"logits/chosen": -0.6204506158828735,
|
|
"logits/rejected": -0.6305866241455078,
|
|
"logps/chosen": -1192.150390625,
|
|
"logps/ref_chosen": -55.87602233886719,
|
|
"logps/ref_rejected": -97.78080749511719,
|
|
"logps/rejected": -1487.0400390625,
|
|
"loss": 1.3723,
|
|
"margin_dpo/margin_mean": 252.98483276367188,
|
|
"margin_dpo/margin_std": 1717.4600830078125,
|
|
"step": 645
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -990.78173828125,
|
|
"KL/mean": -1174.6834716796875,
|
|
"KL/rejected_KL_mean": -1358.585205078125,
|
|
"KL/std": 1137.93994140625,
|
|
"epoch": 0.9486049926578561,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 367.80340576171875,
|
|
"fcm_dpo/q_t": 0.49355489015579224,
|
|
"grad_norm": 5.103117942810059,
|
|
"learning_rate": 4.256725079024553e-09,
|
|
"logits/chosen": -0.6123974919319153,
|
|
"logits/rejected": -0.6034915447235107,
|
|
"logps/chosen": -1052.0574951171875,
|
|
"logps/ref_chosen": -61.275787353515625,
|
|
"logps/ref_rejected": -77.50580596923828,
|
|
"logps/rejected": -1436.091064453125,
|
|
"loss": 1.3629,
|
|
"margin_dpo/margin_mean": 367.80340576171875,
|
|
"margin_dpo/margin_std": 1322.474365234375,
|
|
"step": 646
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -822.7291259765625,
|
|
"KL/mean": -1107.7655029296875,
|
|
"KL/rejected_KL_mean": -1392.802001953125,
|
|
"KL/std": 1028.25341796875,
|
|
"epoch": 0.9500734214390602,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 570.0728759765625,
|
|
"fcm_dpo/q_t": 0.49000051617622375,
|
|
"grad_norm": 10.267186164855957,
|
|
"learning_rate": 4.024152566816791e-09,
|
|
"logits/chosen": -0.5852940678596497,
|
|
"logits/rejected": -0.6047427654266357,
|
|
"logps/chosen": -877.58154296875,
|
|
"logps/ref_chosen": -54.8524169921875,
|
|
"logps/ref_rejected": -93.5194091796875,
|
|
"logps/rejected": -1486.3214111328125,
|
|
"loss": 1.3484,
|
|
"margin_dpo/margin_mean": 570.0728759765625,
|
|
"margin_dpo/margin_std": 1177.3486328125,
|
|
"step": 647
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1093.44384765625,
|
|
"KL/mean": -1523.8251953125,
|
|
"KL/rejected_KL_mean": -1954.206298828125,
|
|
"KL/std": 1495.839111328125,
|
|
"epoch": 0.9515418502202643,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 860.7623291015625,
|
|
"fcm_dpo/q_t": 0.4849778413772583,
|
|
"grad_norm": 10.467401504516602,
|
|
"learning_rate": 3.798061746947995e-09,
|
|
"logits/chosen": -0.6163946390151978,
|
|
"logits/rejected": -0.6602287292480469,
|
|
"logps/chosen": -1147.6153564453125,
|
|
"logps/ref_chosen": -54.17146682739258,
|
|
"logps/ref_rejected": -98.7127914428711,
|
|
"logps/rejected": -2052.9189453125,
|
|
"loss": 1.3305,
|
|
"margin_dpo/margin_mean": 860.7623901367188,
|
|
"margin_dpo/margin_std": 1694.208984375,
|
|
"step": 648
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1002.08935546875,
|
|
"KL/mean": -1291.5301513671875,
|
|
"KL/rejected_KL_mean": -1580.9708251953125,
|
|
"KL/std": 1385.653076171875,
|
|
"epoch": 0.9530102790014684,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 578.8814697265625,
|
|
"fcm_dpo/q_t": 0.4899559020996094,
|
|
"grad_norm": 4.749213218688965,
|
|
"learning_rate": 3.5784585771215235e-09,
|
|
"logits/chosen": -0.6620218753814697,
|
|
"logits/rejected": -0.6723358631134033,
|
|
"logps/chosen": -1064.5697021484375,
|
|
"logps/ref_chosen": -62.480350494384766,
|
|
"logps/ref_rejected": -80.07717895507812,
|
|
"logps/rejected": -1661.048095703125,
|
|
"loss": 1.3503,
|
|
"margin_dpo/margin_mean": 578.8814697265625,
|
|
"margin_dpo/margin_std": 1804.99658203125,
|
|
"step": 649
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1217.0234375,
|
|
"KL/mean": -1500.784423828125,
|
|
"KL/rejected_KL_mean": -1784.54541015625,
|
|
"KL/std": 1650.326416015625,
|
|
"epoch": 0.9544787077826725,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 567.52197265625,
|
|
"fcm_dpo/q_t": 0.4899527430534363,
|
|
"grad_norm": 4.821526527404785,
|
|
"learning_rate": 3.3653488440851253e-09,
|
|
"logits/chosen": -0.6028755903244019,
|
|
"logits/rejected": -0.6270517706871033,
|
|
"logps/chosen": -1273.1163330078125,
|
|
"logps/ref_chosen": -56.09281921386719,
|
|
"logps/ref_rejected": -98.26483917236328,
|
|
"logps/rejected": -1882.810302734375,
|
|
"loss": 1.353,
|
|
"margin_dpo/margin_mean": 567.52197265625,
|
|
"margin_dpo/margin_std": 2182.583984375,
|
|
"step": 650
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -809.0330200195312,
|
|
"KL/mean": -1301.23486328125,
|
|
"KL/rejected_KL_mean": -1793.436767578125,
|
|
"KL/std": 1311.9166259765625,
|
|
"epoch": 0.9559471365638766,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 984.4036865234375,
|
|
"fcm_dpo/q_t": 0.48278218507766724,
|
|
"grad_norm": 13.06442928314209,
|
|
"learning_rate": 3.158738163478475e-09,
|
|
"logits/chosen": -0.647139847278595,
|
|
"logits/rejected": -0.6998910903930664,
|
|
"logps/chosen": -852.45849609375,
|
|
"logps/ref_chosen": -43.42544937133789,
|
|
"logps/ref_rejected": -99.95791625976562,
|
|
"logps/rejected": -1893.3946533203125,
|
|
"loss": 1.3208,
|
|
"margin_dpo/margin_mean": 984.4036865234375,
|
|
"margin_dpo/margin_std": 1392.693359375,
|
|
"step": 651
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -990.5150756835938,
|
|
"KL/mean": -1336.54736328125,
|
|
"KL/rejected_KL_mean": -1682.579833984375,
|
|
"KL/std": 1426.03564453125,
|
|
"epoch": 0.9574155653450808,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 692.0648193359375,
|
|
"fcm_dpo/q_t": 0.4879456162452698,
|
|
"grad_norm": 13.026046752929688,
|
|
"learning_rate": 2.9586319796851555e-09,
|
|
"logits/chosen": -0.6543087959289551,
|
|
"logits/rejected": -0.6866675019264221,
|
|
"logps/chosen": -1053.091796875,
|
|
"logps/ref_chosen": -62.57680892944336,
|
|
"logps/ref_rejected": -111.76779174804688,
|
|
"logps/rejected": -1794.34765625,
|
|
"loss": 1.3424,
|
|
"margin_dpo/margin_mean": 692.0648193359375,
|
|
"margin_dpo/margin_std": 1824.699951171875,
|
|
"step": 652
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1127.3262939453125,
|
|
"KL/mean": -1536.869384765625,
|
|
"KL/rejected_KL_mean": -1946.41259765625,
|
|
"KL/std": 1666.6322021484375,
|
|
"epoch": 0.9588839941262849,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 819.0863647460938,
|
|
"fcm_dpo/q_t": 0.48590990900993347,
|
|
"grad_norm": 12.233861923217773,
|
|
"learning_rate": 2.7650355656892166e-09,
|
|
"logits/chosen": -0.6801096200942993,
|
|
"logits/rejected": -0.7179126143455505,
|
|
"logps/chosen": -1188.439208984375,
|
|
"logps/ref_chosen": -61.11295700073242,
|
|
"logps/ref_rejected": -103.24960327148438,
|
|
"logps/rejected": -2049.662109375,
|
|
"loss": 1.3351,
|
|
"margin_dpo/margin_mean": 819.0863647460938,
|
|
"margin_dpo/margin_std": 2013.414794921875,
|
|
"step": 653
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1065.35546875,
|
|
"KL/mean": -1342.395263671875,
|
|
"KL/rejected_KL_mean": -1619.43505859375,
|
|
"KL/std": 1395.85400390625,
|
|
"epoch": 0.960352422907489,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 554.0797119140625,
|
|
"fcm_dpo/q_t": 0.49033522605895996,
|
|
"grad_norm": 5.433686256408691,
|
|
"learning_rate": 2.577954022936174e-09,
|
|
"logits/chosen": -0.6489402651786804,
|
|
"logits/rejected": -0.6714223623275757,
|
|
"logps/chosen": -1127.08349609375,
|
|
"logps/ref_chosen": -61.7281379699707,
|
|
"logps/ref_rejected": -98.7738037109375,
|
|
"logps/rejected": -1718.208984375,
|
|
"loss": 1.3513,
|
|
"margin_dpo/margin_mean": 554.0797119140625,
|
|
"margin_dpo/margin_std": 1696.0687255859375,
|
|
"step": 654
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -971.6968994140625,
|
|
"KL/mean": -1341.12060546875,
|
|
"KL/rejected_KL_mean": -1710.54443359375,
|
|
"KL/std": 1493.365478515625,
|
|
"epoch": 0.9618208516886931,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 738.8475341796875,
|
|
"fcm_dpo/q_t": 0.4871307611465454,
|
|
"grad_norm": 12.034753799438477,
|
|
"learning_rate": 2.397392281198729e-09,
|
|
"logits/chosen": -0.5706413984298706,
|
|
"logits/rejected": -0.6150772571563721,
|
|
"logps/chosen": -1021.273681640625,
|
|
"logps/ref_chosen": -49.576812744140625,
|
|
"logps/ref_rejected": -98.29183197021484,
|
|
"logps/rejected": -1808.8363037109375,
|
|
"loss": 1.339,
|
|
"margin_dpo/margin_mean": 738.8475341796875,
|
|
"margin_dpo/margin_std": 1762.6845703125,
|
|
"step": 655
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1052.604736328125,
|
|
"KL/mean": -1667.0531005859375,
|
|
"KL/rejected_KL_mean": -2281.50146484375,
|
|
"KL/std": 1618.218505859375,
|
|
"epoch": 0.9632892804698973,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1228.896484375,
|
|
"fcm_dpo/q_t": 0.478542685508728,
|
|
"grad_norm": 15.938714027404785,
|
|
"learning_rate": 2.223355098446622e-09,
|
|
"logits/chosen": -0.5950414538383484,
|
|
"logits/rejected": -0.6505473852157593,
|
|
"logps/chosen": -1105.154296875,
|
|
"logps/ref_chosen": -52.54943084716797,
|
|
"logps/ref_rejected": -113.67464447021484,
|
|
"logps/rejected": -2395.176025390625,
|
|
"loss": 1.3054,
|
|
"margin_dpo/margin_mean": 1228.896484375,
|
|
"margin_dpo/margin_std": 1683.515625,
|
|
"step": 656
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1101.297607421875,
|
|
"KL/mean": -1607.488037109375,
|
|
"KL/rejected_KL_mean": -2113.678466796875,
|
|
"KL/std": 1809.890380859375,
|
|
"epoch": 0.9647577092511013,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1012.3809814453125,
|
|
"fcm_dpo/q_t": 0.4823700189590454,
|
|
"grad_norm": 7.817019939422607,
|
|
"learning_rate": 2.055847060721566e-09,
|
|
"logits/chosen": -0.6291069984436035,
|
|
"logits/rejected": -0.6917046308517456,
|
|
"logps/chosen": -1147.998046875,
|
|
"logps/ref_chosen": -46.700538635253906,
|
|
"logps/ref_rejected": -97.91487121582031,
|
|
"logps/rejected": -2211.59326171875,
|
|
"loss": 1.3215,
|
|
"margin_dpo/margin_mean": 1012.3810424804688,
|
|
"margin_dpo/margin_std": 2023.4796142578125,
|
|
"step": 657
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1130.634033203125,
|
|
"KL/mean": -1422.20703125,
|
|
"KL/rejected_KL_mean": -1713.7802734375,
|
|
"KL/std": 1217.5029296875,
|
|
"epoch": 0.9662261380323054,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 583.1461791992188,
|
|
"fcm_dpo/q_t": 0.4897571802139282,
|
|
"grad_norm": 7.5315752029418945,
|
|
"learning_rate": 1.8948725820160662e-09,
|
|
"logits/chosen": -0.5903108716011047,
|
|
"logits/rejected": -0.6097769737243652,
|
|
"logps/chosen": -1191.59228515625,
|
|
"logps/ref_chosen": -60.95820999145508,
|
|
"logps/ref_rejected": -95.93949127197266,
|
|
"logps/rejected": -1809.7197265625,
|
|
"loss": 1.3492,
|
|
"margin_dpo/margin_mean": 583.1461181640625,
|
|
"margin_dpo/margin_std": 1597.01708984375,
|
|
"step": 658
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1060.837158203125,
|
|
"KL/mean": -1350.650146484375,
|
|
"KL/rejected_KL_mean": -1640.4630126953125,
|
|
"KL/std": 1334.296142578125,
|
|
"epoch": 0.9676945668135095,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 579.6257934570312,
|
|
"fcm_dpo/q_t": 0.48988625407218933,
|
|
"grad_norm": 6.343628883361816,
|
|
"learning_rate": 1.7404359041573723e-09,
|
|
"logits/chosen": -0.6019806861877441,
|
|
"logits/rejected": -0.5888450145721436,
|
|
"logps/chosen": -1137.580078125,
|
|
"logps/ref_chosen": -76.74298095703125,
|
|
"logps/ref_rejected": -87.4709701538086,
|
|
"logps/rejected": -1727.9339599609375,
|
|
"loss": 1.3493,
|
|
"margin_dpo/margin_mean": 579.6257934570312,
|
|
"margin_dpo/margin_std": 1626.3175048828125,
|
|
"step": 659
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1123.677490234375,
|
|
"KL/mean": -1415.98095703125,
|
|
"KL/rejected_KL_mean": -1708.28466796875,
|
|
"KL/std": 1241.296630859375,
|
|
"epoch": 0.9691629955947136,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 584.6071166992188,
|
|
"fcm_dpo/q_t": 0.4897761940956116,
|
|
"grad_norm": 15.733861923217773,
|
|
"learning_rate": 1.592541096695571e-09,
|
|
"logits/chosen": -0.6595280170440674,
|
|
"logits/rejected": -0.6711503267288208,
|
|
"logps/chosen": -1182.725341796875,
|
|
"logps/ref_chosen": -59.04788589477539,
|
|
"logps/ref_rejected": -75.96005249023438,
|
|
"logps/rejected": -1784.24462890625,
|
|
"loss": 1.3485,
|
|
"margin_dpo/margin_mean": 584.6071166992188,
|
|
"margin_dpo/margin_std": 1485.17626953125,
|
|
"step": 660
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -902.3294067382812,
|
|
"KL/mean": -1418.966796875,
|
|
"KL/rejected_KL_mean": -1935.60400390625,
|
|
"KL/std": 2257.097412109375,
|
|
"epoch": 0.9706314243759178,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1033.27490234375,
|
|
"fcm_dpo/q_t": 0.48256513476371765,
|
|
"grad_norm": 8.566376686096191,
|
|
"learning_rate": 1.4511920567963908e-09,
|
|
"logits/chosen": -0.6667207479476929,
|
|
"logits/rejected": -0.7094805240631104,
|
|
"logps/chosen": -953.00341796875,
|
|
"logps/ref_chosen": -50.673973083496094,
|
|
"logps/ref_rejected": -86.00569152832031,
|
|
"logps/rejected": -2021.60986328125,
|
|
"loss": 1.3242,
|
|
"margin_dpo/margin_mean": 1033.27490234375,
|
|
"margin_dpo/margin_std": 2576.364501953125,
|
|
"step": 661
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1044.2470703125,
|
|
"KL/mean": -1371.6861572265625,
|
|
"KL/rejected_KL_mean": -1699.1253662109375,
|
|
"KL/std": 1256.9990234375,
|
|
"epoch": 0.9720998531571219,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 654.878173828125,
|
|
"fcm_dpo/q_t": 0.48855116963386536,
|
|
"grad_norm": 4.699316501617432,
|
|
"learning_rate": 1.3163925091384532e-09,
|
|
"logits/chosen": -0.6130234003067017,
|
|
"logits/rejected": -0.6202554106712341,
|
|
"logps/chosen": -1113.5081787109375,
|
|
"logps/ref_chosen": -69.26106262207031,
|
|
"logps/ref_rejected": -89.05593872070312,
|
|
"logps/rejected": -1788.1812744140625,
|
|
"loss": 1.3441,
|
|
"margin_dpo/margin_mean": 654.8782348632812,
|
|
"margin_dpo/margin_std": 1609.4761962890625,
|
|
"step": 662
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -954.5285034179688,
|
|
"KL/mean": -1455.802001953125,
|
|
"KL/rejected_KL_mean": -1957.0755615234375,
|
|
"KL/std": 1663.22412109375,
|
|
"epoch": 0.973568281938326,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1002.547119140625,
|
|
"fcm_dpo/q_t": 0.48265382647514343,
|
|
"grad_norm": 7.0070013999938965,
|
|
"learning_rate": 1.1881460058152382e-09,
|
|
"logits/chosen": -0.6588038206100464,
|
|
"logits/rejected": -0.7001410126686096,
|
|
"logps/chosen": -1019.4073486328125,
|
|
"logps/ref_chosen": -64.87890625,
|
|
"logps/ref_rejected": -113.92536926269531,
|
|
"logps/rejected": -2071.0009765625,
|
|
"loss": 1.3231,
|
|
"margin_dpo/margin_mean": 1002.5470581054688,
|
|
"margin_dpo/margin_std": 2189.273193359375,
|
|
"step": 663
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1293.694091796875,
|
|
"KL/mean": -1853.238525390625,
|
|
"KL/rejected_KL_mean": -2412.78271484375,
|
|
"KL/std": 2207.93994140625,
|
|
"epoch": 0.9750367107195301,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1119.088623046875,
|
|
"fcm_dpo/q_t": 0.4808417856693268,
|
|
"grad_norm": 5.824970722198486,
|
|
"learning_rate": 1.066455926241383e-09,
|
|
"logits/chosen": -0.6086447238922119,
|
|
"logits/rejected": -0.6509624123573303,
|
|
"logps/chosen": -1354.5826416015625,
|
|
"logps/ref_chosen": -60.88847351074219,
|
|
"logps/ref_rejected": -105.521728515625,
|
|
"logps/rejected": -2518.304443359375,
|
|
"loss": 1.3188,
|
|
"margin_dpo/margin_mean": 1119.088623046875,
|
|
"margin_dpo/margin_std": 2782.05810546875,
|
|
"step": 664
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -998.0809326171875,
|
|
"KL/mean": -1378.9786376953125,
|
|
"KL/rejected_KL_mean": -1759.8763427734375,
|
|
"KL/std": 1507.407470703125,
|
|
"epoch": 0.9765051395007343,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 761.79541015625,
|
|
"fcm_dpo/q_t": 0.4867151081562042,
|
|
"grad_norm": 8.16478443145752,
|
|
"learning_rate": 9.513254770636137e-10,
|
|
"logits/chosen": -0.6453874111175537,
|
|
"logits/rejected": -0.682604193687439,
|
|
"logps/chosen": -1058.6451416015625,
|
|
"logps/ref_chosen": -60.56413269042969,
|
|
"logps/ref_rejected": -84.80882263183594,
|
|
"logps/rejected": -1844.68505859375,
|
|
"loss": 1.3374,
|
|
"margin_dpo/margin_mean": 761.79541015625,
|
|
"margin_dpo/margin_std": 1707.666748046875,
|
|
"step": 665
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1018.7044067382812,
|
|
"KL/mean": -1426.4305419921875,
|
|
"KL/rejected_KL_mean": -1834.1566162109375,
|
|
"KL/std": 1435.4952392578125,
|
|
"epoch": 0.9779735682819384,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 815.4522705078125,
|
|
"fcm_dpo/q_t": 0.48577576875686646,
|
|
"grad_norm": 5.858867645263672,
|
|
"learning_rate": 8.427576920763956e-10,
|
|
"logits/chosen": -0.6097148060798645,
|
|
"logits/rejected": -0.6266220808029175,
|
|
"logps/chosen": -1083.124267578125,
|
|
"logps/ref_chosen": -64.41996002197266,
|
|
"logps/ref_rejected": -95.8916244506836,
|
|
"logps/rejected": -1930.04833984375,
|
|
"loss": 1.3337,
|
|
"margin_dpo/margin_mean": 815.4522705078125,
|
|
"margin_dpo/margin_std": 1725.0411376953125,
|
|
"step": 666
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1350.70068359375,
|
|
"KL/mean": -1672.00048828125,
|
|
"KL/rejected_KL_mean": -1993.3004150390625,
|
|
"KL/std": 1688.798583984375,
|
|
"epoch": 0.9794419970631424,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 642.599853515625,
|
|
"fcm_dpo/q_t": 0.4888229966163635,
|
|
"grad_norm": 10.685932159423828,
|
|
"learning_rate": 7.407554321417764e-10,
|
|
"logits/chosen": -0.5980401039123535,
|
|
"logits/rejected": -0.6029761433601379,
|
|
"logps/chosen": -1419.9775390625,
|
|
"logps/ref_chosen": -69.27702331542969,
|
|
"logps/ref_rejected": -87.83549499511719,
|
|
"logps/rejected": -2081.1357421875,
|
|
"loss": 1.3471,
|
|
"margin_dpo/margin_mean": 642.599853515625,
|
|
"margin_dpo/margin_std": 2042.0830078125,
|
|
"step": 667
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1268.402099609375,
|
|
"KL/mean": -1488.158935546875,
|
|
"KL/rejected_KL_mean": -1707.915771484375,
|
|
"KL/std": 1302.4078369140625,
|
|
"epoch": 0.9809104258443465,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 439.513671875,
|
|
"fcm_dpo/q_t": 0.4923250377178192,
|
|
"grad_norm": 15.58211898803711,
|
|
"learning_rate": 6.453213851142225e-10,
|
|
"logits/chosen": -0.6111631989479065,
|
|
"logits/rejected": -0.6139740943908691,
|
|
"logps/chosen": -1341.006103515625,
|
|
"logps/ref_chosen": -72.60400390625,
|
|
"logps/ref_rejected": -103.73905944824219,
|
|
"logps/rejected": -1811.65478515625,
|
|
"loss": 1.3589,
|
|
"margin_dpo/margin_mean": 439.5137023925781,
|
|
"margin_dpo/margin_std": 1599.6151123046875,
|
|
"step": 668
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -872.9949951171875,
|
|
"KL/mean": -1229.5106201171875,
|
|
"KL/rejected_KL_mean": -1586.026123046875,
|
|
"KL/std": 1136.226318359375,
|
|
"epoch": 0.9823788546255506,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 713.03125,
|
|
"fcm_dpo/q_t": 0.48755186796188354,
|
|
"grad_norm": 4.998262882232666,
|
|
"learning_rate": 5.564580657695939e-10,
|
|
"logits/chosen": -0.6233928203582764,
|
|
"logits/rejected": -0.6421560049057007,
|
|
"logps/chosen": -919.1114501953125,
|
|
"logps/ref_chosen": -46.116416931152344,
|
|
"logps/ref_rejected": -77.92434692382812,
|
|
"logps/rejected": -1663.9505615234375,
|
|
"loss": 1.3394,
|
|
"margin_dpo/margin_mean": 713.0311889648438,
|
|
"margin_dpo/margin_std": 1437.951904296875,
|
|
"step": 669
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1085.6097412109375,
|
|
"KL/mean": -1610.455810546875,
|
|
"KL/rejected_KL_mean": -2135.3017578125,
|
|
"KL/std": 1911.545166015625,
|
|
"epoch": 0.9838472834067548,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1049.692138671875,
|
|
"fcm_dpo/q_t": 0.48183974623680115,
|
|
"grad_norm": 7.568104267120361,
|
|
"learning_rate": 4.741678157389739e-10,
|
|
"logits/chosen": -0.6080718040466309,
|
|
"logits/rejected": -0.6510467529296875,
|
|
"logps/chosen": -1147.9554443359375,
|
|
"logps/ref_chosen": -62.34575271606445,
|
|
"logps/ref_rejected": -96.9405517578125,
|
|
"logps/rejected": -2232.242431640625,
|
|
"loss": 1.3207,
|
|
"margin_dpo/margin_mean": 1049.692138671875,
|
|
"margin_dpo/margin_std": 2298.290283203125,
|
|
"step": 670
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1065.5369873046875,
|
|
"KL/mean": -1437.20361328125,
|
|
"KL/rejected_KL_mean": -1808.870361328125,
|
|
"KL/std": 1385.409423828125,
|
|
"epoch": 0.9853157121879589,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 743.3333129882812,
|
|
"fcm_dpo/q_t": 0.4870268702507019,
|
|
"grad_norm": 8.674588203430176,
|
|
"learning_rate": 3.9845280344705245e-10,
|
|
"logits/chosen": -0.6010315418243408,
|
|
"logits/rejected": -0.6403799057006836,
|
|
"logps/chosen": -1113.537109375,
|
|
"logps/ref_chosen": -48.00010681152344,
|
|
"logps/ref_rejected": -83.81932067871094,
|
|
"logps/rejected": -1892.689697265625,
|
|
"loss": 1.3395,
|
|
"margin_dpo/margin_mean": 743.3333740234375,
|
|
"margin_dpo/margin_std": 1940.243896484375,
|
|
"step": 671
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1323.4586181640625,
|
|
"KL/mean": -1571.2021484375,
|
|
"KL/rejected_KL_mean": -1818.945556640625,
|
|
"KL/std": 1406.629150390625,
|
|
"epoch": 0.986784140969163,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 495.4870910644531,
|
|
"fcm_dpo/q_t": 0.4913468062877655,
|
|
"grad_norm": 15.005202293395996,
|
|
"learning_rate": 3.293150240547549e-10,
|
|
"logits/chosen": -0.6775461435317993,
|
|
"logits/rejected": -0.6908239126205444,
|
|
"logps/chosen": -1382.0418701171875,
|
|
"logps/ref_chosen": -58.58328628540039,
|
|
"logps/ref_rejected": -93.14015197753906,
|
|
"logps/rejected": -1912.085693359375,
|
|
"loss": 1.3548,
|
|
"margin_dpo/margin_mean": 495.4870300292969,
|
|
"margin_dpo/margin_std": 1540.307373046875,
|
|
"step": 672
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1022.36083984375,
|
|
"KL/mean": -1316.231689453125,
|
|
"KL/rejected_KL_mean": -1610.1025390625,
|
|
"KL/std": 1145.38427734375,
|
|
"epoch": 0.9882525697503671,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 587.74169921875,
|
|
"fcm_dpo/q_t": 0.4897216856479645,
|
|
"grad_norm": 6.545974254608154,
|
|
"learning_rate": 2.6675629940689504e-10,
|
|
"logits/chosen": -0.6184306740760803,
|
|
"logits/rejected": -0.6311053037643433,
|
|
"logps/chosen": -1069.083984375,
|
|
"logps/ref_chosen": -46.72320556640625,
|
|
"logps/ref_rejected": -85.29623413085938,
|
|
"logps/rejected": -1695.3988037109375,
|
|
"loss": 1.3476,
|
|
"margin_dpo/margin_mean": 587.74169921875,
|
|
"margin_dpo/margin_std": 1306.4228515625,
|
|
"step": 673
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -847.9764404296875,
|
|
"KL/mean": -1195.31689453125,
|
|
"KL/rejected_KL_mean": -1542.6575927734375,
|
|
"KL/std": 1038.550048828125,
|
|
"epoch": 0.9897209985315712,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 694.6810302734375,
|
|
"fcm_dpo/q_t": 0.4878394603729248,
|
|
"grad_norm": 13.266809463500977,
|
|
"learning_rate": 2.1077827798404725e-10,
|
|
"logits/chosen": -0.6065397262573242,
|
|
"logits/rejected": -0.6238285303115845,
|
|
"logps/chosen": -893.4219970703125,
|
|
"logps/ref_chosen": -45.445526123046875,
|
|
"logps/ref_rejected": -70.04593658447266,
|
|
"logps/rejected": -1612.7034912109375,
|
|
"loss": 1.3403,
|
|
"margin_dpo/margin_mean": 694.6810302734375,
|
|
"margin_dpo/margin_std": 1318.276611328125,
|
|
"step": 674
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1095.146484375,
|
|
"KL/mean": -1370.689697265625,
|
|
"KL/rejected_KL_mean": -1646.23291015625,
|
|
"KL/std": 1590.983154296875,
|
|
"epoch": 0.9911894273127754,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 551.0866088867188,
|
|
"fcm_dpo/q_t": 0.4904383718967438,
|
|
"grad_norm": 9.25955867767334,
|
|
"learning_rate": 1.6138243485910863e-10,
|
|
"logits/chosen": -0.6318482756614685,
|
|
"logits/rejected": -0.6467227935791016,
|
|
"logps/chosen": -1139.32275390625,
|
|
"logps/ref_chosen": -44.17628479003906,
|
|
"logps/ref_rejected": -74.09197998046875,
|
|
"logps/rejected": -1720.324951171875,
|
|
"loss": 1.3534,
|
|
"margin_dpo/margin_mean": 551.0866088867188,
|
|
"margin_dpo/margin_std": 2041.986083984375,
|
|
"step": 675
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1177.5093994140625,
|
|
"KL/mean": -1528.5595703125,
|
|
"KL/rejected_KL_mean": -1879.60986328125,
|
|
"KL/std": 1375.15185546875,
|
|
"epoch": 0.9926578560939795,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 702.1004638671875,
|
|
"fcm_dpo/q_t": 0.4877719581127167,
|
|
"grad_norm": 5.452136516571045,
|
|
"learning_rate": 1.1857007165852472e-10,
|
|
"logits/chosen": -0.6294345259666443,
|
|
"logits/rejected": -0.6576676964759827,
|
|
"logps/chosen": -1248.907958984375,
|
|
"logps/ref_chosen": -71.39852905273438,
|
|
"logps/ref_rejected": -88.3587646484375,
|
|
"logps/rejected": -1967.968505859375,
|
|
"loss": 1.3415,
|
|
"margin_dpo/margin_mean": 702.1004028320312,
|
|
"margin_dpo/margin_std": 1760.55029296875,
|
|
"step": 676
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1092.815185546875,
|
|
"KL/mean": -1425.9197998046875,
|
|
"KL/rejected_KL_mean": -1759.0244140625,
|
|
"KL/std": 1277.3245849609375,
|
|
"epoch": 0.9941262848751835,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 666.209228515625,
|
|
"fcm_dpo/q_t": 0.4884171485900879,
|
|
"grad_norm": 10.614191055297852,
|
|
"learning_rate": 8.23423165278725e-11,
|
|
"logits/chosen": -0.6302313804626465,
|
|
"logits/rejected": -0.6473731994628906,
|
|
"logps/chosen": -1149.342529296875,
|
|
"logps/ref_chosen": -56.527435302734375,
|
|
"logps/ref_rejected": -78.22654724121094,
|
|
"logps/rejected": -1837.2509765625,
|
|
"loss": 1.3443,
|
|
"margin_dpo/margin_mean": 666.209228515625,
|
|
"margin_dpo/margin_std": 1806.208251953125,
|
|
"step": 677
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -861.671630859375,
|
|
"KL/mean": -1364.0057373046875,
|
|
"KL/rejected_KL_mean": -1866.33984375,
|
|
"KL/std": 1516.4210205078125,
|
|
"epoch": 0.9955947136563876,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1004.6680908203125,
|
|
"fcm_dpo/q_t": 0.48245441913604736,
|
|
"grad_norm": 14.39037036895752,
|
|
"learning_rate": 5.270012410216185e-11,
|
|
"logits/chosen": -0.6077277064323425,
|
|
"logits/rejected": -0.6513392925262451,
|
|
"logps/chosen": -907.80615234375,
|
|
"logps/ref_chosen": -46.13447570800781,
|
|
"logps/ref_rejected": -80.60462951660156,
|
|
"logps/rejected": -1946.9443359375,
|
|
"loss": 1.3211,
|
|
"margin_dpo/margin_mean": 1004.6680908203125,
|
|
"margin_dpo/margin_std": 1805.224609375,
|
|
"step": 678
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1052.888671875,
|
|
"KL/mean": -1298.56982421875,
|
|
"KL/rejected_KL_mean": -1544.251220703125,
|
|
"KL/std": 1319.9560546875,
|
|
"epoch": 0.9970631424375918,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 491.36260986328125,
|
|
"fcm_dpo/q_t": 0.4915056824684143,
|
|
"grad_norm": 5.045799732208252,
|
|
"learning_rate": 2.9644275480772416e-11,
|
|
"logits/chosen": -0.6154893636703491,
|
|
"logits/rejected": -0.624566912651062,
|
|
"logps/chosen": -1103.18359375,
|
|
"logps/ref_chosen": -50.294921875,
|
|
"logps/ref_rejected": -76.59813690185547,
|
|
"logps/rejected": -1620.849365234375,
|
|
"loss": 1.356,
|
|
"margin_dpo/margin_mean": 491.36260986328125,
|
|
"margin_dpo/margin_std": 1768.9346923828125,
|
|
"step": 679
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1185.60595703125,
|
|
"KL/mean": -1536.31689453125,
|
|
"KL/rejected_KL_mean": -1887.0277099609375,
|
|
"KL/std": 1552.267822265625,
|
|
"epoch": 0.9985315712187959,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 701.421630859375,
|
|
"fcm_dpo/q_t": 0.4878092408180237,
|
|
"grad_norm": 5.114809989929199,
|
|
"learning_rate": 1.31753782067201e-11,
|
|
"logits/chosen": -0.6257789731025696,
|
|
"logits/rejected": -0.6503081321716309,
|
|
"logps/chosen": -1262.521728515625,
|
|
"logps/ref_chosen": -76.91569519042969,
|
|
"logps/ref_rejected": -112.384765625,
|
|
"logps/rejected": -1999.4124755859375,
|
|
"loss": 1.3422,
|
|
"margin_dpo/margin_mean": 701.421630859375,
|
|
"margin_dpo/margin_std": 1905.763427734375,
|
|
"step": 680
|
|
},
|
|
{
|
|
"KL/chosen_KL_mean": -1252.8399658203125,
|
|
"KL/mean": -1509.438720703125,
|
|
"KL/rejected_KL_mean": -1766.037353515625,
|
|
"KL/std": 1613.639404296875,
|
|
"epoch": 1.0,
|
|
"fcm_dpo/beta": 7.018564065219834e-05,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 513.1973876953125,
|
|
"fcm_dpo/q_t": 0.49122941493988037,
|
|
"grad_norm": 13.061714172363281,
|
|
"learning_rate": 3.2938662507808745e-12,
|
|
"logits/chosen": -0.6694597005844116,
|
|
"logits/rejected": -0.6893618702888489,
|
|
"logps/chosen": -1313.7972412109375,
|
|
"logps/ref_chosen": -60.957279205322266,
|
|
"logps/ref_rejected": -88.55797576904297,
|
|
"logps/rejected": -1854.5953369140625,
|
|
"loss": 1.3573,
|
|
"margin_dpo/margin_mean": 513.1973876953125,
|
|
"margin_dpo/margin_std": 2175.52685546875,
|
|
"step": 681
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 681,
|
|
"total_flos": 0.0,
|
|
"train_loss": 1.3263754091949014,
|
|
"train_runtime": 1741.0925,
|
|
"train_samples_per_second": 25.041,
|
|
"train_steps_per_second": 0.391
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 681,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|