2707 lines
99 KiB
JSON
2707 lines
99 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"eval_steps": 200,
|
||
|
|
"global_step": 681,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.0014684287812041115,
|
||
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
||
|
|
"fcm_dpo/delta": 0.0,
|
||
|
|
"fcm_dpo/margin": -0.02287006378173828,
|
||
|
|
"fcm_dpo/q_t": 0.5005706548690796,
|
||
|
|
"grad_norm": 83.52236938476562,
|
||
|
|
"learning_rate": 0.0,
|
||
|
|
"logits/chosen": -0.4974287748336792,
|
||
|
|
"logits/rejected": -0.43299180269241333,
|
||
|
|
"logps/chosen": -50.1435661315918,
|
||
|
|
"logps/ref_chosen": -50.14883804321289,
|
||
|
|
"logps/ref_rejected": -74.1280517578125,
|
||
|
|
"logps/rejected": -74.09991455078125,
|
||
|
|
"loss": 1.389,
|
||
|
|
"margin_dpo/margin_mean": -0.02287048101425171,
|
||
|
|
"margin_dpo/margin_std": 0.41920793056488037,
|
||
|
|
"step": 1
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.007342143906020558,
|
||
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
||
|
|
"fcm_dpo/delta": 0.0,
|
||
|
|
"fcm_dpo/margin": -0.03249748796224594,
|
||
|
|
"fcm_dpo/q_t": 0.5008123517036438,
|
||
|
|
"grad_norm": 89.65718078613281,
|
||
|
|
"learning_rate": 2.898550724637681e-08,
|
||
|
|
"logits/chosen": -0.4900774359703064,
|
||
|
|
"logits/rejected": -0.453380823135376,
|
||
|
|
"logps/chosen": -56.06619644165039,
|
||
|
|
"logps/ref_chosen": -56.05734634399414,
|
||
|
|
"logps/ref_rejected": -78.69325256347656,
|
||
|
|
"logps/rejected": -78.66961669921875,
|
||
|
|
"loss": 1.3898,
|
||
|
|
"margin_dpo/margin_mean": -0.03249724209308624,
|
||
|
|
"margin_dpo/margin_std": 0.34048572182655334,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.014684287812041116,
|
||
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
||
|
|
"fcm_dpo/delta": 0.0,
|
||
|
|
"fcm_dpo/margin": -0.020858580246567726,
|
||
|
|
"fcm_dpo/q_t": 0.5005213022232056,
|
||
|
|
"grad_norm": 70.88235473632812,
|
||
|
|
"learning_rate": 6.521739130434782e-08,
|
||
|
|
"logits/chosen": -0.5124648809432983,
|
||
|
|
"logits/rejected": -0.4746035933494568,
|
||
|
|
"logps/chosen": -59.54156494140625,
|
||
|
|
"logps/ref_chosen": -59.54457473754883,
|
||
|
|
"logps/ref_rejected": -91.17041778564453,
|
||
|
|
"logps/rejected": -91.14656066894531,
|
||
|
|
"loss": 1.3888,
|
||
|
|
"margin_dpo/margin_mean": -0.02085849642753601,
|
||
|
|
"margin_dpo/margin_std": 0.38151517510414124,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.022026431718061675,
|
||
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
||
|
|
"fcm_dpo/delta": 0.0,
|
||
|
|
"fcm_dpo/margin": -0.008700313977897167,
|
||
|
|
"fcm_dpo/q_t": 0.5002168416976929,
|
||
|
|
"grad_norm": 64.12911987304688,
|
||
|
|
"learning_rate": 1.0144927536231885e-07,
|
||
|
|
"logits/chosen": -0.4861031174659729,
|
||
|
|
"logits/rejected": -0.45661497116088867,
|
||
|
|
"logps/chosen": -58.85633087158203,
|
||
|
|
"logps/ref_chosen": -58.83195877075195,
|
||
|
|
"logps/ref_rejected": -92.93949890136719,
|
||
|
|
"logps/rejected": -92.95516967773438,
|
||
|
|
"loss": 1.3876,
|
||
|
|
"margin_dpo/margin_mean": -0.008700350299477577,
|
||
|
|
"margin_dpo/margin_std": 0.3919045627117157,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.02936857562408223,
|
||
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
||
|
|
"fcm_dpo/delta": 0.0,
|
||
|
|
"fcm_dpo/margin": 0.060948461294174194,
|
||
|
|
"fcm_dpo/q_t": 0.4984763562679291,
|
||
|
|
"grad_norm": 73.97709655761719,
|
||
|
|
"learning_rate": 1.3768115942028986e-07,
|
||
|
|
"logits/chosen": -0.5145794153213501,
|
||
|
|
"logits/rejected": -0.46688389778137207,
|
||
|
|
"logps/chosen": -59.64154815673828,
|
||
|
|
"logps/ref_chosen": -59.6396598815918,
|
||
|
|
"logps/ref_rejected": -82.76026916503906,
|
||
|
|
"logps/rejected": -82.82310485839844,
|
||
|
|
"loss": 1.3805,
|
||
|
|
"margin_dpo/margin_mean": 0.06094840168952942,
|
||
|
|
"margin_dpo/margin_std": 0.3420618772506714,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.03671071953010279,
|
||
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
||
|
|
"fcm_dpo/delta": 0.0,
|
||
|
|
"fcm_dpo/margin": 0.19125637412071228,
|
||
|
|
"fcm_dpo/q_t": 0.49522167444229126,
|
||
|
|
"grad_norm": 74.20327758789062,
|
||
|
|
"learning_rate": 1.7391304347826085e-07,
|
||
|
|
"logits/chosen": -0.5046600103378296,
|
||
|
|
"logits/rejected": -0.47779035568237305,
|
||
|
|
"logps/chosen": -53.17488479614258,
|
||
|
|
"logps/ref_chosen": -53.205284118652344,
|
||
|
|
"logps/ref_rejected": -88.99608612060547,
|
||
|
|
"logps/rejected": -89.15694427490234,
|
||
|
|
"loss": 1.3678,
|
||
|
|
"margin_dpo/margin_mean": 0.19125640392303467,
|
||
|
|
"margin_dpo/margin_std": 0.4390522539615631,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.04405286343612335,
|
||
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
||
|
|
"fcm_dpo/delta": 0.0,
|
||
|
|
"fcm_dpo/margin": 0.465215265750885,
|
||
|
|
"fcm_dpo/q_t": 0.4883840084075928,
|
||
|
|
"grad_norm": 87.46487426757812,
|
||
|
|
"learning_rate": 2.1014492753623187e-07,
|
||
|
|
"logits/chosen": -0.524259626865387,
|
||
|
|
"logits/rejected": -0.4842991232872009,
|
||
|
|
"logps/chosen": -53.4693603515625,
|
||
|
|
"logps/ref_chosen": -53.5526008605957,
|
||
|
|
"logps/ref_rejected": -97.87371826171875,
|
||
|
|
"logps/rejected": -98.25569152832031,
|
||
|
|
"loss": 1.3412,
|
||
|
|
"margin_dpo/margin_mean": 0.4652155041694641,
|
||
|
|
"margin_dpo/margin_std": 0.5838907957077026,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.0513950073421439,
|
||
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
||
|
|
"fcm_dpo/delta": 0.0,
|
||
|
|
"fcm_dpo/margin": 0.7807711362838745,
|
||
|
|
"fcm_dpo/q_t": 0.4805462956428528,
|
||
|
|
"grad_norm": 83.3965072631836,
|
||
|
|
"learning_rate": 2.463768115942029e-07,
|
||
|
|
"logits/chosen": -0.5029053092002869,
|
||
|
|
"logits/rejected": -0.4691234529018402,
|
||
|
|
"logps/chosen": -56.19721221923828,
|
||
|
|
"logps/ref_chosen": -56.3298454284668,
|
||
|
|
"logps/ref_rejected": -91.76858520507812,
|
||
|
|
"logps/rejected": -92.41671752929688,
|
||
|
|
"loss": 1.3116,
|
||
|
|
"margin_dpo/margin_mean": 0.7807710766792297,
|
||
|
|
"margin_dpo/margin_std": 0.820033848285675,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05873715124816446,
|
||
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
||
|
|
"fcm_dpo/delta": 0.0,
|
||
|
|
"fcm_dpo/margin": 1.3780633211135864,
|
||
|
|
"fcm_dpo/q_t": 0.46588778495788574,
|
||
|
|
"grad_norm": 60.05709457397461,
|
||
|
|
"learning_rate": 2.8260869565217386e-07,
|
||
|
|
"logits/chosen": -0.5348217487335205,
|
||
|
|
"logits/rejected": -0.4983762204647064,
|
||
|
|
"logps/chosen": -54.2760009765625,
|
||
|
|
"logps/ref_chosen": -54.38492965698242,
|
||
|
|
"logps/ref_rejected": -82.94353485107422,
|
||
|
|
"logps/rejected": -84.21268463134766,
|
||
|
|
"loss": 1.2592,
|
||
|
|
"margin_dpo/margin_mean": 1.3780632019042969,
|
||
|
|
"margin_dpo/margin_std": 1.4964091777801514,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06607929515418502,
|
||
|
|
"fcm_dpo/beta": 0.13015565276145935,
|
||
|
|
"fcm_dpo/delta": 0.4826194643974304,
|
||
|
|
"fcm_dpo/margin": 2.396104335784912,
|
||
|
|
"fcm_dpo/q_t": 0.42787855863571167,
|
||
|
|
"grad_norm": 84.72518920898438,
|
||
|
|
"learning_rate": 3.188405797101449e-07,
|
||
|
|
"logits/chosen": -0.5097236633300781,
|
||
|
|
"logits/rejected": -0.4814884662628174,
|
||
|
|
"logps/chosen": -54.64501953125,
|
||
|
|
"logps/ref_chosen": -54.862335205078125,
|
||
|
|
"logps/ref_rejected": -98.0264663696289,
|
||
|
|
"logps/rejected": -100.20524597167969,
|
||
|
|
"loss": 1.1198,
|
||
|
|
"margin_dpo/margin_mean": 2.396104335784912,
|
||
|
|
"margin_dpo/margin_std": 2.2702395915985107,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07342143906020558,
|
||
|
|
"fcm_dpo/beta": 0.189666748046875,
|
||
|
|
"fcm_dpo/delta": 0.21794584393501282,
|
||
|
|
"fcm_dpo/margin": 3.3721413612365723,
|
||
|
|
"fcm_dpo/q_t": 0.36233529448509216,
|
||
|
|
"grad_norm": 93.39527130126953,
|
||
|
|
"learning_rate": 3.5507246376811595e-07,
|
||
|
|
"logits/chosen": -0.5606456398963928,
|
||
|
|
"logits/rejected": -0.5093287229537964,
|
||
|
|
"logps/chosen": -58.14433670043945,
|
||
|
|
"logps/ref_chosen": -58.304595947265625,
|
||
|
|
"logps/ref_rejected": -91.69480895996094,
|
||
|
|
"logps/rejected": -94.90668487548828,
|
||
|
|
"loss": 0.9364,
|
||
|
|
"margin_dpo/margin_mean": 3.3721413612365723,
|
||
|
|
"margin_dpo/margin_std": 3.3471221923828125,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08076358296622614,
|
||
|
|
"fcm_dpo/beta": 0.1844942271709442,
|
||
|
|
"fcm_dpo/delta": -0.19296136498451233,
|
||
|
|
"fcm_dpo/margin": 5.56916618347168,
|
||
|
|
"fcm_dpo/q_t": 0.30575454235076904,
|
||
|
|
"grad_norm": 78.50489807128906,
|
||
|
|
"learning_rate": 3.9130434782608694e-07,
|
||
|
|
"logits/chosen": -0.5810452699661255,
|
||
|
|
"logits/rejected": -0.5354865789413452,
|
||
|
|
"logps/chosen": -56.305625915527344,
|
||
|
|
"logps/ref_chosen": -56.06591796875,
|
||
|
|
"logps/ref_rejected": -85.69367980957031,
|
||
|
|
"logps/rejected": -91.50254821777344,
|
||
|
|
"loss": 0.8026,
|
||
|
|
"margin_dpo/margin_mean": 5.5691657066345215,
|
||
|
|
"margin_dpo/margin_std": 5.670819282531738,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.0881057268722467,
|
||
|
|
"fcm_dpo/beta": 0.14939136803150177,
|
||
|
|
"fcm_dpo/delta": -0.18733538687229156,
|
||
|
|
"fcm_dpo/margin": 6.829007148742676,
|
||
|
|
"fcm_dpo/q_t": 0.3050878942012787,
|
||
|
|
"grad_norm": 80.88507080078125,
|
||
|
|
"learning_rate": 4.2753623188405794e-07,
|
||
|
|
"logits/chosen": -0.5981510877609253,
|
||
|
|
"logits/rejected": -0.5482528805732727,
|
||
|
|
"logps/chosen": -61.76525115966797,
|
||
|
|
"logps/ref_chosen": -60.6871337890625,
|
||
|
|
"logps/ref_rejected": -89.72715759277344,
|
||
|
|
"logps/rejected": -97.63427734375,
|
||
|
|
"loss": 0.8231,
|
||
|
|
"margin_dpo/margin_mean": 6.829007148742676,
|
||
|
|
"margin_dpo/margin_std": 7.180797576904297,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09544787077826726,
|
||
|
|
"fcm_dpo/beta": 0.1279543936252594,
|
||
|
|
"fcm_dpo/delta": -0.15479923784732819,
|
||
|
|
"fcm_dpo/margin": 7.741368293762207,
|
||
|
|
"fcm_dpo/q_t": 0.31432515382766724,
|
||
|
|
"grad_norm": 60.72844696044922,
|
||
|
|
"learning_rate": 4.63768115942029e-07,
|
||
|
|
"logits/chosen": -0.6157333850860596,
|
||
|
|
"logits/rejected": -0.5836382508277893,
|
||
|
|
"logps/chosen": -63.23627853393555,
|
||
|
|
"logps/ref_chosen": -61.75325393676758,
|
||
|
|
"logps/ref_rejected": -93.30108642578125,
|
||
|
|
"logps/rejected": -102.52547454833984,
|
||
|
|
"loss": 0.8599,
|
||
|
|
"margin_dpo/margin_mean": 7.741368293762207,
|
||
|
|
"margin_dpo/margin_std": 8.94554615020752,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1027900146842878,
|
||
|
|
"fcm_dpo/beta": 0.11097879707813263,
|
||
|
|
"fcm_dpo/delta": -0.13162511587142944,
|
||
|
|
"fcm_dpo/margin": 8.74174690246582,
|
||
|
|
"fcm_dpo/q_t": 0.31369131803512573,
|
||
|
|
"grad_norm": 56.712982177734375,
|
||
|
|
"learning_rate": 5e-07,
|
||
|
|
"logits/chosen": -0.6086438894271851,
|
||
|
|
"logits/rejected": -0.5690315961837769,
|
||
|
|
"logps/chosen": -61.81795120239258,
|
||
|
|
"logps/ref_chosen": -59.548004150390625,
|
||
|
|
"logps/ref_rejected": -84.01609802246094,
|
||
|
|
"logps/rejected": -95.02778625488281,
|
||
|
|
"loss": 0.8231,
|
||
|
|
"margin_dpo/margin_mean": 8.741745948791504,
|
||
|
|
"margin_dpo/margin_std": 9.22284984588623,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11013215859030837,
|
||
|
|
"fcm_dpo/beta": 0.09055890142917633,
|
||
|
|
"fcm_dpo/delta": -0.27352195978164673,
|
||
|
|
"fcm_dpo/margin": 12.153611183166504,
|
||
|
|
"fcm_dpo/q_t": 0.2983909249305725,
|
||
|
|
"grad_norm": 43.50236129760742,
|
||
|
|
"learning_rate": 4.999176576834721e-07,
|
||
|
|
"logits/chosen": -0.6517193913459778,
|
||
|
|
"logits/rejected": -0.6220844388008118,
|
||
|
|
"logps/chosen": -63.3004264831543,
|
||
|
|
"logps/ref_chosen": -59.86931228637695,
|
||
|
|
"logps/ref_rejected": -98.05613708496094,
|
||
|
|
"logps/rejected": -113.6408462524414,
|
||
|
|
"loss": 0.8066,
|
||
|
|
"margin_dpo/margin_mean": 12.153611183166504,
|
||
|
|
"margin_dpo/margin_std": 13.398017883300781,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11747430249632893,
|
||
|
|
"fcm_dpo/beta": 0.07161784917116165,
|
||
|
|
"fcm_dpo/delta": -0.15674835443496704,
|
||
|
|
"fcm_dpo/margin": 13.832998275756836,
|
||
|
|
"fcm_dpo/q_t": 0.3114686608314514,
|
||
|
|
"grad_norm": 41.86962890625,
|
||
|
|
"learning_rate": 4.996706849759452e-07,
|
||
|
|
"logits/chosen": -0.6776489615440369,
|
||
|
|
"logits/rejected": -0.6389704942703247,
|
||
|
|
"logps/chosen": -60.72328567504883,
|
||
|
|
"logps/ref_chosen": -56.18925857543945,
|
||
|
|
"logps/ref_rejected": -86.42393493652344,
|
||
|
|
"logps/rejected": -104.79096984863281,
|
||
|
|
"loss": 0.834,
|
||
|
|
"margin_dpo/margin_mean": 13.832998275756836,
|
||
|
|
"margin_dpo/margin_std": 14.778533935546875,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12481644640234948,
|
||
|
|
"fcm_dpo/beta": 0.06048274785280228,
|
||
|
|
"fcm_dpo/delta": -0.15613125264644623,
|
||
|
|
"fcm_dpo/margin": 16.303586959838867,
|
||
|
|
"fcm_dpo/q_t": 0.3134708106517792,
|
||
|
|
"grad_norm": 42.009918212890625,
|
||
|
|
"learning_rate": 4.992592445678582e-07,
|
||
|
|
"logits/chosen": -0.6713468432426453,
|
||
|
|
"logits/rejected": -0.6383522748947144,
|
||
|
|
"logps/chosen": -66.5967788696289,
|
||
|
|
"logps/ref_chosen": -60.018287658691406,
|
||
|
|
"logps/ref_rejected": -98.01185607910156,
|
||
|
|
"logps/rejected": -120.89393615722656,
|
||
|
|
"loss": 0.8553,
|
||
|
|
"margin_dpo/margin_mean": 16.303586959838867,
|
||
|
|
"margin_dpo/margin_std": 18.325000762939453,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13215859030837004,
|
||
|
|
"fcm_dpo/beta": 0.055021800100803375,
|
||
|
|
"fcm_dpo/delta": -0.09340845793485641,
|
||
|
|
"fcm_dpo/margin": 16.962265014648438,
|
||
|
|
"fcm_dpo/q_t": 0.3316206634044647,
|
||
|
|
"grad_norm": 42.72884750366211,
|
||
|
|
"learning_rate": 4.986836074908615e-07,
|
||
|
|
"logits/chosen": -0.6945492625236511,
|
||
|
|
"logits/rejected": -0.6738761067390442,
|
||
|
|
"logps/chosen": -69.03782653808594,
|
||
|
|
"logps/ref_chosen": -59.8709831237793,
|
||
|
|
"logps/ref_rejected": -96.78519439697266,
|
||
|
|
"logps/rejected": -122.91429138183594,
|
||
|
|
"loss": 0.9273,
|
||
|
|
"margin_dpo/margin_mean": 16.962265014648438,
|
||
|
|
"margin_dpo/margin_std": 22.208675384521484,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1395007342143906,
|
||
|
|
"fcm_dpo/beta": 0.049969423562288284,
|
||
|
|
"fcm_dpo/delta": -0.06735582649707794,
|
||
|
|
"fcm_dpo/margin": 18.21504783630371,
|
||
|
|
"fcm_dpo/q_t": 0.3252207636833191,
|
||
|
|
"grad_norm": 34.471012115478516,
|
||
|
|
"learning_rate": 4.979441529392784e-07,
|
||
|
|
"logits/chosen": -0.709299623966217,
|
||
|
|
"logits/rejected": -0.6731777787208557,
|
||
|
|
"logps/chosen": -64.95755004882812,
|
||
|
|
"logps/ref_chosen": -55.94385528564453,
|
||
|
|
"logps/ref_rejected": -83.6790542602539,
|
||
|
|
"logps/rejected": -110.90779876708984,
|
||
|
|
"loss": 0.8711,
|
||
|
|
"margin_dpo/margin_mean": 18.215045928955078,
|
||
|
|
"margin_dpo/margin_std": 20.571401596069336,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.14684287812041116,
|
||
|
|
"fcm_dpo/beta": 0.04660476744174957,
|
||
|
|
"fcm_dpo/delta": -0.0532207265496254,
|
||
|
|
"fcm_dpo/margin": 19.219341278076172,
|
||
|
|
"fcm_dpo/q_t": 0.32488057017326355,
|
||
|
|
"grad_norm": 40.7348518371582,
|
||
|
|
"learning_rate": 4.970413680203148e-07,
|
||
|
|
"logits/chosen": -0.670303225517273,
|
||
|
|
"logits/rejected": -0.6368999481201172,
|
||
|
|
"logps/chosen": -66.36493682861328,
|
||
|
|
"logps/ref_chosen": -57.05888748168945,
|
||
|
|
"logps/ref_rejected": -86.11727142333984,
|
||
|
|
"logps/rejected": -114.64266204833984,
|
||
|
|
"loss": 0.8736,
|
||
|
|
"margin_dpo/margin_mean": 19.219341278076172,
|
||
|
|
"margin_dpo/margin_std": 21.64118194580078,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15418502202643172,
|
||
|
|
"fcm_dpo/beta": 0.04551302269101143,
|
||
|
|
"fcm_dpo/delta": -0.15303857624530792,
|
||
|
|
"fcm_dpo/margin": 21.88186264038086,
|
||
|
|
"fcm_dpo/q_t": 0.315370112657547,
|
||
|
|
"grad_norm": 30.820974349975586,
|
||
|
|
"learning_rate": 4.959758474331832e-07,
|
||
|
|
"logits/chosen": -0.7173506021499634,
|
||
|
|
"logits/rejected": -0.6785070896148682,
|
||
|
|
"logps/chosen": -70.07099914550781,
|
||
|
|
"logps/ref_chosen": -59.20774459838867,
|
||
|
|
"logps/ref_rejected": -86.49754333496094,
|
||
|
|
"logps/rejected": -119.24266052246094,
|
||
|
|
"loss": 0.8519,
|
||
|
|
"margin_dpo/margin_mean": 21.88186264038086,
|
||
|
|
"margin_dpo/margin_std": 24.000478744506836,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16152716593245228,
|
||
|
|
"fcm_dpo/beta": 0.03847113624215126,
|
||
|
|
"fcm_dpo/delta": -0.08476176112890244,
|
||
|
|
"fcm_dpo/margin": 24.109981536865234,
|
||
|
|
"fcm_dpo/q_t": 0.3201691508293152,
|
||
|
|
"grad_norm": 30.027286529541016,
|
||
|
|
"learning_rate": 4.947482930773511e-07,
|
||
|
|
"logits/chosen": -0.6687729954719543,
|
||
|
|
"logits/rejected": -0.6251232624053955,
|
||
|
|
"logps/chosen": -72.03107452392578,
|
||
|
|
"logps/ref_chosen": -60.437957763671875,
|
||
|
|
"logps/ref_rejected": -90.83917999267578,
|
||
|
|
"logps/rejected": -126.54228210449219,
|
||
|
|
"loss": 0.8568,
|
||
|
|
"margin_dpo/margin_mean": 24.109981536865234,
|
||
|
|
"margin_dpo/margin_std": 25.9851016998291,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16886930983847284,
|
||
|
|
"fcm_dpo/beta": 0.03624765947461128,
|
||
|
|
"fcm_dpo/delta": -0.01841454766690731,
|
||
|
|
"fcm_dpo/margin": 23.8579044342041,
|
||
|
|
"fcm_dpo/q_t": 0.33425024151802063,
|
||
|
|
"grad_norm": 42.781307220458984,
|
||
|
|
"learning_rate": 4.933595135901732e-07,
|
||
|
|
"logits/chosen": -0.6857893466949463,
|
||
|
|
"logits/rejected": -0.6449792385101318,
|
||
|
|
"logps/chosen": -76.1856918334961,
|
||
|
|
"logps/ref_chosen": -61.7908821105957,
|
||
|
|
"logps/ref_rejected": -85.36943054199219,
|
||
|
|
"logps/rejected": -123.62214660644531,
|
||
|
|
"loss": 0.903,
|
||
|
|
"margin_dpo/margin_mean": 23.8579044342041,
|
||
|
|
"margin_dpo/margin_std": 28.585891723632812,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1762114537444934,
|
||
|
|
"fcm_dpo/beta": 0.0364445336163044,
|
||
|
|
"fcm_dpo/delta": -0.015237887389957905,
|
||
|
|
"fcm_dpo/margin": 23.689233779907227,
|
||
|
|
"fcm_dpo/q_t": 0.33086472749710083,
|
||
|
|
"grad_norm": 33.90849304199219,
|
||
|
|
"learning_rate": 4.918104238142103e-07,
|
||
|
|
"logits/chosen": -0.6950192451477051,
|
||
|
|
"logits/rejected": -0.6637409925460815,
|
||
|
|
"logps/chosen": -81.74562072753906,
|
||
|
|
"logps/ref_chosen": -65.3261489868164,
|
||
|
|
"logps/ref_rejected": -86.75518798828125,
|
||
|
|
"logps/rejected": -126.8638916015625,
|
||
|
|
"loss": 0.8772,
|
||
|
|
"margin_dpo/margin_mean": 23.68923568725586,
|
||
|
|
"margin_dpo/margin_std": 26.572423934936523,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18355359765051396,
|
||
|
|
"fcm_dpo/beta": 0.03069928288459778,
|
||
|
|
"fcm_dpo/delta": -0.22570106387138367,
|
||
|
|
"fcm_dpo/margin": 34.196529388427734,
|
||
|
|
"fcm_dpo/q_t": 0.3057743012905121,
|
||
|
|
"grad_norm": 28.747299194335938,
|
||
|
|
"learning_rate": 4.90102044194588e-07,
|
||
|
|
"logits/chosen": -0.6552901268005371,
|
||
|
|
"logits/rejected": -0.645017147064209,
|
||
|
|
"logps/chosen": -76.18916320800781,
|
||
|
|
"logps/ref_chosen": -58.323204040527344,
|
||
|
|
"logps/ref_rejected": -101.2106704711914,
|
||
|
|
"logps/rejected": -153.27313232421875,
|
||
|
|
"loss": 0.8245,
|
||
|
|
"margin_dpo/margin_mean": 34.196529388427734,
|
||
|
|
"margin_dpo/margin_std": 37.522029876708984,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19089574155653452,
|
||
|
|
"fcm_dpo/beta": 0.02867107465863228,
|
||
|
|
"fcm_dpo/delta": -0.030061205849051476,
|
||
|
|
"fcm_dpo/margin": 30.552875518798828,
|
||
|
|
"fcm_dpo/q_t": 0.33143970370292664,
|
||
|
|
"grad_norm": 29.514667510986328,
|
||
|
|
"learning_rate": 4.882355001067891e-07,
|
||
|
|
"logits/chosen": -0.6441887617111206,
|
||
|
|
"logits/rejected": -0.6197634935379028,
|
||
|
|
"logps/chosen": -75.02465057373047,
|
||
|
|
"logps/ref_chosen": -56.38518524169922,
|
||
|
|
"logps/ref_rejected": -86.15767669677734,
|
||
|
|
"logps/rejected": -135.35000610351562,
|
||
|
|
"loss": 0.9009,
|
||
|
|
"margin_dpo/margin_mean": 30.552875518798828,
|
||
|
|
"margin_dpo/margin_std": 35.57330322265625,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19823788546255505,
|
||
|
|
"fcm_dpo/beta": 0.02630027011036873,
|
||
|
|
"fcm_dpo/delta": -0.08074788004159927,
|
||
|
|
"fcm_dpo/margin": 35.100746154785156,
|
||
|
|
"fcm_dpo/q_t": 0.3213648200035095,
|
||
|
|
"grad_norm": 30.23385238647461,
|
||
|
|
"learning_rate": 4.862120211153265e-07,
|
||
|
|
"logits/chosen": -0.6420992612838745,
|
||
|
|
"logits/rejected": -0.6323195695877075,
|
||
|
|
"logps/chosen": -74.02909851074219,
|
||
|
|
"logps/ref_chosen": -54.59065628051758,
|
||
|
|
"logps/ref_rejected": -95.26080322265625,
|
||
|
|
"logps/rejected": -149.7999725341797,
|
||
|
|
"loss": 0.8502,
|
||
|
|
"margin_dpo/margin_mean": 35.100746154785156,
|
||
|
|
"margin_dpo/margin_std": 38.36650848388672,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2055800293685756,
|
||
|
|
"fcm_dpo/beta": 0.024506056681275368,
|
||
|
|
"fcm_dpo/delta": -0.07339946180582047,
|
||
|
|
"fcm_dpo/margin": 37.39527130126953,
|
||
|
|
"fcm_dpo/q_t": 0.3280564844608307,
|
||
|
|
"grad_norm": 29.89729881286621,
|
||
|
|
"learning_rate": 4.840329401637809e-07,
|
||
|
|
"logits/chosen": -0.6269393563270569,
|
||
|
|
"logits/rejected": -0.6138561367988586,
|
||
|
|
"logps/chosen": -81.1108169555664,
|
||
|
|
"logps/ref_chosen": -56.04347610473633,
|
||
|
|
"logps/ref_rejected": -93.27880859375,
|
||
|
|
"logps/rejected": -155.74142456054688,
|
||
|
|
"loss": 0.8815,
|
||
|
|
"margin_dpo/margin_mean": 37.39527130126953,
|
||
|
|
"margin_dpo/margin_std": 43.980499267578125,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21292217327459617,
|
||
|
|
"fcm_dpo/beta": 0.023285897448658943,
|
||
|
|
"fcm_dpo/delta": -0.010504422709345818,
|
||
|
|
"fcm_dpo/margin": 36.870033264160156,
|
||
|
|
"fcm_dpo/q_t": 0.3340276777744293,
|
||
|
|
"grad_norm": 34.29755401611328,
|
||
|
|
"learning_rate": 4.816996926967401e-07,
|
||
|
|
"logits/chosen": -0.5899958610534668,
|
||
|
|
"logits/rejected": -0.5646438598632812,
|
||
|
|
"logps/chosen": -89.64959716796875,
|
||
|
|
"logps/ref_chosen": -61.4414176940918,
|
||
|
|
"logps/ref_rejected": -86.32813262939453,
|
||
|
|
"logps/rejected": -151.40634155273438,
|
||
|
|
"loss": 0.9115,
|
||
|
|
"margin_dpo/margin_mean": 36.870033264160156,
|
||
|
|
"margin_dpo/margin_std": 44.208961486816406,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22026431718061673,
|
||
|
|
"fcm_dpo/beta": 0.02298033982515335,
|
||
|
|
"fcm_dpo/delta": -0.04273203760385513,
|
||
|
|
"fcm_dpo/margin": 38.58696746826172,
|
||
|
|
"fcm_dpo/q_t": 0.33299776911735535,
|
||
|
|
"grad_norm": 31.706588745117188,
|
||
|
|
"learning_rate": 4.792138157142157e-07,
|
||
|
|
"logits/chosen": -0.5931132435798645,
|
||
|
|
"logits/rejected": -0.5684978365898132,
|
||
|
|
"logps/chosen": -89.40864562988281,
|
||
|
|
"logps/ref_chosen": -57.70451736450195,
|
||
|
|
"logps/ref_rejected": -87.76991271972656,
|
||
|
|
"logps/rejected": -158.06101989746094,
|
||
|
|
"loss": 0.9086,
|
||
|
|
"margin_dpo/margin_mean": 38.58696746826172,
|
||
|
|
"margin_dpo/margin_std": 47.16357421875,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2276064610866373,
|
||
|
|
"fcm_dpo/beta": 0.02123543620109558,
|
||
|
|
"fcm_dpo/delta": -0.0736246332526207,
|
||
|
|
"fcm_dpo/margin": 43.11958694458008,
|
||
|
|
"fcm_dpo/q_t": 0.32444706559181213,
|
||
|
|
"grad_norm": 28.768526077270508,
|
||
|
|
"learning_rate": 4.7657694675916247e-07,
|
||
|
|
"logits/chosen": -0.6274883151054382,
|
||
|
|
"logits/rejected": -0.6081691980361938,
|
||
|
|
"logps/chosen": -90.71160888671875,
|
||
|
|
"logps/ref_chosen": -62.08925247192383,
|
||
|
|
"logps/ref_rejected": -94.79930114746094,
|
||
|
|
"logps/rejected": -166.54122924804688,
|
||
|
|
"loss": 0.8772,
|
||
|
|
"margin_dpo/margin_mean": 43.11958694458008,
|
||
|
|
"margin_dpo/margin_std": 49.604820251464844,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23494860499265785,
|
||
|
|
"fcm_dpo/beta": 0.020709924399852753,
|
||
|
|
"fcm_dpo/delta": -0.028781410306692123,
|
||
|
|
"fcm_dpo/margin": 42.2672004699707,
|
||
|
|
"fcm_dpo/q_t": 0.3394644856452942,
|
||
|
|
"grad_norm": 35.6558837890625,
|
||
|
|
"learning_rate": 4.737908228387656e-07,
|
||
|
|
"logits/chosen": -0.6209047436714172,
|
||
|
|
"logits/rejected": -0.602047324180603,
|
||
|
|
"logps/chosen": -102.9297866821289,
|
||
|
|
"logps/ref_chosen": -67.15288543701172,
|
||
|
|
"logps/ref_rejected": -96.92537689208984,
|
||
|
|
"logps/rejected": -174.969482421875,
|
||
|
|
"loss": 0.95,
|
||
|
|
"margin_dpo/margin_mean": 42.26719665527344,
|
||
|
|
"margin_dpo/margin_std": 56.8797607421875,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2422907488986784,
|
||
|
|
"fcm_dpo/beta": 0.019984444603323936,
|
||
|
|
"fcm_dpo/delta": 0.019107891246676445,
|
||
|
|
"fcm_dpo/margin": 41.58051300048828,
|
||
|
|
"fcm_dpo/q_t": 0.33927860856056213,
|
||
|
|
"grad_norm": 44.44900894165039,
|
||
|
|
"learning_rate": 4.708572792802069e-07,
|
||
|
|
"logits/chosen": -0.5925561189651489,
|
||
|
|
"logits/rejected": -0.5611370801925659,
|
||
|
|
"logps/chosen": -88.99661254882812,
|
||
|
|
"logps/ref_chosen": -57.40401077270508,
|
||
|
|
"logps/ref_rejected": -80.31498718261719,
|
||
|
|
"logps/rejected": -153.48812866210938,
|
||
|
|
"loss": 0.9112,
|
||
|
|
"margin_dpo/margin_mean": 41.58051681518555,
|
||
|
|
"margin_dpo/margin_std": 49.86974334716797,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.24963289280469897,
|
||
|
|
"fcm_dpo/beta": 0.018490394577383995,
|
||
|
|
"fcm_dpo/delta": -0.12243499606847763,
|
||
|
|
"fcm_dpo/margin": 51.66387939453125,
|
||
|
|
"fcm_dpo/q_t": 0.3191189765930176,
|
||
|
|
"grad_norm": 27.767526626586914,
|
||
|
|
"learning_rate": 4.6777824852166437e-07,
|
||
|
|
"logits/chosen": -0.5405124425888062,
|
||
|
|
"logits/rejected": -0.528901994228363,
|
||
|
|
"logps/chosen": -85.2691650390625,
|
||
|
|
"logps/ref_chosen": -52.029144287109375,
|
||
|
|
"logps/ref_rejected": -85.73944091796875,
|
||
|
|
"logps/rejected": -170.64334106445312,
|
||
|
|
"loss": 0.8528,
|
||
|
|
"margin_dpo/margin_mean": 51.66387939453125,
|
||
|
|
"margin_dpo/margin_std": 58.07769775390625,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25697503671071953,
|
||
|
|
"fcm_dpo/beta": 0.01827237568795681,
|
||
|
|
"fcm_dpo/delta": -0.062149811536073685,
|
||
|
|
"fcm_dpo/margin": 49.67750549316406,
|
||
|
|
"fcm_dpo/q_t": 0.32835322618484497,
|
||
|
|
"grad_norm": 37.51227569580078,
|
||
|
|
"learning_rate": 4.645557588393406e-07,
|
||
|
|
"logits/chosen": -0.5422641038894653,
|
||
|
|
"logits/rejected": -0.5189716815948486,
|
||
|
|
"logps/chosen": -106.3938217163086,
|
||
|
|
"logps/ref_chosen": -62.996971130371094,
|
||
|
|
"logps/ref_rejected": -92.98394012451172,
|
||
|
|
"logps/rejected": -186.0583038330078,
|
||
|
|
"loss": 0.8859,
|
||
|
|
"margin_dpo/margin_mean": 49.67750549316406,
|
||
|
|
"margin_dpo/margin_std": 58.46363067626953,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2643171806167401,
|
||
|
|
"fcm_dpo/beta": 0.01582510769367218,
|
||
|
|
"fcm_dpo/delta": -0.1366010159254074,
|
||
|
|
"fcm_dpo/margin": 61.5815315246582,
|
||
|
|
"fcm_dpo/q_t": 0.31790629029273987,
|
||
|
|
"grad_norm": 27.392812728881836,
|
||
|
|
"learning_rate": 4.611919330113591e-07,
|
||
|
|
"logits/chosen": -0.485221803188324,
|
||
|
|
"logits/rejected": -0.48061603307724,
|
||
|
|
"logps/chosen": -105.83013916015625,
|
||
|
|
"logps/ref_chosen": -57.0670280456543,
|
||
|
|
"logps/ref_rejected": -97.1115493774414,
|
||
|
|
"logps/rejected": -207.4561767578125,
|
||
|
|
"loss": 0.8569,
|
||
|
|
"margin_dpo/margin_mean": 61.5815315246582,
|
||
|
|
"margin_dpo/margin_std": 70.3671646118164,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.27165932452276065,
|
||
|
|
"fcm_dpo/beta": 0.015623442828655243,
|
||
|
|
"fcm_dpo/delta": 0.13391907513141632,
|
||
|
|
"fcm_dpo/margin": 46.54853057861328,
|
||
|
|
"fcm_dpo/q_t": 0.3591853976249695,
|
||
|
|
"grad_norm": 36.289039611816406,
|
||
|
|
"learning_rate": 4.5768898691940836e-07,
|
||
|
|
"logits/chosen": -0.5313425064086914,
|
||
|
|
"logits/rejected": -0.5060660243034363,
|
||
|
|
"logps/chosen": -97.86585998535156,
|
||
|
|
"logps/ref_chosen": -54.840736389160156,
|
||
|
|
"logps/ref_rejected": -75.51002502441406,
|
||
|
|
"logps/rejected": -165.0836944580078,
|
||
|
|
"loss": 0.9783,
|
||
|
|
"margin_dpo/margin_mean": 46.54853057861328,
|
||
|
|
"margin_dpo/margin_std": 60.472984313964844,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2790014684287812,
|
||
|
|
"fcm_dpo/beta": 0.01625588908791542,
|
||
|
|
"fcm_dpo/delta": -0.08447099477052689,
|
||
|
|
"fcm_dpo/margin": 56.91190719604492,
|
||
|
|
"fcm_dpo/q_t": 0.3247375786304474,
|
||
|
|
"grad_norm": 40.74492645263672,
|
||
|
|
"learning_rate": 4.5404922808905543e-07,
|
||
|
|
"logits/chosen": -0.5222185254096985,
|
||
|
|
"logits/rejected": -0.49877819418907166,
|
||
|
|
"logps/chosen": -102.94236755371094,
|
||
|
|
"logps/ref_chosen": -57.72148895263672,
|
||
|
|
"logps/ref_rejected": -86.85997009277344,
|
||
|
|
"logps/rejected": -188.99276733398438,
|
||
|
|
"loss": 0.8786,
|
||
|
|
"margin_dpo/margin_mean": 56.91190719604492,
|
||
|
|
"margin_dpo/margin_std": 66.29203033447266,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.28634361233480177,
|
||
|
|
"fcm_dpo/beta": 0.014897446148097515,
|
||
|
|
"fcm_dpo/delta": -0.034334540367126465,
|
||
|
|
"fcm_dpo/margin": 58.907752990722656,
|
||
|
|
"fcm_dpo/q_t": 0.32934439182281494,
|
||
|
|
"grad_norm": 33.52628707885742,
|
||
|
|
"learning_rate": 4.5027505416968985e-07,
|
||
|
|
"logits/chosen": -0.4740094542503357,
|
||
|
|
"logits/rejected": -0.4610044062137604,
|
||
|
|
"logps/chosen": -110.0854263305664,
|
||
|
|
"logps/ref_chosen": -58.26164627075195,
|
||
|
|
"logps/ref_rejected": -89.46485900878906,
|
||
|
|
"logps/rejected": -200.19639587402344,
|
||
|
|
"loss": 0.8883,
|
||
|
|
"margin_dpo/margin_mean": 58.907752990722656,
|
||
|
|
"margin_dpo/margin_std": 67.69029235839844,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2936857562408223,
|
||
|
|
"fcm_dpo/beta": 0.014423737302422523,
|
||
|
|
"fcm_dpo/delta": -0.07692910730838776,
|
||
|
|
"fcm_dpo/margin": 63.713844299316406,
|
||
|
|
"fcm_dpo/q_t": 0.3252788186073303,
|
||
|
|
"grad_norm": 31.517562866210938,
|
||
|
|
"learning_rate": 4.4636895135509966e-07,
|
||
|
|
"logits/chosen": -0.46841588616371155,
|
||
|
|
"logits/rejected": -0.4528680443763733,
|
||
|
|
"logps/chosen": -105.78775787353516,
|
||
|
|
"logps/ref_chosen": -55.71953201293945,
|
||
|
|
"logps/ref_rejected": -83.15235137939453,
|
||
|
|
"logps/rejected": -196.93441772460938,
|
||
|
|
"loss": 0.8697,
|
||
|
|
"margin_dpo/margin_mean": 63.713844299316406,
|
||
|
|
"margin_dpo/margin_std": 73.30004119873047,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2936857562408223,
|
||
|
|
"eval_fcm_dpo/beta": 0.019768698140978813,
|
||
|
|
"eval_fcm_dpo/delta": 0.06340710818767548,
|
||
|
|
"eval_fcm_dpo/margin": 38.271907806396484,
|
||
|
|
"eval_fcm_dpo/q_t": 0.37947365641593933,
|
||
|
|
"eval_logits/chosen": -0.49050527811050415,
|
||
|
|
"eval_logits/rejected": -0.47104698419570923,
|
||
|
|
"eval_logps/chosen": -148.67208862304688,
|
||
|
|
"eval_logps/ref_chosen": -79.05104064941406,
|
||
|
|
"eval_logps/ref_rejected": -86.79793548583984,
|
||
|
|
"eval_logps/rejected": -194.69088745117188,
|
||
|
|
"eval_loss": 0.5858080387115479,
|
||
|
|
"eval_margin_dpo/margin_mean": 38.271907806396484,
|
||
|
|
"eval_margin_dpo/margin_std": 74.73805236816406,
|
||
|
|
"eval_runtime": 39.5337,
|
||
|
|
"eval_samples_per_second": 59.165,
|
||
|
|
"eval_steps_per_second": 1.872,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3010279001468429,
|
||
|
|
"fcm_dpo/beta": 0.018562760204076767,
|
||
|
|
"fcm_dpo/delta": -0.2798296809196472,
|
||
|
|
"fcm_dpo/margin": 58.922264099121094,
|
||
|
|
"fcm_dpo/q_t": 0.30979180335998535,
|
||
|
|
"grad_norm": 39.298316955566406,
|
||
|
|
"learning_rate": 4.4233349274571974e-07,
|
||
|
|
"logits/chosen": -0.4523549973964691,
|
||
|
|
"logits/rejected": -0.43011409044265747,
|
||
|
|
"logps/chosen": -121.86454772949219,
|
||
|
|
"logps/ref_chosen": -65.13258361816406,
|
||
|
|
"logps/ref_rejected": -92.10203552246094,
|
||
|
|
"logps/rejected": -207.75625610351562,
|
||
|
|
"loss": 0.8772,
|
||
|
|
"margin_dpo/margin_mean": 58.9222526550293,
|
||
|
|
"margin_dpo/margin_std": 72.63723754882812,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.30837004405286345,
|
||
|
|
"fcm_dpo/beta": 0.014536149799823761,
|
||
|
|
"fcm_dpo/delta": -0.1799614131450653,
|
||
|
|
"fcm_dpo/margin": 69.11328125,
|
||
|
|
"fcm_dpo/q_t": 0.31058698892593384,
|
||
|
|
"grad_norm": 31.66854476928711,
|
||
|
|
"learning_rate": 4.381713366536311e-07,
|
||
|
|
"logits/chosen": -0.4480575919151306,
|
||
|
|
"logits/rejected": -0.43101295828819275,
|
||
|
|
"logps/chosen": -109.48213195800781,
|
||
|
|
"logps/ref_chosen": -54.52837371826172,
|
||
|
|
"logps/ref_rejected": -87.06227111816406,
|
||
|
|
"logps/rejected": -211.12930297851562,
|
||
|
|
"loss": 0.8362,
|
||
|
|
"margin_dpo/margin_mean": 69.11328125,
|
||
|
|
"margin_dpo/margin_std": 75.44652557373047,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.315712187958884,
|
||
|
|
"fcm_dpo/beta": 0.013478484936058521,
|
||
|
|
"fcm_dpo/delta": -0.06460683792829514,
|
||
|
|
"fcm_dpo/margin": 67.31108093261719,
|
||
|
|
"fcm_dpo/q_t": 0.3255953788757324,
|
||
|
|
"grad_norm": 30.947105407714844,
|
||
|
|
"learning_rate": 4.3388522485142885e-07,
|
||
|
|
"logits/chosen": -0.47888296842575073,
|
||
|
|
"logits/rejected": -0.4555312693119049,
|
||
|
|
"logps/chosen": -116.98878479003906,
|
||
|
|
"logps/ref_chosen": -59.905250549316406,
|
||
|
|
"logps/ref_rejected": -90.25511932373047,
|
||
|
|
"logps/rejected": -214.6497344970703,
|
||
|
|
"loss": 0.8705,
|
||
|
|
"margin_dpo/margin_mean": 67.31107330322266,
|
||
|
|
"margin_dpo/margin_std": 74.4868392944336,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32305433186490456,
|
||
|
|
"fcm_dpo/beta": 0.012834814377129078,
|
||
|
|
"fcm_dpo/delta": 0.019026797264814377,
|
||
|
|
"fcm_dpo/margin": 64.8343505859375,
|
||
|
|
"fcm_dpo/q_t": 0.3348235487937927,
|
||
|
|
"grad_norm": 32.80592346191406,
|
||
|
|
"learning_rate": 4.2947798076611047e-07,
|
||
|
|
"logits/chosen": -0.4836200177669525,
|
||
|
|
"logits/rejected": -0.4622252583503723,
|
||
|
|
"logps/chosen": -118.51663970947266,
|
||
|
|
"logps/ref_chosen": -57.68498611450195,
|
||
|
|
"logps/ref_rejected": -87.72792053222656,
|
||
|
|
"logps/rejected": -213.3939208984375,
|
||
|
|
"loss": 0.89,
|
||
|
|
"margin_dpo/margin_mean": 64.8343505859375,
|
||
|
|
"margin_dpo/margin_std": 71.27854919433594,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3303964757709251,
|
||
|
|
"fcm_dpo/beta": 0.012188979424536228,
|
||
|
|
"fcm_dpo/delta": -0.16104714572429657,
|
||
|
|
"fcm_dpo/margin": 81.43888854980469,
|
||
|
|
"fcm_dpo/q_t": 0.30945101380348206,
|
||
|
|
"grad_norm": 35.1022834777832,
|
||
|
|
"learning_rate": 4.249525076191759e-07,
|
||
|
|
"logits/chosen": -0.40194278955459595,
|
||
|
|
"logits/rejected": -0.3876434564590454,
|
||
|
|
"logps/chosen": -134.11956787109375,
|
||
|
|
"logps/ref_chosen": -54.47245407104492,
|
||
|
|
"logps/ref_rejected": -93.26266479492188,
|
||
|
|
"logps/rejected": -254.3486785888672,
|
||
|
|
"loss": 0.8406,
|
||
|
|
"margin_dpo/margin_mean": 81.43888854980469,
|
||
|
|
"margin_dpo/margin_std": 86.77934265136719,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3377386196769457,
|
||
|
|
"fcm_dpo/beta": 0.01108027994632721,
|
||
|
|
"fcm_dpo/delta": -0.06726758182048798,
|
||
|
|
"fcm_dpo/margin": 82.1200942993164,
|
||
|
|
"fcm_dpo/q_t": 0.3215538561344147,
|
||
|
|
"grad_norm": 34.77726364135742,
|
||
|
|
"learning_rate": 4.203117865141635e-07,
|
||
|
|
"logits/chosen": -0.3820115923881531,
|
||
|
|
"logits/rejected": -0.35805052518844604,
|
||
|
|
"logps/chosen": -139.94625854492188,
|
||
|
|
"logps/ref_chosen": -58.7701301574707,
|
||
|
|
"logps/ref_rejected": -87.85963439941406,
|
||
|
|
"logps/rejected": -251.1558380126953,
|
||
|
|
"loss": 0.865,
|
||
|
|
"margin_dpo/margin_mean": 82.1200942993164,
|
||
|
|
"margin_dpo/margin_std": 89.67250061035156,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34508076358296624,
|
||
|
|
"fcm_dpo/beta": 0.010919634252786636,
|
||
|
|
"fcm_dpo/delta": 0.039671458303928375,
|
||
|
|
"fcm_dpo/margin": 74.35450744628906,
|
||
|
|
"fcm_dpo/q_t": 0.34256863594055176,
|
||
|
|
"grad_norm": 32.15520095825195,
|
||
|
|
"learning_rate": 4.1555887447288255e-07,
|
||
|
|
"logits/chosen": -0.4005228877067566,
|
||
|
|
"logits/rejected": -0.3855542838573456,
|
||
|
|
"logps/chosen": -133.8794403076172,
|
||
|
|
"logps/ref_chosen": -59.0481071472168,
|
||
|
|
"logps/ref_rejected": -91.19654846191406,
|
||
|
|
"logps/rejected": -240.3823699951172,
|
||
|
|
"loss": 0.9162,
|
||
|
|
"margin_dpo/margin_mean": 74.35450744628906,
|
||
|
|
"margin_dpo/margin_std": 88.09162902832031,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3524229074889868,
|
||
|
|
"fcm_dpo/beta": 0.010825184173882008,
|
||
|
|
"fcm_dpo/delta": -0.018980540335178375,
|
||
|
|
"fcm_dpo/margin": 79.97639465332031,
|
||
|
|
"fcm_dpo/q_t": 0.3274964690208435,
|
||
|
|
"grad_norm": 40.340721130371094,
|
||
|
|
"learning_rate": 4.106969024216348e-07,
|
||
|
|
"logits/chosen": -0.3556009829044342,
|
||
|
|
"logits/rejected": -0.34894007444381714,
|
||
|
|
"logps/chosen": -138.43865966796875,
|
||
|
|
"logps/ref_chosen": -55.238983154296875,
|
||
|
|
"logps/ref_rejected": -91.08428955078125,
|
||
|
|
"logps/rejected": -254.26034545898438,
|
||
|
|
"loss": 0.8677,
|
||
|
|
"margin_dpo/margin_mean": 79.97639465332031,
|
||
|
|
"margin_dpo/margin_std": 84.8770523071289,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.35976505139500736,
|
||
|
|
"fcm_dpo/beta": 0.010110095143318176,
|
||
|
|
"fcm_dpo/delta": -0.10127156972885132,
|
||
|
|
"fcm_dpo/margin": 92.84043884277344,
|
||
|
|
"fcm_dpo/q_t": 0.3163699507713318,
|
||
|
|
"grad_norm": 42.48567581176758,
|
||
|
|
"learning_rate": 4.057290731287531e-07,
|
||
|
|
"logits/chosen": -0.35475802421569824,
|
||
|
|
"logits/rejected": -0.3244848847389221,
|
||
|
|
"logps/chosen": -158.22030639648438,
|
||
|
|
"logps/ref_chosen": -65.08844757080078,
|
||
|
|
"logps/ref_rejected": -86.05777740478516,
|
||
|
|
"logps/rejected": -272.03009033203125,
|
||
|
|
"loss": 0.8333,
|
||
|
|
"margin_dpo/margin_mean": 92.8404312133789,
|
||
|
|
"margin_dpo/margin_std": 95.6347885131836,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3671071953010279,
|
||
|
|
"fcm_dpo/beta": 0.00962417759001255,
|
||
|
|
"fcm_dpo/delta": -0.020193621516227722,
|
||
|
|
"fcm_dpo/margin": 90.06238555908203,
|
||
|
|
"fcm_dpo/q_t": 0.3266645073890686,
|
||
|
|
"grad_norm": 54.82107162475586,
|
||
|
|
"learning_rate": 4.006586590948141e-07,
|
||
|
|
"logits/chosen": -0.37928658723831177,
|
||
|
|
"logits/rejected": -0.35434064269065857,
|
||
|
|
"logps/chosen": -154.8458709716797,
|
||
|
|
"logps/ref_chosen": -59.08491897583008,
|
||
|
|
"logps/ref_rejected": -87.36727142333984,
|
||
|
|
"logps/rejected": -273.19061279296875,
|
||
|
|
"loss": 0.8678,
|
||
|
|
"margin_dpo/margin_mean": 90.06238555908203,
|
||
|
|
"margin_dpo/margin_std": 93.82579040527344,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3744493392070485,
|
||
|
|
"fcm_dpo/beta": 0.009607134386897087,
|
||
|
|
"fcm_dpo/delta": 0.0022653634659945965,
|
||
|
|
"fcm_dpo/margin": 88.13050842285156,
|
||
|
|
"fcm_dpo/q_t": 0.3329293131828308,
|
||
|
|
"grad_norm": 58.291961669921875,
|
||
|
|
"learning_rate": 3.954890003969163e-07,
|
||
|
|
"logits/chosen": -0.33103686571121216,
|
||
|
|
"logits/rejected": -0.3057294487953186,
|
||
|
|
"logps/chosen": -170.76805114746094,
|
||
|
|
"logps/ref_chosen": -61.85979461669922,
|
||
|
|
"logps/ref_rejected": -88.32804107666016,
|
||
|
|
"logps/rejected": -285.3667907714844,
|
||
|
|
"loss": 0.9102,
|
||
|
|
"margin_dpo/margin_mean": 88.13050842285156,
|
||
|
|
"margin_dpo/margin_std": 103.13516998291016,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38179148311306904,
|
||
|
|
"fcm_dpo/beta": 0.009414169006049633,
|
||
|
|
"fcm_dpo/delta": -0.016010303050279617,
|
||
|
|
"fcm_dpo/margin": 91.66337585449219,
|
||
|
|
"fcm_dpo/q_t": 0.33001503348350525,
|
||
|
|
"grad_norm": 39.26882553100586,
|
||
|
|
"learning_rate": 3.9022350248844246e-07,
|
||
|
|
"logits/chosen": -0.31099259853363037,
|
||
|
|
"logits/rejected": -0.3064902424812317,
|
||
|
|
"logps/chosen": -154.59693908691406,
|
||
|
|
"logps/ref_chosen": -52.843467712402344,
|
||
|
|
"logps/ref_rejected": -90.4744873046875,
|
||
|
|
"logps/rejected": -283.89129638671875,
|
||
|
|
"loss": 0.8885,
|
||
|
|
"margin_dpo/margin_mean": 91.66337585449219,
|
||
|
|
"margin_dpo/margin_std": 103.34080505371094,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3891336270190896,
|
||
|
|
"fcm_dpo/beta": 0.009381098672747612,
|
||
|
|
"fcm_dpo/delta": -0.015523952431976795,
|
||
|
|
"fcm_dpo/margin": 92.02482604980469,
|
||
|
|
"fcm_dpo/q_t": 0.3260762095451355,
|
||
|
|
"grad_norm": 42.47758865356445,
|
||
|
|
"learning_rate": 3.848656339557562e-07,
|
||
|
|
"logits/chosen": -0.299502432346344,
|
||
|
|
"logits/rejected": -0.27807632088661194,
|
||
|
|
"logps/chosen": -151.35202026367188,
|
||
|
|
"logps/ref_chosen": -59.35320281982422,
|
||
|
|
"logps/ref_rejected": -90.73350524902344,
|
||
|
|
"logps/rejected": -274.75714111328125,
|
||
|
|
"loss": 0.8593,
|
||
|
|
"margin_dpo/margin_mean": 92.02482604980469,
|
||
|
|
"margin_dpo/margin_std": 95.39722442626953,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3964757709251101,
|
||
|
|
"fcm_dpo/beta": 0.009062977507710457,
|
||
|
|
"fcm_dpo/delta": -0.06297405064105988,
|
||
|
|
"fcm_dpo/margin": 100.06938171386719,
|
||
|
|
"fcm_dpo/q_t": 0.32098034024238586,
|
||
|
|
"grad_norm": 51.16044616699219,
|
||
|
|
"learning_rate": 3.794189242333106e-07,
|
||
|
|
"logits/chosen": -0.23682311177253723,
|
||
|
|
"logits/rejected": -0.21265558898448944,
|
||
|
|
"logps/chosen": -184.3603057861328,
|
||
|
|
"logps/ref_chosen": -66.30875396728516,
|
||
|
|
"logps/ref_rejected": -95.4130630493164,
|
||
|
|
"logps/rejected": -313.5339660644531,
|
||
|
|
"loss": 0.8569,
|
||
|
|
"margin_dpo/margin_mean": 100.06938171386719,
|
||
|
|
"margin_dpo/margin_std": 107.111083984375,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.40381791483113066,
|
||
|
|
"fcm_dpo/beta": 0.008018719032406807,
|
||
|
|
"fcm_dpo/delta": -0.12382125854492188,
|
||
|
|
"fcm_dpo/margin": 119.62171936035156,
|
||
|
|
"fcm_dpo/q_t": 0.3118273615837097,
|
||
|
|
"grad_norm": 33.570701599121094,
|
||
|
|
"learning_rate": 3.738869612786737e-07,
|
||
|
|
"logits/chosen": -0.15879306197166443,
|
||
|
|
"logits/rejected": -0.14661014080047607,
|
||
|
|
"logps/chosen": -189.1214599609375,
|
||
|
|
"logps/ref_chosen": -54.69990921020508,
|
||
|
|
"logps/ref_rejected": -92.23838806152344,
|
||
|
|
"logps/rejected": -346.2816467285156,
|
||
|
|
"loss": 0.8306,
|
||
|
|
"margin_dpo/margin_mean": 119.6217269897461,
|
||
|
|
"margin_dpo/margin_std": 122.45941162109375,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4111600587371512,
|
||
|
|
"fcm_dpo/beta": 0.007537333760410547,
|
||
|
|
"fcm_dpo/delta": -0.07704190909862518,
|
||
|
|
"fcm_dpo/margin": 122.15445709228516,
|
||
|
|
"fcm_dpo/q_t": 0.3190959692001343,
|
||
|
|
"grad_norm": 67.1026840209961,
|
||
|
|
"learning_rate": 3.6827338920900253e-07,
|
||
|
|
"logits/chosen": -0.15290072560310364,
|
||
|
|
"logits/rejected": -0.14313052594661713,
|
||
|
|
"logps/chosen": -187.32925415039062,
|
||
|
|
"logps/ref_chosen": -54.64586639404297,
|
||
|
|
"logps/ref_rejected": -88.19416809082031,
|
||
|
|
"logps/rejected": -343.0320129394531,
|
||
|
|
"loss": 0.8467,
|
||
|
|
"margin_dpo/margin_mean": 122.15445709228516,
|
||
|
|
"margin_dpo/margin_std": 127.57193756103516,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4185022026431718,
|
||
|
|
"fcm_dpo/beta": 0.007103285752236843,
|
||
|
|
"fcm_dpo/delta": -0.03963179141283035,
|
||
|
|
"fcm_dpo/margin": 124.67244720458984,
|
||
|
|
"fcm_dpo/q_t": 0.32029491662979126,
|
||
|
|
"grad_norm": 36.61220169067383,
|
||
|
|
"learning_rate": 3.625819059005228e-07,
|
||
|
|
"logits/chosen": -0.09207823872566223,
|
||
|
|
"logits/rejected": -0.07269109785556793,
|
||
|
|
"logps/chosen": -234.484375,
|
||
|
|
"logps/ref_chosen": -63.02496337890625,
|
||
|
|
"logps/ref_rejected": -93.16323852539062,
|
||
|
|
"logps/rejected": -389.29510498046875,
|
||
|
|
"loss": 0.8457,
|
||
|
|
"margin_dpo/margin_mean": 124.67244720458984,
|
||
|
|
"margin_dpo/margin_std": 126.2774429321289,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.42584434654919234,
|
||
|
|
"fcm_dpo/beta": 0.006760433316230774,
|
||
|
|
"fcm_dpo/delta": -0.0626429095864296,
|
||
|
|
"fcm_dpo/margin": 134.1584930419922,
|
||
|
|
"fcm_dpo/q_t": 0.3183870315551758,
|
||
|
|
"grad_norm": 32.6433219909668,
|
||
|
|
"learning_rate": 3.568162605525952e-07,
|
||
|
|
"logits/chosen": -0.07117484509944916,
|
||
|
|
"logits/rejected": -0.06140874698758125,
|
||
|
|
"logps/chosen": -225.1572723388672,
|
||
|
|
"logps/ref_chosen": -58.37105178833008,
|
||
|
|
"logps/ref_rejected": -91.59428405761719,
|
||
|
|
"logps/rejected": -392.53900146484375,
|
||
|
|
"loss": 0.8386,
|
||
|
|
"margin_dpo/margin_mean": 134.1584930419922,
|
||
|
|
"margin_dpo/margin_std": 134.99864196777344,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4331864904552129,
|
||
|
|
"fcm_dpo/beta": 0.006551184691488743,
|
||
|
|
"fcm_dpo/delta": 0.024965018033981323,
|
||
|
|
"fcm_dpo/margin": 126.14031982421875,
|
||
|
|
"fcm_dpo/q_t": 0.3340519070625305,
|
||
|
|
"grad_norm": 46.24834060668945,
|
||
|
|
"learning_rate": 3.509802512179737e-07,
|
||
|
|
"logits/chosen": -0.051733147352933884,
|
||
|
|
"logits/rejected": -0.04382902756333351,
|
||
|
|
"logps/chosen": -215.8157958984375,
|
||
|
|
"logps/ref_chosen": -55.113426208496094,
|
||
|
|
"logps/ref_rejected": -85.29283905029297,
|
||
|
|
"logps/rejected": -372.1355285644531,
|
||
|
|
"loss": 0.8958,
|
||
|
|
"margin_dpo/margin_mean": 126.14030456542969,
|
||
|
|
"margin_dpo/margin_std": 140.62646484375,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44052863436123346,
|
||
|
|
"fcm_dpo/beta": 0.006881644017994404,
|
||
|
|
"fcm_dpo/delta": -0.0010080442298203707,
|
||
|
|
"fcm_dpo/margin": 123.4745101928711,
|
||
|
|
"fcm_dpo/q_t": 0.3347456455230713,
|
||
|
|
"grad_norm": 67.42388916015625,
|
||
|
|
"learning_rate": 3.4507772230088147e-07,
|
||
|
|
"logits/chosen": -0.06820065528154373,
|
||
|
|
"logits/rejected": -0.0680488869547844,
|
||
|
|
"logps/chosen": -241.1220703125,
|
||
|
|
"logps/ref_chosen": -59.46582794189453,
|
||
|
|
"logps/ref_rejected": -97.03690338134766,
|
||
|
|
"logps/rejected": -402.16766357421875,
|
||
|
|
"loss": 0.9105,
|
||
|
|
"margin_dpo/margin_mean": 123.4745101928711,
|
||
|
|
"margin_dpo/margin_std": 146.905029296875,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.447870778267254,
|
||
|
|
"fcm_dpo/beta": 0.006595266051590443,
|
||
|
|
"fcm_dpo/delta": 0.0008342728251591325,
|
||
|
|
"fcm_dpo/margin": 128.62730407714844,
|
||
|
|
"fcm_dpo/q_t": 0.33005213737487793,
|
||
|
|
"grad_norm": 42.539947509765625,
|
||
|
|
"learning_rate": 3.391125620245535e-07,
|
||
|
|
"logits/chosen": -0.09431194514036179,
|
||
|
|
"logits/rejected": -0.08120522648096085,
|
||
|
|
"logps/chosen": -225.65005493164062,
|
||
|
|
"logps/ref_chosen": -62.78144454956055,
|
||
|
|
"logps/ref_rejected": -91.95039367675781,
|
||
|
|
"logps/rejected": -383.4462890625,
|
||
|
|
"loss": 0.8706,
|
||
|
|
"margin_dpo/margin_mean": 128.62728881835938,
|
||
|
|
"margin_dpo/margin_std": 136.1685333251953,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4552129221732746,
|
||
|
|
"fcm_dpo/beta": 0.006700504571199417,
|
||
|
|
"fcm_dpo/delta": -0.0035556063521653414,
|
||
|
|
"fcm_dpo/margin": 127.2578125,
|
||
|
|
"fcm_dpo/q_t": 0.3245096802711487,
|
||
|
|
"grad_norm": 40.4238395690918,
|
||
|
|
"learning_rate": 3.3308869986991487e-07,
|
||
|
|
"logits/chosen": -0.07853604853153229,
|
||
|
|
"logits/rejected": -0.056885600090026855,
|
||
|
|
"logps/chosen": -217.1204071044922,
|
||
|
|
"logps/ref_chosen": -61.359039306640625,
|
||
|
|
"logps/ref_rejected": -82.75496673583984,
|
||
|
|
"logps/rejected": -365.7741394042969,
|
||
|
|
"loss": 0.851,
|
||
|
|
"margin_dpo/margin_mean": 127.2578125,
|
||
|
|
"margin_dpo/margin_std": 124.93436431884766,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46255506607929514,
|
||
|
|
"fcm_dpo/beta": 0.006514403969049454,
|
||
|
|
"fcm_dpo/delta": 0.00254381587728858,
|
||
|
|
"fcm_dpo/margin": 129.9599151611328,
|
||
|
|
"fcm_dpo/q_t": 0.3323710560798645,
|
||
|
|
"grad_norm": 71.91686248779297,
|
||
|
|
"learning_rate": 3.270101039870797e-07,
|
||
|
|
"logits/chosen": 0.050092458724975586,
|
||
|
|
"logits/rejected": 0.06273595988750458,
|
||
|
|
"logps/chosen": -250.1263885498047,
|
||
|
|
"logps/ref_chosen": -51.77602005004883,
|
||
|
|
"logps/ref_rejected": -84.58292388916016,
|
||
|
|
"logps/rejected": -412.8932189941406,
|
||
|
|
"loss": 0.8915,
|
||
|
|
"margin_dpo/margin_mean": 129.9599151611328,
|
||
|
|
"margin_dpo/margin_std": 146.8983612060547,
|
||
|
|
"step": 315
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4698972099853157,
|
||
|
|
"fcm_dpo/beta": 0.006153360940515995,
|
||
|
|
"fcm_dpo/delta": -0.12349160015583038,
|
||
|
|
"fcm_dpo/margin": 156.13824462890625,
|
||
|
|
"fcm_dpo/q_t": 0.3038443326950073,
|
||
|
|
"grad_norm": 37.25245666503906,
|
||
|
|
"learning_rate": 3.208807785813777e-07,
|
||
|
|
"logits/chosen": -0.020375465974211693,
|
||
|
|
"logits/rejected": -0.011311917565762997,
|
||
|
|
"logps/chosen": -244.4154815673828,
|
||
|
|
"logps/ref_chosen": -56.777862548828125,
|
||
|
|
"logps/ref_rejected": -99.26368713378906,
|
||
|
|
"logps/rejected": -443.03955078125,
|
||
|
|
"loss": 0.7846,
|
||
|
|
"margin_dpo/margin_mean": 156.13824462890625,
|
||
|
|
"margin_dpo/margin_std": 138.79611206054688,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.47723935389133626,
|
||
|
|
"fcm_dpo/beta": 0.00587537931278348,
|
||
|
|
"fcm_dpo/delta": 0.004868075251579285,
|
||
|
|
"fcm_dpo/margin": 143.66757202148438,
|
||
|
|
"fcm_dpo/q_t": 0.3283257484436035,
|
||
|
|
"grad_norm": 46.0535774230957,
|
||
|
|
"learning_rate": 3.147047612756302e-07,
|
||
|
|
"logits/chosen": 0.04561225324869156,
|
||
|
|
"logits/rejected": 0.05908801406621933,
|
||
|
|
"logps/chosen": -280.6909484863281,
|
||
|
|
"logps/ref_chosen": -58.28468704223633,
|
||
|
|
"logps/ref_rejected": -83.80326843261719,
|
||
|
|
"logps/rejected": -449.8771057128906,
|
||
|
|
"loss": 0.8735,
|
||
|
|
"margin_dpo/margin_mean": 143.66757202148438,
|
||
|
|
"margin_dpo/margin_std": 146.94345092773438,
|
||
|
|
"step": 325
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4845814977973568,
|
||
|
|
"fcm_dpo/beta": 0.0057464223355054855,
|
||
|
|
"fcm_dpo/delta": -0.0528077557682991,
|
||
|
|
"fcm_dpo/margin": 156.2254638671875,
|
||
|
|
"fcm_dpo/q_t": 0.3170631527900696,
|
||
|
|
"grad_norm": 50.687007904052734,
|
||
|
|
"learning_rate": 3.084861204504122e-07,
|
||
|
|
"logits/chosen": 0.018803680315613747,
|
||
|
|
"logits/rejected": 0.03341676667332649,
|
||
|
|
"logps/chosen": -278.55731201171875,
|
||
|
|
"logps/ref_chosen": -62.75822067260742,
|
||
|
|
"logps/ref_rejected": -94.04203033447266,
|
||
|
|
"logps/rejected": -466.06658935546875,
|
||
|
|
"loss": 0.8302,
|
||
|
|
"margin_dpo/margin_mean": 156.2254638671875,
|
||
|
|
"margin_dpo/margin_std": 149.75978088378906,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4919236417033774,
|
||
|
|
"fcm_dpo/beta": 0.005659957882016897,
|
||
|
|
"fcm_dpo/delta": 0.03993542492389679,
|
||
|
|
"fcm_dpo/margin": 143.60537719726562,
|
||
|
|
"fcm_dpo/q_t": 0.33370915055274963,
|
||
|
|
"grad_norm": 52.71037673950195,
|
||
|
|
"learning_rate": 3.022289525640531e-07,
|
||
|
|
"logits/chosen": 0.07965292036533356,
|
||
|
|
"logits/rejected": 0.0922585278749466,
|
||
|
|
"logps/chosen": -285.34234619140625,
|
||
|
|
"logps/ref_chosen": -58.59650421142578,
|
||
|
|
"logps/ref_rejected": -88.69586944580078,
|
||
|
|
"logps/rejected": -459.047119140625,
|
||
|
|
"loss": 0.8834,
|
||
|
|
"margin_dpo/margin_mean": 143.60537719726562,
|
||
|
|
"margin_dpo/margin_std": 148.41580200195312,
|
||
|
|
"step": 335
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49926578560939794,
|
||
|
|
"fcm_dpo/beta": 0.005560899153351784,
|
||
|
|
"fcm_dpo/delta": -0.058431338518857956,
|
||
|
|
"fcm_dpo/margin": 162.19033813476562,
|
||
|
|
"fcm_dpo/q_t": 0.3149014711380005,
|
||
|
|
"grad_norm": 35.8148193359375,
|
||
|
|
"learning_rate": 2.959373794541426e-07,
|
||
|
|
"logits/chosen": 0.013386556878685951,
|
||
|
|
"logits/rejected": 0.0291533712297678,
|
||
|
|
"logps/chosen": -255.1841278076172,
|
||
|
|
"logps/ref_chosen": -58.18162155151367,
|
||
|
|
"logps/ref_rejected": -94.44358825683594,
|
||
|
|
"logps/rejected": -453.63641357421875,
|
||
|
|
"loss": 0.825,
|
||
|
|
"margin_dpo/margin_mean": 162.1903533935547,
|
||
|
|
"margin_dpo/margin_std": 153.74880981445312,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5066079295154186,
|
||
|
|
"fcm_dpo/beta": 0.0053556510247290134,
|
||
|
|
"fcm_dpo/delta": -0.04881957173347473,
|
||
|
|
"fcm_dpo/margin": 166.8225860595703,
|
||
|
|
"fcm_dpo/q_t": 0.31634849309921265,
|
||
|
|
"grad_norm": 50.1009521484375,
|
||
|
|
"learning_rate": 2.896155456223163e-07,
|
||
|
|
"logits/chosen": -0.0006014652317389846,
|
||
|
|
"logits/rejected": 0.011017178185284138,
|
||
|
|
"logps/chosen": -275.6701354980469,
|
||
|
|
"logps/ref_chosen": -57.9904899597168,
|
||
|
|
"logps/ref_rejected": -99.11092376708984,
|
||
|
|
"logps/rejected": -483.6131896972656,
|
||
|
|
"loss": 0.8223,
|
||
|
|
"margin_dpo/margin_mean": 166.8225860595703,
|
||
|
|
"margin_dpo/margin_std": 155.08956909179688,
|
||
|
|
"step": 345
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5139500734214391,
|
||
|
|
"fcm_dpo/beta": 0.005294440779834986,
|
||
|
|
"fcm_dpo/delta": 0.02646171674132347,
|
||
|
|
"fcm_dpo/margin": 155.80526733398438,
|
||
|
|
"fcm_dpo/q_t": 0.3322368264198303,
|
||
|
|
"grad_norm": 61.714805603027344,
|
||
|
|
"learning_rate": 2.8326761550411346e-07,
|
||
|
|
"logits/chosen": 0.13601693511009216,
|
||
|
|
"logits/rejected": 0.14563313126564026,
|
||
|
|
"logps/chosen": -326.2274169921875,
|
||
|
|
"logps/ref_chosen": -58.29923629760742,
|
||
|
|
"logps/ref_rejected": -89.25711822509766,
|
||
|
|
"logps/rejected": -512.9906005859375,
|
||
|
|
"loss": 0.8883,
|
||
|
|
"margin_dpo/margin_mean": 155.80526733398438,
|
||
|
|
"margin_dpo/margin_std": 168.00064086914062,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5212922173274597,
|
||
|
|
"fcm_dpo/beta": 0.005374093074351549,
|
||
|
|
"fcm_dpo/delta": -0.00456079700961709,
|
||
|
|
"fcm_dpo/margin": 158.74806213378906,
|
||
|
|
"fcm_dpo/q_t": 0.325572669506073,
|
||
|
|
"grad_norm": 50.927425384521484,
|
||
|
|
"learning_rate": 2.7689777072570284e-07,
|
||
|
|
"logits/chosen": 0.049882251769304276,
|
||
|
|
"logits/rejected": 0.07219198346138,
|
||
|
|
"logps/chosen": -304.60711669921875,
|
||
|
|
"logps/ref_chosen": -60.788482666015625,
|
||
|
|
"logps/ref_rejected": -85.94129943847656,
|
||
|
|
"logps/rejected": -488.50799560546875,
|
||
|
|
"loss": 0.8631,
|
||
|
|
"margin_dpo/margin_mean": 158.74806213378906,
|
||
|
|
"margin_dpo/margin_std": 160.3525848388672,
|
||
|
|
"step": 355
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5286343612334802,
|
||
|
|
"fcm_dpo/beta": 0.005639675538986921,
|
||
|
|
"fcm_dpo/delta": 0.06519472599029541,
|
||
|
|
"fcm_dpo/margin": 139.63821411132812,
|
||
|
|
"fcm_dpo/q_t": 0.33998966217041016,
|
||
|
|
"grad_norm": 103.65924835205078,
|
||
|
|
"learning_rate": 2.7051020734928443e-07,
|
||
|
|
"logits/chosen": 0.07553316652774811,
|
||
|
|
"logits/rejected": 0.0942920446395874,
|
||
|
|
"logps/chosen": -303.3082580566406,
|
||
|
|
"logps/ref_chosen": -57.6871337890625,
|
||
|
|
"logps/ref_rejected": -80.62527465820312,
|
||
|
|
"logps/rejected": -465.8846130371094,
|
||
|
|
"loss": 0.9029,
|
||
|
|
"margin_dpo/margin_mean": 139.63821411132812,
|
||
|
|
"margin_dpo/margin_std": 152.2101593017578,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5359765051395007,
|
||
|
|
"fcm_dpo/beta": 0.005498864687979221,
|
||
|
|
"fcm_dpo/delta": -0.058943700045347214,
|
||
|
|
"fcm_dpo/margin": 164.1685028076172,
|
||
|
|
"fcm_dpo/q_t": 0.3161132335662842,
|
||
|
|
"grad_norm": 103.95121002197266,
|
||
|
|
"learning_rate": 2.641091331089811e-07,
|
||
|
|
"logits/chosen": 0.13508400321006775,
|
||
|
|
"logits/rejected": 0.1376177817583084,
|
||
|
|
"logps/chosen": -293.01922607421875,
|
||
|
|
"logps/ref_chosen": -51.490867614746094,
|
||
|
|
"logps/ref_rejected": -91.02871704101562,
|
||
|
|
"logps/rejected": -496.72552490234375,
|
||
|
|
"loss": 0.8274,
|
||
|
|
"margin_dpo/margin_mean": 164.1685028076172,
|
||
|
|
"margin_dpo/margin_std": 157.7042236328125,
|
||
|
|
"step": 365
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5433186490455213,
|
||
|
|
"fcm_dpo/beta": 0.005576250609010458,
|
||
|
|
"fcm_dpo/delta": 0.08467354625463486,
|
||
|
|
"fcm_dpo/margin": 138.50732421875,
|
||
|
|
"fcm_dpo/q_t": 0.3373372554779053,
|
||
|
|
"grad_norm": 51.74324035644531,
|
||
|
|
"learning_rate": 2.5769876463904263e-07,
|
||
|
|
"logits/chosen": 0.10848675668239594,
|
||
|
|
"logits/rejected": 0.11948434263467789,
|
||
|
|
"logps/chosen": -298.34576416015625,
|
||
|
|
"logps/ref_chosen": -58.113502502441406,
|
||
|
|
"logps/ref_rejected": -89.43451690673828,
|
||
|
|
"logps/rejected": -468.1741638183594,
|
||
|
|
"loss": 0.8812,
|
||
|
|
"margin_dpo/margin_mean": 138.50733947753906,
|
||
|
|
"margin_dpo/margin_std": 134.93521118164062,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5506607929515418,
|
||
|
|
"fcm_dpo/beta": 0.0056075444445014,
|
||
|
|
"fcm_dpo/delta": -0.06286319345235825,
|
||
|
|
"fcm_dpo/margin": 161.43272399902344,
|
||
|
|
"fcm_dpo/q_t": 0.3194437623023987,
|
||
|
|
"grad_norm": 49.77079772949219,
|
||
|
|
"learning_rate": 2.512833246961859e-07,
|
||
|
|
"logits/chosen": 0.1206691637635231,
|
||
|
|
"logits/rejected": 0.14743848145008087,
|
||
|
|
"logps/chosen": -326.29534912109375,
|
||
|
|
"logps/ref_chosen": -65.23600769042969,
|
||
|
|
"logps/ref_rejected": -89.24995422363281,
|
||
|
|
"logps/rejected": -511.74200439453125,
|
||
|
|
"loss": 0.8508,
|
||
|
|
"margin_dpo/margin_mean": 161.43270874023438,
|
||
|
|
"margin_dpo/margin_std": 164.4888458251953,
|
||
|
|
"step": 375
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5580029368575624,
|
||
|
|
"fcm_dpo/beta": 0.005182487424463034,
|
||
|
|
"fcm_dpo/delta": -0.11963502317667007,
|
||
|
|
"fcm_dpo/margin": 185.00277709960938,
|
||
|
|
"fcm_dpo/q_t": 0.3068377375602722,
|
||
|
|
"grad_norm": 47.95132064819336,
|
||
|
|
"learning_rate": 2.4486703937790243e-07,
|
||
|
|
"logits/chosen": 0.1993691623210907,
|
||
|
|
"logits/rejected": 0.2027546912431717,
|
||
|
|
"logps/chosen": -336.24005126953125,
|
||
|
|
"logps/ref_chosen": -53.33893966674805,
|
||
|
|
"logps/ref_rejected": -102.15375518798828,
|
||
|
|
"logps/rejected": -570.0576171875,
|
||
|
|
"loss": 0.8158,
|
||
|
|
"margin_dpo/margin_mean": 185.00277709960938,
|
||
|
|
"margin_dpo/margin_std": 178.25515747070312,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5653450807635829,
|
||
|
|
"fcm_dpo/beta": 0.0049522449262440205,
|
||
|
|
"fcm_dpo/delta": 0.019767077639698982,
|
||
|
|
"fcm_dpo/margin": 167.88037109375,
|
||
|
|
"fcm_dpo/q_t": 0.3291183114051819,
|
||
|
|
"grad_norm": 34.479740142822266,
|
||
|
|
"learning_rate": 2.3845413533856514e-07,
|
||
|
|
"logits/chosen": 0.18332220613956451,
|
||
|
|
"logits/rejected": 0.19640536606311798,
|
||
|
|
"logps/chosen": -353.31121826171875,
|
||
|
|
"logps/ref_chosen": -58.36262130737305,
|
||
|
|
"logps/ref_rejected": -89.44685363769531,
|
||
|
|
"logps/rejected": -552.2758178710938,
|
||
|
|
"loss": 0.8731,
|
||
|
|
"margin_dpo/margin_mean": 167.88037109375,
|
||
|
|
"margin_dpo/margin_std": 172.85272216796875,
|
||
|
|
"step": 385
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5726872246696035,
|
||
|
|
"fcm_dpo/beta": 0.004950010217726231,
|
||
|
|
"fcm_dpo/delta": -0.0016832177061587572,
|
||
|
|
"fcm_dpo/margin": 171.85813903808594,
|
||
|
|
"fcm_dpo/q_t": 0.32719507813453674,
|
||
|
|
"grad_norm": 46.603187561035156,
|
||
|
|
"learning_rate": 2.320488370051681e-07,
|
||
|
|
"logits/chosen": 0.12718243896961212,
|
||
|
|
"logits/rejected": 0.1411055475473404,
|
||
|
|
"logps/chosen": -333.3777160644531,
|
||
|
|
"logps/ref_chosen": -56.380653381347656,
|
||
|
|
"logps/ref_rejected": -90.47447204589844,
|
||
|
|
"logps/rejected": -539.3297119140625,
|
||
|
|
"loss": 0.8665,
|
||
|
|
"margin_dpo/margin_mean": 171.85813903808594,
|
||
|
|
"margin_dpo/margin_std": 178.92532348632812,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.580029368575624,
|
||
|
|
"fcm_dpo/beta": 0.005209635477513075,
|
||
|
|
"fcm_dpo/delta": 0.042330484837293625,
|
||
|
|
"fcm_dpo/margin": 155.28634643554688,
|
||
|
|
"fcm_dpo/q_t": 0.33868879079818726,
|
||
|
|
"grad_norm": 44.36903381347656,
|
||
|
|
"learning_rate": 2.2565536379453404e-07,
|
||
|
|
"logits/chosen": 0.03185752406716347,
|
||
|
|
"logits/rejected": 0.039340805262327194,
|
||
|
|
"logps/chosen": -274.38555908203125,
|
||
|
|
"logps/ref_chosen": -55.95304489135742,
|
||
|
|
"logps/ref_rejected": -87.13162994384766,
|
||
|
|
"logps/rejected": -460.8504943847656,
|
||
|
|
"loss": 0.9001,
|
||
|
|
"margin_dpo/margin_mean": 155.28634643554688,
|
||
|
|
"margin_dpo/margin_std": 174.56260681152344,
|
||
|
|
"step": 395
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5873715124816447,
|
||
|
|
"fcm_dpo/beta": 0.005051156505942345,
|
||
|
|
"fcm_dpo/delta": -0.08489851653575897,
|
||
|
|
"fcm_dpo/margin": 183.75982666015625,
|
||
|
|
"fcm_dpo/q_t": 0.31105726957321167,
|
||
|
|
"grad_norm": 40.749473571777344,
|
||
|
|
"learning_rate": 2.192779273338215e-07,
|
||
|
|
"logits/chosen": 0.04275448992848396,
|
||
|
|
"logits/rejected": 0.0653674453496933,
|
||
|
|
"logps/chosen": -283.33929443359375,
|
||
|
|
"logps/ref_chosen": -64.59160614013672,
|
||
|
|
"logps/ref_rejected": -96.700927734375,
|
||
|
|
"logps/rejected": -499.20843505859375,
|
||
|
|
"loss": 0.8159,
|
||
|
|
"margin_dpo/margin_mean": 183.7598419189453,
|
||
|
|
"margin_dpo/margin_std": 173.43460083007812,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5873715124816447,
|
||
|
|
"eval_fcm_dpo/beta": 0.0069984132423996925,
|
||
|
|
"eval_fcm_dpo/delta": 0.07261277735233307,
|
||
|
|
"eval_fcm_dpo/margin": 111.74861907958984,
|
||
|
|
"eval_fcm_dpo/q_t": 0.36032289266586304,
|
||
|
|
"eval_logits/chosen": 0.07495035231113434,
|
||
|
|
"eval_logits/rejected": 0.09724592417478561,
|
||
|
|
"eval_logps/chosen": -360.3431091308594,
|
||
|
|
"eval_logps/ref_chosen": -79.05104064941406,
|
||
|
|
"eval_logps/ref_rejected": -86.79793548583984,
|
||
|
|
"eval_logps/rejected": -479.838623046875,
|
||
|
|
"eval_loss": 0.5361973643302917,
|
||
|
|
"eval_margin_dpo/margin_mean": 111.74861907958984,
|
||
|
|
"eval_margin_dpo/margin_std": 182.2795867919922,
|
||
|
|
"eval_runtime": 39.5126,
|
||
|
|
"eval_samples_per_second": 59.196,
|
||
|
|
"eval_steps_per_second": 1.873,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5947136563876652,
|
||
|
|
"fcm_dpo/beta": 0.006445285864174366,
|
||
|
|
"fcm_dpo/delta": -0.3454572856426239,
|
||
|
|
"fcm_dpo/margin": 178.49400329589844,
|
||
|
|
"fcm_dpo/q_t": 0.29232606291770935,
|
||
|
|
"grad_norm": 75.37237548828125,
|
||
|
|
"learning_rate": 2.129207286861638e-07,
|
||
|
|
"logits/chosen": 0.15050789713859558,
|
||
|
|
"logits/rejected": 0.16446948051452637,
|
||
|
|
"logps/chosen": -286.4220886230469,
|
||
|
|
"logps/ref_chosen": -53.61777877807617,
|
||
|
|
"logps/ref_rejected": -81.28938293457031,
|
||
|
|
"logps/rejected": -492.58770751953125,
|
||
|
|
"loss": 0.8372,
|
||
|
|
"margin_dpo/margin_mean": 178.4940185546875,
|
||
|
|
"margin_dpo/margin_std": 191.11578369140625,
|
||
|
|
"step": 405
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6020558002936858,
|
||
|
|
"fcm_dpo/beta": 0.005619804374873638,
|
||
|
|
"fcm_dpo/delta": -0.025178026407957077,
|
||
|
|
"fcm_dpo/margin": 155.10745239257812,
|
||
|
|
"fcm_dpo/q_t": 0.32710129022598267,
|
||
|
|
"grad_norm": 44.97209167480469,
|
||
|
|
"learning_rate": 2.065879555832674e-07,
|
||
|
|
"logits/chosen": 0.16110701858997345,
|
||
|
|
"logits/rejected": 0.18340006470680237,
|
||
|
|
"logps/chosen": -324.66778564453125,
|
||
|
|
"logps/ref_chosen": -58.9287223815918,
|
||
|
|
"logps/ref_rejected": -85.55818176269531,
|
||
|
|
"logps/rejected": -506.4046936035156,
|
||
|
|
"loss": 0.8828,
|
||
|
|
"margin_dpo/margin_mean": 155.10745239257812,
|
||
|
|
"margin_dpo/margin_std": 172.00360107421875,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6093979441997063,
|
||
|
|
"fcm_dpo/beta": 0.00546681135892868,
|
||
|
|
"fcm_dpo/delta": -0.03474488854408264,
|
||
|
|
"fcm_dpo/margin": 160.98959350585938,
|
||
|
|
"fcm_dpo/q_t": 0.3214709758758545,
|
||
|
|
"grad_norm": 46.68992614746094,
|
||
|
|
"learning_rate": 2.002837796667909e-07,
|
||
|
|
"logits/chosen": 0.17981475591659546,
|
||
|
|
"logits/rejected": 0.1913680136203766,
|
||
|
|
"logps/chosen": -350.0954895019531,
|
||
|
|
"logps/ref_chosen": -58.45662307739258,
|
||
|
|
"logps/ref_rejected": -93.67063903808594,
|
||
|
|
"logps/rejected": -546.299072265625,
|
||
|
|
"loss": 0.848,
|
||
|
|
"margin_dpo/margin_mean": 160.98959350585938,
|
||
|
|
"margin_dpo/margin_std": 161.0213623046875,
|
||
|
|
"step": 415
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6167400881057269,
|
||
|
|
"fcm_dpo/beta": 0.005015296395868063,
|
||
|
|
"fcm_dpo/delta": -0.08632484823465347,
|
||
|
|
"fcm_dpo/margin": 184.5193634033203,
|
||
|
|
"fcm_dpo/q_t": 0.3122255802154541,
|
||
|
|
"grad_norm": 47.35343933105469,
|
||
|
|
"learning_rate": 1.9401235374032425e-07,
|
||
|
|
"logits/chosen": 0.1722276657819748,
|
||
|
|
"logits/rejected": 0.20106978714466095,
|
||
|
|
"logps/chosen": -375.71832275390625,
|
||
|
|
"logps/ref_chosen": -64.2349853515625,
|
||
|
|
"logps/ref_rejected": -84.86299133300781,
|
||
|
|
"logps/rejected": -580.8656616210938,
|
||
|
|
"loss": 0.8229,
|
||
|
|
"margin_dpo/margin_mean": 184.5193634033203,
|
||
|
|
"margin_dpo/margin_std": 177.80752563476562,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6240822320117474,
|
||
|
|
"fcm_dpo/beta": 0.005182936787605286,
|
||
|
|
"fcm_dpo/delta": 0.06720009446144104,
|
||
|
|
"fcm_dpo/margin": 151.88052368164062,
|
||
|
|
"fcm_dpo/q_t": 0.33798202872276306,
|
||
|
|
"grad_norm": 35.061790466308594,
|
||
|
|
"learning_rate": 1.8777780903377732e-07,
|
||
|
|
"logits/chosen": 0.07602418959140778,
|
||
|
|
"logits/rejected": 0.090725377202034,
|
||
|
|
"logps/chosen": -300.1365661621094,
|
||
|
|
"logps/ref_chosen": -56.054161071777344,
|
||
|
|
"logps/ref_rejected": -85.17119598388672,
|
||
|
|
"logps/rejected": -481.1341247558594,
|
||
|
|
"loss": 0.9033,
|
||
|
|
"margin_dpo/margin_mean": 151.88052368164062,
|
||
|
|
"margin_dpo/margin_std": 166.2532958984375,
|
||
|
|
"step": 425
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.631424375917768,
|
||
|
|
"fcm_dpo/beta": 0.005413433536887169,
|
||
|
|
"fcm_dpo/delta": 0.0383293442428112,
|
||
|
|
"fcm_dpo/margin": 150.35128784179688,
|
||
|
|
"fcm_dpo/q_t": 0.3347117006778717,
|
||
|
|
"grad_norm": 38.60581970214844,
|
||
|
|
"learning_rate": 1.8158425248197928e-07,
|
||
|
|
"logits/chosen": 0.00837210938334465,
|
||
|
|
"logits/rejected": 0.03842420130968094,
|
||
|
|
"logps/chosen": -316.62884521484375,
|
||
|
|
"logps/ref_chosen": -69.24568176269531,
|
||
|
|
"logps/ref_rejected": -91.8664321899414,
|
||
|
|
"logps/rejected": -489.60089111328125,
|
||
|
|
"loss": 0.8974,
|
||
|
|
"margin_dpo/margin_mean": 150.35128784179688,
|
||
|
|
"margin_dpo/margin_std": 164.39129638671875,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6387665198237885,
|
||
|
|
"fcm_dpo/beta": 0.0052519310265779495,
|
||
|
|
"fcm_dpo/delta": -0.04517149180173874,
|
||
|
|
"fcm_dpo/margin": 169.1701202392578,
|
||
|
|
"fcm_dpo/q_t": 0.3183755874633789,
|
||
|
|
"grad_norm": 28.78717613220215,
|
||
|
|
"learning_rate": 1.7543576401928218e-07,
|
||
|
|
"logits/chosen": 0.07158392667770386,
|
||
|
|
"logits/rejected": 0.0934370905160904,
|
||
|
|
"logps/chosen": -321.0721130371094,
|
||
|
|
"logps/ref_chosen": -60.03449630737305,
|
||
|
|
"logps/ref_rejected": -90.6872329711914,
|
||
|
|
"logps/rejected": -520.89501953125,
|
||
|
|
"loss": 0.8427,
|
||
|
|
"margin_dpo/margin_mean": 169.1701202392578,
|
||
|
|
"margin_dpo/margin_std": 165.15988159179688,
|
||
|
|
"step": 435
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6461086637298091,
|
||
|
|
"fcm_dpo/beta": 0.00523256603628397,
|
||
|
|
"fcm_dpo/delta": 0.028819337487220764,
|
||
|
|
"fcm_dpo/margin": 157.2620086669922,
|
||
|
|
"fcm_dpo/q_t": 0.33098357915878296,
|
||
|
|
"grad_norm": 44.71712875366211,
|
||
|
|
"learning_rate": 1.6933639389195134e-07,
|
||
|
|
"logits/chosen": 0.05249354988336563,
|
||
|
|
"logits/rejected": 0.08613702654838562,
|
||
|
|
"logps/chosen": -312.62689208984375,
|
||
|
|
"logps/ref_chosen": -65.50349426269531,
|
||
|
|
"logps/ref_rejected": -85.66627502441406,
|
||
|
|
"logps/rejected": -490.05169677734375,
|
||
|
|
"loss": 0.867,
|
||
|
|
"margin_dpo/margin_mean": 157.2620086669922,
|
||
|
|
"margin_dpo/margin_std": 156.13658142089844,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6534508076358296,
|
||
|
|
"fcm_dpo/beta": 0.00520352041348815,
|
||
|
|
"fcm_dpo/delta": -0.07986284047365189,
|
||
|
|
"fcm_dpo/margin": 177.13780212402344,
|
||
|
|
"fcm_dpo/q_t": 0.3184102475643158,
|
||
|
|
"grad_norm": 36.37860107421875,
|
||
|
|
"learning_rate": 1.6329015999011182e-07,
|
||
|
|
"logits/chosen": 0.12497935444116592,
|
||
|
|
"logits/rejected": 0.14361031353473663,
|
||
|
|
"logps/chosen": -362.954345703125,
|
||
|
|
"logps/ref_chosen": -60.72443389892578,
|
||
|
|
"logps/ref_rejected": -89.9255142211914,
|
||
|
|
"logps/rejected": -569.2933349609375,
|
||
|
|
"loss": 0.8507,
|
||
|
|
"margin_dpo/margin_mean": 177.13780212402344,
|
||
|
|
"margin_dpo/margin_std": 186.6912384033203,
|
||
|
|
"step": 445
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6607929515418502,
|
||
|
|
"fcm_dpo/beta": 0.004721947945654392,
|
||
|
|
"fcm_dpo/delta": -0.10506816953420639,
|
||
|
|
"fcm_dpo/margin": 200.17857360839844,
|
||
|
|
"fcm_dpo/q_t": 0.3099059462547302,
|
||
|
|
"grad_norm": 43.16559982299805,
|
||
|
|
"learning_rate": 1.573010452010098e-07,
|
||
|
|
"logits/chosen": 0.18583309650421143,
|
||
|
|
"logits/rejected": 0.19731836020946503,
|
||
|
|
"logps/chosen": -397.82037353515625,
|
||
|
|
"logps/ref_chosen": -59.96248245239258,
|
||
|
|
"logps/ref_rejected": -98.71509552001953,
|
||
|
|
"logps/rejected": -636.7515869140625,
|
||
|
|
"loss": 0.8135,
|
||
|
|
"margin_dpo/margin_mean": 200.17857360839844,
|
||
|
|
"margin_dpo/margin_std": 195.84307861328125,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6681350954478708,
|
||
|
|
"fcm_dpo/beta": 0.004480073228478432,
|
||
|
|
"fcm_dpo/delta": -0.03945715352892876,
|
||
|
|
"fcm_dpo/margin": 197.69662475585938,
|
||
|
|
"fcm_dpo/q_t": 0.31971079111099243,
|
||
|
|
"grad_norm": 35.01383972167969,
|
||
|
|
"learning_rate": 1.5137299478533064e-07,
|
||
|
|
"logits/chosen": 0.23781771957874298,
|
||
|
|
"logits/rejected": 0.259443998336792,
|
||
|
|
"logps/chosen": -383.10321044921875,
|
||
|
|
"logps/ref_chosen": -54.48131561279297,
|
||
|
|
"logps/ref_rejected": -90.6321029663086,
|
||
|
|
"logps/rejected": -616.95068359375,
|
||
|
|
"loss": 0.8442,
|
||
|
|
"margin_dpo/margin_mean": 197.69662475585938,
|
||
|
|
"margin_dpo/margin_std": 198.9961395263672,
|
||
|
|
"step": 455
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6754772393538914,
|
||
|
|
"fcm_dpo/beta": 0.004186900332570076,
|
||
|
|
"fcm_dpo/delta": -0.045056529343128204,
|
||
|
|
"fcm_dpo/margin": 212.65853881835938,
|
||
|
|
"fcm_dpo/q_t": 0.31360098719596863,
|
||
|
|
"grad_norm": 38.242950439453125,
|
||
|
|
"learning_rate": 1.4550991377830423e-07,
|
||
|
|
"logits/chosen": 0.1622854471206665,
|
||
|
|
"logits/rejected": 0.17482970654964447,
|
||
|
|
"logps/chosen": -349.6832580566406,
|
||
|
|
"logps/ref_chosen": -52.97611618041992,
|
||
|
|
"logps/ref_rejected": -95.65971374511719,
|
||
|
|
"logps/rejected": -605.0254516601562,
|
||
|
|
"loss": 0.8071,
|
||
|
|
"margin_dpo/margin_mean": 212.65853881835938,
|
||
|
|
"margin_dpo/margin_std": 186.55628967285156,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6828193832599119,
|
||
|
|
"fcm_dpo/beta": 0.00432776240631938,
|
||
|
|
"fcm_dpo/delta": 0.006079354323446751,
|
||
|
|
"fcm_dpo/margin": 194.74021911621094,
|
||
|
|
"fcm_dpo/q_t": 0.3281356990337372,
|
||
|
|
"grad_norm": 34.140296936035156,
|
||
|
|
"learning_rate": 1.3971566441730714e-07,
|
||
|
|
"logits/chosen": 0.18998193740844727,
|
||
|
|
"logits/rejected": 0.19938889145851135,
|
||
|
|
"logps/chosen": -407.28875732421875,
|
||
|
|
"logps/ref_chosen": -58.2827033996582,
|
||
|
|
"logps/ref_rejected": -94.15567779541016,
|
||
|
|
"logps/rejected": -637.9019775390625,
|
||
|
|
"loss": 0.8641,
|
||
|
|
"margin_dpo/margin_mean": 194.74020385742188,
|
||
|
|
"margin_dpo/margin_std": 195.8119659423828,
|
||
|
|
"step": 465
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6901615271659325,
|
||
|
|
"fcm_dpo/beta": 0.004061209969222546,
|
||
|
|
"fcm_dpo/delta": -0.04996412992477417,
|
||
|
|
"fcm_dpo/margin": 220.33602905273438,
|
||
|
|
"fcm_dpo/q_t": 0.3175013065338135,
|
||
|
|
"grad_norm": 49.6895866394043,
|
||
|
|
"learning_rate": 1.339940635976592e-07,
|
||
|
|
"logits/chosen": 0.18324866890907288,
|
||
|
|
"logits/rejected": 0.2053983509540558,
|
||
|
|
"logps/chosen": -454.7452087402344,
|
||
|
|
"logps/ref_chosen": -62.69774627685547,
|
||
|
|
"logps/ref_rejected": -96.33873748779297,
|
||
|
|
"logps/rejected": -708.7222900390625,
|
||
|
|
"loss": 0.8415,
|
||
|
|
"margin_dpo/margin_mean": 220.33602905273438,
|
||
|
|
"margin_dpo/margin_std": 218.0626220703125,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.697503671071953,
|
||
|
|
"fcm_dpo/beta": 0.004032416269183159,
|
||
|
|
"fcm_dpo/delta": 0.020370058715343475,
|
||
|
|
"fcm_dpo/margin": 205.8577423095703,
|
||
|
|
"fcm_dpo/q_t": 0.33193299174308777,
|
||
|
|
"grad_norm": 61.28792190551758,
|
||
|
|
"learning_rate": 1.2834888035828596e-07,
|
||
|
|
"logits/chosen": 0.18428394198417664,
|
||
|
|
"logits/rejected": 0.20524394512176514,
|
||
|
|
"logps/chosen": -447.81591796875,
|
||
|
|
"logps/ref_chosen": -61.12194061279297,
|
||
|
|
"logps/ref_rejected": -92.42192077636719,
|
||
|
|
"logps/rejected": -684.9736328125,
|
||
|
|
"loss": 0.89,
|
||
|
|
"margin_dpo/margin_mean": 205.8577423095703,
|
||
|
|
"margin_dpo/margin_std": 225.5378875732422,
|
||
|
|
"step": 475
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7048458149779736,
|
||
|
|
"fcm_dpo/beta": 0.004136770032346249,
|
||
|
|
"fcm_dpo/delta": 0.05575358867645264,
|
||
|
|
"fcm_dpo/margin": 192.84674072265625,
|
||
|
|
"fcm_dpo/q_t": 0.3363509476184845,
|
||
|
|
"grad_norm": 75.26780700683594,
|
||
|
|
"learning_rate": 1.227838333989088e-07,
|
||
|
|
"logits/chosen": 0.1994629055261612,
|
||
|
|
"logits/rejected": 0.222102090716362,
|
||
|
|
"logps/chosen": -393.8836364746094,
|
||
|
|
"logps/ref_chosen": -53.550628662109375,
|
||
|
|
"logps/ref_rejected": -82.65167999267578,
|
||
|
|
"logps/rejected": -615.8314208984375,
|
||
|
|
"loss": 0.8956,
|
||
|
|
"margin_dpo/margin_mean": 192.84674072265625,
|
||
|
|
"margin_dpo/margin_std": 205.8238525390625,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7121879588839941,
|
||
|
|
"fcm_dpo/beta": 0.004281387198716402,
|
||
|
|
"fcm_dpo/delta": -0.022714762017130852,
|
||
|
|
"fcm_dpo/margin": 203.22549438476562,
|
||
|
|
"fcm_dpo/q_t": 0.32248061895370483,
|
||
|
|
"grad_norm": 33.045082092285156,
|
||
|
|
"learning_rate": 1.1730258863039347e-07,
|
||
|
|
"logits/chosen": 0.14324723184108734,
|
||
|
|
"logits/rejected": 0.16249491274356842,
|
||
|
|
"logps/chosen": -369.179931640625,
|
||
|
|
"logps/ref_chosen": -60.76704788208008,
|
||
|
|
"logps/ref_rejected": -92.70649719238281,
|
||
|
|
"logps/rejected": -604.3448486328125,
|
||
|
|
"loss": 0.849,
|
||
|
|
"margin_dpo/margin_mean": 203.2254638671875,
|
||
|
|
"margin_dpo/margin_std": 202.1529541015625,
|
||
|
|
"step": 485
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7195301027900147,
|
||
|
|
"fcm_dpo/beta": 0.004233093000948429,
|
||
|
|
"fcm_dpo/delta": 0.0409715473651886,
|
||
|
|
"fcm_dpo/margin": 191.93649291992188,
|
||
|
|
"fcm_dpo/q_t": 0.33092981576919556,
|
||
|
|
"grad_norm": 38.83848571777344,
|
||
|
|
"learning_rate": 1.1190875675987355e-07,
|
||
|
|
"logits/chosen": 0.09927807748317719,
|
||
|
|
"logits/rejected": 0.11963877826929092,
|
||
|
|
"logps/chosen": -345.2209777832031,
|
||
|
|
"logps/ref_chosen": -53.78142547607422,
|
||
|
|
"logps/ref_rejected": -91.67438507080078,
|
||
|
|
"logps/rejected": -575.0504150390625,
|
||
|
|
"loss": 0.8672,
|
||
|
|
"margin_dpo/margin_mean": 191.93650817871094,
|
||
|
|
"margin_dpo/margin_std": 187.42601013183594,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7268722466960352,
|
||
|
|
"fcm_dpo/beta": 0.004637162666767836,
|
||
|
|
"fcm_dpo/delta": 0.03998362645506859,
|
||
|
|
"fcm_dpo/margin": 174.73489379882812,
|
||
|
|
"fcm_dpo/q_t": 0.33525848388671875,
|
||
|
|
"grad_norm": 36.38589096069336,
|
||
|
|
"learning_rate": 1.0660589091223854e-07,
|
||
|
|
"logits/chosen": 0.17240563035011292,
|
||
|
|
"logits/rejected": 0.19218161702156067,
|
||
|
|
"logps/chosen": -392.77386474609375,
|
||
|
|
"logps/ref_chosen": -58.9004020690918,
|
||
|
|
"logps/ref_rejected": -82.15424346923828,
|
||
|
|
"logps/rejected": -590.7625732421875,
|
||
|
|
"loss": 0.905,
|
||
|
|
"margin_dpo/margin_mean": 174.73489379882812,
|
||
|
|
"margin_dpo/margin_std": 193.69952392578125,
|
||
|
|
"step": 495
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7342143906020558,
|
||
|
|
"fcm_dpo/beta": 0.004700102843344212,
|
||
|
|
"fcm_dpo/delta": 0.027824098244309425,
|
||
|
|
"fcm_dpo/margin": 175.03207397460938,
|
||
|
|
"fcm_dpo/q_t": 0.33339887857437134,
|
||
|
|
"grad_norm": 64.69699096679688,
|
||
|
|
"learning_rate": 1.0139748428955333e-07,
|
||
|
|
"logits/chosen": 0.20382651686668396,
|
||
|
|
"logits/rejected": 0.22221975028514862,
|
||
|
|
"logps/chosen": -471.3478088378906,
|
||
|
|
"logps/ref_chosen": -62.13483810424805,
|
||
|
|
"logps/ref_rejected": -87.91773223876953,
|
||
|
|
"logps/rejected": -672.1627197265625,
|
||
|
|
"loss": 0.8955,
|
||
|
|
"margin_dpo/margin_mean": 175.03207397460938,
|
||
|
|
"margin_dpo/margin_std": 190.1815948486328,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7415565345080763,
|
||
|
|
"fcm_dpo/beta": 0.004485036246478558,
|
||
|
|
"fcm_dpo/delta": -0.030254876241087914,
|
||
|
|
"fcm_dpo/margin": 195.27134704589844,
|
||
|
|
"fcm_dpo/q_t": 0.3203888535499573,
|
||
|
|
"grad_norm": 84.53753662109375,
|
||
|
|
"learning_rate": 9.628696786995188e-08,
|
||
|
|
"logits/chosen": 0.2238425463438034,
|
||
|
|
"logits/rejected": 0.2508949339389801,
|
||
|
|
"logps/chosen": -467.5668029785156,
|
||
|
|
"logps/ref_chosen": -62.631813049316406,
|
||
|
|
"logps/ref_rejected": -87.59168243408203,
|
||
|
|
"logps/rejected": -687.7979736328125,
|
||
|
|
"loss": 0.8415,
|
||
|
|
"margin_dpo/margin_mean": 195.2713623046875,
|
||
|
|
"margin_dpo/margin_std": 185.7364959716797,
|
||
|
|
"step": 505
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.748898678414097,
|
||
|
|
"fcm_dpo/beta": 0.004516937769949436,
|
||
|
|
"fcm_dpo/delta": -0.005912508815526962,
|
||
|
|
"fcm_dpo/margin": 189.2612762451172,
|
||
|
|
"fcm_dpo/q_t": 0.32770413160324097,
|
||
|
|
"grad_norm": 47.08674240112305,
|
||
|
|
"learning_rate": 9.127770814751932e-08,
|
||
|
|
"logits/chosen": 0.21628710627555847,
|
||
|
|
"logits/rejected": 0.23013325035572052,
|
||
|
|
"logps/chosen": -452.9044494628906,
|
||
|
|
"logps/ref_chosen": -60.552574157714844,
|
||
|
|
"logps/ref_rejected": -91.0874252319336,
|
||
|
|
"logps/rejected": -672.7006225585938,
|
||
|
|
"loss": 0.8647,
|
||
|
|
"margin_dpo/margin_mean": 189.2612762451172,
|
||
|
|
"margin_dpo/margin_std": 197.85511779785156,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7562408223201175,
|
||
|
|
"fcm_dpo/beta": 0.004476507194340229,
|
||
|
|
"fcm_dpo/delta": 0.04546705633401871,
|
||
|
|
"fcm_dpo/margin": 180.66757202148438,
|
||
|
|
"fcm_dpo/q_t": 0.3367224335670471,
|
||
|
|
"grad_norm": 41.41278076171875,
|
||
|
|
"learning_rate": 8.637300491465272e-08,
|
||
|
|
"logits/chosen": 0.23490139842033386,
|
||
|
|
"logits/rejected": 0.25515562295913696,
|
||
|
|
"logps/chosen": -423.93670654296875,
|
||
|
|
"logps/ref_chosen": -60.9382438659668,
|
||
|
|
"logps/ref_rejected": -83.36767578125,
|
||
|
|
"logps/rejected": -627.0337524414062,
|
||
|
|
"loss": 0.9098,
|
||
|
|
"margin_dpo/margin_mean": 180.66757202148438,
|
||
|
|
"margin_dpo/margin_std": 204.07958984375,
|
||
|
|
"step": 515
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7635829662261381,
|
||
|
|
"fcm_dpo/beta": 0.0045339628122746944,
|
||
|
|
"fcm_dpo/delta": -0.015271050855517387,
|
||
|
|
"fcm_dpo/margin": 190.21194458007812,
|
||
|
|
"fcm_dpo/q_t": 0.32560357451438904,
|
||
|
|
"grad_norm": 33.79608917236328,
|
||
|
|
"learning_rate": 8.15760890883607e-08,
|
||
|
|
"logits/chosen": 0.1384505331516266,
|
||
|
|
"logits/rejected": 0.14931073784828186,
|
||
|
|
"logps/chosen": -399.65496826171875,
|
||
|
|
"logps/ref_chosen": -65.47642517089844,
|
||
|
|
"logps/ref_rejected": -98.10872650146484,
|
||
|
|
"logps/rejected": -622.4992065429688,
|
||
|
|
"loss": 0.8604,
|
||
|
|
"margin_dpo/margin_mean": 190.21194458007812,
|
||
|
|
"margin_dpo/margin_std": 195.6290740966797,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7709251101321586,
|
||
|
|
"fcm_dpo/beta": 0.0047523765824735165,
|
||
|
|
"fcm_dpo/delta": 0.011808094568550587,
|
||
|
|
"fcm_dpo/margin": 176.2508087158203,
|
||
|
|
"fcm_dpo/q_t": 0.3255954086780548,
|
||
|
|
"grad_norm": 39.144676208496094,
|
||
|
|
"learning_rate": 7.689012058193384e-08,
|
||
|
|
"logits/chosen": 0.17502716183662415,
|
||
|
|
"logits/rejected": 0.1978307068347931,
|
||
|
|
"logps/chosen": -376.6075134277344,
|
||
|
|
"logps/ref_chosen": -59.072021484375,
|
||
|
|
"logps/ref_rejected": -82.8821792602539,
|
||
|
|
"logps/rejected": -576.66845703125,
|
||
|
|
"loss": 0.8642,
|
||
|
|
"margin_dpo/margin_mean": 176.2508087158203,
|
||
|
|
"margin_dpo/margin_std": 174.07464599609375,
|
||
|
|
"step": 525
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7782672540381792,
|
||
|
|
"fcm_dpo/beta": 0.0046645584516227245,
|
||
|
|
"fcm_dpo/delta": -0.0016397640574723482,
|
||
|
|
"fcm_dpo/margin": 182.26202392578125,
|
||
|
|
"fcm_dpo/q_t": 0.3263426423072815,
|
||
|
|
"grad_norm": 32.63557434082031,
|
||
|
|
"learning_rate": 7.231818622338822e-08,
|
||
|
|
"logits/chosen": 0.18306271731853485,
|
||
|
|
"logits/rejected": 0.20520134270191193,
|
||
|
|
"logps/chosen": -398.6494445800781,
|
||
|
|
"logps/ref_chosen": -61.11234664916992,
|
||
|
|
"logps/ref_rejected": -87.01112365722656,
|
||
|
|
"logps/rejected": -606.8101806640625,
|
||
|
|
"loss": 0.8608,
|
||
|
|
"margin_dpo/margin_mean": 182.26202392578125,
|
||
|
|
"margin_dpo/margin_std": 181.18548583984375,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7856093979441997,
|
||
|
|
"fcm_dpo/beta": 0.004603945650160313,
|
||
|
|
"fcm_dpo/delta": -0.03259692341089249,
|
||
|
|
"fcm_dpo/margin": 190.7316436767578,
|
||
|
|
"fcm_dpo/q_t": 0.32039114832878113,
|
||
|
|
"grad_norm": 34.20729064941406,
|
||
|
|
"learning_rate": 6.786329772205246e-08,
|
||
|
|
"logits/chosen": 0.19900277256965637,
|
||
|
|
"logits/rejected": 0.2242613583803177,
|
||
|
|
"logps/chosen": -409.9130859375,
|
||
|
|
"logps/ref_chosen": -60.96736526489258,
|
||
|
|
"logps/ref_rejected": -81.727294921875,
|
||
|
|
"logps/rejected": -621.4046630859375,
|
||
|
|
"loss": 0.8464,
|
||
|
|
"margin_dpo/margin_mean": 190.73162841796875,
|
||
|
|
"margin_dpo/margin_std": 186.18397521972656,
|
||
|
|
"step": 535
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7929515418502202,
|
||
|
|
"fcm_dpo/beta": 0.0043141767382621765,
|
||
|
|
"fcm_dpo/delta": -0.06867679208517075,
|
||
|
|
"fcm_dpo/margin": 211.166015625,
|
||
|
|
"fcm_dpo/q_t": 0.3138847351074219,
|
||
|
|
"grad_norm": 36.321292877197266,
|
||
|
|
"learning_rate": 6.352838968463919e-08,
|
||
|
|
"logits/chosen": 0.1988883763551712,
|
||
|
|
"logits/rejected": 0.2228163480758667,
|
||
|
|
"logps/chosen": -403.5604248046875,
|
||
|
|
"logps/ref_chosen": -58.64385986328125,
|
||
|
|
"logps/ref_rejected": -90.40965270996094,
|
||
|
|
"logps/rejected": -646.4922485351562,
|
||
|
|
"loss": 0.8217,
|
||
|
|
"margin_dpo/margin_mean": 211.166015625,
|
||
|
|
"margin_dpo/margin_std": 199.3925323486328,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8002936857562408,
|
||
|
|
"fcm_dpo/beta": 0.004401062615215778,
|
||
|
|
"fcm_dpo/delta": 0.036251001060009,
|
||
|
|
"fcm_dpo/margin": 185.14137268066406,
|
||
|
|
"fcm_dpo/q_t": 0.3344213664531708,
|
||
|
|
"grad_norm": 47.214256286621094,
|
||
|
|
"learning_rate": 5.9316317682106294e-08,
|
||
|
|
"logits/chosen": 0.1989498883485794,
|
||
|
|
"logits/rejected": 0.21334727108478546,
|
||
|
|
"logps/chosen": -444.91400146484375,
|
||
|
|
"logps/ref_chosen": -64.73474884033203,
|
||
|
|
"logps/ref_rejected": -100.44208526611328,
|
||
|
|
"logps/rejected": -665.7626953125,
|
||
|
|
"loss": 0.8888,
|
||
|
|
"margin_dpo/margin_mean": 185.14137268066406,
|
||
|
|
"margin_dpo/margin_std": 199.3173828125,
|
||
|
|
"step": 545
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8076358296622613,
|
||
|
|
"fcm_dpo/beta": 0.0044622840359807014,
|
||
|
|
"fcm_dpo/delta": 0.047435857355594635,
|
||
|
|
"fcm_dpo/margin": 180.65365600585938,
|
||
|
|
"fcm_dpo/q_t": 0.3330041766166687,
|
||
|
|
"grad_norm": 44.434165954589844,
|
||
|
|
"learning_rate": 5.5229856368582376e-08,
|
||
|
|
"logits/chosen": 0.20213136076927185,
|
||
|
|
"logits/rejected": 0.22051355242729187,
|
||
|
|
"logps/chosen": -433.3397521972656,
|
||
|
|
"logps/ref_chosen": -59.13951873779297,
|
||
|
|
"logps/ref_rejected": -87.15635681152344,
|
||
|
|
"logps/rejected": -642.0101928710938,
|
||
|
|
"loss": 0.8799,
|
||
|
|
"margin_dpo/margin_mean": 180.65365600585938,
|
||
|
|
"margin_dpo/margin_std": 186.55726623535156,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8149779735682819,
|
||
|
|
"fcm_dpo/beta": 0.004191819578409195,
|
||
|
|
"fcm_dpo/delta": -0.12720482051372528,
|
||
|
|
"fcm_dpo/margin": 229.26779174804688,
|
||
|
|
"fcm_dpo/q_t": 0.3060959577560425,
|
||
|
|
"grad_norm": 43.85456848144531,
|
||
|
|
"learning_rate": 5.127169765359515e-08,
|
||
|
|
"logits/chosen": 0.1819799244403839,
|
||
|
|
"logits/rejected": 0.20495197176933289,
|
||
|
|
"logps/chosen": -431.9147033691406,
|
||
|
|
"logps/ref_chosen": -62.1995849609375,
|
||
|
|
"logps/ref_rejected": -102.51883697509766,
|
||
|
|
"logps/rejected": -701.501708984375,
|
||
|
|
"loss": 0.8051,
|
||
|
|
"margin_dpo/margin_mean": 229.26779174804688,
|
||
|
|
"margin_dpo/margin_std": 213.53018188476562,
|
||
|
|
"step": 555
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8223201174743024,
|
||
|
|
"fcm_dpo/beta": 0.004056096542626619,
|
||
|
|
"fcm_dpo/delta": 0.024886898696422577,
|
||
|
|
"fcm_dpo/margin": 203.6185760498047,
|
||
|
|
"fcm_dpo/q_t": 0.33151647448539734,
|
||
|
|
"grad_norm": 43.7994270324707,
|
||
|
|
"learning_rate": 4.7444448928806615e-08,
|
||
|
|
"logits/chosen": 0.23396944999694824,
|
||
|
|
"logits/rejected": 0.26040390133857727,
|
||
|
|
"logps/chosen": -446.46905517578125,
|
||
|
|
"logps/ref_chosen": -61.541908264160156,
|
||
|
|
"logps/ref_rejected": -97.37491607666016,
|
||
|
|
"logps/rejected": -685.920654296875,
|
||
|
|
"loss": 0.8683,
|
||
|
|
"margin_dpo/margin_mean": 203.6185760498047,
|
||
|
|
"margin_dpo/margin_std": 203.64950561523438,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8296622613803231,
|
||
|
|
"fcm_dpo/beta": 0.0042213378474116325,
|
||
|
|
"fcm_dpo/delta": 0.0051836492493748665,
|
||
|
|
"fcm_dpo/margin": 199.88539123535156,
|
||
|
|
"fcm_dpo/q_t": 0.3270900845527649,
|
||
|
|
"grad_norm": 34.782867431640625,
|
||
|
|
"learning_rate": 4.375063135042445e-08,
|
||
|
|
"logits/chosen": 0.271693617105484,
|
||
|
|
"logits/rejected": 0.29448002576828003,
|
||
|
|
"logps/chosen": -460.4130859375,
|
||
|
|
"logps/ref_chosen": -62.85475540161133,
|
||
|
|
"logps/ref_rejected": -93.8392105102539,
|
||
|
|
"logps/rejected": -691.2828979492188,
|
||
|
|
"loss": 0.8593,
|
||
|
|
"margin_dpo/margin_mean": 199.88539123535156,
|
||
|
|
"margin_dpo/margin_std": 199.06076049804688,
|
||
|
|
"step": 565
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8370044052863436,
|
||
|
|
"fcm_dpo/beta": 0.004123511724174023,
|
||
|
|
"fcm_dpo/delta": -0.02085905149579048,
|
||
|
|
"fcm_dpo/margin": 210.5379180908203,
|
||
|
|
"fcm_dpo/q_t": 0.32254526019096375,
|
||
|
|
"grad_norm": 37.619651794433594,
|
||
|
|
"learning_rate": 4.019267817841834e-08,
|
||
|
|
"logits/chosen": 0.2957739233970642,
|
||
|
|
"logits/rejected": 0.3121660649776459,
|
||
|
|
"logps/chosen": -445.2286682128906,
|
||
|
|
"logps/ref_chosen": -57.98622512817383,
|
||
|
|
"logps/ref_rejected": -87.91555786132812,
|
||
|
|
"logps/rejected": -685.6959228515625,
|
||
|
|
"loss": 0.857,
|
||
|
|
"margin_dpo/margin_mean": 210.5379180908203,
|
||
|
|
"margin_dpo/margin_std": 210.2278289794922,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8443465491923642,
|
||
|
|
"fcm_dpo/beta": 0.004019825719296932,
|
||
|
|
"fcm_dpo/delta": -0.013523074798285961,
|
||
|
|
"fcm_dpo/margin": 214.1552276611328,
|
||
|
|
"fcm_dpo/q_t": 0.326122909784317,
|
||
|
|
"grad_norm": 44.181026458740234,
|
||
|
|
"learning_rate": 3.677293317363864e-08,
|
||
|
|
"logits/chosen": 0.2930538058280945,
|
||
|
|
"logits/rejected": 0.3044101595878601,
|
||
|
|
"logps/chosen": -452.9761657714844,
|
||
|
|
"logps/ref_chosen": -55.194114685058594,
|
||
|
|
"logps/ref_rejected": -89.68229675292969,
|
||
|
|
"logps/rejected": -701.61962890625,
|
||
|
|
"loss": 0.8759,
|
||
|
|
"margin_dpo/margin_mean": 214.1552276611328,
|
||
|
|
"margin_dpo/margin_std": 227.3217315673828,
|
||
|
|
"step": 575
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8516886930983847,
|
||
|
|
"fcm_dpo/beta": 0.004043132532387972,
|
||
|
|
"fcm_dpo/delta": 0.005774930119514465,
|
||
|
|
"fcm_dpo/margin": 208.61080932617188,
|
||
|
|
"fcm_dpo/q_t": 0.32523873448371887,
|
||
|
|
"grad_norm": 33.26530456542969,
|
||
|
|
"learning_rate": 3.349364905389032e-08,
|
||
|
|
"logits/chosen": 0.2503589987754822,
|
||
|
|
"logits/rejected": 0.2608030438423157,
|
||
|
|
"logps/chosen": -421.0636291503906,
|
||
|
|
"logps/ref_chosen": -54.605796813964844,
|
||
|
|
"logps/ref_rejected": -88.9614486694336,
|
||
|
|
"logps/rejected": -664.0301513671875,
|
||
|
|
"loss": 0.8625,
|
||
|
|
"margin_dpo/margin_mean": 208.61080932617188,
|
||
|
|
"margin_dpo/margin_std": 206.9934539794922,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8590308370044053,
|
||
|
|
"fcm_dpo/beta": 0.004104516003280878,
|
||
|
|
"fcm_dpo/delta": -0.005921521689742804,
|
||
|
|
"fcm_dpo/margin": 208.0009307861328,
|
||
|
|
"fcm_dpo/q_t": 0.3261064291000366,
|
||
|
|
"grad_norm": 40.046321868896484,
|
||
|
|
"learning_rate": 3.035698600998121e-08,
|
||
|
|
"logits/chosen": 0.28162720799446106,
|
||
|
|
"logits/rejected": 0.2981635630130768,
|
||
|
|
"logps/chosen": -438.46337890625,
|
||
|
|
"logps/ref_chosen": -59.03770065307617,
|
||
|
|
"logps/ref_rejected": -94.4029541015625,
|
||
|
|
"logps/rejected": -681.82958984375,
|
||
|
|
"loss": 0.8647,
|
||
|
|
"margin_dpo/margin_mean": 208.0009307861328,
|
||
|
|
"margin_dpo/margin_std": 209.61935424804688,
|
||
|
|
"step": 585
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8663729809104258,
|
||
|
|
"fcm_dpo/beta": 0.00406238529831171,
|
||
|
|
"fcm_dpo/delta": -0.020377200096845627,
|
||
|
|
"fcm_dpo/margin": 213.8004913330078,
|
||
|
|
"fcm_dpo/q_t": 0.32257598638534546,
|
||
|
|
"grad_norm": 41.44359588623047,
|
||
|
|
"learning_rate": 2.736501028272095e-08,
|
||
|
|
"logits/chosen": 0.2646399140357971,
|
||
|
|
"logits/rejected": 0.268844336271286,
|
||
|
|
"logps/chosen": -428.0189514160156,
|
||
|
|
"logps/ref_chosen": -53.5163688659668,
|
||
|
|
"logps/ref_rejected": -99.90290832519531,
|
||
|
|
"logps/rejected": -688.2059326171875,
|
||
|
|
"loss": 0.8486,
|
||
|
|
"margin_dpo/margin_mean": 213.8004913330078,
|
||
|
|
"margin_dpo/margin_std": 211.3090057373047,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8737151248164464,
|
||
|
|
"fcm_dpo/beta": 0.004042100626975298,
|
||
|
|
"fcm_dpo/delta": 0.045332007110118866,
|
||
|
|
"fcm_dpo/margin": 199.73629760742188,
|
||
|
|
"fcm_dpo/q_t": 0.3346654772758484,
|
||
|
|
"grad_norm": 32.38268280029297,
|
||
|
|
"learning_rate": 2.451969280180849e-08,
|
||
|
|
"logits/chosen": 0.30068081617355347,
|
||
|
|
"logits/rejected": 0.32306617498397827,
|
||
|
|
"logps/chosen": -435.3770446777344,
|
||
|
|
"logps/ref_chosen": -51.44538497924805,
|
||
|
|
"logps/ref_rejected": -77.43083190917969,
|
||
|
|
"logps/rejected": -661.0988159179688,
|
||
|
|
"loss": 0.9023,
|
||
|
|
"margin_dpo/margin_mean": 199.73629760742188,
|
||
|
|
"margin_dpo/margin_std": 218.01443481445312,
|
||
|
|
"step": 595
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8810572687224669,
|
||
|
|
"fcm_dpo/beta": 0.0041966065764427185,
|
||
|
|
"fcm_dpo/delta": 0.029329314827919006,
|
||
|
|
"fcm_dpo/margin": 195.78518676757812,
|
||
|
|
"fcm_dpo/q_t": 0.3304395377635956,
|
||
|
|
"grad_norm": 49.09275436401367,
|
||
|
|
"learning_rate": 2.1822907887504932e-08,
|
||
|
|
"logits/chosen": 0.2844870686531067,
|
||
|
|
"logits/rejected": 0.29689234495162964,
|
||
|
|
"logps/chosen": -457.60186767578125,
|
||
|
|
"logps/ref_chosen": -57.161705017089844,
|
||
|
|
"logps/ref_rejected": -87.73274230957031,
|
||
|
|
"logps/rejected": -683.9581298828125,
|
||
|
|
"loss": 0.8894,
|
||
|
|
"margin_dpo/margin_mean": 195.78518676757812,
|
||
|
|
"margin_dpo/margin_std": 207.5529327392578,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8810572687224669,
|
||
|
|
"eval_fcm_dpo/beta": 0.005688491743057966,
|
||
|
|
"eval_fcm_dpo/delta": 0.04913259670138359,
|
||
|
|
"eval_fcm_dpo/margin": 139.8485870361328,
|
||
|
|
"eval_fcm_dpo/q_t": 0.3537566661834717,
|
||
|
|
"eval_logits/chosen": 0.25595730543136597,
|
||
|
|
"eval_logits/rejected": 0.27860310673713684,
|
||
|
|
"eval_logps/chosen": -511.802978515625,
|
||
|
|
"eval_logps/ref_chosen": -79.05104064941406,
|
||
|
|
"eval_logps/ref_rejected": -86.79793548583984,
|
||
|
|
"eval_logps/rejected": -659.3984375,
|
||
|
|
"eval_loss": 0.5130776166915894,
|
||
|
|
"eval_margin_dpo/margin_mean": 139.8485870361328,
|
||
|
|
"eval_margin_dpo/margin_std": 209.58946228027344,
|
||
|
|
"eval_runtime": 39.5413,
|
||
|
|
"eval_samples_per_second": 59.153,
|
||
|
|
"eval_steps_per_second": 1.871,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8883994126284875,
|
||
|
|
"fcm_dpo/beta": 0.005007568746805191,
|
||
|
|
"fcm_dpo/delta": -0.2992556691169739,
|
||
|
|
"fcm_dpo/margin": 222.298583984375,
|
||
|
|
"fcm_dpo/q_t": 0.2881125211715698,
|
||
|
|
"grad_norm": 34.59627914428711,
|
||
|
|
"learning_rate": 1.9276432015946446e-08,
|
||
|
|
"logits/chosen": 0.24366600811481476,
|
||
|
|
"logits/rejected": 0.25845664739608765,
|
||
|
|
"logps/chosen": -441.85162353515625,
|
||
|
|
"logps/ref_chosen": -58.169830322265625,
|
||
|
|
"logps/ref_rejected": -95.36891174316406,
|
||
|
|
"logps/rejected": -701.3492431640625,
|
||
|
|
"loss": 0.7792,
|
||
|
|
"margin_dpo/margin_mean": 222.298583984375,
|
||
|
|
"margin_dpo/margin_std": 220.7966766357422,
|
||
|
|
"step": 605
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.895741556534508,
|
||
|
|
"fcm_dpo/beta": 0.004342931322753429,
|
||
|
|
"fcm_dpo/delta": -0.05808112025260925,
|
||
|
|
"fcm_dpo/margin": 207.65054321289062,
|
||
|
|
"fcm_dpo/q_t": 0.31335949897766113,
|
||
|
|
"grad_norm": 41.44016647338867,
|
||
|
|
"learning_rate": 1.6881942648911074e-08,
|
||
|
|
"logits/chosen": 0.2341168224811554,
|
||
|
|
"logits/rejected": 0.2553400695323944,
|
||
|
|
"logps/chosen": -439.32952880859375,
|
||
|
|
"logps/ref_chosen": -58.97087860107422,
|
||
|
|
"logps/ref_rejected": -89.0286865234375,
|
||
|
|
"logps/rejected": -677.037841796875,
|
||
|
|
"loss": 0.8149,
|
||
|
|
"margin_dpo/margin_mean": 207.65054321289062,
|
||
|
|
"margin_dpo/margin_std": 189.12887573242188,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9030837004405287,
|
||
|
|
"fcm_dpo/beta": 0.00410483730956912,
|
||
|
|
"fcm_dpo/delta": -0.03579559922218323,
|
||
|
|
"fcm_dpo/margin": 214.7685089111328,
|
||
|
|
"fcm_dpo/q_t": 0.3158699870109558,
|
||
|
|
"grad_norm": 38.849464416503906,
|
||
|
|
"learning_rate": 1.4641017128809801e-08,
|
||
|
|
"logits/chosen": 0.2525313198566437,
|
||
|
|
"logits/rejected": 0.26471230387687683,
|
||
|
|
"logps/chosen": -440.29595947265625,
|
||
|
|
"logps/ref_chosen": -58.081878662109375,
|
||
|
|
"logps/ref_rejected": -95.92155456542969,
|
||
|
|
"logps/rejected": -692.904052734375,
|
||
|
|
"loss": 0.8145,
|
||
|
|
"margin_dpo/margin_mean": 214.7685089111328,
|
||
|
|
"margin_dpo/margin_std": 188.67098999023438,
|
||
|
|
"step": 615
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9104258443465492,
|
||
|
|
"fcm_dpo/beta": 0.004342620261013508,
|
||
|
|
"fcm_dpo/delta": 0.06593510508537292,
|
||
|
|
"fcm_dpo/margin": 181.14906311035156,
|
||
|
|
"fcm_dpo/q_t": 0.34116441011428833,
|
||
|
|
"grad_norm": 46.56772232055664,
|
||
|
|
"learning_rate": 1.2555131639630567e-08,
|
||
|
|
"logits/chosen": 0.24702966213226318,
|
||
|
|
"logits/rejected": 0.2741623520851135,
|
||
|
|
"logps/chosen": -464.8133850097656,
|
||
|
|
"logps/ref_chosen": -62.203094482421875,
|
||
|
|
"logps/ref_rejected": -80.53683471679688,
|
||
|
|
"logps/rejected": -664.2962036132812,
|
||
|
|
"loss": 0.9129,
|
||
|
|
"margin_dpo/margin_mean": 181.14906311035156,
|
||
|
|
"margin_dpo/margin_std": 204.39202880859375,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9177679882525698,
|
||
|
|
"fcm_dpo/beta": 0.004249856807291508,
|
||
|
|
"fcm_dpo/delta": -0.03912333771586418,
|
||
|
|
"fcm_dpo/margin": 208.0172576904297,
|
||
|
|
"fcm_dpo/q_t": 0.32065385580062866,
|
||
|
|
"grad_norm": 40.270851135253906,
|
||
|
|
"learning_rate": 1.0625660234518913e-08,
|
||
|
|
"logits/chosen": 0.2736581265926361,
|
||
|
|
"logits/rejected": 0.2979634702205658,
|
||
|
|
"logps/chosen": -449.50860595703125,
|
||
|
|
"logps/ref_chosen": -61.727455139160156,
|
||
|
|
"logps/ref_rejected": -88.4387435913086,
|
||
|
|
"logps/rejected": -684.2371826171875,
|
||
|
|
"loss": 0.8444,
|
||
|
|
"margin_dpo/margin_mean": 208.0172882080078,
|
||
|
|
"margin_dpo/margin_std": 208.2506103515625,
|
||
|
|
"step": 625
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9251101321585903,
|
||
|
|
"fcm_dpo/beta": 0.004256346728652716,
|
||
|
|
"fcm_dpo/delta": 0.025741109624505043,
|
||
|
|
"fcm_dpo/margin": 193.95529174804688,
|
||
|
|
"fcm_dpo/q_t": 0.33398014307022095,
|
||
|
|
"grad_norm": 46.073326110839844,
|
||
|
|
"learning_rate": 8.85387393063622e-09,
|
||
|
|
"logits/chosen": 0.1979989856481552,
|
||
|
|
"logits/rejected": 0.21419593691825867,
|
||
|
|
"logps/chosen": -437.273193359375,
|
||
|
|
"logps/ref_chosen": -61.30865478515625,
|
||
|
|
"logps/ref_rejected": -96.54997253417969,
|
||
|
|
"logps/rejected": -666.4697875976562,
|
||
|
|
"loss": 0.8807,
|
||
|
|
"margin_dpo/margin_mean": 193.95529174804688,
|
||
|
|
"margin_dpo/margin_std": 204.22341918945312,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9324522760646109,
|
||
|
|
"fcm_dpo/beta": 0.004541632253676653,
|
||
|
|
"fcm_dpo/delta": 0.08948247134685516,
|
||
|
|
"fcm_dpo/margin": 168.99508666992188,
|
||
|
|
"fcm_dpo/q_t": 0.3458203673362732,
|
||
|
|
"grad_norm": 41.188926696777344,
|
||
|
|
"learning_rate": 7.240939871891699e-09,
|
||
|
|
"logits/chosen": 0.26888564229011536,
|
||
|
|
"logits/rejected": 0.2919827103614807,
|
||
|
|
"logps/chosen": -468.99005126953125,
|
||
|
|
"logps/ref_chosen": -63.7315673828125,
|
||
|
|
"logps/ref_rejected": -89.66435241699219,
|
||
|
|
"logps/rejected": -663.9180297851562,
|
||
|
|
"loss": 0.944,
|
||
|
|
"margin_dpo/margin_mean": 168.99508666992188,
|
||
|
|
"margin_dpo/margin_std": 206.3029022216797,
|
||
|
|
"step": 635
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9397944199706314,
|
||
|
|
"fcm_dpo/beta": 0.004565252922475338,
|
||
|
|
"fcm_dpo/delta": -0.08675488084554672,
|
||
|
|
"fcm_dpo/margin": 203.2274932861328,
|
||
|
|
"fcm_dpo/q_t": 0.3128082752227783,
|
||
|
|
"grad_norm": 34.060367584228516,
|
||
|
|
"learning_rate": 5.7879205600998296e-09,
|
||
|
|
"logits/chosen": 0.22282154858112335,
|
||
|
|
"logits/rejected": 0.24946781992912292,
|
||
|
|
"logps/chosen": -445.6302185058594,
|
||
|
|
"logps/ref_chosen": -59.17915725708008,
|
||
|
|
"logps/ref_rejected": -88.51210021972656,
|
||
|
|
"logps/rejected": -678.190673828125,
|
||
|
|
"loss": 0.823,
|
||
|
|
"margin_dpo/margin_mean": 203.2274932861328,
|
||
|
|
"margin_dpo/margin_std": 194.40232849121094,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.947136563876652,
|
||
|
|
"fcm_dpo/beta": 0.004437420517206192,
|
||
|
|
"fcm_dpo/delta": 0.06028025597333908,
|
||
|
|
"fcm_dpo/margin": 178.86392211914062,
|
||
|
|
"fcm_dpo/q_t": 0.3435667157173157,
|
||
|
|
"grad_norm": 50.218833923339844,
|
||
|
|
"learning_rate": 4.495773155069299e-09,
|
||
|
|
"logits/chosen": 0.2428874969482422,
|
||
|
|
"logits/rejected": 0.2540218234062195,
|
||
|
|
"logps/chosen": -454.20220947265625,
|
||
|
|
"logps/ref_chosen": -59.50596237182617,
|
||
|
|
"logps/ref_rejected": -93.92404174804688,
|
||
|
|
"logps/rejected": -667.4842529296875,
|
||
|
|
"loss": 0.9275,
|
||
|
|
"margin_dpo/margin_mean": 178.86392211914062,
|
||
|
|
"margin_dpo/margin_std": 214.05029296875,
|
||
|
|
"step": 645
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9544787077826725,
|
||
|
|
"fcm_dpo/beta": 0.004491353873163462,
|
||
|
|
"fcm_dpo/delta": -0.053510040044784546,
|
||
|
|
"fcm_dpo/margin": 199.87738037109375,
|
||
|
|
"fcm_dpo/q_t": 0.3246760070323944,
|
||
|
|
"grad_norm": 47.475616455078125,
|
||
|
|
"learning_rate": 3.3653488440851253e-09,
|
||
|
|
"logits/chosen": 0.27860307693481445,
|
||
|
|
"logits/rejected": 0.2957097887992859,
|
||
|
|
"logps/chosen": -443.88482666015625,
|
||
|
|
"logps/ref_chosen": -57.774566650390625,
|
||
|
|
"logps/ref_rejected": -89.61600494384766,
|
||
|
|
"logps/rejected": -675.6035766601562,
|
||
|
|
"loss": 0.8711,
|
||
|
|
"margin_dpo/margin_mean": 199.87738037109375,
|
||
|
|
"margin_dpo/margin_std": 219.21707153320312,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9618208516886931,
|
||
|
|
"fcm_dpo/beta": 0.004239661619067192,
|
||
|
|
"fcm_dpo/delta": -0.012966930866241455,
|
||
|
|
"fcm_dpo/margin": 203.01885986328125,
|
||
|
|
"fcm_dpo/q_t": 0.3295535147190094,
|
||
|
|
"grad_norm": 43.02524185180664,
|
||
|
|
"learning_rate": 2.397392281198729e-09,
|
||
|
|
"logits/chosen": 0.2716834545135498,
|
||
|
|
"logits/rejected": 0.277962863445282,
|
||
|
|
"logps/chosen": -437.599609375,
|
||
|
|
"logps/ref_chosen": -55.68403244018555,
|
||
|
|
"logps/ref_rejected": -102.4081802368164,
|
||
|
|
"logps/rejected": -687.3425903320312,
|
||
|
|
"loss": 0.8756,
|
||
|
|
"margin_dpo/margin_mean": 203.01885986328125,
|
||
|
|
"margin_dpo/margin_std": 221.51998901367188,
|
||
|
|
"step": 655
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9691629955947136,
|
||
|
|
"fcm_dpo/beta": 0.003964302130043507,
|
||
|
|
"fcm_dpo/delta": -0.10601212829351425,
|
||
|
|
"fcm_dpo/margin": 237.7954559326172,
|
||
|
|
"fcm_dpo/q_t": 0.30317944288253784,
|
||
|
|
"grad_norm": 35.75173568725586,
|
||
|
|
"learning_rate": 1.592541096695571e-09,
|
||
|
|
"logits/chosen": 0.2771255373954773,
|
||
|
|
"logits/rejected": 0.2976723611354828,
|
||
|
|
"logps/chosen": -440.38336181640625,
|
||
|
|
"logps/ref_chosen": -59.19981002807617,
|
||
|
|
"logps/ref_rejected": -94.19200134277344,
|
||
|
|
"logps/rejected": -713.1710815429688,
|
||
|
|
"loss": 0.7768,
|
||
|
|
"margin_dpo/margin_mean": 237.7954559326172,
|
||
|
|
"margin_dpo/margin_std": 198.18447875976562,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9765051395007343,
|
||
|
|
"fcm_dpo/beta": 0.003923547919839621,
|
||
|
|
"fcm_dpo/delta": 0.008545741438865662,
|
||
|
|
"fcm_dpo/margin": 214.4600830078125,
|
||
|
|
"fcm_dpo/q_t": 0.3268297612667084,
|
||
|
|
"grad_norm": 39.92269515991211,
|
||
|
|
"learning_rate": 9.513254770636137e-10,
|
||
|
|
"logits/chosen": 0.2574383616447449,
|
||
|
|
"logits/rejected": 0.2810707688331604,
|
||
|
|
"logps/chosen": -439.8953552246094,
|
||
|
|
"logps/ref_chosen": -61.2533073425293,
|
||
|
|
"logps/ref_rejected": -95.86351013183594,
|
||
|
|
"logps/rejected": -688.9656982421875,
|
||
|
|
"loss": 0.8615,
|
||
|
|
"margin_dpo/margin_mean": 214.46011352539062,
|
||
|
|
"margin_dpo/margin_std": 217.1322021484375,
|
||
|
|
"step": 665
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9838472834067548,
|
||
|
|
"fcm_dpo/beta": 0.003961815033107996,
|
||
|
|
"fcm_dpo/delta": -0.015965379774570465,
|
||
|
|
"fcm_dpo/margin": 218.12954711914062,
|
||
|
|
"fcm_dpo/q_t": 0.31941553950309753,
|
||
|
|
"grad_norm": 31.397859573364258,
|
||
|
|
"learning_rate": 4.741678157389739e-10,
|
||
|
|
"logits/chosen": 0.2319277822971344,
|
||
|
|
"logits/rejected": 0.25498372316360474,
|
||
|
|
"logps/chosen": -442.1675720214844,
|
||
|
|
"logps/ref_chosen": -62.95263671875,
|
||
|
|
"logps/ref_rejected": -92.4662094116211,
|
||
|
|
"logps/rejected": -689.8107299804688,
|
||
|
|
"loss": 0.835,
|
||
|
|
"margin_dpo/margin_mean": 218.12954711914062,
|
||
|
|
"margin_dpo/margin_std": 201.41766357421875,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9911894273127754,
|
||
|
|
"fcm_dpo/beta": 0.00390138803049922,
|
||
|
|
"fcm_dpo/delta": -0.02992972731590271,
|
||
|
|
"fcm_dpo/margin": 224.89852905273438,
|
||
|
|
"fcm_dpo/q_t": 0.31749120354652405,
|
||
|
|
"grad_norm": 32.01641845703125,
|
||
|
|
"learning_rate": 1.6138243485910863e-10,
|
||
|
|
"logits/chosen": 0.2925337255001068,
|
||
|
|
"logits/rejected": 0.3101763129234314,
|
||
|
|
"logps/chosen": -432.8349609375,
|
||
|
|
"logps/ref_chosen": -48.5856819152832,
|
||
|
|
"logps/ref_rejected": -81.27871704101562,
|
||
|
|
"logps/rejected": -690.4266357421875,
|
||
|
|
"loss": 0.8262,
|
||
|
|
"margin_dpo/margin_mean": 224.8985595703125,
|
||
|
|
"margin_dpo/margin_std": 204.8478546142578,
|
||
|
|
"step": 675
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9985315712187959,
|
||
|
|
"fcm_dpo/beta": 0.0037871976383030415,
|
||
|
|
"fcm_dpo/delta": 0.029484611004590988,
|
||
|
|
"fcm_dpo/margin": 217.26602172851562,
|
||
|
|
"fcm_dpo/q_t": 0.32743436098098755,
|
||
|
|
"grad_norm": 36.42646408081055,
|
||
|
|
"learning_rate": 1.31753782067201e-11,
|
||
|
|
"logits/chosen": 0.2187252789735794,
|
||
|
|
"logits/rejected": 0.2422519475221634,
|
||
|
|
"logps/chosen": -443.4412536621094,
|
||
|
|
"logps/ref_chosen": -60.25421905517578,
|
||
|
|
"logps/ref_rejected": -87.23457336425781,
|
||
|
|
"logps/rejected": -687.6876220703125,
|
||
|
|
"loss": 0.8521,
|
||
|
|
"margin_dpo/margin_mean": 217.2659912109375,
|
||
|
|
"margin_dpo/margin_std": 204.62794494628906,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0,
|
||
|
|
"step": 681,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.8947697063024523,
|
||
|
|
"train_runtime": 1846.2605,
|
||
|
|
"train_samples_per_second": 23.614,
|
||
|
|
"train_steps_per_second": 0.369
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 681,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 50,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|