ab10c6b4c0
Model: jackf857/llama-3-8b-base-new-dpo-hh-helpful-s_star0.4-4xh200-batch-64-20260421-214335-rerun Source: Original Platform
2707 lines
99 KiB
JSON
2707 lines
99 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 200,
|
|
"global_step": 681,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0014684287812041115,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.02287006378173828,
|
|
"fcm_dpo/q_t": 0.5005706548690796,
|
|
"grad_norm": 83.51022338867188,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.4974287748336792,
|
|
"logits/rejected": -0.43299180269241333,
|
|
"logps/chosen": -50.1435661315918,
|
|
"logps/ref_chosen": -50.14883804321289,
|
|
"logps/ref_rejected": -74.1280517578125,
|
|
"logps/rejected": -74.09991455078125,
|
|
"loss": 1.389,
|
|
"margin_dpo/margin_mean": -0.02287048101425171,
|
|
"margin_dpo/margin_std": 0.41920793056488037,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.007342143906020558,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.007774516940116882,
|
|
"fcm_dpo/q_t": 0.500194251537323,
|
|
"grad_norm": 89.51734924316406,
|
|
"learning_rate": 2.898550724637681e-08,
|
|
"logits/chosen": -0.4900396764278412,
|
|
"logits/rejected": -0.4533483386039734,
|
|
"logps/chosen": -56.05889892578125,
|
|
"logps/ref_chosen": -56.05734634399414,
|
|
"logps/ref_rejected": -78.69325256347656,
|
|
"logps/rejected": -78.68702697753906,
|
|
"loss": 1.3874,
|
|
"margin_dpo/margin_mean": -0.007774434983730316,
|
|
"margin_dpo/margin_std": 0.37205374240875244,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.014684287812041116,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.017793914303183556,
|
|
"fcm_dpo/q_t": 0.49955543875694275,
|
|
"grad_norm": 70.37712097167969,
|
|
"learning_rate": 6.521739130434782e-08,
|
|
"logits/chosen": -0.49457064270973206,
|
|
"logits/rejected": -0.4558273255825043,
|
|
"logps/chosen": -59.53853225708008,
|
|
"logps/ref_chosen": -59.54457473754883,
|
|
"logps/ref_rejected": -91.17041778564453,
|
|
"logps/rejected": -91.18217468261719,
|
|
"loss": 1.3849,
|
|
"margin_dpo/margin_mean": 0.017794013023376465,
|
|
"margin_dpo/margin_std": 0.40502458810806274,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.022026431718061675,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.013358664698898792,
|
|
"fcm_dpo/q_t": 0.4996658265590668,
|
|
"grad_norm": 64.22943878173828,
|
|
"learning_rate": 1.0144927536231885e-07,
|
|
"logits/chosen": -0.48777151107788086,
|
|
"logits/rejected": -0.4582846164703369,
|
|
"logps/chosen": -58.85475540161133,
|
|
"logps/ref_chosen": -58.83195877075195,
|
|
"logps/ref_rejected": -92.93949890136719,
|
|
"logps/rejected": -92.97565460205078,
|
|
"loss": 1.3854,
|
|
"margin_dpo/margin_mean": 0.01335877738893032,
|
|
"margin_dpo/margin_std": 0.3935544788837433,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.02936857562408223,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.09099045395851135,
|
|
"fcm_dpo/q_t": 0.49772581458091736,
|
|
"grad_norm": 73.8072509765625,
|
|
"learning_rate": 1.3768115942028986e-07,
|
|
"logits/chosen": -0.5034265518188477,
|
|
"logits/rejected": -0.45439839363098145,
|
|
"logps/chosen": -59.618431091308594,
|
|
"logps/ref_chosen": -59.6396598815918,
|
|
"logps/ref_rejected": -82.76026916503906,
|
|
"logps/rejected": -82.83003234863281,
|
|
"loss": 1.3775,
|
|
"margin_dpo/margin_mean": 0.09099055081605911,
|
|
"margin_dpo/margin_std": 0.35805386304855347,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.03671071953010279,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.21624115109443665,
|
|
"fcm_dpo/q_t": 0.49459710717201233,
|
|
"grad_norm": 73.70997619628906,
|
|
"learning_rate": 1.7391304347826085e-07,
|
|
"logits/chosen": -0.5030972361564636,
|
|
"logits/rejected": -0.47600775957107544,
|
|
"logps/chosen": -53.1678352355957,
|
|
"logps/ref_chosen": -53.205284118652344,
|
|
"logps/ref_rejected": -88.99608612060547,
|
|
"logps/rejected": -89.17488098144531,
|
|
"loss": 1.3652,
|
|
"margin_dpo/margin_mean": 0.21624140441417694,
|
|
"margin_dpo/margin_std": 0.4134984016418457,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.04405286343612335,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.4783309996128082,
|
|
"fcm_dpo/q_t": 0.4880555272102356,
|
|
"grad_norm": 87.57738494873047,
|
|
"learning_rate": 2.1014492753623187e-07,
|
|
"logits/chosen": -0.5126634240150452,
|
|
"logits/rejected": -0.4724017083644867,
|
|
"logps/chosen": -53.46284103393555,
|
|
"logps/ref_chosen": -53.5526008605957,
|
|
"logps/ref_rejected": -97.87371826171875,
|
|
"logps/rejected": -98.26229858398438,
|
|
"loss": 1.3399,
|
|
"margin_dpo/margin_mean": 0.47833117842674255,
|
|
"margin_dpo/margin_std": 0.5532158017158508,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.0513950073421439,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.7770208120346069,
|
|
"fcm_dpo/q_t": 0.48065242171287537,
|
|
"grad_norm": 82.80741119384766,
|
|
"learning_rate": 2.463768115942029e-07,
|
|
"logits/chosen": -0.5122802257537842,
|
|
"logits/rejected": -0.4792337417602539,
|
|
"logps/chosen": -56.210533142089844,
|
|
"logps/ref_chosen": -56.3298454284668,
|
|
"logps/ref_rejected": -91.76858520507812,
|
|
"logps/rejected": -92.4262924194336,
|
|
"loss": 1.3121,
|
|
"margin_dpo/margin_mean": 0.7770206332206726,
|
|
"margin_dpo/margin_std": 0.8431359529495239,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.05873715124816446,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.3613355159759521,
|
|
"fcm_dpo/q_t": 0.46629637479782104,
|
|
"grad_norm": 60.205745697021484,
|
|
"learning_rate": 2.8260869565217386e-07,
|
|
"logits/chosen": -0.5219647884368896,
|
|
"logits/rejected": -0.4850676953792572,
|
|
"logps/chosen": -54.278953552246094,
|
|
"logps/ref_chosen": -54.38492965698242,
|
|
"logps/ref_rejected": -82.94353485107422,
|
|
"logps/rejected": -84.19891357421875,
|
|
"loss": 1.2605,
|
|
"margin_dpo/margin_mean": 1.3613355159759521,
|
|
"margin_dpo/margin_std": 1.4631270170211792,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.06607929515418502,
|
|
"fcm_dpo/beta": 0.10669933259487152,
|
|
"fcm_dpo/delta": 0.12330988794565201,
|
|
"fcm_dpo/margin": 2.4058146476745605,
|
|
"fcm_dpo/q_t": 0.4380177855491638,
|
|
"grad_norm": 68.1404800415039,
|
|
"learning_rate": 3.188405797101449e-07,
|
|
"logits/chosen": -0.5121681094169617,
|
|
"logits/rejected": -0.4841387867927551,
|
|
"logps/chosen": -54.63410186767578,
|
|
"logps/ref_chosen": -54.862335205078125,
|
|
"logps/ref_rejected": -98.0264663696289,
|
|
"logps/rejected": -100.20404052734375,
|
|
"loss": 1.1601,
|
|
"margin_dpo/margin_mean": 2.4058141708374023,
|
|
"margin_dpo/margin_std": 2.224526882171631,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.07342143906020558,
|
|
"fcm_dpo/beta": 0.11535553634166718,
|
|
"fcm_dpo/delta": 0.0067912861704826355,
|
|
"fcm_dpo/margin": 3.4076931476593018,
|
|
"fcm_dpo/q_t": 0.4073707163333893,
|
|
"grad_norm": 68.53203582763672,
|
|
"learning_rate": 3.5507246376811595e-07,
|
|
"logits/chosen": -0.562477707862854,
|
|
"logits/rejected": -0.5116110444068909,
|
|
"logps/chosen": -58.12030029296875,
|
|
"logps/ref_chosen": -58.304595947265625,
|
|
"logps/ref_rejected": -91.69480895996094,
|
|
"logps/rejected": -94.91819763183594,
|
|
"loss": 1.0694,
|
|
"margin_dpo/margin_mean": 3.4076931476593018,
|
|
"margin_dpo/margin_std": 3.365309953689575,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.08076358296622614,
|
|
"fcm_dpo/beta": 0.10359964519739151,
|
|
"fcm_dpo/delta": -0.19395628571510315,
|
|
"fcm_dpo/margin": 5.653986930847168,
|
|
"fcm_dpo/q_t": 0.3696584403514862,
|
|
"grad_norm": 52.22880935668945,
|
|
"learning_rate": 3.9130434782608694e-07,
|
|
"logits/chosen": -0.5961982011795044,
|
|
"logits/rejected": -0.5513918399810791,
|
|
"logps/chosen": -56.34511184692383,
|
|
"logps/ref_chosen": -56.06591796875,
|
|
"logps/ref_rejected": -85.69367980957031,
|
|
"logps/rejected": -91.62686157226562,
|
|
"loss": 0.968,
|
|
"margin_dpo/margin_mean": 5.653987407684326,
|
|
"margin_dpo/margin_std": 5.778587818145752,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.0881057268722467,
|
|
"fcm_dpo/beta": 0.08351375162601471,
|
|
"fcm_dpo/delta": -0.2116357386112213,
|
|
"fcm_dpo/margin": 7.191043853759766,
|
|
"fcm_dpo/q_t": 0.3656514883041382,
|
|
"grad_norm": 53.557796478271484,
|
|
"learning_rate": 4.2753623188405794e-07,
|
|
"logits/chosen": -0.614372193813324,
|
|
"logits/rejected": -0.5657563209533691,
|
|
"logps/chosen": -62.0140266418457,
|
|
"logps/ref_chosen": -60.6871337890625,
|
|
"logps/ref_rejected": -89.72715759277344,
|
|
"logps/rejected": -98.24510192871094,
|
|
"loss": 0.9663,
|
|
"margin_dpo/margin_mean": 7.191043853759766,
|
|
"margin_dpo/margin_std": 7.640544891357422,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.09544787077826726,
|
|
"fcm_dpo/beta": 0.06861739605665207,
|
|
"fcm_dpo/delta": -0.19986796379089355,
|
|
"fcm_dpo/margin": 8.604007720947266,
|
|
"fcm_dpo/q_t": 0.37094324827194214,
|
|
"grad_norm": 41.2183723449707,
|
|
"learning_rate": 4.63768115942029e-07,
|
|
"logits/chosen": -0.6187667846679688,
|
|
"logits/rejected": -0.5872984528541565,
|
|
"logps/chosen": -63.8182487487793,
|
|
"logps/ref_chosen": -61.75325393676758,
|
|
"logps/ref_rejected": -93.30108642578125,
|
|
"logps/rejected": -103.9700927734375,
|
|
"loss": 0.9904,
|
|
"margin_dpo/margin_mean": 8.604008674621582,
|
|
"margin_dpo/margin_std": 10.206991195678711,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.1027900146842878,
|
|
"fcm_dpo/beta": 0.05661847069859505,
|
|
"fcm_dpo/delta": -0.1891469657421112,
|
|
"fcm_dpo/margin": 10.259188652038574,
|
|
"fcm_dpo/q_t": 0.3718249201774597,
|
|
"grad_norm": 39.173946380615234,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": -0.6201697587966919,
|
|
"logits/rejected": -0.5813671350479126,
|
|
"logps/chosen": -63.28239059448242,
|
|
"logps/ref_chosen": -59.548004150390625,
|
|
"logps/ref_rejected": -84.01609802246094,
|
|
"logps/rejected": -98.00967407226562,
|
|
"loss": 0.98,
|
|
"margin_dpo/margin_mean": 10.259187698364258,
|
|
"margin_dpo/margin_std": 11.377448081970215,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.11013215859030837,
|
|
"fcm_dpo/beta": 0.04313134402036667,
|
|
"fcm_dpo/delta": -0.354682594537735,
|
|
"fcm_dpo/margin": 17.1324405670166,
|
|
"fcm_dpo/q_t": 0.34925174713134766,
|
|
"grad_norm": 31.45458221435547,
|
|
"learning_rate": 4.999176576834721e-07,
|
|
"logits/chosen": -0.6573521494865417,
|
|
"logits/rejected": -0.6307379603385925,
|
|
"logps/chosen": -67.56647491455078,
|
|
"logps/ref_chosen": -59.86931228637695,
|
|
"logps/ref_rejected": -98.05613708496094,
|
|
"logps/rejected": -122.8857421875,
|
|
"loss": 0.9415,
|
|
"margin_dpo/margin_mean": 17.132436752319336,
|
|
"margin_dpo/margin_std": 21.05853843688965,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.11747430249632893,
|
|
"fcm_dpo/beta": 0.03168341889977455,
|
|
"fcm_dpo/delta": -0.22349825501441956,
|
|
"fcm_dpo/margin": 19.243648529052734,
|
|
"fcm_dpo/q_t": 0.3687431514263153,
|
|
"grad_norm": 28.407453536987305,
|
|
"learning_rate": 4.996706849759452e-07,
|
|
"logits/chosen": -0.6761894822120667,
|
|
"logits/rejected": -0.6410226821899414,
|
|
"logps/chosen": -66.81375122070312,
|
|
"logps/ref_chosen": -56.18925857543945,
|
|
"logps/ref_rejected": -86.42393493652344,
|
|
"logps/rejected": -116.2920913696289,
|
|
"loss": 0.9939,
|
|
"margin_dpo/margin_mean": 19.243648529052734,
|
|
"margin_dpo/margin_std": 23.982927322387695,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12481644640234948,
|
|
"fcm_dpo/beta": 0.025587420910596848,
|
|
"fcm_dpo/delta": -0.18879233300685883,
|
|
"fcm_dpo/margin": 22.462379455566406,
|
|
"fcm_dpo/q_t": 0.3756103515625,
|
|
"grad_norm": 26.530624389648438,
|
|
"learning_rate": 4.992592445678582e-07,
|
|
"logits/chosen": -0.6672359704971313,
|
|
"logits/rejected": -0.6387097835540771,
|
|
"logps/chosen": -73.97752380371094,
|
|
"logps/ref_chosen": -60.018287658691406,
|
|
"logps/ref_rejected": -98.01185607910156,
|
|
"logps/rejected": -134.43350219726562,
|
|
"loss": 1.0175,
|
|
"margin_dpo/margin_mean": 22.462383270263672,
|
|
"margin_dpo/margin_std": 29.141366958618164,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.13215859030837004,
|
|
"fcm_dpo/beta": 0.02217969484627247,
|
|
"fcm_dpo/delta": -0.15027864277362823,
|
|
"fcm_dpo/margin": 24.47287368774414,
|
|
"fcm_dpo/q_t": 0.38759851455688477,
|
|
"grad_norm": 26.1925106048584,
|
|
"learning_rate": 4.986836074908615e-07,
|
|
"logits/chosen": -0.6896823644638062,
|
|
"logits/rejected": -0.6781142354011536,
|
|
"logps/chosen": -78.13359069824219,
|
|
"logps/ref_chosen": -59.8709831237793,
|
|
"logps/ref_rejected": -96.78519439697266,
|
|
"logps/rejected": -139.5206756591797,
|
|
"loss": 1.0705,
|
|
"margin_dpo/margin_mean": 24.47287368774414,
|
|
"margin_dpo/margin_std": 37.618186950683594,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.1395007342143906,
|
|
"fcm_dpo/beta": 0.019353795796632767,
|
|
"fcm_dpo/delta": -0.10450585186481476,
|
|
"fcm_dpo/margin": 25.787700653076172,
|
|
"fcm_dpo/q_t": 0.3899272084236145,
|
|
"grad_norm": 21.816125869750977,
|
|
"learning_rate": 4.979441529392784e-07,
|
|
"logits/chosen": -0.690959095954895,
|
|
"logits/rejected": -0.6617642641067505,
|
|
"logps/chosen": -74.7010269165039,
|
|
"logps/ref_chosen": -55.94385528564453,
|
|
"logps/ref_rejected": -83.6790542602539,
|
|
"logps/rejected": -128.2239227294922,
|
|
"loss": 1.0485,
|
|
"margin_dpo/margin_mean": 25.787700653076172,
|
|
"margin_dpo/margin_std": 34.392967224121094,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.14684287812041116,
|
|
"fcm_dpo/beta": 0.017355434596538544,
|
|
"fcm_dpo/delta": -0.09411351382732391,
|
|
"fcm_dpo/margin": 28.09613037109375,
|
|
"fcm_dpo/q_t": 0.3915005326271057,
|
|
"grad_norm": 23.495582580566406,
|
|
"learning_rate": 4.970413680203148e-07,
|
|
"logits/chosen": -0.6664688587188721,
|
|
"logits/rejected": -0.6419214010238647,
|
|
"logps/chosen": -78.07569885253906,
|
|
"logps/ref_chosen": -57.05888748168945,
|
|
"logps/ref_rejected": -86.11727142333984,
|
|
"logps/rejected": -135.23020935058594,
|
|
"loss": 1.057,
|
|
"margin_dpo/margin_mean": 28.09613037109375,
|
|
"margin_dpo/margin_std": 37.85677719116211,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15418502202643172,
|
|
"fcm_dpo/beta": 0.01605215296149254,
|
|
"fcm_dpo/delta": -0.14830544590950012,
|
|
"fcm_dpo/margin": 32.540122985839844,
|
|
"fcm_dpo/q_t": 0.38781845569610596,
|
|
"grad_norm": 22.918745040893555,
|
|
"learning_rate": 4.959758474331832e-07,
|
|
"logits/chosen": -0.6817123889923096,
|
|
"logits/rejected": -0.6505634188652039,
|
|
"logps/chosen": -84.82267761230469,
|
|
"logps/ref_chosen": -59.20774459838867,
|
|
"logps/ref_rejected": -86.49754333496094,
|
|
"logps/rejected": -144.65260314941406,
|
|
"loss": 1.052,
|
|
"margin_dpo/margin_mean": 32.540122985839844,
|
|
"margin_dpo/margin_std": 44.27156066894531,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.16152716593245228,
|
|
"fcm_dpo/beta": 0.013644491322338581,
|
|
"fcm_dpo/delta": -0.1275637000799179,
|
|
"fcm_dpo/margin": 38.23952102661133,
|
|
"fcm_dpo/q_t": 0.38538408279418945,
|
|
"grad_norm": 18.83212661743164,
|
|
"learning_rate": 4.947482930773511e-07,
|
|
"logits/chosen": -0.6280543804168701,
|
|
"logits/rejected": -0.5978196859359741,
|
|
"logps/chosen": -88.46613311767578,
|
|
"logps/ref_chosen": -60.437957763671875,
|
|
"logps/ref_rejected": -90.83917999267578,
|
|
"logps/rejected": -157.1068878173828,
|
|
"loss": 1.0403,
|
|
"margin_dpo/margin_mean": 38.23952102661133,
|
|
"margin_dpo/margin_std": 50.03200149536133,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.16886930983847284,
|
|
"fcm_dpo/beta": 0.012269817292690277,
|
|
"fcm_dpo/delta": -0.06500422954559326,
|
|
"fcm_dpo/margin": 37.55778121948242,
|
|
"fcm_dpo/q_t": 0.39965710043907166,
|
|
"grad_norm": 27.971012115478516,
|
|
"learning_rate": 4.933595135901732e-07,
|
|
"logits/chosen": -0.6194974184036255,
|
|
"logits/rejected": -0.5893136262893677,
|
|
"logps/chosen": -96.05851745605469,
|
|
"logps/ref_chosen": -61.7908821105957,
|
|
"logps/ref_rejected": -85.36943054199219,
|
|
"logps/rejected": -157.19485473632812,
|
|
"loss": 1.0871,
|
|
"margin_dpo/margin_mean": 37.55778121948242,
|
|
"margin_dpo/margin_std": 56.132057189941406,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.1762114537444934,
|
|
"fcm_dpo/beta": 0.01178265642374754,
|
|
"fcm_dpo/delta": -0.05727796629071236,
|
|
"fcm_dpo/margin": 38.61918640136719,
|
|
"fcm_dpo/q_t": 0.39924171566963196,
|
|
"grad_norm": 21.68465805053711,
|
|
"learning_rate": 4.918104238142103e-07,
|
|
"logits/chosen": -0.6018606424331665,
|
|
"logits/rejected": -0.580163836479187,
|
|
"logps/chosen": -104.28592681884766,
|
|
"logps/ref_chosen": -65.3261489868164,
|
|
"logps/ref_rejected": -86.75518798828125,
|
|
"logps/rejected": -164.33416748046875,
|
|
"loss": 1.0756,
|
|
"margin_dpo/margin_mean": 38.61919021606445,
|
|
"margin_dpo/margin_std": 53.98603057861328,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18355359765051396,
|
|
"fcm_dpo/beta": 0.009986856020987034,
|
|
"fcm_dpo/delta": -0.2121940553188324,
|
|
"fcm_dpo/margin": 59.94220733642578,
|
|
"fcm_dpo/q_t": 0.3725253939628601,
|
|
"grad_norm": 19.076047897338867,
|
|
"learning_rate": 4.90102044194588e-07,
|
|
"logits/chosen": -0.5440986752510071,
|
|
"logits/rejected": -0.5444971323013306,
|
|
"logps/chosen": -102.51204681396484,
|
|
"logps/ref_chosen": -58.323204040527344,
|
|
"logps/ref_rejected": -101.2106704711914,
|
|
"logps/rejected": -205.34170532226562,
|
|
"loss": 1.0102,
|
|
"margin_dpo/margin_mean": 59.94221115112305,
|
|
"margin_dpo/margin_std": 79.08087158203125,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.19089574155653452,
|
|
"fcm_dpo/beta": 0.00899023748934269,
|
|
"fcm_dpo/delta": -0.0574115514755249,
|
|
"fcm_dpo/margin": 50.56972122192383,
|
|
"fcm_dpo/q_t": 0.4008238911628723,
|
|
"grad_norm": 21.011878967285156,
|
|
"learning_rate": 4.882355001067891e-07,
|
|
"logits/chosen": -0.5073991417884827,
|
|
"logits/rejected": -0.49552878737449646,
|
|
"logps/chosen": -103.4196548461914,
|
|
"logps/ref_chosen": -56.38518524169922,
|
|
"logps/ref_rejected": -86.15767669677734,
|
|
"logps/rejected": -183.76187133789062,
|
|
"loss": 1.0949,
|
|
"margin_dpo/margin_mean": 50.569725036621094,
|
|
"margin_dpo/margin_std": 77.31275939941406,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.19823788546255505,
|
|
"fcm_dpo/beta": 0.00818575732409954,
|
|
"fcm_dpo/delta": -0.10119257867336273,
|
|
"fcm_dpo/margin": 60.596702575683594,
|
|
"fcm_dpo/q_t": 0.39124470949172974,
|
|
"grad_norm": 21.952133178710938,
|
|
"learning_rate": 4.862120211153265e-07,
|
|
"logits/chosen": -0.487496942281723,
|
|
"logits/rejected": -0.4898650050163269,
|
|
"logps/chosen": -103.59019470214844,
|
|
"logps/ref_chosen": -54.59065628051758,
|
|
"logps/ref_rejected": -95.26080322265625,
|
|
"logps/rejected": -204.8570556640625,
|
|
"loss": 1.0502,
|
|
"margin_dpo/margin_mean": 60.596702575683594,
|
|
"margin_dpo/margin_std": 81.37339782714844,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.2055800293685756,
|
|
"fcm_dpo/beta": 0.007442955859005451,
|
|
"fcm_dpo/delta": -0.09183903783559799,
|
|
"fcm_dpo/margin": 65.49043273925781,
|
|
"fcm_dpo/q_t": 0.39486831426620483,
|
|
"grad_norm": 20.813894271850586,
|
|
"learning_rate": 4.840329401637809e-07,
|
|
"logits/chosen": -0.4298696517944336,
|
|
"logits/rejected": -0.4265991747379303,
|
|
"logps/chosen": -114.61155700683594,
|
|
"logps/ref_chosen": -56.04347610473633,
|
|
"logps/ref_rejected": -93.27880859375,
|
|
"logps/rejected": -217.3373565673828,
|
|
"loss": 1.064,
|
|
"margin_dpo/margin_mean": 65.49043273925781,
|
|
"margin_dpo/margin_std": 92.36592102050781,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.21292217327459617,
|
|
"fcm_dpo/beta": 0.007036598864942789,
|
|
"fcm_dpo/delta": -0.012454366311430931,
|
|
"fcm_dpo/margin": 58.48834991455078,
|
|
"fcm_dpo/q_t": 0.40865078568458557,
|
|
"grad_norm": 23.93450927734375,
|
|
"learning_rate": 4.816996926967401e-07,
|
|
"logits/chosen": -0.40944308042526245,
|
|
"logits/rejected": -0.39997661113739014,
|
|
"logps/chosen": -129.96218872070312,
|
|
"logps/ref_chosen": -61.4414176940918,
|
|
"logps/ref_rejected": -86.32813262939453,
|
|
"logps/rejected": -213.33724975585938,
|
|
"loss": 1.115,
|
|
"margin_dpo/margin_mean": 58.48834991455078,
|
|
"margin_dpo/margin_std": 92.11347961425781,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.22026431718061673,
|
|
"fcm_dpo/beta": 0.006964278873056173,
|
|
"fcm_dpo/delta": -0.03296690061688423,
|
|
"fcm_dpo/margin": 59.02332305908203,
|
|
"fcm_dpo/q_t": 0.4086775779724121,
|
|
"grad_norm": 18.16049575805664,
|
|
"learning_rate": 4.792138157142157e-07,
|
|
"logits/chosen": -0.44094228744506836,
|
|
"logits/rejected": -0.42822694778442383,
|
|
"logps/chosen": -123.40377044677734,
|
|
"logps/ref_chosen": -57.70451736450195,
|
|
"logps/ref_rejected": -87.76991271972656,
|
|
"logps/rejected": -212.4925079345703,
|
|
"loss": 1.1066,
|
|
"margin_dpo/margin_mean": 59.02332305908203,
|
|
"margin_dpo/margin_std": 87.62327575683594,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.2276064610866373,
|
|
"fcm_dpo/beta": 0.006572508253157139,
|
|
"fcm_dpo/delta": -0.058337144553661346,
|
|
"fcm_dpo/margin": 69.28470611572266,
|
|
"fcm_dpo/q_t": 0.3986854553222656,
|
|
"grad_norm": 19.04676628112793,
|
|
"learning_rate": 4.7657694675916247e-07,
|
|
"logits/chosen": -0.4826192855834961,
|
|
"logits/rejected": -0.4734383523464203,
|
|
"logps/chosen": -126.15814208984375,
|
|
"logps/ref_chosen": -62.08925247192383,
|
|
"logps/ref_rejected": -94.79930114746094,
|
|
"logps/rejected": -228.1529083251953,
|
|
"loss": 1.076,
|
|
"margin_dpo/margin_mean": 69.28469848632812,
|
|
"margin_dpo/margin_std": 97.56148529052734,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.23494860499265785,
|
|
"fcm_dpo/beta": 0.006233358755707741,
|
|
"fcm_dpo/delta": -0.06433447450399399,
|
|
"fcm_dpo/margin": 70.75523376464844,
|
|
"fcm_dpo/q_t": 0.4046873450279236,
|
|
"grad_norm": 23.905847549438477,
|
|
"learning_rate": 4.737908228387656e-07,
|
|
"logits/chosen": -0.37852075695991516,
|
|
"logits/rejected": -0.37446096539497375,
|
|
"logps/chosen": -155.4814453125,
|
|
"logps/ref_chosen": -67.15288543701172,
|
|
"logps/ref_rejected": -96.92537689208984,
|
|
"logps/rejected": -256.0091552734375,
|
|
"loss": 1.1205,
|
|
"margin_dpo/margin_mean": 70.75523376464844,
|
|
"margin_dpo/margin_std": 118.20479583740234,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.2422907488986784,
|
|
"fcm_dpo/beta": 0.005916006397455931,
|
|
"fcm_dpo/delta": -0.016961436718702316,
|
|
"fcm_dpo/margin": 70.23262023925781,
|
|
"fcm_dpo/q_t": 0.4076644480228424,
|
|
"grad_norm": 21.921010971069336,
|
|
"learning_rate": 4.708572792802069e-07,
|
|
"logits/chosen": -0.38327422738075256,
|
|
"logits/rejected": -0.369274765253067,
|
|
"logps/chosen": -138.60523986816406,
|
|
"logps/ref_chosen": -57.40401077270508,
|
|
"logps/ref_rejected": -80.31498718261719,
|
|
"logps/rejected": -231.7488555908203,
|
|
"loss": 1.1054,
|
|
"margin_dpo/margin_mean": 70.23262023925781,
|
|
"margin_dpo/margin_std": 105.99822998046875,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.24963289280469897,
|
|
"fcm_dpo/beta": 0.005466562695801258,
|
|
"fcm_dpo/delta": -0.10773168504238129,
|
|
"fcm_dpo/margin": 87.78046417236328,
|
|
"fcm_dpo/q_t": 0.3945610821247101,
|
|
"grad_norm": 17.49802589416504,
|
|
"learning_rate": 4.6777824852166437e-07,
|
|
"logits/chosen": -0.3227120041847229,
|
|
"logits/rejected": -0.3182465136051178,
|
|
"logps/chosen": -131.82830810546875,
|
|
"logps/ref_chosen": -52.029144287109375,
|
|
"logps/ref_rejected": -85.73944091796875,
|
|
"logps/rejected": -253.31906127929688,
|
|
"loss": 1.0613,
|
|
"margin_dpo/margin_mean": 87.78046417236328,
|
|
"margin_dpo/margin_std": 120.57357025146484,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.25697503671071953,
|
|
"fcm_dpo/beta": 0.0053649842739105225,
|
|
"fcm_dpo/delta": -0.02835622988641262,
|
|
"fcm_dpo/margin": 79.64360809326172,
|
|
"fcm_dpo/q_t": 0.40541133284568787,
|
|
"grad_norm": 29.300512313842773,
|
|
"learning_rate": 4.645557588393406e-07,
|
|
"logits/chosen": -0.3354695737361908,
|
|
"logits/rejected": -0.3250008523464203,
|
|
"logps/chosen": -156.86700439453125,
|
|
"logps/ref_chosen": -62.996971130371094,
|
|
"logps/ref_rejected": -92.98394012451172,
|
|
"logps/rejected": -266.4975891113281,
|
|
"loss": 1.0959,
|
|
"margin_dpo/margin_mean": 79.64360046386719,
|
|
"margin_dpo/margin_std": 116.92179107666016,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.2643171806167401,
|
|
"fcm_dpo/beta": 0.004915366414934397,
|
|
"fcm_dpo/delta": -0.08844007551670074,
|
|
"fcm_dpo/margin": 98.45795440673828,
|
|
"fcm_dpo/q_t": 0.39298662543296814,
|
|
"grad_norm": 18.73993492126465,
|
|
"learning_rate": 4.611919330113591e-07,
|
|
"logits/chosen": -0.2953912019729614,
|
|
"logits/rejected": -0.29680582880973816,
|
|
"logps/chosen": -155.4091796875,
|
|
"logps/ref_chosen": -57.0670280456543,
|
|
"logps/ref_rejected": -97.1115493774414,
|
|
"logps/rejected": -293.9116516113281,
|
|
"loss": 1.0541,
|
|
"margin_dpo/margin_mean": 98.45795440673828,
|
|
"margin_dpo/margin_std": 131.67587280273438,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.27165932452276065,
|
|
"fcm_dpo/beta": 0.00469121104106307,
|
|
"fcm_dpo/delta": -0.007265460677444935,
|
|
"fcm_dpo/margin": 72.3204574584961,
|
|
"fcm_dpo/q_t": 0.423015296459198,
|
|
"grad_norm": 19.846059799194336,
|
|
"learning_rate": 4.5768898691940836e-07,
|
|
"logits/chosen": -0.3211399018764496,
|
|
"logits/rejected": -0.31003421545028687,
|
|
"logps/chosen": -150.46461486816406,
|
|
"logps/ref_chosen": -54.840736389160156,
|
|
"logps/ref_rejected": -75.51002502441406,
|
|
"logps/rejected": -243.45437622070312,
|
|
"loss": 1.1531,
|
|
"margin_dpo/margin_mean": 72.3204574584961,
|
|
"margin_dpo/margin_std": 118.7593994140625,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.2790014684287812,
|
|
"fcm_dpo/beta": 0.004630076698958874,
|
|
"fcm_dpo/delta": -0.02935381792485714,
|
|
"fcm_dpo/margin": 92.37378692626953,
|
|
"fcm_dpo/q_t": 0.4038674235343933,
|
|
"grad_norm": 18.63943862915039,
|
|
"learning_rate": 4.5404922808905543e-07,
|
|
"logits/chosen": -0.3224960267543793,
|
|
"logits/rejected": -0.31152957677841187,
|
|
"logps/chosen": -155.59190368652344,
|
|
"logps/ref_chosen": -57.72148895263672,
|
|
"logps/ref_rejected": -86.85997009277344,
|
|
"logps/rejected": -277.10418701171875,
|
|
"loss": 1.0829,
|
|
"margin_dpo/margin_mean": 92.37378692626953,
|
|
"margin_dpo/margin_std": 125.8380126953125,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.28634361233480177,
|
|
"fcm_dpo/beta": 0.004372724797576666,
|
|
"fcm_dpo/delta": -0.041835661977529526,
|
|
"fcm_dpo/margin": 94.63658142089844,
|
|
"fcm_dpo/q_t": 0.4060741066932678,
|
|
"grad_norm": 19.838130950927734,
|
|
"learning_rate": 4.5027505416968985e-07,
|
|
"logits/chosen": -0.2853700518608093,
|
|
"logits/rejected": -0.2791460156440735,
|
|
"logps/chosen": -171.98052978515625,
|
|
"logps/ref_chosen": -58.26164627075195,
|
|
"logps/ref_rejected": -89.46485900878906,
|
|
"logps/rejected": -297.82037353515625,
|
|
"loss": 1.0971,
|
|
"margin_dpo/margin_mean": 94.63658142089844,
|
|
"margin_dpo/margin_std": 133.46847534179688,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.2936857562408223,
|
|
"fcm_dpo/beta": 0.0042917924001812935,
|
|
"fcm_dpo/delta": -0.03731077164411545,
|
|
"fcm_dpo/margin": 101.45069122314453,
|
|
"fcm_dpo/q_t": 0.40244197845458984,
|
|
"grad_norm": 21.282487869262695,
|
|
"learning_rate": 4.4636895135509966e-07,
|
|
"logits/chosen": -0.29983407258987427,
|
|
"logits/rejected": -0.2885088324546814,
|
|
"logps/chosen": -161.27999877929688,
|
|
"logps/ref_chosen": -55.71953201293945,
|
|
"logps/ref_rejected": -83.15235137939453,
|
|
"logps/rejected": -290.16351318359375,
|
|
"loss": 1.0801,
|
|
"margin_dpo/margin_mean": 101.45067596435547,
|
|
"margin_dpo/margin_std": 138.32688903808594,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2936857562408223,
|
|
"eval_fcm_dpo/beta": 0.004899847786873579,
|
|
"eval_fcm_dpo/delta": 0.03208634629845619,
|
|
"eval_fcm_dpo/margin": 58.008628845214844,
|
|
"eval_fcm_dpo/q_t": 0.4392388164997101,
|
|
"eval_logits/chosen": -0.3472006916999817,
|
|
"eval_logits/rejected": -0.3329962491989136,
|
|
"eval_logps/chosen": -225.8389892578125,
|
|
"eval_logps/ref_chosen": -79.05104064941406,
|
|
"eval_logps/ref_rejected": -86.79793548583984,
|
|
"eval_logps/rejected": -291.5945129394531,
|
|
"eval_loss": 0.6239348649978638,
|
|
"eval_margin_dpo/margin_mean": 58.008628845214844,
|
|
"eval_margin_dpo/margin_std": 147.6545867919922,
|
|
"eval_runtime": 39.6889,
|
|
"eval_samples_per_second": 58.933,
|
|
"eval_steps_per_second": 1.864,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.3010279001468429,
|
|
"fcm_dpo/beta": 0.005112385377287865,
|
|
"fcm_dpo/delta": -0.07373825460672379,
|
|
"fcm_dpo/margin": 91.88821411132812,
|
|
"fcm_dpo/q_t": 0.39707884192466736,
|
|
"grad_norm": 24.034622192382812,
|
|
"learning_rate": 4.4233349274571974e-07,
|
|
"logits/chosen": -0.3471163213253021,
|
|
"logits/rejected": -0.3326929211616516,
|
|
"logps/chosen": -181.0006103515625,
|
|
"logps/ref_chosen": -65.13258361816406,
|
|
"logps/ref_rejected": -92.10203552246094,
|
|
"logps/rejected": -299.8582763671875,
|
|
"loss": 1.0848,
|
|
"margin_dpo/margin_mean": 91.88821411132812,
|
|
"margin_dpo/margin_std": 137.34268188476562,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.30837004405286345,
|
|
"fcm_dpo/beta": 0.00456308713182807,
|
|
"fcm_dpo/delta": -0.12305764853954315,
|
|
"fcm_dpo/margin": 112.89747619628906,
|
|
"fcm_dpo/q_t": 0.3846554160118103,
|
|
"grad_norm": 25.71703338623047,
|
|
"learning_rate": 4.381713366536311e-07,
|
|
"logits/chosen": -0.30507659912109375,
|
|
"logits/rejected": -0.2925286889076233,
|
|
"logps/chosen": -175.38168334960938,
|
|
"logps/ref_chosen": -54.52837371826172,
|
|
"logps/ref_rejected": -87.06227111816406,
|
|
"logps/rejected": -320.81304931640625,
|
|
"loss": 1.0332,
|
|
"margin_dpo/margin_mean": 112.89747619628906,
|
|
"margin_dpo/margin_std": 143.22720336914062,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.315712187958884,
|
|
"fcm_dpo/beta": 0.004155799280852079,
|
|
"fcm_dpo/delta": -0.09338673204183578,
|
|
"fcm_dpo/margin": 117.60514068603516,
|
|
"fcm_dpo/q_t": 0.39112934470176697,
|
|
"grad_norm": 27.700122833251953,
|
|
"learning_rate": 4.3388522485142885e-07,
|
|
"logits/chosen": -0.2812441289424896,
|
|
"logits/rejected": -0.265227735042572,
|
|
"logps/chosen": -197.89137268066406,
|
|
"logps/ref_chosen": -59.905250549316406,
|
|
"logps/ref_rejected": -90.25511932373047,
|
|
"logps/rejected": -345.84637451171875,
|
|
"loss": 1.0502,
|
|
"margin_dpo/margin_mean": 117.60514068603516,
|
|
"margin_dpo/margin_std": 151.13796997070312,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.32305433186490456,
|
|
"fcm_dpo/beta": 0.0038633563090115786,
|
|
"fcm_dpo/delta": -0.0188637413084507,
|
|
"fcm_dpo/margin": 107.99397277832031,
|
|
"fcm_dpo/q_t": 0.406075656414032,
|
|
"grad_norm": 27.14980697631836,
|
|
"learning_rate": 4.2947798076611047e-07,
|
|
"logits/chosen": -0.2910076379776001,
|
|
"logits/rejected": -0.27931129932403564,
|
|
"logps/chosen": -209.11544799804688,
|
|
"logps/ref_chosen": -57.68498611450195,
|
|
"logps/ref_rejected": -87.72792053222656,
|
|
"logps/rejected": -347.15234375,
|
|
"loss": 1.0982,
|
|
"margin_dpo/margin_mean": 107.99397277832031,
|
|
"margin_dpo/margin_std": 155.56427001953125,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.3303964757709251,
|
|
"fcm_dpo/beta": 0.003635880770161748,
|
|
"fcm_dpo/delta": -0.11107522249221802,
|
|
"fcm_dpo/margin": 138.54534912109375,
|
|
"fcm_dpo/q_t": 0.3866313397884369,
|
|
"grad_norm": 21.774181365966797,
|
|
"learning_rate": 4.249525076191759e-07,
|
|
"logits/chosen": -0.26980072259902954,
|
|
"logits/rejected": -0.2639961540699005,
|
|
"logps/chosen": -216.45687866210938,
|
|
"logps/ref_chosen": -54.47245407104492,
|
|
"logps/ref_rejected": -93.26266479492188,
|
|
"logps/rejected": -393.79248046875,
|
|
"loss": 1.0374,
|
|
"margin_dpo/margin_mean": 138.5453643798828,
|
|
"margin_dpo/margin_std": 173.9648895263672,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.3377386196769457,
|
|
"fcm_dpo/beta": 0.003416923340409994,
|
|
"fcm_dpo/delta": -0.05279744789004326,
|
|
"fcm_dpo/margin": 131.7572021484375,
|
|
"fcm_dpo/q_t": 0.3975502550601959,
|
|
"grad_norm": 29.499502182006836,
|
|
"learning_rate": 4.203117865141635e-07,
|
|
"logits/chosen": -0.3394869565963745,
|
|
"logits/rejected": -0.31970351934432983,
|
|
"logps/chosen": -219.03005981445312,
|
|
"logps/ref_chosen": -58.7701301574707,
|
|
"logps/ref_rejected": -87.85963439941406,
|
|
"logps/rejected": -379.87677001953125,
|
|
"loss": 1.0656,
|
|
"margin_dpo/margin_mean": 131.7572021484375,
|
|
"margin_dpo/margin_std": 169.88063049316406,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.34508076358296624,
|
|
"fcm_dpo/beta": 0.003319287672638893,
|
|
"fcm_dpo/delta": -0.0038651395589113235,
|
|
"fcm_dpo/margin": 121.47023010253906,
|
|
"fcm_dpo/q_t": 0.40905800461769104,
|
|
"grad_norm": 27.062564849853516,
|
|
"learning_rate": 4.1555887447288255e-07,
|
|
"logits/chosen": -0.3577764332294464,
|
|
"logits/rejected": -0.3489416241645813,
|
|
"logps/chosen": -223.79031372070312,
|
|
"logps/ref_chosen": -59.0481071472168,
|
|
"logps/ref_rejected": -91.19654846191406,
|
|
"logps/rejected": -377.40899658203125,
|
|
"loss": 1.0973,
|
|
"margin_dpo/margin_mean": 121.47023010253906,
|
|
"margin_dpo/margin_std": 166.64047241210938,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.3524229074889868,
|
|
"fcm_dpo/beta": 0.0032308667432516813,
|
|
"fcm_dpo/delta": -0.02104497328400612,
|
|
"fcm_dpo/margin": 129.8544921875,
|
|
"fcm_dpo/q_t": 0.4038158357143402,
|
|
"grad_norm": 31.66817855834961,
|
|
"learning_rate": 4.106969024216348e-07,
|
|
"logits/chosen": -0.3540635108947754,
|
|
"logits/rejected": -0.35047775506973267,
|
|
"logps/chosen": -231.42440795898438,
|
|
"logps/ref_chosen": -55.238983154296875,
|
|
"logps/ref_rejected": -91.08428955078125,
|
|
"logps/rejected": -397.12420654296875,
|
|
"loss": 1.0799,
|
|
"margin_dpo/margin_mean": 129.85450744628906,
|
|
"margin_dpo/margin_std": 166.92564392089844,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.35976505139500736,
|
|
"fcm_dpo/beta": 0.003068047808483243,
|
|
"fcm_dpo/delta": -0.07209217548370361,
|
|
"fcm_dpo/margin": 152.28811645507812,
|
|
"fcm_dpo/q_t": 0.3937745690345764,
|
|
"grad_norm": 28.177356719970703,
|
|
"learning_rate": 4.057290731287531e-07,
|
|
"logits/chosen": -0.3967796266078949,
|
|
"logits/rejected": -0.37566548585891724,
|
|
"logps/chosen": -252.69760131835938,
|
|
"logps/ref_chosen": -65.08844757080078,
|
|
"logps/ref_rejected": -86.05777740478516,
|
|
"logps/rejected": -425.95501708984375,
|
|
"loss": 1.0455,
|
|
"margin_dpo/margin_mean": 152.28811645507812,
|
|
"margin_dpo/margin_std": 179.84130859375,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.3671071953010279,
|
|
"fcm_dpo/beta": 0.0029506587889045477,
|
|
"fcm_dpo/delta": -0.029526233673095703,
|
|
"fcm_dpo/margin": 144.90142822265625,
|
|
"fcm_dpo/q_t": 0.4018786549568176,
|
|
"grad_norm": 30.520957946777344,
|
|
"learning_rate": 4.006586590948141e-07,
|
|
"logits/chosen": -0.3914863169193268,
|
|
"logits/rejected": -0.3723214566707611,
|
|
"logps/chosen": -273.7362365722656,
|
|
"logps/ref_chosen": -59.08491897583008,
|
|
"logps/ref_rejected": -87.36727142333984,
|
|
"logps/rejected": -446.9200134277344,
|
|
"loss": 1.0748,
|
|
"margin_dpo/margin_mean": 144.9014434814453,
|
|
"margin_dpo/margin_std": 182.38096618652344,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.3744493392070485,
|
|
"fcm_dpo/beta": 0.0029482415411621332,
|
|
"fcm_dpo/delta": 0.010621277615427971,
|
|
"fcm_dpo/margin": 132.09812927246094,
|
|
"fcm_dpo/q_t": 0.4101809859275818,
|
|
"grad_norm": 21.41288185119629,
|
|
"learning_rate": 3.954890003969163e-07,
|
|
"logits/chosen": -0.38516297936439514,
|
|
"logits/rejected": -0.36370107531547546,
|
|
"logps/chosen": -264.5665588378906,
|
|
"logps/ref_chosen": -61.85979461669922,
|
|
"logps/ref_rejected": -88.32804107666016,
|
|
"logps/rejected": -423.1329650878906,
|
|
"loss": 1.1051,
|
|
"margin_dpo/margin_mean": 132.09812927246094,
|
|
"margin_dpo/margin_std": 182.64788818359375,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.38179148311306904,
|
|
"fcm_dpo/beta": 0.0029091238975524902,
|
|
"fcm_dpo/delta": -0.0311284102499485,
|
|
"fcm_dpo/margin": 147.6251678466797,
|
|
"fcm_dpo/q_t": 0.4018276631832123,
|
|
"grad_norm": 31.76065444946289,
|
|
"learning_rate": 3.9022350248844246e-07,
|
|
"logits/chosen": -0.3353942334651947,
|
|
"logits/rejected": -0.33657923340797424,
|
|
"logps/chosen": -256.07330322265625,
|
|
"logps/ref_chosen": -52.843467712402344,
|
|
"logps/ref_rejected": -90.4744873046875,
|
|
"logps/rejected": -441.3294982910156,
|
|
"loss": 1.0767,
|
|
"margin_dpo/margin_mean": 147.6251678466797,
|
|
"margin_dpo/margin_std": 192.7305908203125,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3891336270190896,
|
|
"fcm_dpo/beta": 0.002804348012432456,
|
|
"fcm_dpo/delta": -0.036855168640613556,
|
|
"fcm_dpo/margin": 155.0548095703125,
|
|
"fcm_dpo/q_t": 0.4009890556335449,
|
|
"grad_norm": 26.897104263305664,
|
|
"learning_rate": 3.848656339557562e-07,
|
|
"logits/chosen": -0.3249055743217468,
|
|
"logits/rejected": -0.3102591931819916,
|
|
"logps/chosen": -295.2470703125,
|
|
"logps/ref_chosen": -59.35320281982422,
|
|
"logps/ref_rejected": -90.73350524902344,
|
|
"logps/rejected": -481.68218994140625,
|
|
"loss": 1.0752,
|
|
"margin_dpo/margin_mean": 155.0548095703125,
|
|
"margin_dpo/margin_std": 203.52064514160156,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.3964757709251101,
|
|
"fcm_dpo/beta": 0.002779749920591712,
|
|
"fcm_dpo/delta": 0.0037582286167889833,
|
|
"fcm_dpo/margin": 142.50425720214844,
|
|
"fcm_dpo/q_t": 0.40962162613868713,
|
|
"grad_norm": 36.368804931640625,
|
|
"learning_rate": 3.794189242333106e-07,
|
|
"logits/chosen": -0.3710918426513672,
|
|
"logits/rejected": -0.3549976944923401,
|
|
"logps/chosen": -305.8779602050781,
|
|
"logps/ref_chosen": -66.30875396728516,
|
|
"logps/ref_rejected": -95.4130630493164,
|
|
"logps/rejected": -477.486572265625,
|
|
"loss": 1.1067,
|
|
"margin_dpo/margin_mean": 142.50425720214844,
|
|
"margin_dpo/margin_std": 207.22396850585938,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.40381791483113066,
|
|
"fcm_dpo/beta": 0.002664657775312662,
|
|
"fcm_dpo/delta": -0.05992863327264786,
|
|
"fcm_dpo/margin": 171.36749267578125,
|
|
"fcm_dpo/q_t": 0.3942517042160034,
|
|
"grad_norm": 29.650026321411133,
|
|
"learning_rate": 3.738869612786737e-07,
|
|
"logits/chosen": -0.3030190169811249,
|
|
"logits/rejected": -0.2977046072483063,
|
|
"logps/chosen": -267.1299743652344,
|
|
"logps/ref_chosen": -54.69990921020508,
|
|
"logps/ref_rejected": -92.23838806152344,
|
|
"logps/rejected": -476.03594970703125,
|
|
"loss": 1.0428,
|
|
"margin_dpo/margin_mean": 171.36749267578125,
|
|
"margin_dpo/margin_std": 191.60519409179688,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.4111600587371512,
|
|
"fcm_dpo/beta": 0.002544180955737829,
|
|
"fcm_dpo/delta": -0.059092938899993896,
|
|
"fcm_dpo/margin": 179.50125122070312,
|
|
"fcm_dpo/q_t": 0.395501971244812,
|
|
"grad_norm": 27.062313079833984,
|
|
"learning_rate": 3.6827338920900253e-07,
|
|
"logits/chosen": -0.2739403247833252,
|
|
"logits/rejected": -0.26840710639953613,
|
|
"logps/chosen": -304.5663146972656,
|
|
"logps/ref_chosen": -54.64586639404297,
|
|
"logps/ref_rejected": -88.19416809082031,
|
|
"logps/rejected": -517.6158447265625,
|
|
"loss": 1.053,
|
|
"margin_dpo/margin_mean": 179.5012664794922,
|
|
"margin_dpo/margin_std": 217.075439453125,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.4185022026431718,
|
|
"fcm_dpo/beta": 0.002449630293995142,
|
|
"fcm_dpo/delta": -0.01916874758899212,
|
|
"fcm_dpo/margin": 170.6898956298828,
|
|
"fcm_dpo/q_t": 0.40265077352523804,
|
|
"grad_norm": 27.657169342041016,
|
|
"learning_rate": 3.625819059005228e-07,
|
|
"logits/chosen": -0.3483652174472809,
|
|
"logits/rejected": -0.33659619092941284,
|
|
"logps/chosen": -324.3221130371094,
|
|
"logps/ref_chosen": -63.02496337890625,
|
|
"logps/ref_rejected": -93.16323852539062,
|
|
"logps/rejected": -525.1502685546875,
|
|
"loss": 1.0686,
|
|
"margin_dpo/margin_mean": 170.68988037109375,
|
|
"margin_dpo/margin_std": 199.1321563720703,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.42584434654919234,
|
|
"fcm_dpo/beta": 0.0023679626174271107,
|
|
"fcm_dpo/delta": -0.05114912986755371,
|
|
"fcm_dpo/margin": 189.60928344726562,
|
|
"fcm_dpo/q_t": 0.39646488428115845,
|
|
"grad_norm": 24.63570785522461,
|
|
"learning_rate": 3.568162605525952e-07,
|
|
"logits/chosen": -0.30854520201683044,
|
|
"logits/rejected": -0.30314746499061584,
|
|
"logps/chosen": -322.7059631347656,
|
|
"logps/ref_chosen": -58.37105178833008,
|
|
"logps/ref_rejected": -91.59428405761719,
|
|
"logps/rejected": -545.5384521484375,
|
|
"loss": 1.0504,
|
|
"margin_dpo/margin_mean": 189.60928344726562,
|
|
"margin_dpo/margin_std": 216.8871307373047,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.4331864904552129,
|
|
"fcm_dpo/beta": 0.00226641446352005,
|
|
"fcm_dpo/delta": -0.026808306574821472,
|
|
"fcm_dpo/margin": 187.75306701660156,
|
|
"fcm_dpo/q_t": 0.4028984606266022,
|
|
"grad_norm": 37.57793045043945,
|
|
"learning_rate": 3.509802512179737e-07,
|
|
"logits/chosen": -0.3241910934448242,
|
|
"logits/rejected": -0.31886667013168335,
|
|
"logps/chosen": -314.74871826171875,
|
|
"logps/ref_chosen": -55.113426208496094,
|
|
"logps/ref_rejected": -85.29283905029297,
|
|
"logps/rejected": -532.68115234375,
|
|
"loss": 1.0814,
|
|
"margin_dpo/margin_mean": 187.75306701660156,
|
|
"margin_dpo/margin_std": 250.9693603515625,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.44052863436123346,
|
|
"fcm_dpo/beta": 0.0022166285198181868,
|
|
"fcm_dpo/delta": -0.03155837580561638,
|
|
"fcm_dpo/margin": 183.79110717773438,
|
|
"fcm_dpo/q_t": 0.40807557106018066,
|
|
"grad_norm": 29.412609100341797,
|
|
"learning_rate": 3.4507772230088147e-07,
|
|
"logits/chosen": -0.37595048546791077,
|
|
"logits/rejected": -0.3784215748310089,
|
|
"logps/chosen": -391.8227844238281,
|
|
"logps/ref_chosen": -59.46582794189453,
|
|
"logps/ref_rejected": -97.03690338134766,
|
|
"logps/rejected": -613.1849975585938,
|
|
"loss": 1.107,
|
|
"margin_dpo/margin_mean": 183.79110717773438,
|
|
"margin_dpo/margin_std": 271.849853515625,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.447870778267254,
|
|
"fcm_dpo/beta": 0.0021604837384074926,
|
|
"fcm_dpo/delta": 0.016617389395833015,
|
|
"fcm_dpo/margin": 177.69085693359375,
|
|
"fcm_dpo/q_t": 0.41120848059654236,
|
|
"grad_norm": 26.565410614013672,
|
|
"learning_rate": 3.391125620245535e-07,
|
|
"logits/chosen": -0.4044717252254486,
|
|
"logits/rejected": -0.3968522846698761,
|
|
"logps/chosen": -303.1658935546875,
|
|
"logps/ref_chosen": -62.78144454956055,
|
|
"logps/ref_rejected": -91.95039367675781,
|
|
"logps/rejected": -510.02569580078125,
|
|
"loss": 1.0938,
|
|
"margin_dpo/margin_mean": 177.69085693359375,
|
|
"margin_dpo/margin_std": 218.7647247314453,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.4552129221732746,
|
|
"fcm_dpo/beta": 0.0022380652371793985,
|
|
"fcm_dpo/delta": 0.01699766516685486,
|
|
"fcm_dpo/margin": 171.28904724121094,
|
|
"fcm_dpo/q_t": 0.4098632335662842,
|
|
"grad_norm": 25.57453155517578,
|
|
"learning_rate": 3.3308869986991487e-07,
|
|
"logits/chosen": -0.3871304988861084,
|
|
"logits/rejected": -0.3712696135044098,
|
|
"logps/chosen": -297.95452880859375,
|
|
"logps/ref_chosen": -61.359039306640625,
|
|
"logps/ref_rejected": -82.75496673583984,
|
|
"logps/rejected": -490.6394958496094,
|
|
"loss": 1.0861,
|
|
"margin_dpo/margin_mean": 171.2890625,
|
|
"margin_dpo/margin_std": 196.2567138671875,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.46255506607929514,
|
|
"fcm_dpo/beta": 0.0022226418368518353,
|
|
"fcm_dpo/delta": -0.002875490579754114,
|
|
"fcm_dpo/margin": 181.02659606933594,
|
|
"fcm_dpo/q_t": 0.4087609350681305,
|
|
"grad_norm": 23.077590942382812,
|
|
"learning_rate": 3.270101039870797e-07,
|
|
"logits/chosen": -0.3677603602409363,
|
|
"logits/rejected": -0.36049968004226685,
|
|
"logps/chosen": -327.0533447265625,
|
|
"logps/ref_chosen": -51.77602005004883,
|
|
"logps/ref_rejected": -84.58292388916016,
|
|
"logps/rejected": -540.8868408203125,
|
|
"loss": 1.0964,
|
|
"margin_dpo/margin_mean": 181.02658081054688,
|
|
"margin_dpo/margin_std": 249.53970336914062,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.4698972099853157,
|
|
"fcm_dpo/beta": 0.002118379110470414,
|
|
"fcm_dpo/delta": -0.09106224775314331,
|
|
"fcm_dpo/margin": 229.5352325439453,
|
|
"fcm_dpo/q_t": 0.38847747445106506,
|
|
"grad_norm": 26.46944236755371,
|
|
"learning_rate": 3.208807785813777e-07,
|
|
"logits/chosen": -0.40023693442344666,
|
|
"logits/rejected": -0.3959748148918152,
|
|
"logps/chosen": -333.487548828125,
|
|
"logps/ref_chosen": -56.777862548828125,
|
|
"logps/ref_rejected": -99.26368713378906,
|
|
"logps/rejected": -605.5086059570312,
|
|
"loss": 1.0268,
|
|
"margin_dpo/margin_mean": 229.5352325439453,
|
|
"margin_dpo/margin_std": 254.9644317626953,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.47723935389133626,
|
|
"fcm_dpo/beta": 0.002015223726630211,
|
|
"fcm_dpo/delta": 0.0017573073273524642,
|
|
"fcm_dpo/margin": 197.41522216796875,
|
|
"fcm_dpo/q_t": 0.4082733690738678,
|
|
"grad_norm": 27.207300186157227,
|
|
"learning_rate": 3.147047612756302e-07,
|
|
"logits/chosen": -0.4213026165962219,
|
|
"logits/rejected": -0.4106159210205078,
|
|
"logps/chosen": -381.36676025390625,
|
|
"logps/ref_chosen": -58.28468704223633,
|
|
"logps/ref_rejected": -83.80326843261719,
|
|
"logps/rejected": -604.3004760742188,
|
|
"loss": 1.0923,
|
|
"margin_dpo/margin_mean": 197.41522216796875,
|
|
"margin_dpo/margin_std": 251.11337280273438,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.4845814977973568,
|
|
"fcm_dpo/beta": 0.002033454831689596,
|
|
"fcm_dpo/delta": -0.02132582850754261,
|
|
"fcm_dpo/margin": 206.72314453125,
|
|
"fcm_dpo/q_t": 0.40213847160339355,
|
|
"grad_norm": 28.69426727294922,
|
|
"learning_rate": 3.084861204504122e-07,
|
|
"logits/chosen": -0.42850977182388306,
|
|
"logits/rejected": -0.42162972688674927,
|
|
"logps/chosen": -349.00677490234375,
|
|
"logps/ref_chosen": -62.75822067260742,
|
|
"logps/ref_rejected": -94.04203033447266,
|
|
"logps/rejected": -587.0137939453125,
|
|
"loss": 1.0667,
|
|
"margin_dpo/margin_mean": 206.72314453125,
|
|
"margin_dpo/margin_std": 237.6807403564453,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.4919236417033774,
|
|
"fcm_dpo/beta": 0.0019993516616523266,
|
|
"fcm_dpo/delta": 0.017332084476947784,
|
|
"fcm_dpo/margin": 191.62435913085938,
|
|
"fcm_dpo/q_t": 0.41195163130760193,
|
|
"grad_norm": 32.573978424072266,
|
|
"learning_rate": 3.022289525640531e-07,
|
|
"logits/chosen": -0.46553975343704224,
|
|
"logits/rejected": -0.454815149307251,
|
|
"logps/chosen": -367.4043273925781,
|
|
"logps/ref_chosen": -58.59650421142578,
|
|
"logps/ref_rejected": -88.69586944580078,
|
|
"logps/rejected": -589.1280517578125,
|
|
"loss": 1.1035,
|
|
"margin_dpo/margin_mean": 191.62435913085938,
|
|
"margin_dpo/margin_std": 254.1148223876953,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.49926578560939794,
|
|
"fcm_dpo/beta": 0.001957540400326252,
|
|
"fcm_dpo/delta": -0.052527785301208496,
|
|
"fcm_dpo/margin": 229.53335571289062,
|
|
"fcm_dpo/q_t": 0.3968558609485626,
|
|
"grad_norm": 28.016321182250977,
|
|
"learning_rate": 2.959373794541426e-07,
|
|
"logits/chosen": -0.45491886138916016,
|
|
"logits/rejected": -0.44332224130630493,
|
|
"logps/chosen": -372.17889404296875,
|
|
"logps/ref_chosen": -58.18162155151367,
|
|
"logps/ref_rejected": -94.44358825683594,
|
|
"logps/rejected": -637.9742431640625,
|
|
"loss": 1.0561,
|
|
"margin_dpo/margin_mean": 229.53335571289062,
|
|
"margin_dpo/margin_std": 272.4295959472656,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.5066079295154186,
|
|
"fcm_dpo/beta": 0.0019028617534786463,
|
|
"fcm_dpo/delta": -0.032430313527584076,
|
|
"fcm_dpo/margin": 226.35147094726562,
|
|
"fcm_dpo/q_t": 0.4004201292991638,
|
|
"grad_norm": 25.25508689880371,
|
|
"learning_rate": 2.896155456223163e-07,
|
|
"logits/chosen": -0.44945335388183594,
|
|
"logits/rejected": -0.4429987072944641,
|
|
"logps/chosen": -372.24151611328125,
|
|
"logps/ref_chosen": -57.9904899597168,
|
|
"logps/ref_rejected": -99.11092376708984,
|
|
"logps/rejected": -639.7134399414062,
|
|
"loss": 1.0594,
|
|
"margin_dpo/margin_mean": 226.3514862060547,
|
|
"margin_dpo/margin_std": 255.4004364013672,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.5139500734214391,
|
|
"fcm_dpo/beta": 0.0018661676440387964,
|
|
"fcm_dpo/delta": 0.005433606915175915,
|
|
"fcm_dpo/margin": 211.39328002929688,
|
|
"fcm_dpo/q_t": 0.40922412276268005,
|
|
"grad_norm": 30.192829132080078,
|
|
"learning_rate": 2.8326761550411346e-07,
|
|
"logits/chosen": -0.442319393157959,
|
|
"logits/rejected": -0.4401112198829651,
|
|
"logps/chosen": -384.77960205078125,
|
|
"logps/ref_chosen": -58.29923629760742,
|
|
"logps/ref_rejected": -89.25711822509766,
|
|
"logps/rejected": -627.1307373046875,
|
|
"loss": 1.0959,
|
|
"margin_dpo/margin_mean": 211.3932647705078,
|
|
"margin_dpo/margin_std": 280.6878967285156,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.5212922173274597,
|
|
"fcm_dpo/beta": 0.001845445716753602,
|
|
"fcm_dpo/delta": -0.019822590053081512,
|
|
"fcm_dpo/margin": 216.8295135498047,
|
|
"fcm_dpo/q_t": 0.4076092839241028,
|
|
"grad_norm": 37.005271911621094,
|
|
"learning_rate": 2.7689777072570284e-07,
|
|
"logits/chosen": -0.4728454649448395,
|
|
"logits/rejected": -0.46268409490585327,
|
|
"logps/chosen": -382.65460205078125,
|
|
"logps/ref_chosen": -60.788482666015625,
|
|
"logps/ref_rejected": -85.94129943847656,
|
|
"logps/rejected": -624.6368408203125,
|
|
"loss": 1.0868,
|
|
"margin_dpo/margin_mean": 216.8295135498047,
|
|
"margin_dpo/margin_std": 270.96405029296875,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.5286343612334802,
|
|
"fcm_dpo/beta": 0.0018649225821718574,
|
|
"fcm_dpo/delta": 0.030804917216300964,
|
|
"fcm_dpo/margin": 183.98890686035156,
|
|
"fcm_dpo/q_t": 0.4203580319881439,
|
|
"grad_norm": 36.66780471801758,
|
|
"learning_rate": 2.7051020734928443e-07,
|
|
"logits/chosen": -0.4746428430080414,
|
|
"logits/rejected": -0.4617390036582947,
|
|
"logps/chosen": -384.0989074707031,
|
|
"logps/ref_chosen": -57.6871337890625,
|
|
"logps/ref_rejected": -80.62527465820312,
|
|
"logps/rejected": -591.0260009765625,
|
|
"loss": 1.1282,
|
|
"margin_dpo/margin_mean": 183.98892211914062,
|
|
"margin_dpo/margin_std": 254.2589569091797,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.5359765051395007,
|
|
"fcm_dpo/beta": 0.0018712872406467795,
|
|
"fcm_dpo/delta": -0.026755044236779213,
|
|
"fcm_dpo/margin": 227.2862091064453,
|
|
"fcm_dpo/q_t": 0.40148013830184937,
|
|
"grad_norm": 26.54566764831543,
|
|
"learning_rate": 2.641091331089811e-07,
|
|
"logits/chosen": -0.473797470331192,
|
|
"logits/rejected": -0.47837966680526733,
|
|
"logps/chosen": -337.5777893066406,
|
|
"logps/ref_chosen": -51.490867614746094,
|
|
"logps/ref_rejected": -91.02871704101562,
|
|
"logps/rejected": -604.4019165039062,
|
|
"loss": 1.0628,
|
|
"margin_dpo/margin_mean": 227.28622436523438,
|
|
"margin_dpo/margin_std": 260.75958251953125,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.5433186490455213,
|
|
"fcm_dpo/beta": 0.0018645223462954164,
|
|
"fcm_dpo/delta": 0.038537509739398956,
|
|
"fcm_dpo/margin": 194.5872039794922,
|
|
"fcm_dpo/q_t": 0.41541823744773865,
|
|
"grad_norm": 38.50527572631836,
|
|
"learning_rate": 2.5769876463904263e-07,
|
|
"logits/chosen": -0.47979289293289185,
|
|
"logits/rejected": -0.4759935438632965,
|
|
"logps/chosen": -354.23822021484375,
|
|
"logps/ref_chosen": -58.113502502441406,
|
|
"logps/ref_rejected": -89.43451690673828,
|
|
"logps/rejected": -580.1464233398438,
|
|
"loss": 1.1143,
|
|
"margin_dpo/margin_mean": 194.5872039794922,
|
|
"margin_dpo/margin_std": 256.3455505371094,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.5506607929515418,
|
|
"fcm_dpo/beta": 0.0018788978923112154,
|
|
"fcm_dpo/delta": -0.028660928830504417,
|
|
"fcm_dpo/margin": 227.169677734375,
|
|
"fcm_dpo/q_t": 0.403217077255249,
|
|
"grad_norm": 42.09442901611328,
|
|
"learning_rate": 2.512833246961859e-07,
|
|
"logits/chosen": -0.5087894201278687,
|
|
"logits/rejected": -0.494729608297348,
|
|
"logps/chosen": -409.12677001953125,
|
|
"logps/ref_chosen": -65.23600769042969,
|
|
"logps/ref_rejected": -89.24995422363281,
|
|
"logps/rejected": -660.3104248046875,
|
|
"loss": 1.0839,
|
|
"margin_dpo/margin_mean": 227.169677734375,
|
|
"margin_dpo/margin_std": 307.71136474609375,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.5580029368575624,
|
|
"fcm_dpo/beta": 0.001791507238522172,
|
|
"fcm_dpo/delta": -0.07822184264659882,
|
|
"fcm_dpo/margin": 264.9973449707031,
|
|
"fcm_dpo/q_t": 0.39022326469421387,
|
|
"grad_norm": 27.727758407592773,
|
|
"learning_rate": 2.4486703937790243e-07,
|
|
"logits/chosen": -0.4783501625061035,
|
|
"logits/rejected": -0.49099501967430115,
|
|
"logps/chosen": -376.74932861328125,
|
|
"logps/ref_chosen": -53.33893966674805,
|
|
"logps/ref_rejected": -102.15375518798828,
|
|
"logps/rejected": -690.5615234375,
|
|
"loss": 1.034,
|
|
"margin_dpo/margin_mean": 264.9973449707031,
|
|
"margin_dpo/margin_std": 298.83721923828125,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5653450807635829,
|
|
"fcm_dpo/beta": 0.0017382428050041199,
|
|
"fcm_dpo/delta": 0.007922522723674774,
|
|
"fcm_dpo/margin": 225.6304931640625,
|
|
"fcm_dpo/q_t": 0.40880244970321655,
|
|
"grad_norm": 39.414588928222656,
|
|
"learning_rate": 2.3845413533856514e-07,
|
|
"logits/chosen": -0.4913402497768402,
|
|
"logits/rejected": -0.4843069911003113,
|
|
"logps/chosen": -414.2498474121094,
|
|
"logps/ref_chosen": -58.36262130737305,
|
|
"logps/ref_rejected": -89.44685363769531,
|
|
"logps/rejected": -670.964599609375,
|
|
"loss": 1.0946,
|
|
"margin_dpo/margin_mean": 225.6304931640625,
|
|
"margin_dpo/margin_std": 290.14666748046875,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.5726872246696035,
|
|
"fcm_dpo/beta": 0.0017035829368978739,
|
|
"fcm_dpo/delta": -0.029813114553689957,
|
|
"fcm_dpo/margin": 251.324462890625,
|
|
"fcm_dpo/q_t": 0.4024105966091156,
|
|
"grad_norm": 26.369205474853516,
|
|
"learning_rate": 2.320488370051681e-07,
|
|
"logits/chosen": -0.486684650182724,
|
|
"logits/rejected": -0.4810120165348053,
|
|
"logps/chosen": -435.18115234375,
|
|
"logps/ref_chosen": -56.380653381347656,
|
|
"logps/ref_rejected": -90.47447204589844,
|
|
"logps/rejected": -720.5994873046875,
|
|
"loss": 1.0741,
|
|
"margin_dpo/margin_mean": 251.324462890625,
|
|
"margin_dpo/margin_std": 322.6702575683594,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.580029368575624,
|
|
"fcm_dpo/beta": 0.0017207658383995295,
|
|
"fcm_dpo/delta": 0.012584102340042591,
|
|
"fcm_dpo/margin": 225.0916748046875,
|
|
"fcm_dpo/q_t": 0.41174229979515076,
|
|
"grad_norm": 31.417821884155273,
|
|
"learning_rate": 2.2565536379453404e-07,
|
|
"logits/chosen": -0.47851499915122986,
|
|
"logits/rejected": -0.4759146273136139,
|
|
"logps/chosen": -402.8694763183594,
|
|
"logps/ref_chosen": -55.95304489135742,
|
|
"logps/ref_rejected": -87.13162994384766,
|
|
"logps/rejected": -659.1397705078125,
|
|
"loss": 1.1036,
|
|
"margin_dpo/margin_mean": 225.0916748046875,
|
|
"margin_dpo/margin_std": 308.83575439453125,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.5873715124816447,
|
|
"fcm_dpo/beta": 0.0016688309842720628,
|
|
"fcm_dpo/delta": -0.05934460088610649,
|
|
"fcm_dpo/margin": 273.747314453125,
|
|
"fcm_dpo/q_t": 0.39453184604644775,
|
|
"grad_norm": 22.06127166748047,
|
|
"learning_rate": 2.192779273338215e-07,
|
|
"logits/chosen": -0.48663657903671265,
|
|
"logits/rejected": -0.4787791669368744,
|
|
"logps/chosen": -424.73455810546875,
|
|
"logps/ref_chosen": -64.59160614013672,
|
|
"logps/ref_rejected": -96.700927734375,
|
|
"logps/rejected": -730.5912475585938,
|
|
"loss": 1.0433,
|
|
"margin_dpo/margin_mean": 273.747314453125,
|
|
"margin_dpo/margin_std": 306.27801513671875,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5873715124816447,
|
|
"eval_fcm_dpo/beta": 0.0018981158500537276,
|
|
"eval_fcm_dpo/delta": 0.03762635216116905,
|
|
"eval_fcm_dpo/margin": 156.90065002441406,
|
|
"eval_fcm_dpo/q_t": 0.43355923891067505,
|
|
"eval_logits/chosen": -0.5229986310005188,
|
|
"eval_logits/rejected": -0.5119041800498962,
|
|
"eval_logps/chosen": -532.2332763671875,
|
|
"eval_logps/ref_chosen": -79.05104064941406,
|
|
"eval_logps/ref_rejected": -86.79793548583984,
|
|
"eval_logps/rejected": -696.880859375,
|
|
"eval_loss": 0.6033138036727905,
|
|
"eval_margin_dpo/margin_mean": 156.90061950683594,
|
|
"eval_margin_dpo/margin_std": 328.45404052734375,
|
|
"eval_runtime": 39.7397,
|
|
"eval_samples_per_second": 58.858,
|
|
"eval_steps_per_second": 1.862,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5947136563876652,
|
|
"fcm_dpo/beta": 0.001960620516911149,
|
|
"fcm_dpo/delta": -0.1365078240633011,
|
|
"fcm_dpo/margin": 260.3297119140625,
|
|
"fcm_dpo/q_t": 0.3874695897102356,
|
|
"grad_norm": 38.00107955932617,
|
|
"learning_rate": 2.129207286861638e-07,
|
|
"logits/chosen": -0.505179226398468,
|
|
"logits/rejected": -0.5002223253250122,
|
|
"logps/chosen": -406.5534973144531,
|
|
"logps/ref_chosen": -53.61777877807617,
|
|
"logps/ref_rejected": -81.28938293457031,
|
|
"logps/rejected": -694.5548095703125,
|
|
"loss": 1.0528,
|
|
"margin_dpo/margin_mean": 260.3297119140625,
|
|
"margin_dpo/margin_std": 351.9696044921875,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.6020558002936858,
|
|
"fcm_dpo/beta": 0.0018232803558930755,
|
|
"fcm_dpo/delta": -0.01396404393017292,
|
|
"fcm_dpo/margin": 216.00399780273438,
|
|
"fcm_dpo/q_t": 0.4089049696922302,
|
|
"grad_norm": 22.69110679626465,
|
|
"learning_rate": 2.065879555832674e-07,
|
|
"logits/chosen": -0.5198922157287598,
|
|
"logits/rejected": -0.508955717086792,
|
|
"logps/chosen": -396.2334899902344,
|
|
"logps/ref_chosen": -58.9287223815918,
|
|
"logps/ref_rejected": -85.55818176269531,
|
|
"logps/rejected": -638.8670043945312,
|
|
"loss": 1.096,
|
|
"margin_dpo/margin_mean": 216.00399780273438,
|
|
"margin_dpo/margin_std": 284.52081298828125,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.6093979441997063,
|
|
"fcm_dpo/beta": 0.0017951425397768617,
|
|
"fcm_dpo/delta": -0.023637911304831505,
|
|
"fcm_dpo/margin": 235.01473999023438,
|
|
"fcm_dpo/q_t": 0.4024273753166199,
|
|
"grad_norm": 26.751333236694336,
|
|
"learning_rate": 2.002837796667909e-07,
|
|
"logits/chosen": -0.4968322813510895,
|
|
"logits/rejected": -0.5035668015480042,
|
|
"logps/chosen": -417.4717712402344,
|
|
"logps/ref_chosen": -58.45662307739258,
|
|
"logps/ref_rejected": -93.67063903808594,
|
|
"logps/rejected": -687.7005615234375,
|
|
"loss": 1.0778,
|
|
"margin_dpo/margin_mean": 235.01473999023438,
|
|
"margin_dpo/margin_std": 297.6578063964844,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.6167400881057269,
|
|
"fcm_dpo/beta": 0.001714545302093029,
|
|
"fcm_dpo/delta": -0.04269330948591232,
|
|
"fcm_dpo/margin": 256.39959716796875,
|
|
"fcm_dpo/q_t": 0.39849138259887695,
|
|
"grad_norm": 58.22108459472656,
|
|
"learning_rate": 1.9401235374032425e-07,
|
|
"logits/chosen": -0.47421860694885254,
|
|
"logits/rejected": -0.46059077978134155,
|
|
"logps/chosen": -428.48443603515625,
|
|
"logps/ref_chosen": -64.2349853515625,
|
|
"logps/ref_rejected": -84.86299133300781,
|
|
"logps/rejected": -705.5120849609375,
|
|
"loss": 1.062,
|
|
"margin_dpo/margin_mean": 256.39959716796875,
|
|
"margin_dpo/margin_std": 306.3155822753906,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.6240822320117474,
|
|
"fcm_dpo/beta": 0.0017159763956442475,
|
|
"fcm_dpo/delta": 0.010966436937451363,
|
|
"fcm_dpo/margin": 226.81314086914062,
|
|
"fcm_dpo/q_t": 0.4094250798225403,
|
|
"grad_norm": 45.61664962768555,
|
|
"learning_rate": 1.8777780903377732e-07,
|
|
"logits/chosen": -0.5051223635673523,
|
|
"logits/rejected": -0.4999346137046814,
|
|
"logps/chosen": -442.1536560058594,
|
|
"logps/ref_chosen": -56.054161071777344,
|
|
"logps/ref_rejected": -85.17119598388672,
|
|
"logps/rejected": -698.0838012695312,
|
|
"loss": 1.1005,
|
|
"margin_dpo/margin_mean": 226.81314086914062,
|
|
"margin_dpo/margin_std": 304.298583984375,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.631424375917768,
|
|
"fcm_dpo/beta": 0.0017323114443570375,
|
|
"fcm_dpo/delta": 0.009447330608963966,
|
|
"fcm_dpo/margin": 213.61093139648438,
|
|
"fcm_dpo/q_t": 0.414214551448822,
|
|
"grad_norm": 40.23195266723633,
|
|
"learning_rate": 1.8158425248197928e-07,
|
|
"logits/chosen": -0.5632457733154297,
|
|
"logits/rejected": -0.5552536249160767,
|
|
"logps/chosen": -472.23919677734375,
|
|
"logps/ref_chosen": -69.24568176269531,
|
|
"logps/ref_rejected": -91.8664321899414,
|
|
"logps/rejected": -708.4708862304688,
|
|
"loss": 1.1182,
|
|
"margin_dpo/margin_mean": 213.61093139648438,
|
|
"margin_dpo/margin_std": 301.84185791015625,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.6387665198237885,
|
|
"fcm_dpo/beta": 0.0017033193726092577,
|
|
"fcm_dpo/delta": -0.027609745040535927,
|
|
"fcm_dpo/margin": 249.95333862304688,
|
|
"fcm_dpo/q_t": 0.4024946093559265,
|
|
"grad_norm": 34.37434768676758,
|
|
"learning_rate": 1.7543576401928218e-07,
|
|
"logits/chosen": -0.618207573890686,
|
|
"logits/rejected": -0.6184796690940857,
|
|
"logps/chosen": -466.94012451171875,
|
|
"logps/ref_chosen": -60.03449630737305,
|
|
"logps/ref_rejected": -90.6872329711914,
|
|
"logps/rejected": -747.5462036132812,
|
|
"loss": 1.0796,
|
|
"margin_dpo/margin_mean": 249.95333862304688,
|
|
"margin_dpo/margin_std": 326.78277587890625,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.6461086637298091,
|
|
"fcm_dpo/beta": 0.0016936672618612647,
|
|
"fcm_dpo/delta": -0.009997454471886158,
|
|
"fcm_dpo/margin": 227.880126953125,
|
|
"fcm_dpo/q_t": 0.41129302978515625,
|
|
"grad_norm": 52.6486930847168,
|
|
"learning_rate": 1.6933639389195134e-07,
|
|
"logits/chosen": -0.647782027721405,
|
|
"logits/rejected": -0.6332479119300842,
|
|
"logps/chosen": -471.6587829589844,
|
|
"logps/ref_chosen": -65.50349426269531,
|
|
"logps/ref_rejected": -85.66627502441406,
|
|
"logps/rejected": -719.70166015625,
|
|
"loss": 1.1022,
|
|
"margin_dpo/margin_mean": 227.88015747070312,
|
|
"margin_dpo/margin_std": 307.6328125,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.6534508076358296,
|
|
"fcm_dpo/beta": 0.001662442460656166,
|
|
"fcm_dpo/delta": -0.03005291521549225,
|
|
"fcm_dpo/margin": 245.63253784179688,
|
|
"fcm_dpo/q_t": 0.4081791043281555,
|
|
"grad_norm": 32.25020980834961,
|
|
"learning_rate": 1.6329015999011182e-07,
|
|
"logits/chosen": -0.6261739730834961,
|
|
"logits/rejected": -0.6163330674171448,
|
|
"logps/chosen": -485.8421936035156,
|
|
"logps/ref_chosen": -60.72443389892578,
|
|
"logps/ref_rejected": -89.9255142211914,
|
|
"logps/rejected": -760.6757202148438,
|
|
"loss": 1.1036,
|
|
"margin_dpo/margin_mean": 245.63253784179688,
|
|
"margin_dpo/margin_std": 356.2879943847656,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.6607929515418502,
|
|
"fcm_dpo/beta": 0.0015721891541033983,
|
|
"fcm_dpo/delta": -0.07007580995559692,
|
|
"fcm_dpo/margin": 297.0084533691406,
|
|
"fcm_dpo/q_t": 0.3933987319469452,
|
|
"grad_norm": 37.675498962402344,
|
|
"learning_rate": 1.573010452010098e-07,
|
|
"logits/chosen": -0.674242377281189,
|
|
"logits/rejected": -0.6845596432685852,
|
|
"logps/chosen": -466.7557067871094,
|
|
"logps/ref_chosen": -59.96248245239258,
|
|
"logps/ref_rejected": -98.71509552001953,
|
|
"logps/rejected": -802.5167846679688,
|
|
"loss": 1.0453,
|
|
"margin_dpo/margin_mean": 297.0084533691406,
|
|
"margin_dpo/margin_std": 355.6662292480469,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.6681350954478708,
|
|
"fcm_dpo/beta": 0.0015043766470625997,
|
|
"fcm_dpo/delta": -0.04557330161333084,
|
|
"fcm_dpo/margin": 295.05352783203125,
|
|
"fcm_dpo/q_t": 0.3984234929084778,
|
|
"grad_norm": 46.07245635986328,
|
|
"learning_rate": 1.5137299478533064e-07,
|
|
"logits/chosen": -0.7228642702102661,
|
|
"logits/rejected": -0.7254277467727661,
|
|
"logps/chosen": -525.2667846679688,
|
|
"logps/ref_chosen": -54.48131561279297,
|
|
"logps/ref_rejected": -90.6321029663086,
|
|
"logps/rejected": -856.4710693359375,
|
|
"loss": 1.0728,
|
|
"margin_dpo/margin_mean": 295.05352783203125,
|
|
"margin_dpo/margin_std": 389.12469482421875,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.6754772393538914,
|
|
"fcm_dpo/beta": 0.0013938735937699676,
|
|
"fcm_dpo/delta": -0.0647268071770668,
|
|
"fcm_dpo/margin": 331.0070495605469,
|
|
"fcm_dpo/q_t": 0.3938430845737457,
|
|
"grad_norm": 38.70931625366211,
|
|
"learning_rate": 1.4550991377830423e-07,
|
|
"logits/chosen": -0.6967940926551819,
|
|
"logits/rejected": -0.7114230394363403,
|
|
"logps/chosen": -526.4164428710938,
|
|
"logps/ref_chosen": -52.97611618041992,
|
|
"logps/ref_rejected": -95.65971374511719,
|
|
"logps/rejected": -900.1070556640625,
|
|
"loss": 1.0402,
|
|
"margin_dpo/margin_mean": 331.0070495605469,
|
|
"margin_dpo/margin_std": 373.8094177246094,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6828193832599119,
|
|
"fcm_dpo/beta": 0.0013582499232143164,
|
|
"fcm_dpo/delta": -0.040646448731422424,
|
|
"fcm_dpo/margin": 323.32489013671875,
|
|
"fcm_dpo/q_t": 0.3995057940483093,
|
|
"grad_norm": 29.232101440429688,
|
|
"learning_rate": 1.3971566441730714e-07,
|
|
"logits/chosen": -0.7399168014526367,
|
|
"logits/rejected": -0.7570127248764038,
|
|
"logps/chosen": -553.7732543945312,
|
|
"logps/ref_chosen": -58.2827033996582,
|
|
"logps/ref_rejected": -94.15567779541016,
|
|
"logps/rejected": -912.9710693359375,
|
|
"loss": 1.0657,
|
|
"margin_dpo/margin_mean": 323.32489013671875,
|
|
"margin_dpo/margin_std": 396.70892333984375,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.6901615271659325,
|
|
"fcm_dpo/beta": 0.0012759026139974594,
|
|
"fcm_dpo/delta": -0.050003357231616974,
|
|
"fcm_dpo/margin": 350.7779235839844,
|
|
"fcm_dpo/q_t": 0.3979727625846863,
|
|
"grad_norm": 27.062070846557617,
|
|
"learning_rate": 1.339940635976592e-07,
|
|
"logits/chosen": -0.7901474833488464,
|
|
"logits/rejected": -0.7927638292312622,
|
|
"logps/chosen": -599.5657348632812,
|
|
"logps/ref_chosen": -62.69774627685547,
|
|
"logps/ref_rejected": -96.33873748779297,
|
|
"logps/rejected": -983.9846801757812,
|
|
"loss": 1.0702,
|
|
"margin_dpo/margin_mean": 350.7779846191406,
|
|
"margin_dpo/margin_std": 457.7704162597656,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.697503671071953,
|
|
"fcm_dpo/beta": 0.0012454693205654621,
|
|
"fcm_dpo/delta": -0.008181336335837841,
|
|
"fcm_dpo/margin": 326.77447509765625,
|
|
"fcm_dpo/q_t": 0.41024675965309143,
|
|
"grad_norm": 34.630069732666016,
|
|
"learning_rate": 1.2834888035828596e-07,
|
|
"logits/chosen": -0.9200963973999023,
|
|
"logits/rejected": -0.9176486730575562,
|
|
"logps/chosen": -705.766845703125,
|
|
"logps/ref_chosen": -61.12194061279297,
|
|
"logps/ref_rejected": -92.42192077636719,
|
|
"logps/rejected": -1063.84130859375,
|
|
"loss": 1.1387,
|
|
"margin_dpo/margin_mean": 326.7744140625,
|
|
"margin_dpo/margin_std": 567.3615112304688,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.7048458149779736,
|
|
"fcm_dpo/beta": 0.0012077607680112123,
|
|
"fcm_dpo/delta": -0.04966871812939644,
|
|
"fcm_dpo/margin": 349.90106201171875,
|
|
"fcm_dpo/q_t": 0.4062713086605072,
|
|
"grad_norm": 62.070411682128906,
|
|
"learning_rate": 1.227838333989088e-07,
|
|
"logits/chosen": -0.9689761996269226,
|
|
"logits/rejected": -0.9785006642341614,
|
|
"logps/chosen": -730.4820556640625,
|
|
"logps/ref_chosen": -53.550628662109375,
|
|
"logps/ref_rejected": -82.65167999267578,
|
|
"logps/rejected": -1109.484130859375,
|
|
"loss": 1.111,
|
|
"margin_dpo/margin_mean": 349.90106201171875,
|
|
"margin_dpo/margin_std": 551.5509033203125,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.7121879588839941,
|
|
"fcm_dpo/beta": 0.001141466898843646,
|
|
"fcm_dpo/delta": -0.044795047491788864,
|
|
"fcm_dpo/margin": 370.09796142578125,
|
|
"fcm_dpo/q_t": 0.404446542263031,
|
|
"grad_norm": 33.84178924560547,
|
|
"learning_rate": 1.1730258863039347e-07,
|
|
"logits/chosen": -0.9285829663276672,
|
|
"logits/rejected": -0.9435798525810242,
|
|
"logps/chosen": -689.6338500976562,
|
|
"logps/ref_chosen": -60.76704788208008,
|
|
"logps/ref_rejected": -92.70649719238281,
|
|
"logps/rejected": -1091.671142578125,
|
|
"loss": 1.0989,
|
|
"margin_dpo/margin_mean": 370.09796142578125,
|
|
"margin_dpo/margin_std": 542.5574951171875,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.7195301027900147,
|
|
"fcm_dpo/beta": 0.0011192201636731625,
|
|
"fcm_dpo/delta": 0.0058375876396894455,
|
|
"fcm_dpo/margin": 352.23004150390625,
|
|
"fcm_dpo/q_t": 0.4108501374721527,
|
|
"grad_norm": 50.5614128112793,
|
|
"learning_rate": 1.1190875675987355e-07,
|
|
"logits/chosen": -0.9131494760513306,
|
|
"logits/rejected": -0.9286991953849792,
|
|
"logps/chosen": -628.578369140625,
|
|
"logps/ref_chosen": -53.78142547607422,
|
|
"logps/ref_rejected": -91.67438507080078,
|
|
"logps/rejected": -1018.7013549804688,
|
|
"loss": 1.1102,
|
|
"margin_dpo/margin_mean": 352.2300109863281,
|
|
"margin_dpo/margin_std": 520.4669189453125,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.7268722466960352,
|
|
"fcm_dpo/beta": 0.0011620967416092753,
|
|
"fcm_dpo/delta": 0.019140174612402916,
|
|
"fcm_dpo/margin": 309.65850830078125,
|
|
"fcm_dpo/q_t": 0.418142706155777,
|
|
"grad_norm": 35.54931640625,
|
|
"learning_rate": 1.0660589091223854e-07,
|
|
"logits/chosen": -0.9482976794242859,
|
|
"logits/rejected": -0.9485372304916382,
|
|
"logps/chosen": -638.1903076171875,
|
|
"logps/ref_chosen": -58.9004020690918,
|
|
"logps/ref_rejected": -82.15424346923828,
|
|
"logps/rejected": -971.1026611328125,
|
|
"loss": 1.1356,
|
|
"margin_dpo/margin_mean": 309.6585388183594,
|
|
"margin_dpo/margin_std": 488.75909423828125,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.7342143906020558,
|
|
"fcm_dpo/beta": 0.0011855836492031813,
|
|
"fcm_dpo/delta": 0.024493711069226265,
|
|
"fcm_dpo/margin": 291.8643798828125,
|
|
"fcm_dpo/q_t": 0.42091307044029236,
|
|
"grad_norm": 30.435544967651367,
|
|
"learning_rate": 1.0139748428955333e-07,
|
|
"logits/chosen": -0.9160359501838684,
|
|
"logits/rejected": -0.91932213306427,
|
|
"logps/chosen": -699.1871948242188,
|
|
"logps/ref_chosen": -62.13483810424805,
|
|
"logps/ref_rejected": -87.91773223876953,
|
|
"logps/rejected": -1016.83447265625,
|
|
"loss": 1.1519,
|
|
"margin_dpo/margin_mean": 291.8643798828125,
|
|
"margin_dpo/margin_std": 487.0228576660156,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.7415565345080763,
|
|
"fcm_dpo/beta": 0.0011691536055877805,
|
|
"fcm_dpo/delta": 0.006966643966734409,
|
|
"fcm_dpo/margin": 336.22186279296875,
|
|
"fcm_dpo/q_t": 0.41082048416137695,
|
|
"grad_norm": 31.930696487426758,
|
|
"learning_rate": 9.628696786995188e-08,
|
|
"logits/chosen": -0.8171493411064148,
|
|
"logits/rejected": -0.8128056526184082,
|
|
"logps/chosen": -598.7544555664062,
|
|
"logps/ref_chosen": -62.631813049316406,
|
|
"logps/ref_rejected": -87.59168243408203,
|
|
"logps/rejected": -959.9362182617188,
|
|
"loss": 1.1017,
|
|
"margin_dpo/margin_mean": 336.2218322753906,
|
|
"margin_dpo/margin_std": 463.255615234375,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.748898678414097,
|
|
"fcm_dpo/beta": 0.0012091896496713161,
|
|
"fcm_dpo/delta": 0.028515011072158813,
|
|
"fcm_dpo/margin": 307.8777770996094,
|
|
"fcm_dpo/q_t": 0.4152832627296448,
|
|
"grad_norm": 37.425514221191406,
|
|
"learning_rate": 9.127770814751932e-08,
|
|
"logits/chosen": -0.7757030129432678,
|
|
"logits/rejected": -0.7864035367965698,
|
|
"logps/chosen": -574.5384521484375,
|
|
"logps/ref_chosen": -60.552574157714844,
|
|
"logps/ref_rejected": -91.0874252319336,
|
|
"logps/rejected": -912.951171875,
|
|
"loss": 1.1211,
|
|
"margin_dpo/margin_mean": 307.8777770996094,
|
|
"margin_dpo/margin_std": 457.5035705566406,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.7562408223201175,
|
|
"fcm_dpo/beta": 0.0012351378099992871,
|
|
"fcm_dpo/delta": 0.04284387826919556,
|
|
"fcm_dpo/margin": 290.5696105957031,
|
|
"fcm_dpo/q_t": 0.41743749380111694,
|
|
"grad_norm": 42.793312072753906,
|
|
"learning_rate": 8.637300491465272e-08,
|
|
"logits/chosen": -0.7973334789276123,
|
|
"logits/rejected": -0.8029496073722839,
|
|
"logps/chosen": -538.7898559570312,
|
|
"logps/ref_chosen": -60.9382438659668,
|
|
"logps/ref_rejected": -83.36767578125,
|
|
"logps/rejected": -851.7888793945312,
|
|
"loss": 1.1273,
|
|
"margin_dpo/margin_mean": 290.5696105957031,
|
|
"margin_dpo/margin_std": 429.46759033203125,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.7635829662261381,
|
|
"fcm_dpo/beta": 0.0012580296024680138,
|
|
"fcm_dpo/delta": 0.001387386815622449,
|
|
"fcm_dpo/margin": 316.6277770996094,
|
|
"fcm_dpo/q_t": 0.4091448187828064,
|
|
"grad_norm": 41.04801559448242,
|
|
"learning_rate": 8.15760890883607e-08,
|
|
"logits/chosen": -0.8406698107719421,
|
|
"logits/rejected": -0.8473473787307739,
|
|
"logps/chosen": -598.2073974609375,
|
|
"logps/ref_chosen": -65.47642517089844,
|
|
"logps/ref_rejected": -98.10872650146484,
|
|
"logps/rejected": -947.4674072265625,
|
|
"loss": 1.1012,
|
|
"margin_dpo/margin_mean": 316.6277770996094,
|
|
"margin_dpo/margin_std": 440.81640625,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.7709251101321586,
|
|
"fcm_dpo/beta": 0.0012629171833395958,
|
|
"fcm_dpo/delta": -0.03817005455493927,
|
|
"fcm_dpo/margin": 345.6103820800781,
|
|
"fcm_dpo/q_t": 0.4016591012477875,
|
|
"grad_norm": 33.16600799560547,
|
|
"learning_rate": 7.689012058193384e-08,
|
|
"logits/chosen": -0.9183955192565918,
|
|
"logits/rejected": -0.9257911443710327,
|
|
"logps/chosen": -636.746337890625,
|
|
"logps/ref_chosen": -59.072021484375,
|
|
"logps/ref_rejected": -82.8821792602539,
|
|
"logps/rejected": -1006.1668701171875,
|
|
"loss": 1.0892,
|
|
"margin_dpo/margin_mean": 345.6103515625,
|
|
"margin_dpo/margin_std": 494.670654296875,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.7782672540381792,
|
|
"fcm_dpo/beta": 0.001211377326399088,
|
|
"fcm_dpo/delta": -0.020752830430865288,
|
|
"fcm_dpo/margin": 330.8208923339844,
|
|
"fcm_dpo/q_t": 0.4099728465080261,
|
|
"grad_norm": 49.311241149902344,
|
|
"learning_rate": 7.231818622338822e-08,
|
|
"logits/chosen": -0.9991754293441772,
|
|
"logits/rejected": -0.9904956817626953,
|
|
"logps/chosen": -721.9043579101562,
|
|
"logps/ref_chosen": -61.11234664916992,
|
|
"logps/ref_rejected": -87.01112365722656,
|
|
"logps/rejected": -1078.6241455078125,
|
|
"loss": 1.1306,
|
|
"margin_dpo/margin_mean": 330.8208923339844,
|
|
"margin_dpo/margin_std": 549.4489135742188,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.7856093979441997,
|
|
"fcm_dpo/beta": 0.001194678363390267,
|
|
"fcm_dpo/delta": -0.01992225833237171,
|
|
"fcm_dpo/margin": 334.68017578125,
|
|
"fcm_dpo/q_t": 0.4102866053581238,
|
|
"grad_norm": 28.87725067138672,
|
|
"learning_rate": 6.786329772205246e-08,
|
|
"logits/chosen": -0.9631765484809875,
|
|
"logits/rejected": -0.9504965543746948,
|
|
"logps/chosen": -703.7914428710938,
|
|
"logps/ref_chosen": -60.96736526489258,
|
|
"logps/ref_rejected": -81.727294921875,
|
|
"logps/rejected": -1059.231689453125,
|
|
"loss": 1.1238,
|
|
"margin_dpo/margin_mean": 334.68017578125,
|
|
"margin_dpo/margin_std": 534.3411865234375,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.7929515418502202,
|
|
"fcm_dpo/beta": 0.001148082548752427,
|
|
"fcm_dpo/delta": -0.046564746648073196,
|
|
"fcm_dpo/margin": 386.79718017578125,
|
|
"fcm_dpo/q_t": 0.4005100131034851,
|
|
"grad_norm": 40.99845886230469,
|
|
"learning_rate": 6.352838968463919e-08,
|
|
"logits/chosen": -0.8846408128738403,
|
|
"logits/rejected": -0.8896158337593079,
|
|
"logps/chosen": -599.884765625,
|
|
"logps/ref_chosen": -58.64385986328125,
|
|
"logps/ref_rejected": -90.40965270996094,
|
|
"logps/rejected": -1018.4476318359375,
|
|
"loss": 1.0741,
|
|
"margin_dpo/margin_mean": 386.7971496582031,
|
|
"margin_dpo/margin_std": 522.71826171875,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.8002936857562408,
|
|
"fcm_dpo/beta": 0.0011082093697041273,
|
|
"fcm_dpo/delta": -0.027572477236390114,
|
|
"fcm_dpo/margin": 361.2355651855469,
|
|
"fcm_dpo/q_t": 0.4120791554450989,
|
|
"grad_norm": 31.77776527404785,
|
|
"learning_rate": 5.9316317682106294e-08,
|
|
"logits/chosen": -0.9308038949966431,
|
|
"logits/rejected": -0.9519888758659363,
|
|
"logps/chosen": -702.5291137695312,
|
|
"logps/ref_chosen": -64.73474884033203,
|
|
"logps/ref_rejected": -100.44208526611328,
|
|
"logps/rejected": -1099.471923828125,
|
|
"loss": 1.1263,
|
|
"margin_dpo/margin_mean": 361.2355651855469,
|
|
"margin_dpo/margin_std": 592.0833740234375,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.8076358296622613,
|
|
"fcm_dpo/beta": 0.0011139644775539637,
|
|
"fcm_dpo/delta": 0.027384355664253235,
|
|
"fcm_dpo/margin": 318.26593017578125,
|
|
"fcm_dpo/q_t": 0.42001286149024963,
|
|
"grad_norm": 41.989044189453125,
|
|
"learning_rate": 5.5229856368582376e-08,
|
|
"logits/chosen": -0.9026668667793274,
|
|
"logits/rejected": -0.9054571390151978,
|
|
"logps/chosen": -672.8770751953125,
|
|
"logps/ref_chosen": -59.13951873779297,
|
|
"logps/ref_rejected": -87.15635681152344,
|
|
"logps/rejected": -1019.1598510742188,
|
|
"loss": 1.1374,
|
|
"margin_dpo/margin_mean": 318.26593017578125,
|
|
"margin_dpo/margin_std": 509.5707092285156,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.8149779735682819,
|
|
"fcm_dpo/beta": 0.0010551275918260217,
|
|
"fcm_dpo/delta": -0.08985067903995514,
|
|
"fcm_dpo/margin": 458.0889587402344,
|
|
"fcm_dpo/q_t": 0.39341944456100464,
|
|
"grad_norm": 37.768455505371094,
|
|
"learning_rate": 5.127169765359515e-08,
|
|
"logits/chosen": -0.9196065664291382,
|
|
"logits/rejected": -0.9442132115364075,
|
|
"logps/chosen": -678.6885986328125,
|
|
"logps/ref_chosen": -62.1995849609375,
|
|
"logps/ref_rejected": -102.51883697509766,
|
|
"logps/rejected": -1177.096923828125,
|
|
"loss": 1.0596,
|
|
"margin_dpo/margin_mean": 458.0889587402344,
|
|
"margin_dpo/margin_std": 621.8514404296875,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.8223201174743024,
|
|
"fcm_dpo/beta": 0.001013588160276413,
|
|
"fcm_dpo/delta": -0.03401713818311691,
|
|
"fcm_dpo/margin": 426.43377685546875,
|
|
"fcm_dpo/q_t": 0.4025075435638428,
|
|
"grad_norm": 32.85615158081055,
|
|
"learning_rate": 4.7444448928806615e-08,
|
|
"logits/chosen": -0.9791582226753235,
|
|
"logits/rejected": -0.987125039100647,
|
|
"logps/chosen": -729.31787109375,
|
|
"logps/ref_chosen": -61.541908264160156,
|
|
"logps/ref_rejected": -97.37491607666016,
|
|
"logps/rejected": -1191.584716796875,
|
|
"loss": 1.0859,
|
|
"margin_dpo/margin_mean": 426.43377685546875,
|
|
"margin_dpo/margin_std": 586.039794921875,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.8296622613803231,
|
|
"fcm_dpo/beta": 0.0010072619188576937,
|
|
"fcm_dpo/delta": 0.004320817068219185,
|
|
"fcm_dpo/margin": 365.18194580078125,
|
|
"fcm_dpo/q_t": 0.41634687781333923,
|
|
"grad_norm": 38.367191314697266,
|
|
"learning_rate": 4.375063135042445e-08,
|
|
"logits/chosen": -1.0174771547317505,
|
|
"logits/rejected": -1.0257703065872192,
|
|
"logps/chosen": -805.9225463867188,
|
|
"logps/ref_chosen": -62.85475540161133,
|
|
"logps/ref_rejected": -93.8392105102539,
|
|
"logps/rejected": -1202.0889892578125,
|
|
"loss": 1.1532,
|
|
"margin_dpo/margin_mean": 365.18194580078125,
|
|
"margin_dpo/margin_std": 646.3736572265625,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.8370044052863436,
|
|
"fcm_dpo/beta": 0.000991632230579853,
|
|
"fcm_dpo/delta": -0.003437476698309183,
|
|
"fcm_dpo/margin": 406.2334899902344,
|
|
"fcm_dpo/q_t": 0.40820297598838806,
|
|
"grad_norm": 32.0954475402832,
|
|
"learning_rate": 4.019267817841834e-08,
|
|
"logits/chosen": -1.057476282119751,
|
|
"logits/rejected": -1.0744601488113403,
|
|
"logps/chosen": -756.536865234375,
|
|
"logps/ref_chosen": -57.98622512817383,
|
|
"logps/ref_rejected": -87.91555786132812,
|
|
"logps/rejected": -1192.6998291015625,
|
|
"loss": 1.108,
|
|
"margin_dpo/margin_mean": 406.2334899902344,
|
|
"margin_dpo/margin_std": 592.2410278320312,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.8443465491923642,
|
|
"fcm_dpo/beta": 0.000984189915470779,
|
|
"fcm_dpo/delta": -0.0074859424494206905,
|
|
"fcm_dpo/margin": 413.28948974609375,
|
|
"fcm_dpo/q_t": 0.4089413583278656,
|
|
"grad_norm": 39.946693420410156,
|
|
"learning_rate": 3.677293317363864e-08,
|
|
"logits/chosen": -1.012502908706665,
|
|
"logits/rejected": -1.0302174091339111,
|
|
"logps/chosen": -763.88623046875,
|
|
"logps/ref_chosen": -55.194114685058594,
|
|
"logps/ref_rejected": -89.68229675292969,
|
|
"logps/rejected": -1211.663818359375,
|
|
"loss": 1.1112,
|
|
"margin_dpo/margin_mean": 413.28948974609375,
|
|
"margin_dpo/margin_std": 633.7578735351562,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.8516886930983847,
|
|
"fcm_dpo/beta": 0.0010015666484832764,
|
|
"fcm_dpo/delta": 0.005687057971954346,
|
|
"fcm_dpo/margin": 373.99749755859375,
|
|
"fcm_dpo/q_t": 0.41545066237449646,
|
|
"grad_norm": 36.99066925048828,
|
|
"learning_rate": 3.349364905389032e-08,
|
|
"logits/chosen": -0.9414408802986145,
|
|
"logits/rejected": -0.9566031694412231,
|
|
"logps/chosen": -681.4942626953125,
|
|
"logps/ref_chosen": -54.605796813964844,
|
|
"logps/ref_rejected": -88.9614486694336,
|
|
"logps/rejected": -1089.847412109375,
|
|
"loss": 1.1256,
|
|
"margin_dpo/margin_mean": 373.99749755859375,
|
|
"margin_dpo/margin_std": 568.8309326171875,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.8590308370044053,
|
|
"fcm_dpo/beta": 0.0009860583813861012,
|
|
"fcm_dpo/delta": -0.024760950356721878,
|
|
"fcm_dpo/margin": 374.35772705078125,
|
|
"fcm_dpo/q_t": 0.4164944589138031,
|
|
"grad_norm": 30.979644775390625,
|
|
"learning_rate": 3.035698600998121e-08,
|
|
"logits/chosen": -1.005885362625122,
|
|
"logits/rejected": -1.0213685035705566,
|
|
"logps/chosen": -722.11083984375,
|
|
"logps/ref_chosen": -59.03770065307617,
|
|
"logps/ref_rejected": -94.4029541015625,
|
|
"logps/rejected": -1131.833984375,
|
|
"loss": 1.1365,
|
|
"margin_dpo/margin_mean": 374.35772705078125,
|
|
"margin_dpo/margin_std": 598.1573486328125,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.8663729809104258,
|
|
"fcm_dpo/beta": 0.0009923167526721954,
|
|
"fcm_dpo/delta": 0.020330144092440605,
|
|
"fcm_dpo/margin": 382.85650634765625,
|
|
"fcm_dpo/q_t": 0.41268691420555115,
|
|
"grad_norm": 26.06329917907715,
|
|
"learning_rate": 2.736501028272095e-08,
|
|
"logits/chosen": -1.001636266708374,
|
|
"logits/rejected": -1.0243651866912842,
|
|
"logps/chosen": -699.3173828125,
|
|
"logps/ref_chosen": -53.5163688659668,
|
|
"logps/ref_rejected": -99.90290832519531,
|
|
"logps/rejected": -1128.5604248046875,
|
|
"loss": 1.1114,
|
|
"margin_dpo/margin_mean": 382.85650634765625,
|
|
"margin_dpo/margin_std": 544.3261108398438,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.8737151248164464,
|
|
"fcm_dpo/beta": 0.0009959937306120992,
|
|
"fcm_dpo/delta": 0.008491059765219688,
|
|
"fcm_dpo/margin": 356.68133544921875,
|
|
"fcm_dpo/q_t": 0.4196888506412506,
|
|
"grad_norm": 51.35660934448242,
|
|
"learning_rate": 2.451969280180849e-08,
|
|
"logits/chosen": -0.9872266054153442,
|
|
"logits/rejected": -0.9868671298027039,
|
|
"logps/chosen": -701.3692626953125,
|
|
"logps/ref_chosen": -51.44538497924805,
|
|
"logps/ref_rejected": -77.43083190917969,
|
|
"logps/rejected": -1084.035888671875,
|
|
"loss": 1.1503,
|
|
"margin_dpo/margin_mean": 356.68133544921875,
|
|
"margin_dpo/margin_std": 606.1689453125,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.8810572687224669,
|
|
"fcm_dpo/beta": 0.0010021307971328497,
|
|
"fcm_dpo/delta": 0.01705138385295868,
|
|
"fcm_dpo/margin": 358.8981018066406,
|
|
"fcm_dpo/q_t": 0.418224960565567,
|
|
"grad_norm": 36.674869537353516,
|
|
"learning_rate": 2.1822907887504932e-08,
|
|
"logits/chosen": -0.9832857251167297,
|
|
"logits/rejected": -0.9899972677230835,
|
|
"logps/chosen": -721.6881103515625,
|
|
"logps/ref_chosen": -57.161705017089844,
|
|
"logps/ref_rejected": -87.73274230957031,
|
|
"logps/rejected": -1111.1573486328125,
|
|
"loss": 1.1393,
|
|
"margin_dpo/margin_mean": 358.89813232421875,
|
|
"margin_dpo/margin_std": 566.36669921875,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8810572687224669,
|
|
"eval_fcm_dpo/beta": 0.0011259522289037704,
|
|
"eval_fcm_dpo/delta": 0.023851953446865082,
|
|
"eval_fcm_dpo/margin": 270.7278747558594,
|
|
"eval_fcm_dpo/q_t": 0.4331645667552948,
|
|
"eval_logits/chosen": -1.0197498798370361,
|
|
"eval_logits/rejected": -1.0176793336868286,
|
|
"eval_logps/chosen": -823.232421875,
|
|
"eval_logps/ref_chosen": -79.05104064941406,
|
|
"eval_logps/ref_rejected": -86.79793548583984,
|
|
"eval_logps/rejected": -1101.70703125,
|
|
"eval_loss": 0.6074439883232117,
|
|
"eval_margin_dpo/margin_mean": 270.7278747558594,
|
|
"eval_margin_dpo/margin_std": 601.7490234375,
|
|
"eval_runtime": 39.7407,
|
|
"eval_samples_per_second": 58.856,
|
|
"eval_steps_per_second": 1.862,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8883994126284875,
|
|
"fcm_dpo/beta": 0.0011384881800040603,
|
|
"fcm_dpo/delta": -0.09710332006216049,
|
|
"fcm_dpo/margin": 432.0416564941406,
|
|
"fcm_dpo/q_t": 0.39160969853401184,
|
|
"grad_norm": 36.93719482421875,
|
|
"learning_rate": 1.9276432015946446e-08,
|
|
"logits/chosen": -0.9972618222236633,
|
|
"logits/rejected": -1.01743483543396,
|
|
"logps/chosen": -690.4810791015625,
|
|
"logps/ref_chosen": -58.169830322265625,
|
|
"logps/ref_rejected": -95.36891174316406,
|
|
"logps/rejected": -1159.7216796875,
|
|
"loss": 1.0561,
|
|
"margin_dpo/margin_mean": 432.0416564941406,
|
|
"margin_dpo/margin_std": 594.4193725585938,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.895741556534508,
|
|
"fcm_dpo/beta": 0.0010870896512642503,
|
|
"fcm_dpo/delta": -0.01724116876721382,
|
|
"fcm_dpo/margin": 382.8441467285156,
|
|
"fcm_dpo/q_t": 0.4077000617980957,
|
|
"grad_norm": 48.549068450927734,
|
|
"learning_rate": 1.6881942648911074e-08,
|
|
"logits/chosen": -0.9739276766777039,
|
|
"logits/rejected": -0.978052020072937,
|
|
"logps/chosen": -703.338623046875,
|
|
"logps/ref_chosen": -58.97087860107422,
|
|
"logps/ref_rejected": -89.0286865234375,
|
|
"logps/rejected": -1116.2406005859375,
|
|
"loss": 1.102,
|
|
"margin_dpo/margin_mean": 382.8440856933594,
|
|
"margin_dpo/margin_std": 576.8626708984375,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.9030837004405287,
|
|
"fcm_dpo/beta": 0.001023916294798255,
|
|
"fcm_dpo/delta": -0.07400115579366684,
|
|
"fcm_dpo/margin": 458.7464294433594,
|
|
"fcm_dpo/q_t": 0.39545583724975586,
|
|
"grad_norm": 26.998958587646484,
|
|
"learning_rate": 1.4641017128809801e-08,
|
|
"logits/chosen": -0.9687691926956177,
|
|
"logits/rejected": -1.0111548900604248,
|
|
"logps/chosen": -723.0494384765625,
|
|
"logps/ref_chosen": -58.081878662109375,
|
|
"logps/ref_rejected": -95.92155456542969,
|
|
"logps/rejected": -1219.635498046875,
|
|
"loss": 1.0677,
|
|
"margin_dpo/margin_mean": 458.7464904785156,
|
|
"margin_dpo/margin_std": 639.427734375,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.9104258443465492,
|
|
"fcm_dpo/beta": 0.0010129540460184216,
|
|
"fcm_dpo/delta": 0.012466437183320522,
|
|
"fcm_dpo/margin": 362.10418701171875,
|
|
"fcm_dpo/q_t": 0.4198682904243469,
|
|
"grad_norm": 31.997928619384766,
|
|
"learning_rate": 1.2555131639630567e-08,
|
|
"logits/chosen": -0.9928318858146667,
|
|
"logits/rejected": -0.9981027841567993,
|
|
"logps/chosen": -801.2032470703125,
|
|
"logps/ref_chosen": -62.203094482421875,
|
|
"logps/ref_rejected": -80.53683471679688,
|
|
"logps/rejected": -1181.641357421875,
|
|
"loss": 1.1594,
|
|
"margin_dpo/margin_mean": 362.10418701171875,
|
|
"margin_dpo/margin_std": 658.1754150390625,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.9177679882525698,
|
|
"fcm_dpo/beta": 0.0009900578297674656,
|
|
"fcm_dpo/delta": -0.04029404744505882,
|
|
"fcm_dpo/margin": 442.16802978515625,
|
|
"fcm_dpo/q_t": 0.4008522033691406,
|
|
"grad_norm": 67.80354309082031,
|
|
"learning_rate": 1.0625660234518913e-08,
|
|
"logits/chosen": -0.9926285743713379,
|
|
"logits/rejected": -1.0111424922943115,
|
|
"logps/chosen": -743.4299926757812,
|
|
"logps/ref_chosen": -61.727455139160156,
|
|
"logps/ref_rejected": -88.4387435913086,
|
|
"logps/rejected": -1212.309326171875,
|
|
"loss": 1.0786,
|
|
"margin_dpo/margin_mean": 442.1680603027344,
|
|
"margin_dpo/margin_std": 596.7000732421875,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.9251101321585903,
|
|
"fcm_dpo/beta": 0.0009705528500489891,
|
|
"fcm_dpo/delta": -0.0021920218132436275,
|
|
"fcm_dpo/margin": 385.10406494140625,
|
|
"fcm_dpo/q_t": 0.41799044609069824,
|
|
"grad_norm": 57.19618225097656,
|
|
"learning_rate": 8.85387393063622e-09,
|
|
"logits/chosen": -0.981094479560852,
|
|
"logits/rejected": -0.999189019203186,
|
|
"logps/chosen": -764.3931884765625,
|
|
"logps/ref_chosen": -61.30865478515625,
|
|
"logps/ref_rejected": -96.54997253417969,
|
|
"logps/rejected": -1184.7386474609375,
|
|
"loss": 1.1426,
|
|
"margin_dpo/margin_mean": 385.10406494140625,
|
|
"margin_dpo/margin_std": 646.9403076171875,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.9324522760646109,
|
|
"fcm_dpo/beta": 0.000991217908449471,
|
|
"fcm_dpo/delta": 0.04439081624150276,
|
|
"fcm_dpo/margin": 321.2152099609375,
|
|
"fcm_dpo/q_t": 0.42787250876426697,
|
|
"grad_norm": 41.87347412109375,
|
|
"learning_rate": 7.240939871891699e-09,
|
|
"logits/chosen": -1.0177139043807983,
|
|
"logits/rejected": -1.0183391571044922,
|
|
"logps/chosen": -819.9927978515625,
|
|
"logps/ref_chosen": -63.7315673828125,
|
|
"logps/ref_rejected": -89.66435241699219,
|
|
"logps/rejected": -1167.1407470703125,
|
|
"loss": 1.1768,
|
|
"margin_dpo/margin_mean": 321.2152404785156,
|
|
"margin_dpo/margin_std": 599.1561889648438,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.9397944199706314,
|
|
"fcm_dpo/beta": 0.0009926257189363241,
|
|
"fcm_dpo/delta": -0.05684474855661392,
|
|
"fcm_dpo/margin": 380.9329528808594,
|
|
"fcm_dpo/q_t": 0.41502299904823303,
|
|
"grad_norm": 40.166175842285156,
|
|
"learning_rate": 5.7879205600998296e-09,
|
|
"logits/chosen": -0.9638550877571106,
|
|
"logits/rejected": -0.9688556790351868,
|
|
"logps/chosen": -797.8259887695312,
|
|
"logps/ref_chosen": -59.17915725708008,
|
|
"logps/ref_rejected": -88.51210021972656,
|
|
"logps/rejected": -1208.0919189453125,
|
|
"loss": 1.1439,
|
|
"margin_dpo/margin_mean": 380.9329528808594,
|
|
"margin_dpo/margin_std": 651.566162109375,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.947136563876652,
|
|
"fcm_dpo/beta": 0.0009744317503646016,
|
|
"fcm_dpo/delta": 0.0341024175286293,
|
|
"fcm_dpo/margin": 321.48162841796875,
|
|
"fcm_dpo/q_t": 0.43180274963378906,
|
|
"grad_norm": 79.06712341308594,
|
|
"learning_rate": 4.495773155069299e-09,
|
|
"logits/chosen": -0.9938424229621887,
|
|
"logits/rejected": -1.0045819282531738,
|
|
"logps/chosen": -791.4851684570312,
|
|
"logps/ref_chosen": -59.50596237182617,
|
|
"logps/ref_rejected": -93.92404174804688,
|
|
"logps/rejected": -1147.3848876953125,
|
|
"loss": 1.1994,
|
|
"margin_dpo/margin_mean": 321.48162841796875,
|
|
"margin_dpo/margin_std": 670.0167846679688,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.9544787077826725,
|
|
"fcm_dpo/beta": 0.0009998297318816185,
|
|
"fcm_dpo/delta": 0.0030700929928570986,
|
|
"fcm_dpo/margin": 377.16595458984375,
|
|
"fcm_dpo/q_t": 0.41648992896080017,
|
|
"grad_norm": 39.774078369140625,
|
|
"learning_rate": 3.3653488440851253e-09,
|
|
"logits/chosen": -0.994001567363739,
|
|
"logits/rejected": -1.0127675533294678,
|
|
"logps/chosen": -719.2337036132812,
|
|
"logps/ref_chosen": -57.774566650390625,
|
|
"logps/ref_rejected": -89.61600494384766,
|
|
"logps/rejected": -1128.2410888671875,
|
|
"loss": 1.1354,
|
|
"margin_dpo/margin_mean": 377.16595458984375,
|
|
"margin_dpo/margin_std": 618.4581298828125,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.9618208516886931,
|
|
"fcm_dpo/beta": 0.0009819023543968797,
|
|
"fcm_dpo/delta": 0.001074020517989993,
|
|
"fcm_dpo/margin": 406.14300537109375,
|
|
"fcm_dpo/q_t": 0.41048040986061096,
|
|
"grad_norm": 35.741294860839844,
|
|
"learning_rate": 2.397392281198729e-09,
|
|
"logits/chosen": -1.0151567459106445,
|
|
"logits/rejected": -1.0480637550354004,
|
|
"logps/chosen": -682.8621826171875,
|
|
"logps/ref_chosen": -55.68403244018555,
|
|
"logps/ref_rejected": -102.4081802368164,
|
|
"logps/rejected": -1135.7293701171875,
|
|
"loss": 1.1068,
|
|
"margin_dpo/margin_mean": 406.1430358886719,
|
|
"margin_dpo/margin_std": 598.8343505859375,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.9691629955947136,
|
|
"fcm_dpo/beta": 0.0009497703285887837,
|
|
"fcm_dpo/delta": -0.056020140647888184,
|
|
"fcm_dpo/margin": 475.952880859375,
|
|
"fcm_dpo/q_t": 0.39766284823417664,
|
|
"grad_norm": 36.36381149291992,
|
|
"learning_rate": 1.592541096695571e-09,
|
|
"logits/chosen": -0.988078773021698,
|
|
"logits/rejected": -1.017830491065979,
|
|
"logps/chosen": -716.3291625976562,
|
|
"logps/ref_chosen": -59.19981002807617,
|
|
"logps/ref_rejected": -94.19200134277344,
|
|
"logps/rejected": -1227.274169921875,
|
|
"loss": 1.0617,
|
|
"margin_dpo/margin_mean": 475.95281982421875,
|
|
"margin_dpo/margin_std": 605.3031005859375,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.9765051395007343,
|
|
"fcm_dpo/beta": 0.0009469041833654046,
|
|
"fcm_dpo/delta": 0.0041985055431723595,
|
|
"fcm_dpo/margin": 417.9197692871094,
|
|
"fcm_dpo/q_t": 0.41134023666381836,
|
|
"grad_norm": 40.71209716796875,
|
|
"learning_rate": 9.513254770636137e-10,
|
|
"logits/chosen": -1.0182468891143799,
|
|
"logits/rejected": -1.0391613245010376,
|
|
"logps/chosen": -698.357421875,
|
|
"logps/ref_chosen": -61.2533073425293,
|
|
"logps/ref_rejected": -95.86351013183594,
|
|
"logps/rejected": -1150.887451171875,
|
|
"loss": 1.1093,
|
|
"margin_dpo/margin_mean": 417.9197692871094,
|
|
"margin_dpo/margin_std": 626.8549194335938,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.9838472834067548,
|
|
"fcm_dpo/beta": 0.0009450761717744172,
|
|
"fcm_dpo/delta": -0.012223686091601849,
|
|
"fcm_dpo/margin": 435.3887634277344,
|
|
"fcm_dpo/q_t": 0.40688472986221313,
|
|
"grad_norm": 29.223600387573242,
|
|
"learning_rate": 4.741678157389739e-10,
|
|
"logits/chosen": -0.9618457555770874,
|
|
"logits/rejected": -0.9741120338439941,
|
|
"logps/chosen": -724.4750366210938,
|
|
"logps/ref_chosen": -62.95263671875,
|
|
"logps/ref_rejected": -92.4662094116211,
|
|
"logps/rejected": -1189.3773193359375,
|
|
"loss": 1.1,
|
|
"margin_dpo/margin_mean": 435.3887634277344,
|
|
"margin_dpo/margin_std": 627.5145263671875,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.9911894273127754,
|
|
"fcm_dpo/beta": 0.000940955535043031,
|
|
"fcm_dpo/delta": -0.006233499385416508,
|
|
"fcm_dpo/margin": 431.116455078125,
|
|
"fcm_dpo/q_t": 0.4094417095184326,
|
|
"grad_norm": 23.441814422607422,
|
|
"learning_rate": 1.6138243485910863e-10,
|
|
"logits/chosen": -1.0078109502792358,
|
|
"logits/rejected": -1.02387535572052,
|
|
"logps/chosen": -741.0542602539062,
|
|
"logps/ref_chosen": -48.5856819152832,
|
|
"logps/ref_rejected": -81.27871704101562,
|
|
"logps/rejected": -1204.8636474609375,
|
|
"loss": 1.1069,
|
|
"margin_dpo/margin_mean": 431.116455078125,
|
|
"margin_dpo/margin_std": 644.87255859375,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.9985315712187959,
|
|
"fcm_dpo/beta": 0.0009158365428447723,
|
|
"fcm_dpo/delta": -0.01913156732916832,
|
|
"fcm_dpo/margin": 456.44354248046875,
|
|
"fcm_dpo/q_t": 0.40588778257369995,
|
|
"grad_norm": 31.682743072509766,
|
|
"learning_rate": 1.31753782067201e-11,
|
|
"logits/chosen": -0.9820996522903442,
|
|
"logits/rejected": -0.9966181516647339,
|
|
"logps/chosen": -744.6751708984375,
|
|
"logps/ref_chosen": -60.25421905517578,
|
|
"logps/ref_rejected": -87.23457336425781,
|
|
"logps/rejected": -1228.09912109375,
|
|
"loss": 1.0916,
|
|
"margin_dpo/margin_mean": 456.44354248046875,
|
|
"margin_dpo/margin_std": 645.1212768554688,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 681,
|
|
"total_flos": 0.0,
|
|
"train_loss": 1.1011297496229893,
|
|
"train_runtime": 1861.3017,
|
|
"train_samples_per_second": 23.423,
|
|
"train_steps_per_second": 0.366
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 681,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|