765dad5674
Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-s_star0.6-4xh200-batch-64-20260421-213851 Source: Original Platform
2631 lines
96 KiB
JSON
2631 lines
96 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.999244142101285,
|
|
"eval_steps": 200,
|
|
"global_step": 661,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0015117157974300832,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.0013532638549804688,
|
|
"fcm_dpo/q_t": 0.5000336766242981,
|
|
"grad_norm": 28.21952247619629,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": 0.13337239623069763,
|
|
"logits/rejected": 0.12492948770523071,
|
|
"logps/chosen": -64.5841293334961,
|
|
"logps/ref_chosen": -64.61280822753906,
|
|
"logps/ref_rejected": -64.17195129394531,
|
|
"logps/rejected": -64.14192199707031,
|
|
"loss": 1.3866,
|
|
"margin_dpo/margin_mean": -0.0013527870178222656,
|
|
"margin_dpo/margin_std": 0.2561596930027008,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.007558578987150416,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.6003847122192383e-05,
|
|
"fcm_dpo/q_t": 0.4999997317790985,
|
|
"grad_norm": 29.559865951538086,
|
|
"learning_rate": 2.9850746268656714e-08,
|
|
"logits/chosen": 0.0971517488360405,
|
|
"logits/rejected": 0.0687209814786911,
|
|
"logps/chosen": -65.35282897949219,
|
|
"logps/ref_chosen": -65.34695434570312,
|
|
"logps/ref_rejected": -79.315673828125,
|
|
"logps/rejected": -79.32157135009766,
|
|
"loss": 1.3866,
|
|
"margin_dpo/margin_mean": 1.5869736671447754e-05,
|
|
"margin_dpo/margin_std": 0.3168628215789795,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.015117157974300832,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.012913530692458153,
|
|
"fcm_dpo/q_t": 0.5003229975700378,
|
|
"grad_norm": 29.694316864013672,
|
|
"learning_rate": 6.71641791044776e-08,
|
|
"logits/chosen": 0.10982687771320343,
|
|
"logits/rejected": 0.07068048417568207,
|
|
"logps/chosen": -56.66190719604492,
|
|
"logps/ref_chosen": -56.65692901611328,
|
|
"logps/ref_rejected": -80.12786865234375,
|
|
"logps/rejected": -80.11993408203125,
|
|
"loss": 1.3879,
|
|
"margin_dpo/margin_mean": -0.012913751415908337,
|
|
"margin_dpo/margin_std": 0.32729893922805786,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.022675736961451247,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.008519763126969337,
|
|
"fcm_dpo/q_t": 0.49978700280189514,
|
|
"grad_norm": 33.27705001831055,
|
|
"learning_rate": 1.044776119402985e-07,
|
|
"logits/chosen": 0.08304646611213684,
|
|
"logits/rejected": 0.05341915413737297,
|
|
"logps/chosen": -60.09071731567383,
|
|
"logps/ref_chosen": -60.09392166137695,
|
|
"logps/ref_rejected": -78.99056243896484,
|
|
"logps/rejected": -78.99589538574219,
|
|
"loss": 1.3857,
|
|
"margin_dpo/margin_mean": 0.008519655093550682,
|
|
"margin_dpo/margin_std": 0.3333361744880676,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.030234315948601664,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.03206338733434677,
|
|
"fcm_dpo/q_t": 0.49919837713241577,
|
|
"grad_norm": 29.28687858581543,
|
|
"learning_rate": 1.4179104477611938e-07,
|
|
"logits/chosen": 0.10171808302402496,
|
|
"logits/rejected": 0.07373008131980896,
|
|
"logps/chosen": -55.454063415527344,
|
|
"logps/ref_chosen": -55.464561462402344,
|
|
"logps/ref_rejected": -77.40013122558594,
|
|
"logps/rejected": -77.42169189453125,
|
|
"loss": 1.3833,
|
|
"margin_dpo/margin_mean": 0.03206341713666916,
|
|
"margin_dpo/margin_std": 0.2828105092048645,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.03779289493575208,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.018508916720747948,
|
|
"fcm_dpo/q_t": 0.5004625916481018,
|
|
"grad_norm": 29.7989559173584,
|
|
"learning_rate": 1.7910447761194027e-07,
|
|
"logits/chosen": 0.10036797821521759,
|
|
"logits/rejected": 0.06887850165367126,
|
|
"logps/chosen": -60.73334503173828,
|
|
"logps/ref_chosen": -60.711814880371094,
|
|
"logps/ref_rejected": -82.71756744384766,
|
|
"logps/rejected": -82.72056579589844,
|
|
"loss": 1.3884,
|
|
"margin_dpo/margin_mean": -0.01850915513932705,
|
|
"margin_dpo/margin_std": 0.28522950410842896,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.045351473922902494,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.015936514362692833,
|
|
"fcm_dpo/q_t": 0.4996017813682556,
|
|
"grad_norm": 30.30830955505371,
|
|
"learning_rate": 2.1641791044776117e-07,
|
|
"logits/chosen": 0.10710334777832031,
|
|
"logits/rejected": 0.082417331635952,
|
|
"logps/chosen": -60.89277267456055,
|
|
"logps/ref_chosen": -60.880210876464844,
|
|
"logps/ref_rejected": -78.44148254394531,
|
|
"logps/rejected": -78.46997833251953,
|
|
"loss": 1.385,
|
|
"margin_dpo/margin_mean": 0.01593649946153164,
|
|
"margin_dpo/margin_std": 0.3193960189819336,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.05291005291005291,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.03081861510872841,
|
|
"fcm_dpo/q_t": 0.4992298483848572,
|
|
"grad_norm": 27.896162033081055,
|
|
"learning_rate": 2.537313432835821e-07,
|
|
"logits/chosen": 0.07303156703710556,
|
|
"logits/rejected": 0.04675190895795822,
|
|
"logps/chosen": -62.2817268371582,
|
|
"logps/ref_chosen": -62.248138427734375,
|
|
"logps/ref_rejected": -79.56475830078125,
|
|
"logps/rejected": -79.62915802001953,
|
|
"loss": 1.3835,
|
|
"margin_dpo/margin_mean": 0.030818644911050797,
|
|
"margin_dpo/margin_std": 0.3220459818840027,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.06046863189720333,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.05380158871412277,
|
|
"fcm_dpo/q_t": 0.49865588545799255,
|
|
"grad_norm": 31.392417907714844,
|
|
"learning_rate": 2.9104477611940296e-07,
|
|
"logits/chosen": 0.10881145298480988,
|
|
"logits/rejected": 0.06401894986629486,
|
|
"logps/chosen": -58.942840576171875,
|
|
"logps/ref_chosen": -58.87812423706055,
|
|
"logps/ref_rejected": -84.22982025146484,
|
|
"logps/rejected": -84.34834289550781,
|
|
"loss": 1.3813,
|
|
"margin_dpo/margin_mean": 0.0538015179336071,
|
|
"margin_dpo/margin_std": 0.36063262820243835,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.06802721088435375,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.06139056012034416,
|
|
"fcm_dpo/q_t": 0.4984658360481262,
|
|
"grad_norm": 31.959802627563477,
|
|
"learning_rate": 3.2835820895522385e-07,
|
|
"logits/chosen": 0.06842530518770218,
|
|
"logits/rejected": 0.041332852095365524,
|
|
"logps/chosen": -66.02344512939453,
|
|
"logps/ref_chosen": -65.88298034667969,
|
|
"logps/ref_rejected": -83.87881469726562,
|
|
"logps/rejected": -84.08067321777344,
|
|
"loss": 1.3805,
|
|
"margin_dpo/margin_mean": 0.06139082834124565,
|
|
"margin_dpo/margin_std": 0.3803684115409851,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.07558578987150416,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.11904527992010117,
|
|
"fcm_dpo/q_t": 0.49702557921409607,
|
|
"grad_norm": 28.045490264892578,
|
|
"learning_rate": 3.6567164179104475e-07,
|
|
"logits/chosen": 0.09588325768709183,
|
|
"logits/rejected": 0.06175718456506729,
|
|
"logps/chosen": -55.35811233520508,
|
|
"logps/ref_chosen": -55.172386169433594,
|
|
"logps/ref_rejected": -69.63300323486328,
|
|
"logps/rejected": -69.9377670288086,
|
|
"loss": 1.3749,
|
|
"margin_dpo/margin_mean": 0.1190454512834549,
|
|
"margin_dpo/margin_std": 0.4321363866329193,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.08314436885865457,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.20462636649608612,
|
|
"fcm_dpo/q_t": 0.4948909282684326,
|
|
"grad_norm": 31.350805282592773,
|
|
"learning_rate": 4.0298507462686564e-07,
|
|
"logits/chosen": 0.07329290360212326,
|
|
"logits/rejected": 0.03742387518286705,
|
|
"logps/chosen": -57.520660400390625,
|
|
"logps/ref_chosen": -57.193580627441406,
|
|
"logps/ref_rejected": -79.69940948486328,
|
|
"logps/rejected": -80.23112487792969,
|
|
"loss": 1.3667,
|
|
"margin_dpo/margin_mean": 0.2046263962984085,
|
|
"margin_dpo/margin_std": 0.5350508093833923,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.09070294784580499,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.21316950023174286,
|
|
"fcm_dpo/q_t": 0.494695246219635,
|
|
"grad_norm": 29.479215621948242,
|
|
"learning_rate": 4.4029850746268654e-07,
|
|
"logits/chosen": 0.11556963622570038,
|
|
"logits/rejected": 0.08139903843402863,
|
|
"logps/chosen": -60.59169387817383,
|
|
"logps/ref_chosen": -60.068870544433594,
|
|
"logps/ref_rejected": -74.41178894042969,
|
|
"logps/rejected": -75.14778137207031,
|
|
"loss": 1.3669,
|
|
"margin_dpo/margin_mean": 0.2131694257259369,
|
|
"margin_dpo/margin_std": 0.7954167127609253,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.0982615268329554,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.2978232204914093,
|
|
"fcm_dpo/q_t": 0.4925784468650818,
|
|
"grad_norm": 30.698984146118164,
|
|
"learning_rate": 4.776119402985074e-07,
|
|
"logits/chosen": 0.142142653465271,
|
|
"logits/rejected": 0.11199666559696198,
|
|
"logps/chosen": -58.89265060424805,
|
|
"logps/ref_chosen": -58.1558952331543,
|
|
"logps/ref_rejected": -76.06512451171875,
|
|
"logps/rejected": -77.09969329833984,
|
|
"loss": 1.3592,
|
|
"margin_dpo/margin_mean": 0.2978229522705078,
|
|
"margin_dpo/margin_std": 0.9595780372619629,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.10582010582010581,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.3882128596305847,
|
|
"fcm_dpo/q_t": 0.4903312623500824,
|
|
"grad_norm": 28.7775936126709,
|
|
"learning_rate": 4.999860140229787e-07,
|
|
"logits/chosen": 0.09354963898658752,
|
|
"logits/rejected": 0.0587652213871479,
|
|
"logps/chosen": -68.39994812011719,
|
|
"logps/ref_chosen": -67.35506439208984,
|
|
"logps/ref_rejected": -82.24962615966797,
|
|
"logps/rejected": -83.68272399902344,
|
|
"loss": 1.351,
|
|
"margin_dpo/margin_mean": 0.38821321725845337,
|
|
"margin_dpo/margin_std": 1.0907655954360962,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.11337868480725624,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.5082494020462036,
|
|
"fcm_dpo/q_t": 0.4873982071876526,
|
|
"grad_norm": 26.24626922607422,
|
|
"learning_rate": 4.998286897523808e-07,
|
|
"logits/chosen": 0.11840841919183731,
|
|
"logits/rejected": 0.08255789428949356,
|
|
"logps/chosen": -58.29798126220703,
|
|
"logps/ref_chosen": -56.86763381958008,
|
|
"logps/ref_rejected": -72.56938934326172,
|
|
"logps/rejected": -74.50798797607422,
|
|
"loss": 1.3414,
|
|
"margin_dpo/margin_mean": 0.5082495212554932,
|
|
"margin_dpo/margin_std": 1.3911728858947754,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.12093726379440665,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.6494876742362976,
|
|
"fcm_dpo/q_t": 0.48396021127700806,
|
|
"grad_norm": 28.429441452026367,
|
|
"learning_rate": 4.994966691179711e-07,
|
|
"logits/chosen": 0.15457963943481445,
|
|
"logits/rejected": 0.10858809947967529,
|
|
"logps/chosen": -59.45546340942383,
|
|
"logps/ref_chosen": -57.687095642089844,
|
|
"logps/ref_rejected": -78.06813049316406,
|
|
"logps/rejected": -80.48597717285156,
|
|
"loss": 1.3301,
|
|
"margin_dpo/margin_mean": 0.6494876146316528,
|
|
"margin_dpo/margin_std": 1.6857645511627197,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12849584278155707,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.0505290031433105,
|
|
"fcm_dpo/q_t": 0.47403591871261597,
|
|
"grad_norm": 26.629859924316406,
|
|
"learning_rate": 4.989901842900325e-07,
|
|
"logits/chosen": 0.15848767757415771,
|
|
"logits/rejected": 0.11160705238580704,
|
|
"logps/chosen": -59.09556198120117,
|
|
"logps/ref_chosen": -56.96040725708008,
|
|
"logps/ref_rejected": -75.22166442871094,
|
|
"logps/rejected": -78.40734100341797,
|
|
"loss": 1.2925,
|
|
"margin_dpo/margin_mean": 1.0505287647247314,
|
|
"margin_dpo/margin_std": 1.8001108169555664,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.1360544217687075,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.128682255744934,
|
|
"fcm_dpo/q_t": 0.47230347990989685,
|
|
"grad_norm": 29.16546058654785,
|
|
"learning_rate": 4.983095894354857e-07,
|
|
"logits/chosen": 0.19552312791347504,
|
|
"logits/rejected": 0.1472882330417633,
|
|
"logps/chosen": -60.3253059387207,
|
|
"logps/ref_chosen": -57.41730499267578,
|
|
"logps/ref_rejected": -80.87986755371094,
|
|
"logps/rejected": -84.91656494140625,
|
|
"loss": 1.2924,
|
|
"margin_dpo/margin_mean": 1.1286823749542236,
|
|
"margin_dpo/margin_std": 2.453193426132202,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.1436130007558579,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.432558298110962,
|
|
"fcm_dpo/q_t": 0.46547260880470276,
|
|
"grad_norm": 29.277822494506836,
|
|
"learning_rate": 4.974553604702332e-07,
|
|
"logits/chosen": 0.20299391448497772,
|
|
"logits/rejected": 0.15903373062610626,
|
|
"logps/chosen": -57.57347869873047,
|
|
"logps/ref_chosen": -54.08087158203125,
|
|
"logps/ref_rejected": -76.15860748291016,
|
|
"logps/rejected": -81.0837631225586,
|
|
"loss": 1.2756,
|
|
"margin_dpo/margin_mean": 1.432558298110962,
|
|
"margin_dpo/margin_std": 3.3028881549835205,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.15117157974300832,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.2184693813323975,
|
|
"fcm_dpo/q_t": 0.47156065702438354,
|
|
"grad_norm": 32.97772216796875,
|
|
"learning_rate": 4.964280947263676e-07,
|
|
"logits/chosen": 0.21789178252220154,
|
|
"logits/rejected": 0.18518556654453278,
|
|
"logps/chosen": -68.62821197509766,
|
|
"logps/ref_chosen": -63.875038146972656,
|
|
"logps/ref_rejected": -82.077880859375,
|
|
"logps/rejected": -88.04951477050781,
|
|
"loss": 1.322,
|
|
"margin_dpo/margin_mean": 1.218469262123108,
|
|
"margin_dpo/margin_std": 4.569883346557617,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15873015873015872,
|
|
"fcm_dpo/beta": 0.10632991790771484,
|
|
"fcm_dpo/delta": 0.10528752952814102,
|
|
"fcm_dpo/margin": 1.7072112560272217,
|
|
"fcm_dpo/q_t": 0.45956483483314514,
|
|
"grad_norm": 34.242095947265625,
|
|
"learning_rate": 4.952285105344791e-07,
|
|
"logits/chosen": 0.23205998539924622,
|
|
"logits/rejected": 0.18055513501167297,
|
|
"logps/chosen": -67.48213195800781,
|
|
"logps/ref_chosen": -62.572479248046875,
|
|
"logps/ref_rejected": -80.93415069580078,
|
|
"logps/rejected": -87.55101013183594,
|
|
"loss": 1.2809,
|
|
"margin_dpo/margin_mean": 1.7072112560272217,
|
|
"margin_dpo/margin_std": 4.9097795486450195,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.16628873771730915,
|
|
"fcm_dpo/beta": 0.1157844066619873,
|
|
"fcm_dpo/delta": 0.08823363482952118,
|
|
"fcm_dpo/margin": 1.5210223197937012,
|
|
"fcm_dpo/q_t": 0.46007877588272095,
|
|
"grad_norm": 55.85200881958008,
|
|
"learning_rate": 4.938574467213517e-07,
|
|
"logits/chosen": 0.19681879878044128,
|
|
"logits/rejected": 0.1736396998167038,
|
|
"logps/chosen": -74.27507781982422,
|
|
"logps/ref_chosen": -68.67534637451172,
|
|
"logps/ref_rejected": -78.82028198242188,
|
|
"logps/rejected": -85.94103240966797,
|
|
"loss": 1.3155,
|
|
"margin_dpo/margin_mean": 1.5210223197937012,
|
|
"margin_dpo/margin_std": 5.531653881072998,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.17384731670445955,
|
|
"fcm_dpo/beta": 0.13739340007305145,
|
|
"fcm_dpo/delta": 0.2502194941043854,
|
|
"fcm_dpo/margin": 1.9611819982528687,
|
|
"fcm_dpo/q_t": 0.441068172454834,
|
|
"grad_norm": 44.24903106689453,
|
|
"learning_rate": 4.923158620234019e-07,
|
|
"logits/chosen": 0.23888036608695984,
|
|
"logits/rejected": 0.18526621162891388,
|
|
"logps/chosen": -63.93567657470703,
|
|
"logps/ref_chosen": -58.65370559692383,
|
|
"logps/ref_rejected": -81.89688873291016,
|
|
"logps/rejected": -89.1400375366211,
|
|
"loss": 1.2282,
|
|
"margin_dpo/margin_mean": 1.9611822366714478,
|
|
"margin_dpo/margin_std": 4.587324142456055,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.18140589569160998,
|
|
"fcm_dpo/beta": 0.1672566831111908,
|
|
"fcm_dpo/delta": 0.12861795723438263,
|
|
"fcm_dpo/margin": 2.2945079803466797,
|
|
"fcm_dpo/q_t": 0.41543060541152954,
|
|
"grad_norm": 44.8306884765625,
|
|
"learning_rate": 4.906048344162676e-07,
|
|
"logits/chosen": 0.24071356654167175,
|
|
"logits/rejected": 0.19102147221565247,
|
|
"logps/chosen": -61.27440643310547,
|
|
"logps/ref_chosen": -56.16423797607422,
|
|
"logps/ref_rejected": -75.87689971923828,
|
|
"logps/rejected": -83.28157806396484,
|
|
"loss": 1.1491,
|
|
"margin_dpo/margin_mean": 2.2945079803466797,
|
|
"margin_dpo/margin_std": 4.038813591003418,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.1889644746787604,
|
|
"fcm_dpo/beta": 0.18768858909606934,
|
|
"fcm_dpo/delta": 0.08637391030788422,
|
|
"fcm_dpo/margin": 2.755068302154541,
|
|
"fcm_dpo/q_t": 0.39426344633102417,
|
|
"grad_norm": 50.769744873046875,
|
|
"learning_rate": 4.887255603610184e-07,
|
|
"logits/chosen": 0.2529417872428894,
|
|
"logits/rejected": 0.19630905985832214,
|
|
"logps/chosen": -65.41935729980469,
|
|
"logps/ref_chosen": -59.744285583496094,
|
|
"logps/ref_rejected": -86.77314758300781,
|
|
"logps/rejected": -95.20329284667969,
|
|
"loss": 1.0755,
|
|
"margin_dpo/margin_mean": 2.755068302154541,
|
|
"margin_dpo/margin_std": 4.354895114898682,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.1965230536659108,
|
|
"fcm_dpo/beta": 0.2045096904039383,
|
|
"fcm_dpo/delta": 0.08989689499139786,
|
|
"fcm_dpo/margin": 2.5075488090515137,
|
|
"fcm_dpo/q_t": 0.3945372402667999,
|
|
"grad_norm": 59.820316314697266,
|
|
"learning_rate": 4.866793539675126e-07,
|
|
"logits/chosen": 0.17989113926887512,
|
|
"logits/rejected": 0.14967137575149536,
|
|
"logps/chosen": -70.37138366699219,
|
|
"logps/ref_chosen": -64.15296936035156,
|
|
"logps/ref_rejected": -75.17271423339844,
|
|
"logps/rejected": -83.89866638183594,
|
|
"loss": 1.1195,
|
|
"margin_dpo/margin_mean": 2.5075488090515137,
|
|
"margin_dpo/margin_std": 4.384979248046875,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.20408163265306123,
|
|
"fcm_dpo/beta": 0.20533373951911926,
|
|
"fcm_dpo/delta": -0.03491649031639099,
|
|
"fcm_dpo/margin": 3.0766525268554688,
|
|
"fcm_dpo/q_t": 0.37100949883461,
|
|
"grad_norm": 47.41992950439453,
|
|
"learning_rate": 4.844676460754862e-07,
|
|
"logits/chosen": 0.22673170268535614,
|
|
"logits/rejected": 0.18738315999507904,
|
|
"logps/chosen": -63.47684860229492,
|
|
"logps/ref_chosen": -57.006690979003906,
|
|
"logps/ref_rejected": -73.71768188476562,
|
|
"logps/rejected": -83.26448822021484,
|
|
"loss": 1.0509,
|
|
"margin_dpo/margin_mean": 3.076652765274048,
|
|
"margin_dpo/margin_std": 4.7008538246154785,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.21164021164021163,
|
|
"fcm_dpo/beta": 0.19947479665279388,
|
|
"fcm_dpo/delta": -0.0736149474978447,
|
|
"fcm_dpo/margin": 3.355564594268799,
|
|
"fcm_dpo/q_t": 0.3690628409385681,
|
|
"grad_norm": 58.970130920410156,
|
|
"learning_rate": 4.820919832540181e-07,
|
|
"logits/chosen": 0.20238320529460907,
|
|
"logits/rejected": 0.15974575281143188,
|
|
"logps/chosen": -71.1889419555664,
|
|
"logps/ref_chosen": -63.36246871948242,
|
|
"logps/ref_rejected": -79.62621307373047,
|
|
"logps/rejected": -90.8082504272461,
|
|
"loss": 1.0837,
|
|
"margin_dpo/margin_mean": 3.355564594268799,
|
|
"margin_dpo/margin_std": 5.568287372589111,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.21919879062736206,
|
|
"fcm_dpo/beta": 0.18338093161582947,
|
|
"fcm_dpo/delta": -0.04409990459680557,
|
|
"fcm_dpo/margin": 3.4895870685577393,
|
|
"fcm_dpo/q_t": 0.3738616108894348,
|
|
"grad_norm": 56.26732635498047,
|
|
"learning_rate": 4.795540267200686e-07,
|
|
"logits/chosen": 0.21315772831439972,
|
|
"logits/rejected": 0.19285325706005096,
|
|
"logps/chosen": -73.33685302734375,
|
|
"logps/ref_chosen": -65.01470184326172,
|
|
"logps/ref_rejected": -80.49073791503906,
|
|
"logps/rejected": -92.30247497558594,
|
|
"loss": 1.1029,
|
|
"margin_dpo/margin_mean": 3.4895870685577393,
|
|
"margin_dpo/margin_std": 6.0685014724731445,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.22675736961451248,
|
|
"fcm_dpo/beta": 0.17459313571453094,
|
|
"fcm_dpo/delta": -0.12236909568309784,
|
|
"fcm_dpo/margin": 4.1057610511779785,
|
|
"fcm_dpo/q_t": 0.35746586322784424,
|
|
"grad_norm": 39.50666046142578,
|
|
"learning_rate": 4.768555511768486e-07,
|
|
"logits/chosen": 0.2202187478542328,
|
|
"logits/rejected": 0.17578254640102386,
|
|
"logps/chosen": -67.75491333007812,
|
|
"logps/ref_chosen": -59.19135284423828,
|
|
"logps/ref_rejected": -74.0339126586914,
|
|
"logps/rejected": -86.70323181152344,
|
|
"loss": 1.0186,
|
|
"margin_dpo/margin_mean": 4.105760097503662,
|
|
"margin_dpo/margin_std": 5.879034519195557,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.23431594860166288,
|
|
"fcm_dpo/beta": 0.1496421992778778,
|
|
"fcm_dpo/delta": -0.08264624327421188,
|
|
"fcm_dpo/margin": 4.5022430419921875,
|
|
"fcm_dpo/q_t": 0.36458662152290344,
|
|
"grad_norm": 41.55534362792969,
|
|
"learning_rate": 4.7399844357283393e-07,
|
|
"logits/chosen": 0.23970897495746613,
|
|
"logits/rejected": 0.20661358535289764,
|
|
"logps/chosen": -69.90528869628906,
|
|
"logps/ref_chosen": -60.93949508666992,
|
|
"logps/ref_rejected": -74.51151275634766,
|
|
"logps/rejected": -87.97954559326172,
|
|
"loss": 1.04,
|
|
"margin_dpo/margin_mean": 4.5022430419921875,
|
|
"margin_dpo/margin_std": 6.7144455909729,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.2418745275888133,
|
|
"fcm_dpo/beta": 0.1405264288187027,
|
|
"fcm_dpo/delta": -0.07398034632205963,
|
|
"fcm_dpo/margin": 4.7481889724731445,
|
|
"fcm_dpo/q_t": 0.3655697703361511,
|
|
"grad_norm": 35.43960189819336,
|
|
"learning_rate": 4.7098470178228755e-07,
|
|
"logits/chosen": 0.22334730625152588,
|
|
"logits/rejected": 0.186327263712883,
|
|
"logps/chosen": -68.7593002319336,
|
|
"logps/ref_chosen": -58.763816833496094,
|
|
"logps/ref_rejected": -74.94743347167969,
|
|
"logps/rejected": -89.69109344482422,
|
|
"loss": 1.0324,
|
|
"margin_dpo/margin_mean": 4.7481889724731445,
|
|
"margin_dpo/margin_std": 6.990658760070801,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.2494331065759637,
|
|
"fcm_dpo/beta": 0.12699048221111298,
|
|
"fcm_dpo/delta": -0.14860393106937408,
|
|
"fcm_dpo/margin": 5.813238143920898,
|
|
"fcm_dpo/q_t": 0.34892064332962036,
|
|
"grad_norm": 32.44923400878906,
|
|
"learning_rate": 4.678164332082175e-07,
|
|
"logits/chosen": 0.26088446378707886,
|
|
"logits/rejected": 0.20988254249095917,
|
|
"logps/chosen": -67.06852722167969,
|
|
"logps/ref_chosen": -55.70417022705078,
|
|
"logps/ref_rejected": -76.59439849853516,
|
|
"logps/rejected": -93.77200317382812,
|
|
"loss": 0.9805,
|
|
"margin_dpo/margin_mean": 5.813237190246582,
|
|
"margin_dpo/margin_std": 7.511562347412109,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.25699168556311414,
|
|
"fcm_dpo/beta": 0.11718620359897614,
|
|
"fcm_dpo/delta": -0.03449869900941849,
|
|
"fcm_dpo/margin": 5.392855644226074,
|
|
"fcm_dpo/q_t": 0.3719424307346344,
|
|
"grad_norm": 28.188493728637695,
|
|
"learning_rate": 4.6449585330874425e-07,
|
|
"logits/chosen": 0.23041145503520966,
|
|
"logits/rejected": 0.19097033143043518,
|
|
"logps/chosen": -72.28368377685547,
|
|
"logps/ref_chosen": -61.169105529785156,
|
|
"logps/ref_rejected": -77.21674346923828,
|
|
"logps/rejected": -93.72418212890625,
|
|
"loss": 1.0359,
|
|
"margin_dpo/margin_mean": 5.392855644226074,
|
|
"margin_dpo/margin_std": 7.895319938659668,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.26455026455026454,
|
|
"fcm_dpo/beta": 0.1070084348320961,
|
|
"fcm_dpo/delta": -0.1262674331665039,
|
|
"fcm_dpo/margin": 6.6942877769470215,
|
|
"fcm_dpo/q_t": 0.35919734835624695,
|
|
"grad_norm": 29.46046257019043,
|
|
"learning_rate": 4.6102528404790965e-07,
|
|
"logits/chosen": 0.2935723066329956,
|
|
"logits/rejected": 0.23458893597126007,
|
|
"logps/chosen": -71.26114654541016,
|
|
"logps/ref_chosen": -59.24176788330078,
|
|
"logps/ref_rejected": -81.80384826660156,
|
|
"logps/rejected": -100.51751708984375,
|
|
"loss": 1.0369,
|
|
"margin_dpo/margin_mean": 6.6942877769470215,
|
|
"margin_dpo/margin_std": 9.981403350830078,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.272108843537415,
|
|
"fcm_dpo/beta": 0.09683612734079361,
|
|
"fcm_dpo/delta": -0.05345671623945236,
|
|
"fcm_dpo/margin": 6.68777322769165,
|
|
"fcm_dpo/q_t": 0.3701228201389313,
|
|
"grad_norm": 29.12032699584961,
|
|
"learning_rate": 4.5740715227200897e-07,
|
|
"logits/chosen": 0.25504690408706665,
|
|
"logits/rejected": 0.2136983871459961,
|
|
"logps/chosen": -75.70313262939453,
|
|
"logps/ref_chosen": -63.24883270263672,
|
|
"logps/ref_rejected": -79.00736236572266,
|
|
"logps/rejected": -98.14942932128906,
|
|
"loss": 1.0588,
|
|
"margin_dpo/margin_mean": 6.687772274017334,
|
|
"margin_dpo/margin_std": 10.323027610778809,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.2796674225245654,
|
|
"fcm_dpo/beta": 0.09016869217157364,
|
|
"fcm_dpo/delta": -0.05902116373181343,
|
|
"fcm_dpo/margin": 7.227444648742676,
|
|
"fcm_dpo/q_t": 0.36848214268684387,
|
|
"grad_norm": 25.792259216308594,
|
|
"learning_rate": 4.5364398801258394e-07,
|
|
"logits/chosen": 0.29077380895614624,
|
|
"logits/rejected": 0.24018768966197968,
|
|
"logps/chosen": -68.09630584716797,
|
|
"logps/ref_chosen": -56.390625,
|
|
"logps/ref_rejected": -76.81001281738281,
|
|
"logps/rejected": -95.74314880371094,
|
|
"loss": 1.0435,
|
|
"margin_dpo/margin_mean": 7.227444648742676,
|
|
"margin_dpo/margin_std": 10.630643844604492,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.2872260015117158,
|
|
"fcm_dpo/beta": 0.08473627269268036,
|
|
"fcm_dpo/delta": -0.11705086380243301,
|
|
"fcm_dpo/margin": 8.364612579345703,
|
|
"fcm_dpo/q_t": 0.35912564396858215,
|
|
"grad_norm": 29.976728439331055,
|
|
"learning_rate": 4.4973842271726024e-07,
|
|
"logits/chosen": 0.28480634093284607,
|
|
"logits/rejected": 0.23106631636619568,
|
|
"logps/chosen": -79.46668243408203,
|
|
"logps/ref_chosen": -68.25389099121094,
|
|
"logps/ref_rejected": -86.461181640625,
|
|
"logps/rejected": -106.03858947753906,
|
|
"loss": 1.0203,
|
|
"margin_dpo/margin_mean": 8.364612579345703,
|
|
"margin_dpo/margin_std": 12.093205451965332,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.2947845804988662,
|
|
"fcm_dpo/beta": 0.07972662150859833,
|
|
"fcm_dpo/delta": -0.029689926654100418,
|
|
"fcm_dpo/margin": 7.86168909072876,
|
|
"fcm_dpo/q_t": 0.37187460064888,
|
|
"grad_norm": 26.973175048828125,
|
|
"learning_rate": 4.4569318740967043e-07,
|
|
"logits/chosen": 0.300616979598999,
|
|
"logits/rejected": 0.27948108315467834,
|
|
"logps/chosen": -75.24478912353516,
|
|
"logps/ref_chosen": -62.1484260559082,
|
|
"logps/ref_rejected": -71.33458709716797,
|
|
"logps/rejected": -92.2926254272461,
|
|
"loss": 1.056,
|
|
"margin_dpo/margin_mean": 7.86168909072876,
|
|
"margin_dpo/margin_std": 11.976266860961914,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.30234315948601664,
|
|
"fcm_dpo/beta": 0.07483974099159241,
|
|
"fcm_dpo/delta": -0.10010068118572235,
|
|
"fcm_dpo/margin": 9.267468452453613,
|
|
"fcm_dpo/q_t": 0.3587101101875305,
|
|
"grad_norm": 22.538490295410156,
|
|
"learning_rate": 4.415111107797445e-07,
|
|
"logits/chosen": 0.3558996617794037,
|
|
"logits/rejected": 0.2972090244293213,
|
|
"logps/chosen": -69.53649139404297,
|
|
"logps/ref_chosen": -56.950096130371094,
|
|
"logps/ref_rejected": -78.66989135742188,
|
|
"logps/rejected": -100.52375793457031,
|
|
"loss": 1.0309,
|
|
"margin_dpo/margin_mean": 9.267467498779297,
|
|
"margin_dpo/margin_std": 13.319124221801758,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.30234315948601664,
|
|
"eval_fcm_dpo/beta": 0.0705723762512207,
|
|
"eval_fcm_dpo/delta": -0.007554640062153339,
|
|
"eval_fcm_dpo/margin": 8.440375328063965,
|
|
"eval_fcm_dpo/q_t": 0.38098761439323425,
|
|
"eval_logits/chosen": 0.3429543077945709,
|
|
"eval_logits/rejected": 0.29528242349624634,
|
|
"eval_logps/chosen": -88.24712371826172,
|
|
"eval_logps/ref_chosen": -74.85946655273438,
|
|
"eval_logps/ref_rejected": -79.54898834228516,
|
|
"eval_logps/rejected": -101.37702178955078,
|
|
"eval_loss": 0.5478904247283936,
|
|
"eval_margin_dpo/margin_mean": 8.440375328063965,
|
|
"eval_margin_dpo/margin_std": 14.14255428314209,
|
|
"eval_runtime": 38.387,
|
|
"eval_samples_per_second": 59.994,
|
|
"eval_steps_per_second": 1.876,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.30990173847316704,
|
|
"fcm_dpo/beta": 0.06560994684696198,
|
|
"fcm_dpo/delta": -0.028526384383440018,
|
|
"fcm_dpo/margin": 9.536072731018066,
|
|
"fcm_dpo/q_t": 0.3714202344417572,
|
|
"grad_norm": 28.681243896484375,
|
|
"learning_rate": 4.3719511720570814e-07,
|
|
"logits/chosen": 0.35043153166770935,
|
|
"logits/rejected": 0.2849184572696686,
|
|
"logps/chosen": -72.63653564453125,
|
|
"logps/ref_chosen": -57.99428176879883,
|
|
"logps/ref_rejected": -83.5367431640625,
|
|
"logps/rejected": -107.71504974365234,
|
|
"loss": 1.0472,
|
|
"margin_dpo/margin_mean": 9.536072731018066,
|
|
"margin_dpo/margin_std": 14.235984802246094,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.31746031746031744,
|
|
"fcm_dpo/beta": 0.06596539914608002,
|
|
"fcm_dpo/delta": -0.020397888496518135,
|
|
"fcm_dpo/margin": 9.372965812683105,
|
|
"fcm_dpo/q_t": 0.3780348598957062,
|
|
"grad_norm": 22.18747901916504,
|
|
"learning_rate": 4.327482247091679e-07,
|
|
"logits/chosen": 0.3586636185646057,
|
|
"logits/rejected": 0.3060414493083954,
|
|
"logps/chosen": -80.08374786376953,
|
|
"logps/ref_chosen": -63.77195358276367,
|
|
"logps/ref_rejected": -82.56491088867188,
|
|
"logps/rejected": -108.24967956542969,
|
|
"loss": 1.0977,
|
|
"margin_dpo/margin_mean": 9.372964859008789,
|
|
"margin_dpo/margin_std": 15.807058334350586,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.3250188964474679,
|
|
"fcm_dpo/beta": 0.0596235916018486,
|
|
"fcm_dpo/delta": -0.1393919438123703,
|
|
"fcm_dpo/margin": 12.262106895446777,
|
|
"fcm_dpo/q_t": 0.35417699813842773,
|
|
"grad_norm": 21.598207473754883,
|
|
"learning_rate": 4.281735428447157e-07,
|
|
"logits/chosen": 0.3513588309288025,
|
|
"logits/rejected": 0.28718453645706177,
|
|
"logps/chosen": -74.48625183105469,
|
|
"logps/ref_chosen": -60.27800750732422,
|
|
"logps/ref_rejected": -83.91607666015625,
|
|
"logps/rejected": -110.38641357421875,
|
|
"loss": 0.9949,
|
|
"margin_dpo/margin_mean": 12.262106895446777,
|
|
"margin_dpo/margin_std": 16.72055435180664,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.3325774754346183,
|
|
"fcm_dpo/beta": 0.05317453667521477,
|
|
"fcm_dpo/delta": -0.02484332025051117,
|
|
"fcm_dpo/margin": 11.684633255004883,
|
|
"fcm_dpo/q_t": 0.3738202452659607,
|
|
"grad_norm": 20.055500030517578,
|
|
"learning_rate": 4.234742705255272e-07,
|
|
"logits/chosen": 0.356646329164505,
|
|
"logits/rejected": 0.29619479179382324,
|
|
"logps/chosen": -76.64110565185547,
|
|
"logps/ref_chosen": -60.88572311401367,
|
|
"logps/ref_rejected": -80.1805191040039,
|
|
"logps/rejected": -107.62054443359375,
|
|
"loss": 1.055,
|
|
"margin_dpo/margin_mean": 11.684633255004883,
|
|
"margin_dpo/margin_std": 17.75079917907715,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.3401360544217687,
|
|
"fcm_dpo/beta": 0.05095481872558594,
|
|
"fcm_dpo/delta": -0.07625424116849899,
|
|
"fcm_dpo/margin": 13.132394790649414,
|
|
"fcm_dpo/q_t": 0.3655363917350769,
|
|
"grad_norm": 22.153240203857422,
|
|
"learning_rate": 4.186536937864752e-07,
|
|
"logits/chosen": 0.43087267875671387,
|
|
"logits/rejected": 0.3594560921192169,
|
|
"logps/chosen": -76.80330657958984,
|
|
"logps/ref_chosen": -61.02507781982422,
|
|
"logps/ref_rejected": -91.92439270019531,
|
|
"logps/rejected": -120.83500671386719,
|
|
"loss": 1.0157,
|
|
"margin_dpo/margin_mean": 13.132394790649414,
|
|
"margin_dpo/margin_std": 18.37344741821289,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.3476946334089191,
|
|
"fcm_dpo/beta": 0.0496797040104866,
|
|
"fcm_dpo/delta": -0.028627533465623856,
|
|
"fcm_dpo/margin": 12.607888221740723,
|
|
"fcm_dpo/q_t": 0.37395650148391724,
|
|
"grad_norm": 20.35274314880371,
|
|
"learning_rate": 4.137151834863213e-07,
|
|
"logits/chosen": 0.469480037689209,
|
|
"logits/rejected": 0.4215773642063141,
|
|
"logps/chosen": -70.93717956542969,
|
|
"logps/ref_chosen": -54.49797821044922,
|
|
"logps/ref_rejected": -71.96363830566406,
|
|
"logps/rejected": -101.01072692871094,
|
|
"loss": 1.076,
|
|
"margin_dpo/margin_mean": 12.607889175415039,
|
|
"margin_dpo/margin_std": 20.359996795654297,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.35525321239606955,
|
|
"fcm_dpo/beta": 0.04417480155825615,
|
|
"fcm_dpo/delta": -0.1501634567975998,
|
|
"fcm_dpo/margin": 14.744340896606445,
|
|
"fcm_dpo/q_t": 0.36673247814178467,
|
|
"grad_norm": 18.327699661254883,
|
|
"learning_rate": 4.08662192950594e-07,
|
|
"logits/chosen": 0.4181364178657532,
|
|
"logits/rejected": 0.3918091058731079,
|
|
"logps/chosen": -82.67752075195312,
|
|
"logps/ref_chosen": -63.250282287597656,
|
|
"logps/ref_rejected": -73.09049987792969,
|
|
"logps/rejected": -107.26206970214844,
|
|
"loss": 1.0368,
|
|
"margin_dpo/margin_mean": 14.744340896606445,
|
|
"margin_dpo/margin_std": 20.939937591552734,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.36281179138321995,
|
|
"fcm_dpo/beta": 0.040509648621082306,
|
|
"fcm_dpo/delta": -0.09375976026058197,
|
|
"fcm_dpo/margin": 17.00118637084961,
|
|
"fcm_dpo/q_t": 0.35856300592422485,
|
|
"grad_norm": 15.43008041381836,
|
|
"learning_rate": 4.0349825555680045e-07,
|
|
"logits/chosen": 0.4488718509674072,
|
|
"logits/rejected": 0.3973858654499054,
|
|
"logps/chosen": -88.68736267089844,
|
|
"logps/ref_chosen": -65.26150512695312,
|
|
"logps/ref_rejected": -87.60311126708984,
|
|
"logps/rejected": -128.0301513671875,
|
|
"loss": 1.0096,
|
|
"margin_dpo/margin_mean": 17.00118637084961,
|
|
"margin_dpo/margin_std": 23.233928680419922,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.37037037037037035,
|
|
"fcm_dpo/beta": 0.03684638813138008,
|
|
"fcm_dpo/delta": -0.12539520859718323,
|
|
"fcm_dpo/margin": 19.555816650390625,
|
|
"fcm_dpo/q_t": 0.3527432084083557,
|
|
"grad_norm": 34.119014739990234,
|
|
"learning_rate": 3.982269822636601e-07,
|
|
"logits/chosen": 0.4494594633579254,
|
|
"logits/rejected": 0.41574639081954956,
|
|
"logps/chosen": -89.88227844238281,
|
|
"logps/ref_chosen": -65.73170471191406,
|
|
"logps/ref_rejected": -75.19642639160156,
|
|
"logps/rejected": -118.90281677246094,
|
|
"loss": 0.9743,
|
|
"margin_dpo/margin_mean": 19.555816650390625,
|
|
"margin_dpo/margin_std": 24.575748443603516,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.3779289493575208,
|
|
"fcm_dpo/beta": 0.033488351851701736,
|
|
"fcm_dpo/delta": -0.029287120327353477,
|
|
"fcm_dpo/margin": 17.215007781982422,
|
|
"fcm_dpo/q_t": 0.383605420589447,
|
|
"grad_norm": 19.24226188659668,
|
|
"learning_rate": 3.9285205908608934e-07,
|
|
"logits/chosen": 0.5213881731033325,
|
|
"logits/rejected": 0.5142917633056641,
|
|
"logps/chosen": -104.99214172363281,
|
|
"logps/ref_chosen": -70.71224212646484,
|
|
"logps/ref_rejected": -76.12723541259766,
|
|
"logps/rejected": -127.62215423583984,
|
|
"loss": 1.1187,
|
|
"margin_dpo/margin_mean": 17.215011596679688,
|
|
"margin_dpo/margin_std": 29.802139282226562,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.3854875283446712,
|
|
"fcm_dpo/beta": 0.033009327948093414,
|
|
"fcm_dpo/delta": -0.005891552660614252,
|
|
"fcm_dpo/margin": 16.929611206054688,
|
|
"fcm_dpo/q_t": 0.3879493176937103,
|
|
"grad_norm": 22.598548889160156,
|
|
"learning_rate": 3.873772445177015e-07,
|
|
"logits/chosen": 0.5174540281295776,
|
|
"logits/rejected": 0.4729521870613098,
|
|
"logps/chosen": -98.29107666015625,
|
|
"logps/ref_chosen": -61.767662048339844,
|
|
"logps/ref_rejected": -77.38813018798828,
|
|
"logps/rejected": -130.8411407470703,
|
|
"loss": 1.1283,
|
|
"margin_dpo/margin_mean": 16.929611206054688,
|
|
"margin_dpo/margin_std": 30.323169708251953,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.3930461073318216,
|
|
"fcm_dpo/beta": 0.030517011880874634,
|
|
"fcm_dpo/delta": -0.11403815448284149,
|
|
"fcm_dpo/margin": 23.142879486083984,
|
|
"fcm_dpo/q_t": 0.358590304851532,
|
|
"grad_norm": 20.97575569152832,
|
|
"learning_rate": 3.818063669026256e-07,
|
|
"logits/chosen": 0.5084746479988098,
|
|
"logits/rejected": 0.43368783593177795,
|
|
"logps/chosen": -100.66755676269531,
|
|
"logps/ref_chosen": -61.57584762573242,
|
|
"logps/ref_rejected": -91.87513732910156,
|
|
"logps/rejected": -154.1096954345703,
|
|
"loss": 1.0132,
|
|
"margin_dpo/margin_mean": 23.142879486083984,
|
|
"margin_dpo/margin_std": 32.55268096923828,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.40060468631897206,
|
|
"fcm_dpo/beta": 0.029083365574479103,
|
|
"fcm_dpo/delta": -0.01140249241143465,
|
|
"fcm_dpo/margin": 20.991168975830078,
|
|
"fcm_dpo/q_t": 0.37674590945243835,
|
|
"grad_norm": 18.634904861450195,
|
|
"learning_rate": 3.7614332175848027e-07,
|
|
"logits/chosen": 0.47952961921691895,
|
|
"logits/rejected": 0.42895132303237915,
|
|
"logps/chosen": -102.08247375488281,
|
|
"logps/ref_chosen": -65.75422668457031,
|
|
"logps/ref_rejected": -77.9569320678711,
|
|
"logps/rejected": -135.27633666992188,
|
|
"loss": 1.0721,
|
|
"margin_dpo/margin_mean": 20.991168975830078,
|
|
"margin_dpo/margin_std": 33.38361740112305,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.40816326530612246,
|
|
"fcm_dpo/beta": 0.02850642241537571,
|
|
"fcm_dpo/delta": -0.009921222925186157,
|
|
"fcm_dpo/margin": 21.323711395263672,
|
|
"fcm_dpo/q_t": 0.3749794363975525,
|
|
"grad_norm": 17.992469787597656,
|
|
"learning_rate": 3.7039206905237656e-07,
|
|
"logits/chosen": 0.5241806507110596,
|
|
"logits/rejected": 0.4762350022792816,
|
|
"logps/chosen": -94.45357513427734,
|
|
"logps/ref_chosen": -62.27649688720703,
|
|
"logps/ref_rejected": -76.56950378417969,
|
|
"logps/rejected": -130.07029724121094,
|
|
"loss": 1.0539,
|
|
"margin_dpo/margin_mean": 21.323711395263672,
|
|
"margin_dpo/margin_std": 31.970006942749023,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.41572184429327286,
|
|
"fcm_dpo/beta": 0.02760036289691925,
|
|
"fcm_dpo/delta": -0.06361333280801773,
|
|
"fcm_dpo/margin": 23.88420295715332,
|
|
"fcm_dpo/q_t": 0.368137389421463,
|
|
"grad_norm": 17.147911071777344,
|
|
"learning_rate": 3.645566304318526e-07,
|
|
"logits/chosen": 0.5007555484771729,
|
|
"logits/rejected": 0.45732545852661133,
|
|
"logps/chosen": -96.15426635742188,
|
|
"logps/ref_chosen": -61.854393005371094,
|
|
"logps/ref_rejected": -77.22246551513672,
|
|
"logps/rejected": -135.4065399169922,
|
|
"loss": 1.0454,
|
|
"margin_dpo/margin_mean": 23.88420295715332,
|
|
"margin_dpo/margin_std": 35.828495025634766,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.42328042328042326,
|
|
"fcm_dpo/beta": 0.02634030021727085,
|
|
"fcm_dpo/delta": -0.08169054239988327,
|
|
"fcm_dpo/margin": 25.740819931030273,
|
|
"fcm_dpo/q_t": 0.3601287007331848,
|
|
"grad_norm": 15.523616790771484,
|
|
"learning_rate": 3.586410864126781e-07,
|
|
"logits/chosen": 0.5158032178878784,
|
|
"logits/rejected": 0.4866085648536682,
|
|
"logps/chosen": -96.00532531738281,
|
|
"logps/ref_chosen": -61.29896926879883,
|
|
"logps/ref_rejected": -73.35762023925781,
|
|
"logps/rejected": -133.80477905273438,
|
|
"loss": 0.9899,
|
|
"margin_dpo/margin_mean": 25.740819931030273,
|
|
"margin_dpo/margin_std": 32.85602951049805,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.4308390022675737,
|
|
"fcm_dpo/beta": 0.02367565594613552,
|
|
"fcm_dpo/delta": -0.0362996831536293,
|
|
"fcm_dpo/margin": 26.715778350830078,
|
|
"fcm_dpo/q_t": 0.3702591061592102,
|
|
"grad_norm": 19.280351638793945,
|
|
"learning_rate": 3.5264957352549375e-07,
|
|
"logits/chosen": 0.5398426651954651,
|
|
"logits/rejected": 0.4950627386569977,
|
|
"logps/chosen": -106.26362609863281,
|
|
"logps/ref_chosen": -63.435462951660156,
|
|
"logps/ref_rejected": -79.73661804199219,
|
|
"logps/rejected": -149.28054809570312,
|
|
"loss": 1.0277,
|
|
"margin_dpo/margin_mean": 26.715778350830078,
|
|
"margin_dpo/margin_std": 37.812416076660156,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.4383975812547241,
|
|
"fcm_dpo/beta": 0.02178608998656273,
|
|
"fcm_dpo/delta": -0.09696368873119354,
|
|
"fcm_dpo/margin": 31.538326263427734,
|
|
"fcm_dpo/q_t": 0.35935452580451965,
|
|
"grad_norm": 20.562551498413086,
|
|
"learning_rate": 3.465862814232821e-07,
|
|
"logits/chosen": 0.5788189172744751,
|
|
"logits/rejected": 0.5170078277587891,
|
|
"logps/chosen": -110.50838470458984,
|
|
"logps/ref_chosen": -57.696876525878906,
|
|
"logps/ref_rejected": -79.78132629394531,
|
|
"logps/rejected": -164.13116455078125,
|
|
"loss": 0.9951,
|
|
"margin_dpo/margin_mean": 31.538326263427734,
|
|
"margin_dpo/margin_std": 42.166587829589844,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.4459561602418745,
|
|
"fcm_dpo/beta": 0.02080090157687664,
|
|
"fcm_dpo/delta": -0.02840259112417698,
|
|
"fcm_dpo/margin": 30.041790008544922,
|
|
"fcm_dpo/q_t": 0.37336617708206177,
|
|
"grad_norm": 20.723968505859375,
|
|
"learning_rate": 3.4045544995169125e-07,
|
|
"logits/chosen": 0.6665645837783813,
|
|
"logits/rejected": 0.5920735597610474,
|
|
"logps/chosen": -118.08138275146484,
|
|
"logps/ref_chosen": -55.430633544921875,
|
|
"logps/ref_rejected": -78.1390151977539,
|
|
"logps/rejected": -170.83157348632812,
|
|
"loss": 1.0454,
|
|
"margin_dpo/margin_mean": 30.041790008544922,
|
|
"margin_dpo/margin_std": 44.420555114746094,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.45351473922902497,
|
|
"fcm_dpo/beta": 0.020768912509083748,
|
|
"fcm_dpo/delta": 0.014924813993275166,
|
|
"fcm_dpo/margin": 28.187641143798828,
|
|
"fcm_dpo/q_t": 0.381052166223526,
|
|
"grad_norm": 17.88654327392578,
|
|
"learning_rate": 3.3426136618426043e-07,
|
|
"logits/chosen": 0.6102429628372192,
|
|
"logits/rejected": 0.556972861289978,
|
|
"logps/chosen": -131.4066925048828,
|
|
"logps/ref_chosen": -61.207069396972656,
|
|
"logps/ref_rejected": -75.23294067382812,
|
|
"logps/rejected": -173.62020874023438,
|
|
"loss": 1.0841,
|
|
"margin_dpo/margin_mean": 28.187641143798828,
|
|
"margin_dpo/margin_std": 45.52865982055664,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.46107331821617537,
|
|
"fcm_dpo/beta": 0.02134864404797554,
|
|
"fcm_dpo/delta": -0.011247384361922741,
|
|
"fcm_dpo/margin": 28.5620059967041,
|
|
"fcm_dpo/q_t": 0.3759697377681732,
|
|
"grad_norm": 18.560199737548828,
|
|
"learning_rate": 3.280083614246217e-07,
|
|
"logits/chosen": 0.588545024394989,
|
|
"logits/rejected": 0.5514200925827026,
|
|
"logps/chosen": -128.7273406982422,
|
|
"logps/ref_chosen": -63.06663131713867,
|
|
"logps/ref_rejected": -78.45845031738281,
|
|
"logps/rejected": -172.68116760253906,
|
|
"loss": 1.0782,
|
|
"margin_dpo/margin_mean": 28.5620059967041,
|
|
"margin_dpo/margin_std": 45.59451675415039,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.46863189720332576,
|
|
"fcm_dpo/beta": 0.022000327706336975,
|
|
"fcm_dpo/delta": 0.11391662061214447,
|
|
"fcm_dpo/margin": 22.363637924194336,
|
|
"fcm_dpo/q_t": 0.40100401639938354,
|
|
"grad_norm": 20.21632194519043,
|
|
"learning_rate": 3.2170080817777257e-07,
|
|
"logits/chosen": 0.6432641744613647,
|
|
"logits/rejected": 0.5922056436538696,
|
|
"logps/chosen": -123.6684799194336,
|
|
"logps/ref_chosen": -63.60908889770508,
|
|
"logps/ref_rejected": -74.06394958496094,
|
|
"logps/rejected": -156.48696899414062,
|
|
"loss": 1.1536,
|
|
"margin_dpo/margin_mean": 22.363637924194336,
|
|
"margin_dpo/margin_std": 42.26737976074219,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.47619047619047616,
|
|
"fcm_dpo/beta": 0.022742778062820435,
|
|
"fcm_dpo/delta": -0.050774771720170975,
|
|
"fcm_dpo/margin": 28.4006290435791,
|
|
"fcm_dpo/q_t": 0.3682178854942322,
|
|
"grad_norm": 15.559822082519531,
|
|
"learning_rate": 3.1534311709253723e-07,
|
|
"logits/chosen": 0.5904372334480286,
|
|
"logits/rejected": 0.5356141328811646,
|
|
"logps/chosen": -112.68599700927734,
|
|
"logps/ref_chosen": -62.31493377685547,
|
|
"logps/ref_rejected": -75.07472229003906,
|
|
"logps/rejected": -153.84640502929688,
|
|
"loss": 1.0239,
|
|
"margin_dpo/margin_mean": 28.4006290435791,
|
|
"margin_dpo/margin_std": 40.05788040161133,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.4837490551776266,
|
|
"fcm_dpo/beta": 0.02097228541970253,
|
|
"fcm_dpo/delta": -0.10140693187713623,
|
|
"fcm_dpo/margin": 31.283824920654297,
|
|
"fcm_dpo/q_t": 0.3660925030708313,
|
|
"grad_norm": 17.410728454589844,
|
|
"learning_rate": 3.0893973387735683e-07,
|
|
"logits/chosen": 0.5705456137657166,
|
|
"logits/rejected": 0.5071772336959839,
|
|
"logps/chosen": -102.21686553955078,
|
|
"logps/ref_chosen": -55.336036682128906,
|
|
"logps/ref_rejected": -80.05536651611328,
|
|
"logps/rejected": -158.2200164794922,
|
|
"loss": 1.0213,
|
|
"margin_dpo/margin_mean": 31.283824920654297,
|
|
"margin_dpo/margin_std": 43.42279052734375,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.491307634164777,
|
|
"fcm_dpo/beta": 0.019703637808561325,
|
|
"fcm_dpo/delta": -0.03309765085577965,
|
|
"fcm_dpo/margin": 31.972396850585938,
|
|
"fcm_dpo/q_t": 0.36921393871307373,
|
|
"grad_norm": 15.720254898071289,
|
|
"learning_rate": 3.0249513619156206e-07,
|
|
"logits/chosen": 0.5923582315444946,
|
|
"logits/rejected": 0.5292683243751526,
|
|
"logps/chosen": -112.33642578125,
|
|
"logps/ref_chosen": -57.90629959106445,
|
|
"logps/ref_rejected": -74.2243881225586,
|
|
"logps/rejected": -160.62692260742188,
|
|
"loss": 1.0317,
|
|
"margin_dpo/margin_mean": 31.972393035888672,
|
|
"margin_dpo/margin_std": 45.57465744018555,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.4988662131519274,
|
|
"fcm_dpo/beta": 0.02025691233575344,
|
|
"fcm_dpo/delta": 0.007672114763408899,
|
|
"fcm_dpo/margin": 29.19550132751465,
|
|
"fcm_dpo/q_t": 0.381369411945343,
|
|
"grad_norm": 16.641584396362305,
|
|
"learning_rate": 2.9601383051430505e-07,
|
|
"logits/chosen": 0.5546280145645142,
|
|
"logits/rejected": 0.516182005405426,
|
|
"logps/chosen": -129.70407104492188,
|
|
"logps/ref_chosen": -65.17555236816406,
|
|
"logps/ref_rejected": -78.53681182861328,
|
|
"logps/rejected": -172.26084899902344,
|
|
"loss": 1.093,
|
|
"margin_dpo/margin_mean": 29.19550132751465,
|
|
"margin_dpo/margin_std": 48.34088134765625,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.5064247921390779,
|
|
"fcm_dpo/beta": 0.01816193014383316,
|
|
"fcm_dpo/delta": -0.08961853384971619,
|
|
"fcm_dpo/margin": 37.44696807861328,
|
|
"fcm_dpo/q_t": 0.36152511835098267,
|
|
"grad_norm": 21.882186889648438,
|
|
"learning_rate": 2.895003489933375e-07,
|
|
"logits/chosen": 0.6239126324653625,
|
|
"logits/rejected": 0.5766108632087708,
|
|
"logps/chosen": -128.90843200683594,
|
|
"logps/ref_chosen": -62.62797927856445,
|
|
"logps/ref_rejected": -79.9095458984375,
|
|
"logps/rejected": -183.636962890625,
|
|
"loss": 1.0065,
|
|
"margin_dpo/margin_mean": 37.44696807861328,
|
|
"margin_dpo/margin_std": 51.33563232421875,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.5139833711262283,
|
|
"fcm_dpo/beta": 0.018349742516875267,
|
|
"fcm_dpo/delta": 0.032100338488817215,
|
|
"fcm_dpo/margin": 31.00588035583496,
|
|
"fcm_dpo/q_t": 0.38639387488365173,
|
|
"grad_norm": 17.62742042541504,
|
|
"learning_rate": 2.8295924627584004e-07,
|
|
"logits/chosen": 0.5883735418319702,
|
|
"logits/rejected": 0.5494518280029297,
|
|
"logps/chosen": -136.67315673828125,
|
|
"logps/ref_chosen": -61.1064567565918,
|
|
"logps/ref_rejected": -76.71846008300781,
|
|
"logps/rejected": -183.29104614257812,
|
|
"loss": 1.0985,
|
|
"margin_dpo/margin_mean": 31.00588035583496,
|
|
"margin_dpo/margin_std": 52.399314880371094,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.5215419501133787,
|
|
"fcm_dpo/beta": 0.01758972555398941,
|
|
"fcm_dpo/delta": -0.06735783815383911,
|
|
"fcm_dpo/margin": 35.31150436401367,
|
|
"fcm_dpo/q_t": 0.37285932898521423,
|
|
"grad_norm": 16.583221435546875,
|
|
"learning_rate": 2.7639509632351927e-07,
|
|
"logits/chosen": 0.616736888885498,
|
|
"logits/rejected": 0.5733811855316162,
|
|
"logps/chosen": -128.89810180664062,
|
|
"logps/ref_chosen": -60.12370681762695,
|
|
"logps/ref_rejected": -78.58574676513672,
|
|
"logps/rejected": -182.67166137695312,
|
|
"loss": 1.0437,
|
|
"margin_dpo/margin_mean": 35.311500549316406,
|
|
"margin_dpo/margin_std": 51.527427673339844,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.5291005291005291,
|
|
"fcm_dpo/beta": 0.016854315996170044,
|
|
"fcm_dpo/delta": -0.06805837154388428,
|
|
"fcm_dpo/margin": 39.38313674926758,
|
|
"fcm_dpo/q_t": 0.3659011721611023,
|
|
"grad_norm": 17.990488052368164,
|
|
"learning_rate": 2.698124892141971e-07,
|
|
"logits/chosen": 0.6850234866142273,
|
|
"logits/rejected": 0.6155174970626831,
|
|
"logps/chosen": -119.6955337524414,
|
|
"logps/ref_chosen": -55.104461669921875,
|
|
"logps/ref_rejected": -80.63292694091797,
|
|
"logps/rejected": -184.60714721679688,
|
|
"loss": 1.013,
|
|
"margin_dpo/margin_mean": 39.38313674926758,
|
|
"margin_dpo/margin_std": 54.5691032409668,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.5366591080876795,
|
|
"fcm_dpo/beta": 0.015808746218681335,
|
|
"fcm_dpo/delta": -0.05140886455774307,
|
|
"fcm_dpo/margin": 40.9948616027832,
|
|
"fcm_dpo/q_t": 0.364651083946228,
|
|
"grad_norm": 17.801734924316406,
|
|
"learning_rate": 2.632160279321328e-07,
|
|
"logits/chosen": 0.646989643573761,
|
|
"logits/rejected": 0.5661012530326843,
|
|
"logps/chosen": -116.357666015625,
|
|
"logps/ref_chosen": -54.87224197387695,
|
|
"logps/ref_rejected": -77.01316833496094,
|
|
"logps/rejected": -179.4934844970703,
|
|
"loss": 1.0215,
|
|
"margin_dpo/margin_mean": 40.9948616027832,
|
|
"margin_dpo/margin_std": 57.23894500732422,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.54421768707483,
|
|
"fcm_dpo/beta": 0.015229565091431141,
|
|
"fcm_dpo/delta": -0.004581466782838106,
|
|
"fcm_dpo/margin": 33.99236297607422,
|
|
"fcm_dpo/q_t": 0.39209577441215515,
|
|
"grad_norm": 15.181863784790039,
|
|
"learning_rate": 2.5661032514931834e-07,
|
|
"logits/chosen": 0.5896913409233093,
|
|
"logits/rejected": 0.5458609461784363,
|
|
"logps/chosen": -131.3365478515625,
|
|
"logps/ref_chosen": -60.75285720825195,
|
|
"logps/ref_rejected": -75.21507263183594,
|
|
"logps/rejected": -179.79110717773438,
|
|
"loss": 1.1095,
|
|
"margin_dpo/margin_mean": 33.99236297607422,
|
|
"margin_dpo/margin_std": 56.61328887939453,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.5517762660619804,
|
|
"fcm_dpo/beta": 0.01466338336467743,
|
|
"fcm_dpo/delta": -0.07188267260789871,
|
|
"fcm_dpo/margin": 45.488773345947266,
|
|
"fcm_dpo/q_t": 0.3608033359050751,
|
|
"grad_norm": 13.636979103088379,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": 0.6325476765632629,
|
|
"logits/rejected": 0.5525530576705933,
|
|
"logps/chosen": -128.69789123535156,
|
|
"logps/ref_chosen": -58.56513595581055,
|
|
"logps/ref_rejected": -84.06403350830078,
|
|
"logps/rejected": -199.68553161621094,
|
|
"loss": 0.9882,
|
|
"margin_dpo/margin_mean": 45.488773345947266,
|
|
"margin_dpo/margin_std": 58.70988082885742,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.5593348450491308,
|
|
"fcm_dpo/beta": 0.014749327674508095,
|
|
"fcm_dpo/delta": 0.050389669835567474,
|
|
"fcm_dpo/margin": 37.40172576904297,
|
|
"fcm_dpo/q_t": 0.3838474154472351,
|
|
"grad_norm": 15.615440368652344,
|
|
"learning_rate": 2.4338967485068164e-07,
|
|
"logits/chosen": 0.598479151725769,
|
|
"logits/rejected": 0.5479581356048584,
|
|
"logps/chosen": -129.9058837890625,
|
|
"logps/ref_chosen": -59.443138122558594,
|
|
"logps/ref_rejected": -75.80937194824219,
|
|
"logps/rejected": -183.67385864257812,
|
|
"loss": 1.082,
|
|
"margin_dpo/margin_mean": 37.40172576904297,
|
|
"margin_dpo/margin_std": 59.58678436279297,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.5668934240362812,
|
|
"fcm_dpo/beta": 0.014894585125148296,
|
|
"fcm_dpo/delta": -0.010102972388267517,
|
|
"fcm_dpo/margin": 38.15407180786133,
|
|
"fcm_dpo/q_t": 0.3826829791069031,
|
|
"grad_norm": 17.60791015625,
|
|
"learning_rate": 2.3678397206786715e-07,
|
|
"logits/chosen": 0.641754150390625,
|
|
"logits/rejected": 0.5882676839828491,
|
|
"logps/chosen": -131.53549194335938,
|
|
"logps/ref_chosen": -58.59185028076172,
|
|
"logps/ref_rejected": -73.7529525756836,
|
|
"logps/rejected": -184.85067749023438,
|
|
"loss": 1.0957,
|
|
"margin_dpo/margin_mean": 38.15407180786133,
|
|
"margin_dpo/margin_std": 62.604209899902344,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.5744520030234316,
|
|
"fcm_dpo/beta": 0.014794014394283295,
|
|
"fcm_dpo/delta": -0.023977603763341904,
|
|
"fcm_dpo/margin": 42.04156494140625,
|
|
"fcm_dpo/q_t": 0.3720186650753021,
|
|
"grad_norm": 14.933080673217773,
|
|
"learning_rate": 2.3018751078580283e-07,
|
|
"logits/chosen": 0.6510765552520752,
|
|
"logits/rejected": 0.5984392166137695,
|
|
"logps/chosen": -130.0701904296875,
|
|
"logps/ref_chosen": -58.93424606323242,
|
|
"logps/ref_rejected": -76.27055358886719,
|
|
"logps/rejected": -189.4480438232422,
|
|
"loss": 1.0408,
|
|
"margin_dpo/margin_mean": 42.04156494140625,
|
|
"margin_dpo/margin_std": 61.29801559448242,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.582010582010582,
|
|
"fcm_dpo/beta": 0.014798015356063843,
|
|
"fcm_dpo/delta": 0.008013037964701653,
|
|
"fcm_dpo/margin": 36.892799377441406,
|
|
"fcm_dpo/q_t": 0.3861129879951477,
|
|
"grad_norm": 14.123330116271973,
|
|
"learning_rate": 2.2360490367648084e-07,
|
|
"logits/chosen": 0.6028137803077698,
|
|
"logits/rejected": 0.5752580761909485,
|
|
"logps/chosen": -141.14915466308594,
|
|
"logps/ref_chosen": -66.42684173583984,
|
|
"logps/ref_rejected": -76.96304321289062,
|
|
"logps/rejected": -188.5781707763672,
|
|
"loss": 1.0783,
|
|
"margin_dpo/margin_mean": 36.892799377441406,
|
|
"margin_dpo/margin_std": 57.62492752075195,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.5895691609977324,
|
|
"fcm_dpo/beta": 0.014910424128174782,
|
|
"fcm_dpo/delta": 0.034819699823856354,
|
|
"fcm_dpo/margin": 37.98401641845703,
|
|
"fcm_dpo/q_t": 0.3820473253726959,
|
|
"grad_norm": 14.307579040527344,
|
|
"learning_rate": 2.170407537241599e-07,
|
|
"logits/chosen": 0.6292208433151245,
|
|
"logits/rejected": 0.5711486339569092,
|
|
"logps/chosen": -135.46531677246094,
|
|
"logps/ref_chosen": -60.984214782714844,
|
|
"logps/ref_rejected": -79.54056549072266,
|
|
"logps/rejected": -192.00570678710938,
|
|
"loss": 1.0604,
|
|
"margin_dpo/margin_mean": 37.98401641845703,
|
|
"margin_dpo/margin_std": 57.268310546875,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.5971277399848829,
|
|
"fcm_dpo/beta": 0.014500841498374939,
|
|
"fcm_dpo/delta": -0.03668956086039543,
|
|
"fcm_dpo/margin": 43.64439010620117,
|
|
"fcm_dpo/q_t": 0.3682101368904114,
|
|
"grad_norm": 18.736286163330078,
|
|
"learning_rate": 2.104996510066625e-07,
|
|
"logits/chosen": 0.6559888124465942,
|
|
"logits/rejected": 0.5947400331497192,
|
|
"logps/chosen": -128.33447265625,
|
|
"logps/ref_chosen": -58.30937957763672,
|
|
"logps/ref_rejected": -80.09587097167969,
|
|
"logps/rejected": -193.765380859375,
|
|
"loss": 1.0149,
|
|
"margin_dpo/margin_mean": 43.64439010620117,
|
|
"margin_dpo/margin_std": 59.09160232543945,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.6046863189720333,
|
|
"fcm_dpo/beta": 0.014643101021647453,
|
|
"fcm_dpo/delta": 0.00806088000535965,
|
|
"fcm_dpo/margin": 40.27904510498047,
|
|
"fcm_dpo/q_t": 0.37856990098953247,
|
|
"grad_norm": 23.970535278320312,
|
|
"learning_rate": 2.0398616948569493e-07,
|
|
"logits/chosen": 0.6093511581420898,
|
|
"logits/rejected": 0.5225298404693604,
|
|
"logps/chosen": -140.3079833984375,
|
|
"logps/ref_chosen": -61.39867401123047,
|
|
"logps/ref_rejected": -89.0177993774414,
|
|
"logps/rejected": -208.20614624023438,
|
|
"loss": 1.058,
|
|
"margin_dpo/margin_mean": 40.27904510498047,
|
|
"margin_dpo/margin_std": 59.58887481689453,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.6046863189720333,
|
|
"eval_fcm_dpo/beta": 0.015623129904270172,
|
|
"eval_fcm_dpo/delta": 0.00515873683616519,
|
|
"eval_fcm_dpo/margin": 37.30812072753906,
|
|
"eval_fcm_dpo/q_t": 0.3823583722114563,
|
|
"eval_logits/chosen": 0.6411985158920288,
|
|
"eval_logits/rejected": 0.5883597731590271,
|
|
"eval_logps/chosen": -151.68788146972656,
|
|
"eval_logps/ref_chosen": -74.85946655273438,
|
|
"eval_logps/ref_rejected": -79.54898834228516,
|
|
"eval_logps/rejected": -193.68551635742188,
|
|
"eval_loss": 0.5441513061523438,
|
|
"eval_margin_dpo/margin_mean": 37.30812454223633,
|
|
"eval_margin_dpo/margin_std": 61.142723083496094,
|
|
"eval_runtime": 38.3592,
|
|
"eval_samples_per_second": 60.038,
|
|
"eval_steps_per_second": 1.877,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.6122448979591837,
|
|
"fcm_dpo/beta": 0.014417248778045177,
|
|
"fcm_dpo/delta": -0.06634485721588135,
|
|
"fcm_dpo/margin": 45.74829864501953,
|
|
"fcm_dpo/q_t": 0.365024596452713,
|
|
"grad_norm": 14.021449089050293,
|
|
"learning_rate": 1.975048638084379e-07,
|
|
"logits/chosen": 0.6662113070487976,
|
|
"logits/rejected": 0.5925288796424866,
|
|
"logps/chosen": -130.4202880859375,
|
|
"logps/ref_chosen": -55.953521728515625,
|
|
"logps/ref_rejected": -77.67539978027344,
|
|
"logps/rejected": -197.8904571533203,
|
|
"loss": 1.0027,
|
|
"margin_dpo/margin_mean": 45.74829864501953,
|
|
"margin_dpo/margin_std": 60.512489318847656,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.6198034769463341,
|
|
"fcm_dpo/beta": 0.013882627710700035,
|
|
"fcm_dpo/delta": -0.022442538291215897,
|
|
"fcm_dpo/margin": 44.60148239135742,
|
|
"fcm_dpo/q_t": 0.36986368894577026,
|
|
"grad_norm": 13.81552505493164,
|
|
"learning_rate": 1.9106026612264315e-07,
|
|
"logits/chosen": 0.6225928068161011,
|
|
"logits/rejected": 0.5533093214035034,
|
|
"logps/chosen": -144.36842346191406,
|
|
"logps/ref_chosen": -63.40419387817383,
|
|
"logps/ref_rejected": -80.85710144042969,
|
|
"logps/rejected": -206.4228057861328,
|
|
"loss": 1.016,
|
|
"margin_dpo/margin_mean": 44.601478576660156,
|
|
"margin_dpo/margin_std": 59.1302375793457,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.6273620559334845,
|
|
"fcm_dpo/beta": 0.013873199000954628,
|
|
"fcm_dpo/delta": 0.0005700096371583641,
|
|
"fcm_dpo/margin": 43.196556091308594,
|
|
"fcm_dpo/q_t": 0.37501293420791626,
|
|
"grad_norm": 14.810785293579102,
|
|
"learning_rate": 1.846568829074628e-07,
|
|
"logits/chosen": 0.6255594491958618,
|
|
"logits/rejected": 0.5735424757003784,
|
|
"logps/chosen": -148.52664184570312,
|
|
"logps/ref_chosen": -57.6942024230957,
|
|
"logps/ref_rejected": -71.74036407470703,
|
|
"logps/rejected": -205.7693634033203,
|
|
"loss": 1.0421,
|
|
"margin_dpo/margin_mean": 43.19655227661133,
|
|
"margin_dpo/margin_std": 62.83301544189453,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.6349206349206349,
|
|
"fcm_dpo/beta": 0.013408740982413292,
|
|
"fcm_dpo/delta": -0.017916694283485413,
|
|
"fcm_dpo/margin": 39.38436508178711,
|
|
"fcm_dpo/q_t": 0.3907091021537781,
|
|
"grad_norm": 16.092344284057617,
|
|
"learning_rate": 1.782991918222275e-07,
|
|
"logits/chosen": 0.6533733606338501,
|
|
"logits/rejected": 0.6073211431503296,
|
|
"logps/chosen": -156.04934692382812,
|
|
"logps/ref_chosen": -59.169517517089844,
|
|
"logps/ref_rejected": -69.47721099853516,
|
|
"logps/rejected": -205.7414093017578,
|
|
"loss": 1.1125,
|
|
"margin_dpo/margin_mean": 39.38436508178711,
|
|
"margin_dpo/margin_std": 66.23831176757812,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.6424792139077853,
|
|
"fcm_dpo/beta": 0.013325588777661324,
|
|
"fcm_dpo/delta": -0.042791374027729034,
|
|
"fcm_dpo/margin": 47.987789154052734,
|
|
"fcm_dpo/q_t": 0.36810848116874695,
|
|
"grad_norm": 16.099363327026367,
|
|
"learning_rate": 1.7199163857537824e-07,
|
|
"logits/chosen": 0.639710545539856,
|
|
"logits/rejected": 0.5940824747085571,
|
|
"logps/chosen": -149.20211791992188,
|
|
"logps/ref_chosen": -58.09320831298828,
|
|
"logps/ref_rejected": -73.98226165771484,
|
|
"logps/rejected": -213.0789794921875,
|
|
"loss": 1.0223,
|
|
"margin_dpo/margin_mean": 47.987789154052734,
|
|
"margin_dpo/margin_std": 66.80766296386719,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.6500377928949358,
|
|
"fcm_dpo/beta": 0.013703095726668835,
|
|
"fcm_dpo/delta": 0.015814904123544693,
|
|
"fcm_dpo/margin": 42.556365966796875,
|
|
"fcm_dpo/q_t": 0.3806520104408264,
|
|
"grad_norm": 15.242733001708984,
|
|
"learning_rate": 1.6573863381573954e-07,
|
|
"logits/chosen": 0.5981285572052002,
|
|
"logits/rejected": 0.5805391073226929,
|
|
"logps/chosen": -165.95809936523438,
|
|
"logps/ref_chosen": -62.7039909362793,
|
|
"logps/ref_rejected": -74.52284240722656,
|
|
"logps/rejected": -220.3333282470703,
|
|
"loss": 1.0933,
|
|
"margin_dpo/margin_mean": 42.556365966796875,
|
|
"margin_dpo/margin_std": 70.32002258300781,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.6575963718820862,
|
|
"fcm_dpo/beta": 0.013115659356117249,
|
|
"fcm_dpo/delta": -0.03992351144552231,
|
|
"fcm_dpo/margin": 48.57158660888672,
|
|
"fcm_dpo/q_t": 0.3666061460971832,
|
|
"grad_norm": 17.644805908203125,
|
|
"learning_rate": 1.5954455004830878e-07,
|
|
"logits/chosen": 0.7035324573516846,
|
|
"logits/rejected": 0.6597825884819031,
|
|
"logps/chosen": -146.27212524414062,
|
|
"logps/ref_chosen": -56.12516403198242,
|
|
"logps/ref_rejected": -74.36073303222656,
|
|
"logps/rejected": -213.0792999267578,
|
|
"loss": 1.025,
|
|
"margin_dpo/margin_mean": 48.57158660888672,
|
|
"margin_dpo/margin_std": 67.80796813964844,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.6651549508692366,
|
|
"fcm_dpo/beta": 0.012779777869582176,
|
|
"fcm_dpo/delta": -0.02733975648880005,
|
|
"fcm_dpo/margin": 48.86384201049805,
|
|
"fcm_dpo/q_t": 0.3719489872455597,
|
|
"grad_norm": 22.34282684326172,
|
|
"learning_rate": 1.534137185767178e-07,
|
|
"logits/chosen": 0.6284548044204712,
|
|
"logits/rejected": 0.548454225063324,
|
|
"logps/chosen": -148.30421447753906,
|
|
"logps/ref_chosen": -55.67548751831055,
|
|
"logps/ref_rejected": -76.62055206298828,
|
|
"logps/rejected": -218.11312866210938,
|
|
"loss": 1.0416,
|
|
"margin_dpo/margin_mean": 48.86384201049805,
|
|
"margin_dpo/margin_std": 70.98736572265625,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.672713529856387,
|
|
"fcm_dpo/beta": 0.011895755305886269,
|
|
"fcm_dpo/delta": -0.0644448846578598,
|
|
"fcm_dpo/margin": 51.74351119995117,
|
|
"fcm_dpo/q_t": 0.3690551221370697,
|
|
"grad_norm": 16.176162719726562,
|
|
"learning_rate": 1.473504264745062e-07,
|
|
"logits/chosen": 0.6225286722183228,
|
|
"logits/rejected": 0.5559119582176208,
|
|
"logps/chosen": -155.0991973876953,
|
|
"logps/ref_chosen": -59.903411865234375,
|
|
"logps/ref_rejected": -82.02873229980469,
|
|
"logps/rejected": -228.96804809570312,
|
|
"loss": 1.0151,
|
|
"margin_dpo/margin_mean": 51.74351119995117,
|
|
"margin_dpo/margin_std": 68.27033996582031,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.6802721088435374,
|
|
"fcm_dpo/beta": 0.011266094632446766,
|
|
"fcm_dpo/delta": -0.007983550429344177,
|
|
"fcm_dpo/margin": 50.21624755859375,
|
|
"fcm_dpo/q_t": 0.3788827955722809,
|
|
"grad_norm": 16.09433937072754,
|
|
"learning_rate": 1.4135891358732205e-07,
|
|
"logits/chosen": 0.643526554107666,
|
|
"logits/rejected": 0.5692092776298523,
|
|
"logps/chosen": -150.8534698486328,
|
|
"logps/ref_chosen": -55.83526611328125,
|
|
"logps/ref_rejected": -79.63658142089844,
|
|
"logps/rejected": -224.8710479736328,
|
|
"loss": 1.0399,
|
|
"margin_dpo/margin_mean": 50.21624755859375,
|
|
"margin_dpo/margin_std": 67.95020294189453,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.6878306878306878,
|
|
"fcm_dpo/beta": 0.01194181852042675,
|
|
"fcm_dpo/delta": 0.029494991526007652,
|
|
"fcm_dpo/margin": 47.85871505737305,
|
|
"fcm_dpo/q_t": 0.3804067373275757,
|
|
"grad_norm": 14.716300964355469,
|
|
"learning_rate": 1.354433695681474e-07,
|
|
"logits/chosen": 0.586762547492981,
|
|
"logits/rejected": 0.5374394655227661,
|
|
"logps/chosen": -152.26715087890625,
|
|
"logps/ref_chosen": -60.59226608276367,
|
|
"logps/ref_rejected": -73.37936401367188,
|
|
"logps/rejected": -212.9129638671875,
|
|
"loss": 1.0577,
|
|
"margin_dpo/margin_mean": 47.85871505737305,
|
|
"margin_dpo/margin_std": 71.9669189453125,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.6953892668178382,
|
|
"fcm_dpo/beta": 0.011895488947629929,
|
|
"fcm_dpo/delta": -0.04346683993935585,
|
|
"fcm_dpo/margin": 53.85084915161133,
|
|
"fcm_dpo/q_t": 0.36482182145118713,
|
|
"grad_norm": 14.025132179260254,
|
|
"learning_rate": 1.2960793094762345e-07,
|
|
"logits/chosen": 0.6461644172668457,
|
|
"logits/rejected": 0.5591436624526978,
|
|
"logps/chosen": -149.276611328125,
|
|
"logps/ref_chosen": -56.21283721923828,
|
|
"logps/ref_rejected": -83.02075958251953,
|
|
"logps/rejected": -229.93539428710938,
|
|
"loss": 0.9967,
|
|
"margin_dpo/margin_mean": 53.85084915161133,
|
|
"margin_dpo/margin_std": 68.7022476196289,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.7029478458049887,
|
|
"fcm_dpo/beta": 0.01112604420632124,
|
|
"fcm_dpo/delta": -0.010392392054200172,
|
|
"fcm_dpo/margin": 54.66693115234375,
|
|
"fcm_dpo/q_t": 0.3700573742389679,
|
|
"grad_norm": 17.04851722717285,
|
|
"learning_rate": 1.238566782415197e-07,
|
|
"logits/chosen": 0.6725102066993713,
|
|
"logits/rejected": 0.6229192614555359,
|
|
"logps/chosen": -153.4409942626953,
|
|
"logps/ref_chosen": -59.0674934387207,
|
|
"logps/ref_rejected": -74.53498840332031,
|
|
"logps/rejected": -223.5753936767578,
|
|
"loss": 1.0133,
|
|
"margin_dpo/margin_mean": 54.66693115234375,
|
|
"margin_dpo/margin_std": 70.4762191772461,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.7105064247921391,
|
|
"fcm_dpo/beta": 0.011645983904600143,
|
|
"fcm_dpo/delta": 0.009741531684994698,
|
|
"fcm_dpo/margin": 50.60006332397461,
|
|
"fcm_dpo/q_t": 0.37510764598846436,
|
|
"grad_norm": 14.018816947937012,
|
|
"learning_rate": 1.1819363309737438e-07,
|
|
"logits/chosen": 0.6600295305252075,
|
|
"logits/rejected": 0.6102507710456848,
|
|
"logps/chosen": -155.27413940429688,
|
|
"logps/ref_chosen": -58.3397331237793,
|
|
"logps/ref_rejected": -74.33660125732422,
|
|
"logps/rejected": -221.87109375,
|
|
"loss": 1.0271,
|
|
"margin_dpo/margin_mean": 50.600067138671875,
|
|
"margin_dpo/margin_std": 68.35984802246094,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.7180650037792895,
|
|
"fcm_dpo/beta": 0.01119951345026493,
|
|
"fcm_dpo/delta": -0.040131840854883194,
|
|
"fcm_dpo/margin": 56.87284469604492,
|
|
"fcm_dpo/q_t": 0.3679283559322357,
|
|
"grad_norm": 11.378620147705078,
|
|
"learning_rate": 1.126227554822985e-07,
|
|
"logits/chosen": 0.66913902759552,
|
|
"logits/rejected": 0.5974291563034058,
|
|
"logps/chosen": -144.87710571289062,
|
|
"logps/ref_chosen": -54.60407638549805,
|
|
"logps/ref_rejected": -79.94635009765625,
|
|
"logps/rejected": -227.0922088623047,
|
|
"loss": 1.0247,
|
|
"margin_dpo/margin_mean": 56.87284469604492,
|
|
"margin_dpo/margin_std": 79.54670715332031,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.7256235827664399,
|
|
"fcm_dpo/beta": 0.011514843441545963,
|
|
"fcm_dpo/delta": 0.07255177199840546,
|
|
"fcm_dpo/margin": 46.03236389160156,
|
|
"fcm_dpo/q_t": 0.3892512619495392,
|
|
"grad_norm": 18.64552879333496,
|
|
"learning_rate": 1.0714794091391072e-07,
|
|
"logits/chosen": 0.6036539077758789,
|
|
"logits/rejected": 0.582791268825531,
|
|
"logps/chosen": -167.0009002685547,
|
|
"logps/ref_chosen": -63.0672492980957,
|
|
"logps/ref_rejected": -68.59602355957031,
|
|
"logps/rejected": -218.5620574951172,
|
|
"loss": 1.1136,
|
|
"margin_dpo/margin_mean": 46.0323600769043,
|
|
"margin_dpo/margin_std": 79.20867919921875,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.7331821617535903,
|
|
"fcm_dpo/beta": 0.011711874976754189,
|
|
"fcm_dpo/delta": 0.006459618918597698,
|
|
"fcm_dpo/margin": 50.647342681884766,
|
|
"fcm_dpo/q_t": 0.37608593702316284,
|
|
"grad_norm": 16.108354568481445,
|
|
"learning_rate": 1.0177301773633992e-07,
|
|
"logits/chosen": 0.6584078073501587,
|
|
"logits/rejected": 0.5911266207695007,
|
|
"logps/chosen": -159.75131225585938,
|
|
"logps/ref_chosen": -58.75799560546875,
|
|
"logps/ref_rejected": -79.72233581542969,
|
|
"logps/rejected": -231.3629608154297,
|
|
"loss": 1.0454,
|
|
"margin_dpo/margin_mean": 50.647342681884766,
|
|
"margin_dpo/margin_std": 73.55042266845703,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.7407407407407407,
|
|
"fcm_dpo/beta": 0.012160698883235455,
|
|
"fcm_dpo/delta": 0.057119082659482956,
|
|
"fcm_dpo/margin": 44.87133026123047,
|
|
"fcm_dpo/q_t": 0.3892955183982849,
|
|
"grad_norm": 16.322307586669922,
|
|
"learning_rate": 9.650174444319956e-08,
|
|
"logits/chosen": 0.685616135597229,
|
|
"logits/rejected": 0.6174665689468384,
|
|
"logps/chosen": -169.23971557617188,
|
|
"logps/ref_chosen": -61.394195556640625,
|
|
"logps/ref_rejected": -81.1914291381836,
|
|
"logps/rejected": -233.9082489013672,
|
|
"loss": 1.1167,
|
|
"margin_dpo/margin_mean": 44.871334075927734,
|
|
"margin_dpo/margin_std": 78.93081665039062,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.7482993197278912,
|
|
"fcm_dpo/beta": 0.012708587571978569,
|
|
"fcm_dpo/delta": 0.024855732917785645,
|
|
"fcm_dpo/margin": 45.255348205566406,
|
|
"fcm_dpo/q_t": 0.3839653432369232,
|
|
"grad_norm": 21.597793579101562,
|
|
"learning_rate": 9.133780704940594e-08,
|
|
"logits/chosen": 0.6221252679824829,
|
|
"logits/rejected": 0.5517234206199646,
|
|
"logps/chosen": -163.41339111328125,
|
|
"logps/ref_chosen": -59.85382843017578,
|
|
"logps/ref_rejected": -80.63748931884766,
|
|
"logps/rejected": -229.452392578125,
|
|
"loss": 1.1054,
|
|
"margin_dpo/margin_mean": 45.255348205566406,
|
|
"margin_dpo/margin_std": 77.06855773925781,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.7558578987150416,
|
|
"fcm_dpo/beta": 0.012341910973191261,
|
|
"fcm_dpo/delta": -0.06186755746603012,
|
|
"fcm_dpo/margin": 53.241127014160156,
|
|
"fcm_dpo/q_t": 0.3657708168029785,
|
|
"grad_norm": 20.970151901245117,
|
|
"learning_rate": 8.628481651367875e-08,
|
|
"logits/chosen": 0.6411687135696411,
|
|
"logits/rejected": 0.5881538391113281,
|
|
"logps/chosen": -168.67385864257812,
|
|
"logps/ref_chosen": -66.17753601074219,
|
|
"logps/ref_rejected": -83.75955200195312,
|
|
"logps/rejected": -239.49697875976562,
|
|
"loss": 1.0368,
|
|
"margin_dpo/margin_mean": 53.241127014160156,
|
|
"margin_dpo/margin_std": 78.86491394042969,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.763416477702192,
|
|
"fcm_dpo/beta": 0.012122569605708122,
|
|
"fcm_dpo/delta": -0.005300647113472223,
|
|
"fcm_dpo/margin": 49.808876037597656,
|
|
"fcm_dpo/q_t": 0.3728008270263672,
|
|
"grad_norm": 17.493913650512695,
|
|
"learning_rate": 8.134630621352483e-08,
|
|
"logits/chosen": 0.629217267036438,
|
|
"logits/rejected": 0.6063399314880371,
|
|
"logps/chosen": -159.32550048828125,
|
|
"logps/ref_chosen": -62.11005401611328,
|
|
"logps/ref_rejected": -74.64705657958984,
|
|
"logps/rejected": -221.67138671875,
|
|
"loss": 1.0325,
|
|
"margin_dpo/margin_mean": 49.80887222290039,
|
|
"margin_dpo/margin_std": 68.94776153564453,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.7709750566893424,
|
|
"fcm_dpo/beta": 0.012009757570922375,
|
|
"fcm_dpo/delta": -0.01045228075236082,
|
|
"fcm_dpo/margin": 50.718772888183594,
|
|
"fcm_dpo/q_t": 0.3756389617919922,
|
|
"grad_norm": 16.645915985107422,
|
|
"learning_rate": 7.652572947447272e-08,
|
|
"logits/chosen": 0.6274048686027527,
|
|
"logits/rejected": 0.5604012608528137,
|
|
"logps/chosen": -164.4465789794922,
|
|
"logps/ref_chosen": -64.42265319824219,
|
|
"logps/ref_rejected": -87.00096130371094,
|
|
"logps/rejected": -237.74365234375,
|
|
"loss": 1.0613,
|
|
"margin_dpo/margin_mean": 50.718772888183594,
|
|
"margin_dpo/margin_std": 78.1541976928711,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.7785336356764928,
|
|
"fcm_dpo/beta": 0.011663762852549553,
|
|
"fcm_dpo/delta": -0.026412179693579674,
|
|
"fcm_dpo/margin": 53.50532150268555,
|
|
"fcm_dpo/q_t": 0.3693719506263733,
|
|
"grad_norm": 17.784894943237305,
|
|
"learning_rate": 7.182645715528435e-08,
|
|
"logits/chosen": 0.6600545644760132,
|
|
"logits/rejected": 0.6091629862785339,
|
|
"logps/chosen": -156.29165649414062,
|
|
"logps/ref_chosen": -58.284393310546875,
|
|
"logps/ref_rejected": -79.09356689453125,
|
|
"logps/rejected": -230.6061553955078,
|
|
"loss": 1.0248,
|
|
"margin_dpo/margin_mean": 53.50532150268555,
|
|
"margin_dpo/margin_std": 74.59442901611328,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.7860922146636432,
|
|
"fcm_dpo/beta": 0.01241991762071848,
|
|
"fcm_dpo/delta": 0.12249796092510223,
|
|
"fcm_dpo/margin": 38.903263092041016,
|
|
"fcm_dpo/q_t": 0.40102332830429077,
|
|
"grad_norm": 20.327253341674805,
|
|
"learning_rate": 6.725177529083209e-08,
|
|
"logits/chosen": 0.6678498983383179,
|
|
"logits/rejected": 0.623926043510437,
|
|
"logps/chosen": -163.60305786132812,
|
|
"logps/ref_chosen": -61.03638458251953,
|
|
"logps/ref_rejected": -72.15824890136719,
|
|
"logps/rejected": -213.628173828125,
|
|
"loss": 1.1589,
|
|
"margin_dpo/margin_mean": 38.903263092041016,
|
|
"margin_dpo/margin_std": 75.42561340332031,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.7936507936507936,
|
|
"fcm_dpo/beta": 0.012936798855662346,
|
|
"fcm_dpo/delta": 0.010262085124850273,
|
|
"fcm_dpo/margin": 45.59959411621094,
|
|
"fcm_dpo/q_t": 0.37764960527420044,
|
|
"grad_norm": 21.973684310913086,
|
|
"learning_rate": 6.280488279429185e-08,
|
|
"logits/chosen": 0.5573867559432983,
|
|
"logits/rejected": 0.5065245032310486,
|
|
"logps/chosen": -167.7998046875,
|
|
"logps/ref_chosen": -68.02732849121094,
|
|
"logps/ref_rejected": -85.41429901123047,
|
|
"logps/rejected": -230.786376953125,
|
|
"loss": 1.0859,
|
|
"margin_dpo/margin_mean": 45.59960174560547,
|
|
"margin_dpo/margin_std": 74.41303253173828,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.8012093726379441,
|
|
"fcm_dpo/beta": 0.013767753727734089,
|
|
"fcm_dpo/delta": 0.014420712366700172,
|
|
"fcm_dpo/margin": 42.34447479248047,
|
|
"fcm_dpo/q_t": 0.3815748989582062,
|
|
"grad_norm": 23.141691207885742,
|
|
"learning_rate": 5.848888922025552e-08,
|
|
"logits/chosen": 0.6335071325302124,
|
|
"logits/rejected": 0.5626540184020996,
|
|
"logps/chosen": -162.35891723632812,
|
|
"logps/ref_chosen": -58.67436599731445,
|
|
"logps/ref_rejected": -79.38807678222656,
|
|
"logps/rejected": -225.41708374023438,
|
|
"loss": 1.111,
|
|
"margin_dpo/margin_mean": 42.34447479248047,
|
|
"margin_dpo/margin_std": 71.12373352050781,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.8087679516250945,
|
|
"fcm_dpo/beta": 0.013212913647294044,
|
|
"fcm_dpo/delta": -0.020001692697405815,
|
|
"fcm_dpo/margin": 46.78357696533203,
|
|
"fcm_dpo/q_t": 0.3747491240501404,
|
|
"grad_norm": 20.625534057617188,
|
|
"learning_rate": 5.430681259032957e-08,
|
|
"logits/chosen": 0.624588131904602,
|
|
"logits/rejected": 0.5618466138839722,
|
|
"logps/chosen": -157.4933319091797,
|
|
"logps/ref_chosen": -57.640098571777344,
|
|
"logps/ref_rejected": -77.25399780273438,
|
|
"logps/rejected": -223.8908233642578,
|
|
"loss": 1.0721,
|
|
"margin_dpo/margin_mean": 46.78357696533203,
|
|
"margin_dpo/margin_std": 74.61866760253906,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.8163265306122449,
|
|
"fcm_dpo/beta": 0.012540744617581367,
|
|
"fcm_dpo/delta": -0.0733686089515686,
|
|
"fcm_dpo/margin": 53.17271041870117,
|
|
"fcm_dpo/q_t": 0.3620644211769104,
|
|
"grad_norm": 17.126371383666992,
|
|
"learning_rate": 5.026157728273966e-08,
|
|
"logits/chosen": 0.6427196860313416,
|
|
"logits/rejected": 0.5600071549415588,
|
|
"logps/chosen": -163.82118225097656,
|
|
"logps/ref_chosen": -60.17341995239258,
|
|
"logps/ref_rejected": -85.50316619873047,
|
|
"logps/rejected": -242.32363891601562,
|
|
"loss": 0.989,
|
|
"margin_dpo/margin_mean": 53.17271041870117,
|
|
"margin_dpo/margin_std": 68.73289489746094,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.8238851095993953,
|
|
"fcm_dpo/beta": 0.011654900386929512,
|
|
"fcm_dpo/delta": -0.05292552709579468,
|
|
"fcm_dpo/margin": 55.58978271484375,
|
|
"fcm_dpo/q_t": 0.3632128834724426,
|
|
"grad_norm": 15.992572784423828,
|
|
"learning_rate": 4.635601198741607e-08,
|
|
"logits/chosen": 0.6579868793487549,
|
|
"logits/rejected": 0.6033874750137329,
|
|
"logps/chosen": -153.53955078125,
|
|
"logps/ref_chosen": -56.985809326171875,
|
|
"logps/ref_rejected": -73.21353912353516,
|
|
"logps/rejected": -225.3570556640625,
|
|
"loss": 1.0015,
|
|
"margin_dpo/margin_mean": 55.58978271484375,
|
|
"margin_dpo/margin_std": 72.88048553466797,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.8314436885865457,
|
|
"fcm_dpo/beta": 0.011842542327940464,
|
|
"fcm_dpo/delta": 0.04592089354991913,
|
|
"fcm_dpo/margin": 43.93280792236328,
|
|
"fcm_dpo/q_t": 0.39010632038116455,
|
|
"grad_norm": 18.3736629486084,
|
|
"learning_rate": 4.259284772799099e-08,
|
|
"logits/chosen": 0.6481934189796448,
|
|
"logits/rejected": 0.5963156223297119,
|
|
"logps/chosen": -161.9949188232422,
|
|
"logps/ref_chosen": -59.600929260253906,
|
|
"logps/ref_rejected": -75.24870300292969,
|
|
"logps/rejected": -221.5755157470703,
|
|
"loss": 1.1012,
|
|
"margin_dpo/margin_mean": 43.93280792236328,
|
|
"margin_dpo/margin_std": 72.24934387207031,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.8390022675736961,
|
|
"fcm_dpo/beta": 0.012416016310453415,
|
|
"fcm_dpo/delta": 0.019648931920528412,
|
|
"fcm_dpo/margin": 46.74137496948242,
|
|
"fcm_dpo/q_t": 0.3777768015861511,
|
|
"grad_norm": 18.58342170715332,
|
|
"learning_rate": 3.89747159520904e-08,
|
|
"logits/chosen": 0.62873375415802,
|
|
"logits/rejected": 0.576077401638031,
|
|
"logps/chosen": -169.8580322265625,
|
|
"logps/ref_chosen": -63.578895568847656,
|
|
"logps/ref_rejected": -78.87867736816406,
|
|
"logps/rejected": -231.899169921875,
|
|
"loss": 1.0686,
|
|
"margin_dpo/margin_mean": 46.74137496948242,
|
|
"margin_dpo/margin_std": 72.33360290527344,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.8465608465608465,
|
|
"fcm_dpo/beta": 0.012055947445333004,
|
|
"fcm_dpo/delta": -0.05154518038034439,
|
|
"fcm_dpo/margin": 53.696868896484375,
|
|
"fcm_dpo/q_t": 0.3679019510746002,
|
|
"grad_norm": 16.838607788085938,
|
|
"learning_rate": 3.550414669125573e-08,
|
|
"logits/chosen": 0.6742048263549805,
|
|
"logits/rejected": 0.6106564402580261,
|
|
"logps/chosen": -165.71080017089844,
|
|
"logps/ref_chosen": -58.651512145996094,
|
|
"logps/ref_rejected": -78.67181396484375,
|
|
"logps/rejected": -239.427978515625,
|
|
"loss": 1.0164,
|
|
"margin_dpo/margin_mean": 53.696868896484375,
|
|
"margin_dpo/margin_std": 73.68684387207031,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.854119425547997,
|
|
"fcm_dpo/beta": 0.011463714763522148,
|
|
"fcm_dpo/delta": -0.03745627775788307,
|
|
"fcm_dpo/margin": 55.323936462402344,
|
|
"fcm_dpo/q_t": 0.3676683008670807,
|
|
"grad_norm": 15.158285140991211,
|
|
"learning_rate": 3.218356679178252e-08,
|
|
"logits/chosen": 0.6555038690567017,
|
|
"logits/rejected": 0.6098966002464294,
|
|
"logps/chosen": -162.03512573242188,
|
|
"logps/ref_chosen": -60.3114128112793,
|
|
"logps/ref_rejected": -78.25270080566406,
|
|
"logps/rejected": -235.3003692626953,
|
|
"loss": 1.0158,
|
|
"margin_dpo/margin_mean": 55.323936462402344,
|
|
"margin_dpo/margin_std": 75.325927734375,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.8616780045351474,
|
|
"fcm_dpo/beta": 0.011363612487912178,
|
|
"fcm_dpo/delta": -0.02258976735174656,
|
|
"fcm_dpo/margin": 54.51459503173828,
|
|
"fcm_dpo/q_t": 0.3701397478580475,
|
|
"grad_norm": 13.35988998413086,
|
|
"learning_rate": 2.9015298217712453e-08,
|
|
"logits/chosen": 0.635320782661438,
|
|
"logits/rejected": 0.5687986612319946,
|
|
"logps/chosen": -158.89346313476562,
|
|
"logps/ref_chosen": -57.752410888671875,
|
|
"logps/ref_rejected": -76.99858093261719,
|
|
"logps/rejected": -232.6542205810547,
|
|
"loss": 1.0243,
|
|
"margin_dpo/margin_mean": 54.51459884643555,
|
|
"margin_dpo/margin_std": 73.74528503417969,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.8692365835222978,
|
|
"fcm_dpo/beta": 0.01154360268265009,
|
|
"fcm_dpo/delta": 0.05699888616800308,
|
|
"fcm_dpo/margin": 47.14950180053711,
|
|
"fcm_dpo/q_t": 0.3850955665111542,
|
|
"grad_norm": 16.961835861206055,
|
|
"learning_rate": 2.600155642716606e-08,
|
|
"logits/chosen": 0.6573041677474976,
|
|
"logits/rejected": 0.6165328025817871,
|
|
"logps/chosen": -169.44454956054688,
|
|
"logps/ref_chosen": -63.61958694458008,
|
|
"logps/ref_rejected": -79.51353454589844,
|
|
"logps/rejected": -232.48800659179688,
|
|
"loss": 1.0768,
|
|
"margin_dpo/margin_mean": 47.14950180053711,
|
|
"margin_dpo/margin_std": 72.31434631347656,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.8767951625094482,
|
|
"fcm_dpo/beta": 0.011755215004086494,
|
|
"fcm_dpo/delta": 0.009137720800936222,
|
|
"fcm_dpo/margin": 50.234230041503906,
|
|
"fcm_dpo/q_t": 0.3770310878753662,
|
|
"grad_norm": 17.833290100097656,
|
|
"learning_rate": 2.3144448823151392e-08,
|
|
"logits/chosen": 0.6418476700782776,
|
|
"logits/rejected": 0.592076301574707,
|
|
"logps/chosen": -157.65115356445312,
|
|
"logps/ref_chosen": -57.3541145324707,
|
|
"logps/ref_rejected": -73.14434051513672,
|
|
"logps/rejected": -223.6756134033203,
|
|
"loss": 1.056,
|
|
"margin_dpo/margin_mean": 50.234230041503906,
|
|
"margin_dpo/margin_std": 74.81739807128906,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.8843537414965986,
|
|
"fcm_dpo/beta": 0.011965381912887096,
|
|
"fcm_dpo/delta": 0.03466043993830681,
|
|
"fcm_dpo/margin": 47.38968276977539,
|
|
"fcm_dpo/q_t": 0.3831913471221924,
|
|
"grad_norm": 16.188016891479492,
|
|
"learning_rate": 2.044597327993153e-08,
|
|
"logits/chosen": 0.692867636680603,
|
|
"logits/rejected": 0.6268293261528015,
|
|
"logps/chosen": -160.28024291992188,
|
|
"logps/ref_chosen": -56.0127067565918,
|
|
"logps/ref_rejected": -77.16522216796875,
|
|
"logps/rejected": -228.8224334716797,
|
|
"loss": 1.0892,
|
|
"margin_dpo/margin_mean": 47.38968276977539,
|
|
"margin_dpo/margin_std": 77.33354187011719,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.891912320483749,
|
|
"fcm_dpo/beta": 0.01196371577680111,
|
|
"fcm_dpo/delta": -0.05104018375277519,
|
|
"fcm_dpo/margin": 54.1452751159668,
|
|
"fcm_dpo/q_t": 0.36301979422569275,
|
|
"grad_norm": 15.689112663269043,
|
|
"learning_rate": 1.7908016745981856e-08,
|
|
"logits/chosen": 0.6593881249427795,
|
|
"logits/rejected": 0.6252022981643677,
|
|
"logps/chosen": -164.38351440429688,
|
|
"logps/ref_chosen": -60.5894660949707,
|
|
"logps/ref_rejected": -74.34771728515625,
|
|
"logps/rejected": -232.2870635986328,
|
|
"loss": 1.0103,
|
|
"margin_dpo/margin_mean": 54.1452751159668,
|
|
"margin_dpo/margin_std": 73.66951751708984,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.8994708994708994,
|
|
"fcm_dpo/beta": 0.011334089562296867,
|
|
"fcm_dpo/delta": -0.034378211945295334,
|
|
"fcm_dpo/margin": 55.64763259887695,
|
|
"fcm_dpo/q_t": 0.3687277138233185,
|
|
"grad_norm": 15.465320587158203,
|
|
"learning_rate": 1.553235392451377e-08,
|
|
"logits/chosen": 0.6757582426071167,
|
|
"logits/rejected": 0.5952944159507751,
|
|
"logps/chosen": -151.85220336914062,
|
|
"logps/ref_chosen": -54.77838897705078,
|
|
"logps/ref_rejected": -78.102783203125,
|
|
"logps/rejected": -230.82424926757812,
|
|
"loss": 1.0279,
|
|
"margin_dpo/margin_mean": 55.64763259887695,
|
|
"margin_dpo/margin_std": 77.98216247558594,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.9070294784580499,
|
|
"fcm_dpo/beta": 0.012342174537479877,
|
|
"fcm_dpo/delta": 0.13277825713157654,
|
|
"fcm_dpo/margin": 38.14574432373047,
|
|
"fcm_dpo/q_t": 0.4049660563468933,
|
|
"grad_norm": 20.803640365600586,
|
|
"learning_rate": 1.3320646032487393e-08,
|
|
"logits/chosen": 0.668152928352356,
|
|
"logits/rejected": 0.6297565698623657,
|
|
"logps/chosen": -166.8023681640625,
|
|
"logps/ref_chosen": -58.45500564575195,
|
|
"logps/ref_rejected": -70.7367172241211,
|
|
"logps/rejected": -217.22982788085938,
|
|
"loss": 1.1526,
|
|
"margin_dpo/margin_mean": 38.14574432373047,
|
|
"margin_dpo/margin_std": 72.89844512939453,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.9070294784580499,
|
|
"eval_fcm_dpo/beta": 0.012666304595768452,
|
|
"eval_fcm_dpo/delta": -0.008464168757200241,
|
|
"eval_fcm_dpo/margin": 46.96513366699219,
|
|
"eval_fcm_dpo/q_t": 0.3796946406364441,
|
|
"eval_logits/chosen": 0.6470546722412109,
|
|
"eval_logits/rejected": 0.5976328253746033,
|
|
"eval_logps/chosen": -176.9747314453125,
|
|
"eval_logps/ref_chosen": -74.85946655273438,
|
|
"eval_logps/ref_rejected": -79.54898834228516,
|
|
"eval_logps/rejected": -228.62939453125,
|
|
"eval_loss": 0.5427461266517639,
|
|
"eval_margin_dpo/margin_mean": 46.96513366699219,
|
|
"eval_margin_dpo/margin_std": 76.42948150634766,
|
|
"eval_runtime": 38.2644,
|
|
"eval_samples_per_second": 60.187,
|
|
"eval_steps_per_second": 1.882,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.9145880574452003,
|
|
"fcm_dpo/beta": 0.01174403727054596,
|
|
"fcm_dpo/delta": -0.022130563855171204,
|
|
"fcm_dpo/margin": 52.78753662109375,
|
|
"fcm_dpo/q_t": 0.37097370624542236,
|
|
"grad_norm": 17.305164337158203,
|
|
"learning_rate": 1.1274439638981532e-08,
|
|
"logits/chosen": 0.651776909828186,
|
|
"logits/rejected": 0.6054097414016724,
|
|
"logps/chosen": -154.97470092773438,
|
|
"logps/ref_chosen": -59.87483596801758,
|
|
"logps/ref_rejected": -75.75318908691406,
|
|
"logps/rejected": -223.6405792236328,
|
|
"loss": 1.0357,
|
|
"margin_dpo/margin_mean": 52.78753662109375,
|
|
"margin_dpo/margin_std": 76.32707214355469,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.9221466364323507,
|
|
"fcm_dpo/beta": 0.011598466895520687,
|
|
"fcm_dpo/delta": -0.03721737116575241,
|
|
"fcm_dpo/margin": 54.65764236450195,
|
|
"fcm_dpo/q_t": 0.3674970269203186,
|
|
"grad_norm": 14.85545825958252,
|
|
"learning_rate": 9.395165583732379e-09,
|
|
"logits/chosen": 0.6490997672080994,
|
|
"logits/rejected": 0.6031283140182495,
|
|
"logps/chosen": -160.9161376953125,
|
|
"logps/ref_chosen": -60.35883712768555,
|
|
"logps/ref_rejected": -81.3543930053711,
|
|
"logps/rejected": -236.5693359375,
|
|
"loss": 1.0079,
|
|
"margin_dpo/margin_mean": 54.65764236450195,
|
|
"margin_dpo/margin_std": 72.39617919921875,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.9297052154195011,
|
|
"fcm_dpo/beta": 0.011334126815199852,
|
|
"fcm_dpo/delta": -0.02821548655629158,
|
|
"fcm_dpo/margin": 55.235130310058594,
|
|
"fcm_dpo/q_t": 0.36660271883010864,
|
|
"grad_norm": 16.21509552001953,
|
|
"learning_rate": 7.684137976598088e-09,
|
|
"logits/chosen": 0.6232366561889648,
|
|
"logits/rejected": 0.5847585797309875,
|
|
"logps/chosen": -159.66122436523438,
|
|
"logps/ref_chosen": -59.17219161987305,
|
|
"logps/ref_rejected": -79.92167663574219,
|
|
"logps/rejected": -235.64584350585938,
|
|
"loss": 1.0121,
|
|
"margin_dpo/margin_mean": 55.235130310058594,
|
|
"margin_dpo/margin_std": 73.58882141113281,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.9372637944066515,
|
|
"fcm_dpo/beta": 0.011340516619384289,
|
|
"fcm_dpo/delta": 0.013897893019020557,
|
|
"fcm_dpo/margin": 48.57066345214844,
|
|
"fcm_dpo/q_t": 0.38371890783309937,
|
|
"grad_norm": 18.236835479736328,
|
|
"learning_rate": 6.142553278648238e-09,
|
|
"logits/chosen": 0.6844276189804077,
|
|
"logits/rejected": 0.6169918775558472,
|
|
"logps/chosen": -159.17074584960938,
|
|
"logps/ref_chosen": -58.052696228027344,
|
|
"logps/ref_rejected": -78.37252807617188,
|
|
"logps/rejected": -228.0612335205078,
|
|
"loss": 1.0546,
|
|
"margin_dpo/margin_mean": 48.57066345214844,
|
|
"margin_dpo/margin_std": 69.87725830078125,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.9448223733938019,
|
|
"fcm_dpo/beta": 0.011483758687973022,
|
|
"fcm_dpo/delta": 0.0366099514067173,
|
|
"fcm_dpo/margin": 49.20580291748047,
|
|
"fcm_dpo/q_t": 0.38207700848579407,
|
|
"grad_norm": 15.810871124267578,
|
|
"learning_rate": 4.7714894655209174e-09,
|
|
"logits/chosen": 0.6805665493011475,
|
|
"logits/rejected": 0.5953696966171265,
|
|
"logps/chosen": -160.55422973632812,
|
|
"logps/ref_chosen": -56.957862854003906,
|
|
"logps/ref_rejected": -82.68255615234375,
|
|
"logps/rejected": -235.48471069335938,
|
|
"loss": 1.0589,
|
|
"margin_dpo/margin_mean": 49.20580291748047,
|
|
"margin_dpo/margin_std": 73.78382873535156,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.9523809523809523,
|
|
"fcm_dpo/beta": 0.011226166971027851,
|
|
"fcm_dpo/delta": -0.060714829713106155,
|
|
"fcm_dpo/margin": 58.395973205566406,
|
|
"fcm_dpo/q_t": 0.3628186285495758,
|
|
"grad_norm": 12.203248023986816,
|
|
"learning_rate": 3.5719052736323806e-09,
|
|
"logits/chosen": 0.6524108648300171,
|
|
"logits/rejected": 0.5797510743141174,
|
|
"logps/chosen": -154.64805603027344,
|
|
"logps/ref_chosen": -56.71510696411133,
|
|
"logps/ref_rejected": -82.94544219970703,
|
|
"logps/rejected": -239.2743682861328,
|
|
"loss": 1.0058,
|
|
"margin_dpo/margin_mean": 58.395965576171875,
|
|
"margin_dpo/margin_std": 77.0825424194336,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.9599395313681028,
|
|
"fcm_dpo/beta": 0.010768270120024681,
|
|
"fcm_dpo/delta": 0.03129608556628227,
|
|
"fcm_dpo/margin": 49.29106521606445,
|
|
"fcm_dpo/q_t": 0.38956043124198914,
|
|
"grad_norm": 15.734051704406738,
|
|
"learning_rate": 2.5446395297668287e-09,
|
|
"logits/chosen": 0.6613369584083557,
|
|
"logits/rejected": 0.6100578904151917,
|
|
"logps/chosen": -160.97386169433594,
|
|
"logps/ref_chosen": -59.33793258666992,
|
|
"logps/ref_rejected": -75.01703643798828,
|
|
"logps/rejected": -225.9440155029297,
|
|
"loss": 1.0885,
|
|
"margin_dpo/margin_mean": 49.29106521606445,
|
|
"margin_dpo/margin_std": 76.83003234863281,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.9674981103552532,
|
|
"fcm_dpo/beta": 0.011434816755354404,
|
|
"fcm_dpo/delta": 0.013064498081803322,
|
|
"fcm_dpo/margin": 51.36224365234375,
|
|
"fcm_dpo/q_t": 0.3775227963924408,
|
|
"grad_norm": 15.347606658935547,
|
|
"learning_rate": 1.690410564514244e-09,
|
|
"logits/chosen": 0.6572569012641907,
|
|
"logits/rejected": 0.5778030753135681,
|
|
"logps/chosen": -165.37149047851562,
|
|
"logps/ref_chosen": -58.1605339050293,
|
|
"logps/ref_rejected": -79.85365295410156,
|
|
"logps/rejected": -238.42684936523438,
|
|
"loss": 1.0496,
|
|
"margin_dpo/margin_mean": 51.36224365234375,
|
|
"margin_dpo/margin_std": 76.05950927734375,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.9750566893424036,
|
|
"fcm_dpo/beta": 0.011937716044485569,
|
|
"fcm_dpo/delta": 0.06634578108787537,
|
|
"fcm_dpo/margin": 44.95128631591797,
|
|
"fcm_dpo/q_t": 0.3872304856777191,
|
|
"grad_norm": 16.9814453125,
|
|
"learning_rate": 1.0098157099674987e-09,
|
|
"logits/chosen": 0.6469696760177612,
|
|
"logits/rejected": 0.6165892481803894,
|
|
"logps/chosen": -168.5662384033203,
|
|
"logps/ref_chosen": -63.45180130004883,
|
|
"logps/ref_rejected": -74.18285369873047,
|
|
"logps/rejected": -224.2485809326172,
|
|
"loss": 1.0905,
|
|
"margin_dpo/margin_mean": 44.95128631591797,
|
|
"margin_dpo/margin_std": 72.61418914794922,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.982615268329554,
|
|
"fcm_dpo/beta": 0.011954518966376781,
|
|
"fcm_dpo/delta": -0.033706896007061005,
|
|
"fcm_dpo/margin": 52.76036834716797,
|
|
"fcm_dpo/q_t": 0.36990886926651,
|
|
"grad_norm": 17.623476028442383,
|
|
"learning_rate": 5.033308820289184e-10,
|
|
"logits/chosen": 0.6857269406318665,
|
|
"logits/rejected": 0.6130499243736267,
|
|
"logps/chosen": -168.08609008789062,
|
|
"logps/ref_chosen": -59.75496292114258,
|
|
"logps/ref_rejected": -84.31481170654297,
|
|
"logps/rejected": -245.4062957763672,
|
|
"loss": 1.0462,
|
|
"margin_dpo/margin_mean": 52.76036834716797,
|
|
"margin_dpo/margin_std": 78.340087890625,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.9901738473167044,
|
|
"fcm_dpo/beta": 0.012021342292428017,
|
|
"fcm_dpo/delta": -0.0002914905489888042,
|
|
"fcm_dpo/margin": 49.76955032348633,
|
|
"fcm_dpo/q_t": 0.37682992219924927,
|
|
"grad_norm": 14.043972969055176,
|
|
"learning_rate": 1.7131024761923852e-10,
|
|
"logits/chosen": 0.6599605679512024,
|
|
"logits/rejected": 0.5831471681594849,
|
|
"logps/chosen": -160.75221252441406,
|
|
"logps/ref_chosen": -57.817848205566406,
|
|
"logps/ref_rejected": -79.81755065917969,
|
|
"logps/rejected": -232.521484375,
|
|
"loss": 1.0505,
|
|
"margin_dpo/margin_mean": 49.76955032348633,
|
|
"margin_dpo/margin_std": 72.32919311523438,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.9977324263038548,
|
|
"fcm_dpo/beta": 0.01167194452136755,
|
|
"fcm_dpo/delta": -0.019878769293427467,
|
|
"fcm_dpo/margin": 52.92683029174805,
|
|
"fcm_dpo/q_t": 0.370964378118515,
|
|
"grad_norm": 16.102828979492188,
|
|
"learning_rate": 1.3985977021235829e-11,
|
|
"logits/chosen": 0.7206791639328003,
|
|
"logits/rejected": 0.6511392593383789,
|
|
"logps/chosen": -165.01748657226562,
|
|
"logps/ref_chosen": -59.12651443481445,
|
|
"logps/ref_rejected": -79.42085266113281,
|
|
"logps/rejected": -238.2386474609375,
|
|
"loss": 1.0408,
|
|
"margin_dpo/margin_mean": 52.92683029174805,
|
|
"margin_dpo/margin_std": 77.05123138427734,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.999244142101285,
|
|
"step": 661,
|
|
"total_flos": 0.0,
|
|
"train_loss": 1.103111317586971,
|
|
"train_runtime": 1749.744,
|
|
"train_samples_per_second": 24.196,
|
|
"train_steps_per_second": 0.378
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 661,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|