jackf857/llama-3-8b-base-new-dpo-hh-helpful-s_star1.0-4xh200-batch-64-20260421-233802
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
fd963e80d1c4743e23926fdbc074e6178e6e9b64
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC fd963e80d1 初始化项目，由ModelHub XC社区提供模型 
Model: jackf857/llama-3-8b-base-new-dpo-hh-helpful-s_star1.0-4xh200-batch-64-20260421-233802
Source: Original Platform
2026-05-30 03:50:21 +08:00

2707 lines
99 KiB
JSON
Raw Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.51715850830078,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000894069672,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.01767297089099884,
"fcm_dpo/q_t": 0.5004419088363647,
"grad_norm": 89.5003433227539,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.4901275634765625,
"logits/rejected": -0.45340436697006226,
"logps/chosen": -56.052879333496094,
"logps/ref_chosen": -56.05734634399414,
"logps/ref_rejected": -78.69325256347656,
"logps/rejected": -78.6711196899414,
"loss": 1.3883,
"margin_dpo/margin_mean": -0.017673060297966003,
"margin_dpo/margin_std": 0.3324979245662689,
"step": 5
},
{
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000894069672,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.011697113513946533,
"fcm_dpo/q_t": 0.5002920031547546,
"grad_norm": 70.89155578613281,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.5073713660240173,
"logits/rejected": -0.4694274961948395,
"logps/chosen": -59.539703369140625,
"logps/ref_chosen": -59.54457473754883,
"logps/ref_rejected": -91.17041778564453,
"logps/rejected": -91.15385437011719,
"loss": 1.3879,
"margin_dpo/margin_mean": -0.011697125621140003,
"margin_dpo/margin_std": 0.3836909830570221,
"step": 10
},
{
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000894069672,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0060859620571136475,
"fcm_dpo/q_t": 0.4998478293418884,
"grad_norm": 64.05209350585938,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.4684832990169525,
"logits/rejected": -0.43799877166748047,
"logps/chosen": -58.84619140625,
"logps/ref_chosen": -58.83195877075195,
"logps/ref_rejected": -92.93949890136719,
"logps/rejected": -92.95982360839844,
"loss": 1.3861,
"margin_dpo/margin_mean": 0.006086033768951893,
"margin_dpo/margin_std": 0.40277695655822754,
"step": 15
},
{
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000894069672,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04989497736096382,
"fcm_dpo/q_t": 0.4987525939941406,
"grad_norm": 73.78077697753906,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5167207717895508,
"logits/rejected": -0.46918559074401855,
"logps/chosen": -59.63946533203125,
"logps/ref_chosen": -59.6396598815918,
"logps/ref_rejected": -82.76026916503906,
"logps/rejected": -82.80996704101562,
"loss": 1.3816,
"margin_dpo/margin_mean": 0.04989498108625412,
"margin_dpo/margin_std": 0.3380239009857178,
"step": 20
},
{
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000894069672,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20749202370643616,
"fcm_dpo/q_t": 0.4948153495788574,
"grad_norm": 73.6851577758789,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.49890923500061035,
"logits/rejected": -0.47215452790260315,
"logps/chosen": -53.1756591796875,
"logps/ref_chosen": -53.205284118652344,
"logps/ref_rejected": -88.99608612060547,
"logps/rejected": -89.1739501953125,
"loss": 1.3661,
"margin_dpo/margin_mean": 0.20749172568321228,
"margin_dpo/margin_std": 0.39785444736480713,
"step": 25
},
{
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000894069672,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4946414828300476,
"fcm_dpo/q_t": 0.4876500964164734,
"grad_norm": 87.87821960449219,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.5192720890045166,
"logits/rejected": -0.4791105389595032,
"logps/chosen": -53.45599365234375,
"logps/ref_chosen": -53.5526008605957,
"logps/ref_rejected": -97.87371826171875,
"logps/rejected": -98.2717514038086,
"loss": 1.3383,
"margin_dpo/margin_mean": 0.4946417212486267,
"margin_dpo/margin_std": 0.575846791267395,
"step": 30
},
{
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000894069672,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7729043960571289,
"fcm_dpo/q_t": 0.4807465672492981,
"grad_norm": 83.32990264892578,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.48504266142845154,
"logits/rejected": -0.45020541548728943,
"logps/chosen": -56.183860778808594,
"logps/ref_chosen": -56.3298454284668,
"logps/ref_rejected": -91.76858520507812,
"logps/rejected": -92.39549255371094,
"loss": 1.3125,
"margin_dpo/margin_mean": 0.7729043960571289,
"margin_dpo/margin_std": 0.8287278413772583,
"step": 35
},
{
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000894069672,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3533049821853638,
"fcm_dpo/q_t": 0.46647366881370544,
"grad_norm": 60.212345123291016,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5429744720458984,
"logits/rejected": -0.5070551037788391,
"logps/chosen": -54.291542053222656,
"logps/ref_chosen": -54.38492965698242,
"logps/ref_rejected": -82.94353485107422,
"logps/rejected": -84.20346069335938,
"loss": 1.2611,
"margin_dpo/margin_mean": 1.3533051013946533,
"margin_dpo/margin_std": 1.4385766983032227,
"step": 40
},
{
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.13899105787277222,
"fcm_dpo/delta": 0.597256600856781,
"fcm_dpo/margin": 2.4335668087005615,
"fcm_dpo/q_t": 0.4230595529079437,
"grad_norm": 92.142822265625,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.5021598935127258,
"logits/rejected": -0.4736817479133606,
"logps/chosen": -54.635047912597656,
"logps/ref_chosen": -54.862335205078125,
"logps/ref_rejected": -98.0264663696289,
"logps/rejected": -100.23272705078125,
"loss": 1.1013,
"margin_dpo/margin_mean": 2.4335670471191406,
"margin_dpo/margin_std": 2.2672231197357178,
"step": 45
},
{
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.22112059593200684,
"fcm_dpo/delta": 0.2686576247215271,
"fcm_dpo/margin": 3.3537793159484863,
"fcm_dpo/q_t": 0.34648868441581726,
"grad_norm": 102.62498474121094,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5621516108512878,
"logits/rejected": -0.5111474394798279,
"logps/chosen": -58.14952850341797,
"logps/ref_chosen": -58.304595947265625,
"logps/ref_rejected": -91.69480895996094,
"logps/rejected": -94.89351654052734,
"loss": 0.8932,
"margin_dpo/margin_mean": 3.3537793159484863,
"margin_dpo/margin_std": 3.3193202018737793,
"step": 50
},
{
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.21576957404613495,
"fcm_dpo/delta": -0.21076150238513947,
"fcm_dpo/margin": 5.520001411437988,
"fcm_dpo/q_t": 0.2881360948085785,
"grad_norm": 89.94622039794922,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.5873750448226929,
"logits/rejected": -0.5421477556228638,
"logps/chosen": -56.29509735107422,
"logps/ref_chosen": -56.06591796875,
"logps/ref_rejected": -85.69367980957031,
"logps/rejected": -91.44287109375,
"loss": 0.7614,
"margin_dpo/margin_mean": 5.520001411437988,
"margin_dpo/margin_std": 5.638136863708496,
"step": 55
},
{
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.17349031567573547,
"fcm_dpo/delta": -0.17873340845108032,
"fcm_dpo/margin": 6.684614658355713,
"fcm_dpo/q_t": 0.28984424471855164,
"grad_norm": 94.9372329711914,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.6055638194084167,
"logits/rejected": -0.5562660098075867,
"logps/chosen": -61.69890213012695,
"logps/ref_chosen": -60.6871337890625,
"logps/ref_rejected": -89.72715759277344,
"logps/rejected": -97.42354583740234,
"loss": 0.7907,
"margin_dpo/margin_mean": 6.684614658355713,
"margin_dpo/margin_std": 7.005400657653809,
"step": 60
},
{
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.1508425772190094,
"fcm_dpo/delta": -0.1438160538673401,
"fcm_dpo/margin": 7.480526924133301,
"fcm_dpo/q_t": 0.29980701208114624,
"grad_norm": 67.76667022705078,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.5985899567604065,
"logits/rejected": -0.5659457445144653,
"logps/chosen": -63.0954475402832,
"logps/ref_chosen": -61.75325393676758,
"logps/ref_rejected": -93.30108642578125,
"logps/rejected": -102.12379455566406,
"loss": 0.8297,
"margin_dpo/margin_mean": 7.480526924133301,
"margin_dpo/margin_std": 8.598801612854004,
"step": 65
},
{
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.13242055475711823,
"fcm_dpo/delta": -0.1166788712143898,
"fcm_dpo/margin": 8.34177017211914,
"fcm_dpo/q_t": 0.2980775237083435,
"grad_norm": 65.84673309326172,
"learning_rate": 5e-07,
"logits/chosen": -0.6093414425849915,
"logits/rejected": -0.5704778432846069,
"logps/chosen": -61.53076934814453,
"logps/ref_chosen": -59.548004150390625,
"logps/ref_rejected": -84.01609802246094,
"logps/rejected": -94.34062194824219,
"loss": 0.7832,
"margin_dpo/margin_mean": 8.34177017211914,
"margin_dpo/margin_std": 8.739455223083496,
"step": 70
},
{
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.10968731343746185,
"fcm_dpo/delta": -0.25056737661361694,
"fcm_dpo/margin": 11.169679641723633,
"fcm_dpo/q_t": 0.283592164516449,
"grad_norm": 47.23740005493164,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6356680989265442,
"logits/rejected": -0.6052416563034058,
"logps/chosen": -62.6329460144043,
"logps/ref_chosen": -59.86931228637695,
"logps/ref_rejected": -98.05613708496094,
"logps/rejected": -111.98944091796875,
"loss": 0.7689,
"margin_dpo/margin_mean": 11.169679641723633,
"margin_dpo/margin_std": 11.989259719848633,
"step": 75
},
{
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.08976603299379349,
"fcm_dpo/delta": -0.12075959146022797,
"fcm_dpo/margin": 12.325640678405762,
"fcm_dpo/q_t": 0.29587632417678833,
"grad_norm": 47.55595397949219,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.6552311182022095,
"logits/rejected": -0.6144415736198425,
"logps/chosen": -59.39103317260742,
"logps/ref_chosen": -56.18925857543945,
"logps/ref_rejected": -86.42393493652344,
"logps/rejected": -101.95135498046875,
"loss": 0.7882,
"margin_dpo/margin_mean": 12.325640678405762,
"margin_dpo/margin_std": 12.580961227416992,
"step": 80
},
{
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.0777033269405365,
"fcm_dpo/delta": -0.14079028367996216,
"fcm_dpo/margin": 14.418505668640137,
"fcm_dpo/q_t": 0.2961280643939972,
"grad_norm": 51.494895935058594,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6709647178649902,
"logits/rejected": -0.6374942064285278,
"logps/chosen": -64.86982727050781,
"logps/ref_chosen": -60.018287658691406,
"logps/ref_rejected": -98.01185607910156,
"logps/rejected": -117.28189849853516,
"loss": 0.808,
"margin_dpo/margin_mean": 14.418505668640137,
"margin_dpo/margin_std": 15.550962448120117,
"step": 85
},
{
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.07141976058483124,
"fcm_dpo/delta": -0.08756770193576813,
"fcm_dpo/margin": 15.070363998413086,
"fcm_dpo/q_t": 0.31418612599372864,
"grad_norm": 49.612510681152344,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.6960055232048035,
"logits/rejected": -0.6736189723014832,
"logps/chosen": -67.00579071044922,
"logps/ref_chosen": -59.8709831237793,
"logps/ref_rejected": -96.78519439697266,
"logps/rejected": -118.99037170410156,
"loss": 0.8807,
"margin_dpo/margin_mean": 15.070365905761719,
"margin_dpo/margin_std": 18.86634063720703,
"step": 90
},
{
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.06466929614543915,
"fcm_dpo/delta": -0.06842272728681564,
"fcm_dpo/margin": 16.387496948242188,
"fcm_dpo/q_t": 0.3058759272098541,
"grad_norm": 39.34260559082031,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.7063266634941101,
"logits/rejected": -0.6684988737106323,
"logps/chosen": -63.07061004638672,
"logps/ref_chosen": -55.94385528564453,
"logps/ref_rejected": -83.6790542602539,
"logps/rejected": -107.19331359863281,
"loss": 0.82,
"margin_dpo/margin_mean": 16.387496948242188,
"margin_dpo/margin_std": 17.77635955810547,
"step": 95
},
{
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.0604403130710125,
"fcm_dpo/delta": -0.049155093729496,
"fcm_dpo/margin": 17.22031021118164,
"fcm_dpo/q_t": 0.3058018684387207,
"grad_norm": 48.312198638916016,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.680860698223114,
"logits/rejected": -0.6464991569519043,
"logps/chosen": -64.37776184082031,
"logps/ref_chosen": -57.05888748168945,
"logps/ref_rejected": -86.11727142333984,
"logps/rejected": -110.65645599365234,
"loss": 0.8241,
"margin_dpo/margin_mean": 17.22031021118164,
"margin_dpo/margin_std": 18.75308609008789,
"step": 100
},
{
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.05940910056233406,
"fcm_dpo/delta": -0.1656493991613388,
"fcm_dpo/margin": 19.471708297729492,
"fcm_dpo/q_t": 0.29566460847854614,
"grad_norm": 33.41164016723633,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.7165222764015198,
"logits/rejected": -0.6761705875396729,
"logps/chosen": -67.66845703125,
"logps/ref_chosen": -59.20774459838867,
"logps/ref_rejected": -86.49754333496094,
"logps/rejected": -114.4299545288086,
"loss": 0.8003,
"margin_dpo/margin_mean": 19.471708297729492,
"margin_dpo/margin_std": 20.485849380493164,
"step": 105
},
{
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.05002494528889656,
"fcm_dpo/delta": -0.06976357847452164,
"fcm_dpo/margin": 21.24454116821289,
"fcm_dpo/q_t": 0.3023180663585663,
"grad_norm": 33.63520050048828,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.6657849550247192,
"logits/rejected": -0.6188558340072632,
"logps/chosen": -69.47651672363281,
"logps/ref_chosen": -60.437957763671875,
"logps/ref_rejected": -90.83917999267578,
"logps/rejected": -121.12227630615234,
"loss": 0.8078,
"margin_dpo/margin_mean": 21.24454116821289,
"margin_dpo/margin_std": 22.012073516845703,
"step": 110
},
{
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.04783306270837784,
"fcm_dpo/delta": -0.005023984704166651,
"fcm_dpo/margin": 20.951387405395508,
"fcm_dpo/q_t": 0.31595808267593384,
"grad_norm": 48.63422393798828,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.7007675766944885,
"logits/rejected": -0.6588722467422485,
"logps/chosen": -73.18070983886719,
"logps/ref_chosen": -61.7908821105957,
"logps/ref_rejected": -85.36943054199219,
"logps/rejected": -117.71064758300781,
"loss": 0.8539,
"margin_dpo/margin_mean": 20.951387405395508,
"margin_dpo/margin_std": 24.009220123291016,
"step": 115
},
{
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.04857005923986435,
"fcm_dpo/delta": -0.008426040410995483,
"fcm_dpo/margin": 20.727174758911133,
"fcm_dpo/q_t": 0.31097596883773804,
"grad_norm": 37.845821380615234,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.6924178600311279,
"logits/rejected": -0.6578742265701294,
"logps/chosen": -78.19169616699219,
"logps/ref_chosen": -65.3261489868164,
"logps/ref_rejected": -86.75518798828125,
"logps/rejected": -120.3479232788086,
"loss": 0.8219,
"margin_dpo/margin_mean": 20.727174758911133,
"margin_dpo/margin_std": 22.264766693115234,
"step": 120
},
{
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.040757957845926285,
"fcm_dpo/delta": -0.2243596315383911,
"fcm_dpo/margin": 29.304885864257812,
"fcm_dpo/q_t": 0.28754281997680664,
"grad_norm": 35.820945739746094,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.6601894497871399,
"logits/rejected": -0.6469351053237915,
"logps/chosen": -72.17552947998047,
"logps/ref_chosen": -58.323204040527344,
"logps/ref_rejected": -101.2106704711914,
"logps/rejected": -144.36788940429688,
"loss": 0.7748,
"margin_dpo/margin_mean": 29.304885864257812,
"margin_dpo/margin_std": 30.770614624023438,
"step": 125
},
{
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.038607753813266754,
"fcm_dpo/delta": -0.023931104689836502,
"fcm_dpo/margin": 26.40726661682129,
"fcm_dpo/q_t": 0.31171610951423645,
"grad_norm": 32.028621673583984,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.6508277058601379,
"logits/rejected": -0.6233072280883789,
"logps/chosen": -70.66328430175781,
"logps/ref_chosen": -56.38518524169922,
"logps/ref_rejected": -86.15767669677734,
"logps/rejected": -126.8430404663086,
"loss": 0.8458,
"margin_dpo/margin_mean": 26.40726661682129,
"margin_dpo/margin_std": 28.96893310546875,
"step": 130
},
{
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.03552158921957016,
"fcm_dpo/delta": -0.07289578765630722,
"fcm_dpo/margin": 29.98138427734375,
"fcm_dpo/q_t": 0.30137279629707336,
"grad_norm": 33.19447708129883,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.6592376828193665,
"logits/rejected": -0.6462143659591675,
"logps/chosen": -69.44171905517578,
"logps/ref_chosen": -54.59065628051758,
"logps/ref_rejected": -95.26080322265625,
"logps/rejected": -140.09323120117188,
"loss": 0.7941,
"margin_dpo/margin_mean": 29.98138427734375,
"margin_dpo/margin_std": 31.074054718017578,
"step": 135
},
{
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.033399857580661774,
"fcm_dpo/delta": -0.06322337687015533,
"fcm_dpo/margin": 31.61517906188965,
"fcm_dpo/q_t": 0.3098406195640564,
"grad_norm": 34.031742095947266,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.6483560800552368,
"logits/rejected": -0.6316757202148438,
"logps/chosen": -75.58506774902344,
"logps/ref_chosen": -56.04347610473633,
"logps/ref_rejected": -93.27880859375,
"logps/rejected": -144.43556213378906,
"loss": 0.8325,
"margin_dpo/margin_mean": 31.61517906188965,
"margin_dpo/margin_std": 35.689491271972656,
"step": 140
},
{
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.031947266310453415,
"fcm_dpo/delta": -0.00863505620509386,
"fcm_dpo/margin": 31.503803253173828,
"fcm_dpo/q_t": 0.3131418824195862,
"grad_norm": 36.38671112060547,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.6257671117782593,
"logits/rejected": -0.5978158116340637,
"logps/chosen": -82.8645248413086,
"logps/ref_chosen": -61.4414176940918,
"logps/ref_rejected": -86.32813262939453,
"logps/rejected": -139.25503540039062,
"loss": 0.8557,
"margin_dpo/margin_mean": 31.503803253173828,
"margin_dpo/margin_std": 35.5254020690918,
"step": 145
},
{
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.03161965310573578,
"fcm_dpo/delta": -0.0391874685883522,
"fcm_dpo/margin": 32.65111541748047,
"fcm_dpo/q_t": 0.3133260905742645,
"grad_norm": 36.64207077026367,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.6312776803970337,
"logits/rejected": -0.6036733984947205,
"logps/chosen": -81.85924530029297,
"logps/ref_chosen": -57.70451736450195,
"logps/ref_rejected": -87.76991271972656,
"logps/rejected": -144.57577514648438,
"loss": 0.8533,
"margin_dpo/margin_mean": 32.65111541748047,
"margin_dpo/margin_std": 37.58909225463867,
"step": 150
},
{
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.029207954183220863,
"fcm_dpo/delta": -0.06974439322948456,
"fcm_dpo/margin": 36.313804626464844,
"fcm_dpo/q_t": 0.3046707212924957,
"grad_norm": 31.77535629272461,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.6591686010360718,
"logits/rejected": -0.6376873850822449,
"logps/chosen": -83.7581787109375,
"logps/ref_chosen": -62.08925247192383,
"logps/ref_rejected": -94.79930114746094,
"logps/rejected": -152.7820281982422,
"loss": 0.824,
"margin_dpo/margin_mean": 36.313804626464844,
"margin_dpo/margin_std": 39.620338439941406,
"step": 155
},
{
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.028771813958883286,
"fcm_dpo/delta": -0.017074979841709137,
"fcm_dpo/margin": 35.24787139892578,
"fcm_dpo/q_t": 0.32106462121009827,
"grad_norm": 35.20014572143555,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.6497837901115417,
"logits/rejected": -0.6263854503631592,
"logps/chosen": -93.72312927246094,
"logps/ref_chosen": -67.15288543701172,
"logps/ref_rejected": -96.92537689208984,
"logps/rejected": -158.7434844970703,
"loss": 0.8984,
"margin_dpo/margin_mean": 35.24787139892578,
"margin_dpo/margin_std": 44.607139587402344,
"step": 160
},
{
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.0279733594506979,
"fcm_dpo/delta": 0.028953036293387413,
"fcm_dpo/margin": 34.75352478027344,
"fcm_dpo/q_t": 0.3189981281757355,
"grad_norm": 43.92022705078125,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.6410833597183228,
"logits/rejected": -0.6081478595733643,
"logps/chosen": -80.7635269165039,
"logps/ref_chosen": -57.40401077270508,
"logps/ref_rejected": -80.31498718261719,
"logps/rejected": -138.42803955078125,
"loss": 0.8543,
"margin_dpo/margin_mean": 34.7535285949707,
"margin_dpo/margin_std": 39.33875274658203,
"step": 165
},
{
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.025926318019628525,
"fcm_dpo/delta": -0.12728442251682281,
"fcm_dpo/margin": 42.69086456298828,
"fcm_dpo/q_t": 0.29947513341903687,
"grad_norm": 28.440479278564453,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.5913532376289368,
"logits/rejected": -0.576996922492981,
"logps/chosen": -76.37390899658203,
"logps/ref_chosen": -52.029144287109375,
"logps/ref_rejected": -85.73944091796875,
"logps/rejected": -152.7750701904297,
"loss": 0.802,
"margin_dpo/margin_mean": 42.69086456298828,
"margin_dpo/margin_std": 46.09825897216797,
"step": 170
},
{
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.025716686621308327,
"fcm_dpo/delta": -0.06026681512594223,
"fcm_dpo/margin": 41.025394439697266,
"fcm_dpo/q_t": 0.30824658274650574,
"grad_norm": 40.217952728271484,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.6109501123428345,
"logits/rejected": -0.5846124887466431,
"logps/chosen": -94.55276489257812,
"logps/ref_chosen": -62.996971130371094,
"logps/ref_rejected": -92.98394012451172,
"logps/rejected": -165.5651397705078,
"loss": 0.8288,
"margin_dpo/margin_mean": 41.025394439697266,
"margin_dpo/margin_std": 45.7181396484375,
"step": 175
},
{
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.021963346749544144,
"fcm_dpo/delta": -0.16025081276893616,
"fcm_dpo/margin": 52.09009552001953,
"fcm_dpo/q_t": 0.2960316240787506,
"grad_norm": 28.27334976196289,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.5492862462997437,
"logits/rejected": -0.5417381525039673,
"logps/chosen": -94.24219512939453,
"logps/ref_chosen": -57.0670280456543,
"logps/ref_rejected": -97.1115493774414,
"logps/rejected": -186.37680053710938,
"loss": 0.7974,
"margin_dpo/margin_mean": 52.09009552001953,
"margin_dpo/margin_std": 56.285667419433594,
"step": 180
},
{
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.021468380466103554,
"fcm_dpo/delta": 0.1422530859708786,
"fcm_dpo/margin": 40.60783386230469,
"fcm_dpo/q_t": 0.33963409066200256,
"grad_norm": 41.11544418334961,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.5713580250740051,
"logits/rejected": -0.5433865785598755,
"logps/chosen": -89.1233901977539,
"logps/ref_chosen": -54.840736389160156,
"logps/ref_rejected": -75.51002502441406,
"logps/rejected": -150.4005126953125,
"loss": 0.9251,
"margin_dpo/margin_mean": 40.60783767700195,
"margin_dpo/margin_std": 49.8930778503418,
"step": 185
},
{
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.02240898832678795,
"fcm_dpo/delta": -0.08936971426010132,
"fcm_dpo/margin": 48.08588790893555,
"fcm_dpo/q_t": 0.3056829571723938,
"grad_norm": 36.971946716308594,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.5829431414604187,
"logits/rejected": -0.5550256371498108,
"logps/chosen": -92.63221740722656,
"logps/ref_chosen": -57.72148895263672,
"logps/ref_rejected": -86.85997009277344,
"logps/rejected": -169.85658264160156,
"loss": 0.8272,
"margin_dpo/margin_mean": 48.08588790893555,
"margin_dpo/margin_std": 53.611724853515625,
"step": 190
},
{
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.020605169236660004,
"fcm_dpo/delta": -0.03353874385356903,
"fcm_dpo/margin": 49.802146911621094,
"fcm_dpo/q_t": 0.30824965238571167,
"grad_norm": 41.08122253417969,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.5500985383987427,
"logits/rejected": -0.5345898270606995,
"logps/chosen": -97.18587493896484,
"logps/ref_chosen": -58.26164627075195,
"logps/ref_rejected": -89.46485900878906,
"logps/rejected": -178.19125366210938,
"loss": 0.829,
"margin_dpo/margin_mean": 49.802146911621094,
"margin_dpo/margin_std": 54.20235061645508,
"step": 195
},
{
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.019669184461236,
"fcm_dpo/delta": -0.09520086646080017,
"fcm_dpo/margin": 55.09596633911133,
"fcm_dpo/q_t": 0.30649372935295105,
"grad_norm": 37.28245162963867,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.5128642320632935,
"logits/rejected": -0.4953628480434418,
"logps/chosen": -96.5008316040039,
"logps/ref_chosen": -55.71953201293945,
"logps/ref_rejected": -83.15235137939453,
"logps/rejected": -179.0296173095703,
"loss": 0.8238,
"margin_dpo/margin_mean": 55.09596633911133,
"margin_dpo/margin_std": 62.59626007080078,
"step": 200
},
{
"epoch": 0.2936857562408223,
"eval_fcm_dpo/beta": 0.02702677994966507,
"eval_fcm_dpo/delta": 0.059787724167108536,
"eval_fcm_dpo/margin": 34.20482635498047,
"eval_fcm_dpo/q_t": 0.36632612347602844,
"eval_logits/chosen": -0.5095491409301758,
"eval_logits/rejected": -0.4882952570915222,
"eval_logps/chosen": -136.1259002685547,
"eval_logps/ref_chosen": -79.05104064941406,
"eval_logps/ref_rejected": -86.79793548583984,
"eval_logps/rejected": -178.07763671875,
"eval_loss": 0.5817958116531372,
"eval_margin_dpo/margin_mean": 34.20482635498047,
"eval_margin_dpo/margin_std": 64.29653930664062,
"eval_runtime": 39.5418,
"eval_samples_per_second": 59.153,
"eval_steps_per_second": 1.871,
"step": 200
},
{
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.02430490031838417,
"fcm_dpo/delta": -0.2789553105831146,
"fcm_dpo/margin": 50.88113784790039,
"fcm_dpo/q_t": 0.2960304617881775,
"grad_norm": 43.3216552734375,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.5082226395606995,
"logits/rejected": -0.484395831823349,
"logps/chosen": -109.9628677368164,
"logps/ref_chosen": -65.13258361816406,
"logps/ref_rejected": -92.10203552246094,
"logps/rejected": -187.81346130371094,
"loss": 0.8422,
"margin_dpo/margin_mean": 50.881141662597656,
"margin_dpo/margin_std": 61.578651428222656,
"step": 205
},
{
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.019351085647940636,
"fcm_dpo/delta": -0.15747056901454926,
"fcm_dpo/margin": 58.509986877441406,
"fcm_dpo/q_t": 0.2959654927253723,
"grad_norm": 34.101295471191406,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.5159139633178711,
"logits/rejected": -0.49807208776474,
"logps/chosen": -96.82698822021484,
"logps/ref_chosen": -54.52837371826172,
"logps/ref_rejected": -87.06227111816406,
"logps/rejected": -187.87086486816406,
"loss": 0.7896,
"margin_dpo/margin_mean": 58.509986877441406,
"margin_dpo/margin_std": 61.90760040283203,
"step": 210
},
{
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.018195774406194687,
"fcm_dpo/delta": -0.06633646786212921,
"fcm_dpo/margin": 58.11163330078125,
"fcm_dpo/q_t": 0.3067888617515564,
"grad_norm": 33.01030731201172,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.5397628545761108,
"logits/rejected": -0.5149329900741577,
"logps/chosen": -104.81040954589844,
"logps/ref_chosen": -59.905250549316406,
"logps/ref_rejected": -90.25511932373047,
"logps/rejected": -193.27191162109375,
"loss": 0.8198,
"margin_dpo/margin_mean": 58.11163330078125,
"margin_dpo/margin_std": 62.161781311035156,
"step": 215
},
{
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.017347611486911774,
"fcm_dpo/delta": 0.028919270262122154,
"fcm_dpo/margin": 56.12737274169922,
"fcm_dpo/q_t": 0.3152950406074524,
"grad_norm": 38.29086685180664,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.5171216130256653,
"logits/rejected": -0.49278515577316284,
"logps/chosen": -107.46978759765625,
"logps/ref_chosen": -57.68498611450195,
"logps/ref_rejected": -87.72792053222656,
"logps/rejected": -193.6400909423828,
"loss": 0.8374,
"margin_dpo/margin_mean": 56.12738037109375,
"margin_dpo/margin_std": 59.3189582824707,
"step": 220
},
{
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.016515303403139114,
"fcm_dpo/delta": -0.17107121646404266,
"fcm_dpo/margin": 69.53387451171875,
"fcm_dpo/q_t": 0.2908400893211365,
"grad_norm": 38.22285079956055,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.4581502377986908,
"logits/rejected": -0.43911489844322205,
"logps/chosen": -117.43310546875,
"logps/ref_chosen": -54.47245407104492,
"logps/ref_rejected": -93.26266479492188,
"logps/rejected": -225.75717163085938,
"loss": 0.7955,
"margin_dpo/margin_mean": 69.53387451171875,
"margin_dpo/margin_std": 72.02003479003906,
"step": 225
},
{
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.014946828596293926,
"fcm_dpo/delta": -0.07315317541360855,
"fcm_dpo/margin": 71.19001770019531,
"fcm_dpo/q_t": 0.3010673224925995,
"grad_norm": 36.011512756347656,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.44661521911621094,
"logits/rejected": -0.4208187162876129,
"logps/chosen": -121.6751708984375,
"logps/ref_chosen": -58.7701301574707,
"logps/ref_rejected": -87.85963439941406,
"logps/rejected": -221.9547119140625,
"loss": 0.8123,
"margin_dpo/margin_mean": 71.19001770019531,
"margin_dpo/margin_std": 74.79989624023438,
"step": 230
},
{
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.014678606763482094,
"fcm_dpo/delta": 0.03617086634039879,
"fcm_dpo/margin": 65.72369384765625,
"fcm_dpo/q_t": 0.32347989082336426,
"grad_norm": 40.8921012878418,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.4440877437591553,
"logits/rejected": -0.42741116881370544,
"logps/chosen": -120.95252990722656,
"logps/ref_chosen": -59.0481071472168,
"logps/ref_rejected": -91.19654846191406,
"logps/rejected": -218.8246612548828,
"loss": 0.868,
"margin_dpo/margin_mean": 65.72369384765625,
"margin_dpo/margin_std": 76.00477600097656,
"step": 235
},
{
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.014482825994491577,
"fcm_dpo/delta": -0.015936076641082764,
"fcm_dpo/margin": 69.90579223632812,
"fcm_dpo/q_t": 0.30879801511764526,
"grad_norm": 37.091163635253906,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.39641961455345154,
"logits/rejected": -0.3881601393222809,
"logps/chosen": -124.786376953125,
"logps/ref_chosen": -55.238983154296875,
"logps/ref_rejected": -91.08428955078125,
"logps/rejected": -230.53744506835938,
"loss": 0.8191,
"margin_dpo/margin_mean": 69.90579223632812,
"margin_dpo/margin_std": 73.0508041381836,
"step": 240
},
{
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.013445289805531502,
"fcm_dpo/delta": -0.10347223281860352,
"fcm_dpo/margin": 80.9016342163086,
"fcm_dpo/q_t": 0.2974715530872345,
"grad_norm": 57.8762092590332,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.4064258635044098,
"logits/rejected": -0.37399476766586304,
"logps/chosen": -140.6631622314453,
"logps/ref_chosen": -65.08844757080078,
"logps/ref_rejected": -86.05777740478516,
"logps/rejected": -242.53414916992188,
"loss": 0.7822,
"margin_dpo/margin_mean": 80.9016342163086,
"margin_dpo/margin_std": 81.34479522705078,
"step": 245
},
{
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.012752078473567963,
"fcm_dpo/delta": -0.024351056665182114,
"fcm_dpo/margin": 79.96165466308594,
"fcm_dpo/q_t": 0.3058565557003021,
"grad_norm": 55.92478942871094,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.4105095863342285,
"logits/rejected": -0.38465723395347595,
"logps/chosen": -140.99496459960938,
"logps/ref_chosen": -59.08491897583008,
"logps/ref_rejected": -87.36727142333984,
"logps/rejected": -249.239013671875,
"loss": 0.812,
"margin_dpo/margin_mean": 79.96165466308594,
"margin_dpo/margin_std": 80.3189468383789,
"step": 250
},
{
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.012846621684730053,
"fcm_dpo/delta": 0.003509513335302472,
"fcm_dpo/margin": 77.48503112792969,
"fcm_dpo/q_t": 0.31272420287132263,
"grad_norm": 71.0317611694336,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.3824227452278137,
"logits/rejected": -0.35425513982772827,
"logps/chosen": -150.32997131347656,
"logps/ref_chosen": -61.85979461669922,
"logps/ref_rejected": -88.32804107666016,
"logps/rejected": -254.28323364257812,
"loss": 0.8651,
"margin_dpo/margin_mean": 77.48503112792969,
"margin_dpo/margin_std": 87.42262268066406,
"step": 255
},
{
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.012445470318198204,
"fcm_dpo/delta": -0.015315398573875427,
"fcm_dpo/margin": 81.26345825195312,
"fcm_dpo/q_t": 0.3129083514213562,
"grad_norm": 48.482181549072266,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.32099491357803345,
"logits/rejected": -0.314310759305954,
"logps/chosen": -142.50636291503906,
"logps/ref_chosen": -52.843467712402344,
"logps/ref_rejected": -90.4744873046875,
"logps/rejected": -261.40081787109375,
"loss": 0.8509,
"margin_dpo/margin_mean": 81.26345825195312,
"margin_dpo/margin_std": 91.2393569946289,
"step": 260
},
{
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.012557732872664928,
"fcm_dpo/delta": -0.006850825157016516,
"fcm_dpo/margin": 80.0453872680664,
"fcm_dpo/q_t": 0.3058241605758667,
"grad_norm": 58.8089714050293,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.33507245779037476,
"logits/rejected": -0.3110835552215576,
"logps/chosen": -132.56455993652344,
"logps/ref_chosen": -59.35320281982422,
"logps/ref_rejected": -90.73350524902344,
"logps/rejected": -243.99026489257812,
"loss": 0.8031,
"margin_dpo/margin_mean": 80.0453872680664,
"margin_dpo/margin_std": 79.68402862548828,
"step": 265
},
{
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.012217868119478226,
"fcm_dpo/delta": -0.05243998020887375,
"fcm_dpo/margin": 85.6143798828125,
"fcm_dpo/q_t": 0.301820307970047,
"grad_norm": 61.461185455322266,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.2629415690898895,
"logits/rejected": -0.23643343150615692,
"logps/chosen": -162.1711883544922,
"logps/ref_chosen": -66.30875396728516,
"logps/ref_rejected": -95.4130630493164,
"logps/rejected": -276.88983154296875,
"loss": 0.8022,
"margin_dpo/margin_mean": 85.6143798828125,
"margin_dpo/margin_std": 87.26689910888672,
"step": 270
},
{
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.010901778005063534,
"fcm_dpo/delta": -0.12498165667057037,
"fcm_dpo/margin": 101.67488098144531,
"fcm_dpo/q_t": 0.293281227350235,
"grad_norm": 37.275238037109375,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.18267172574996948,
"logits/rejected": -0.16967333853244781,
"logps/chosen": -166.1475830078125,
"logps/ref_chosen": -54.69990921020508,
"logps/ref_rejected": -92.23838806152344,
"logps/rejected": -305.3609313964844,
"loss": 0.7829,
"margin_dpo/margin_mean": 101.67487335205078,
"margin_dpo/margin_std": 101.45657348632812,
"step": 275
},
{
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.01014392077922821,
"fcm_dpo/delta": -0.10269641876220703,
"fcm_dpo/margin": 107.79083251953125,
"fcm_dpo/q_t": 0.2967400550842285,
"grad_norm": 61.96967315673828,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.18761689960956573,
"logits/rejected": -0.1775480955839157,
"logps/chosen": -165.2073974609375,
"logps/ref_chosen": -54.64586639404297,
"logps/ref_rejected": -88.19416809082031,
"logps/rejected": -306.5465393066406,
"loss": 0.7896,
"margin_dpo/margin_mean": 107.79083251953125,
"margin_dpo/margin_std": 108.4886474609375,
"step": 280
},
{
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.009512031450867653,
"fcm_dpo/delta": -0.016698762774467468,
"fcm_dpo/margin": 106.60298156738281,
"fcm_dpo/q_t": 0.3010271191596985,
"grad_norm": 49.41031265258789,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.18153177201747894,
"logits/rejected": -0.16163872182369232,
"logps/chosen": -202.77694702148438,
"logps/ref_chosen": -63.02496337890625,
"logps/ref_rejected": -93.16323852539062,
"logps/rejected": -339.5182189941406,
"loss": 0.7905,
"margin_dpo/margin_mean": 106.60298156738281,
"margin_dpo/margin_std": 102.84651947021484,
"step": 285
},
{
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.009123164229094982,
"fcm_dpo/delta": -0.08528591692447662,
"fcm_dpo/margin": 118.04217529296875,
"fcm_dpo/q_t": 0.29781579971313477,
"grad_norm": 41.782135009765625,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.10961093753576279,
"logits/rejected": -0.09905137121677399,
"logps/chosen": -217.45785522460938,
"logps/ref_chosen": -58.37105178833008,
"logps/ref_rejected": -91.59428405761719,
"logps/rejected": -368.72332763671875,
"loss": 0.7905,
"margin_dpo/margin_mean": 118.04217529296875,
"margin_dpo/margin_std": 118.39399719238281,
"step": 290
},
{
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.008718650788068771,
"fcm_dpo/delta": 0.0468270368874073,
"fcm_dpo/margin": 109.77592468261719,
"fcm_dpo/q_t": 0.3172770142555237,
"grad_norm": 67.40666198730469,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.11454031616449356,
"logits/rejected": -0.10916812717914581,
"logps/chosen": -198.9319610595703,
"logps/ref_chosen": -55.113426208496094,
"logps/ref_rejected": -85.29283905029297,
"logps/rejected": -338.88726806640625,
"loss": 0.8541,
"margin_dpo/margin_mean": 109.77592468261719,
"margin_dpo/margin_std": 119.68659973144531,
"step": 295
},
{
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.00930819846689701,
"fcm_dpo/delta": -0.025993477553129196,
"fcm_dpo/margin": 109.86900329589844,
"fcm_dpo/q_t": 0.3122258186340332,
"grad_norm": 76.4415054321289,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.16808557510375977,
"logits/rejected": -0.16672025620937347,
"logps/chosen": -212.42636108398438,
"logps/ref_chosen": -59.46582794189453,
"logps/ref_rejected": -97.03690338134766,
"logps/rejected": -359.866455078125,
"loss": 0.8634,
"margin_dpo/margin_mean": 109.86900329589844,
"margin_dpo/margin_std": 126.25106048583984,
"step": 300
},
{
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.008549506776034832,
"fcm_dpo/delta": -0.015921849757432938,
"fcm_dpo/margin": 118.4684829711914,
"fcm_dpo/q_t": 0.3066399097442627,
"grad_norm": 48.3686408996582,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.10468463599681854,
"logits/rejected": -0.09179846942424774,
"logps/chosen": -234.199462890625,
"logps/ref_chosen": -62.78144454956055,
"logps/ref_rejected": -91.95039367675781,
"logps/rejected": -381.8368835449219,
"loss": 0.8036,
"margin_dpo/margin_mean": 118.46849060058594,
"margin_dpo/margin_std": 118.95068359375,
"step": 305
},
{
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.0087824622169137,
"fcm_dpo/delta": 0.021257583051919937,
"fcm_dpo/margin": 111.6076889038086,
"fcm_dpo/q_t": 0.3067692816257477,
"grad_norm": 86.68696594238281,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.07544278353452682,
"logits/rejected": -0.053475070744752884,
"logps/chosen": -223.5658721923828,
"logps/ref_chosen": -61.359039306640625,
"logps/ref_rejected": -82.75496673583984,
"logps/rejected": -356.5694885253906,
"loss": 0.8047,
"margin_dpo/margin_mean": 111.6076889038086,
"margin_dpo/margin_std": 107.86360931396484,
"step": 310
},
{
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.008732178248465061,
"fcm_dpo/delta": 0.03534591943025589,
"fcm_dpo/margin": 110.84236145019531,
"fcm_dpo/q_t": 0.3149026334285736,
"grad_norm": 103.87069702148438,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.07893811911344528,
"logits/rejected": -0.06525443494319916,
"logps/chosen": -205.45095825195312,
"logps/ref_chosen": -51.77602005004883,
"logps/ref_rejected": -84.58292388916016,
"logps/rejected": -349.1002502441406,
"loss": 0.8408,
"margin_dpo/margin_mean": 110.84236145019531,
"margin_dpo/margin_std": 118.48417663574219,
"step": 315
},
{
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.008424820378422737,
"fcm_dpo/delta": -0.15117022395133972,
"fcm_dpo/margin": 134.76324462890625,
"fcm_dpo/q_t": 0.28069472312927246,
"grad_norm": 42.092960357666016,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.07524473965167999,
"logits/rejected": -0.06744035333395004,
"logps/chosen": -215.2461700439453,
"logps/ref_chosen": -56.777862548828125,
"logps/ref_rejected": -99.26368713378906,
"logps/rejected": -392.4952392578125,
"loss": 0.7257,
"margin_dpo/margin_mean": 134.7632598876953,
"margin_dpo/margin_std": 118.01239013671875,
"step": 320
},
{
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.007868105545639992,
"fcm_dpo/delta": 0.016687119379639626,
"fcm_dpo/margin": 124.93754577636719,
"fcm_dpo/q_t": 0.3126205801963806,
"grad_norm": 62.89131164550781,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": 0.0841856449842453,
"logits/rejected": 0.0996657982468605,
"logps/chosen": -293.92755126953125,
"logps/ref_chosen": -58.28468704223633,
"logps/ref_rejected": -83.80326843261719,
"logps/rejected": -444.3836364746094,
"loss": 0.8489,
"margin_dpo/margin_mean": 124.93754577636719,
"margin_dpo/margin_std": 132.80572509765625,
"step": 325
},
{
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.007621816359460354,
"fcm_dpo/delta": -0.08121494203805923,
"fcm_dpo/margin": 140.6494140625,
"fcm_dpo/q_t": 0.29238396883010864,
"grad_norm": 76.2926025390625,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": 0.003629236714914441,
"logits/rejected": 0.02080373466014862,
"logps/chosen": -265.3781433105469,
"logps/ref_chosen": -62.75822067260742,
"logps/ref_rejected": -94.04203033447266,
"logps/rejected": -437.31134033203125,
"loss": 0.7637,
"margin_dpo/margin_mean": 140.6494140625,
"margin_dpo/margin_std": 128.74356079101562,
"step": 330
},
{
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.007474665530025959,
"fcm_dpo/delta": 0.060898564755916595,
"fcm_dpo/margin": 126.42668151855469,
"fcm_dpo/q_t": 0.3141075074672699,
"grad_norm": 97.99361419677734,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": 0.01387687586247921,
"logits/rejected": 0.026036038994789124,
"logps/chosen": -255.6044921875,
"logps/ref_chosen": -58.59650421142578,
"logps/ref_rejected": -88.69586944580078,
"logps/rejected": -412.13055419921875,
"loss": 0.8295,
"margin_dpo/margin_mean": 126.42668151855469,
"margin_dpo/margin_std": 124.67610931396484,
"step": 335
},
{
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.007420250680297613,
"fcm_dpo/delta": -0.07010959088802338,
"fcm_dpo/margin": 143.0354766845703,
"fcm_dpo/q_t": 0.29213762283325195,
"grad_norm": 47.06379318237305,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": 0.0238631684333086,
"logits/rejected": 0.039838775992393494,
"logps/chosen": -247.1556854248047,
"logps/ref_chosen": -58.18162155151367,
"logps/ref_rejected": -94.44358825683594,
"logps/rejected": -426.453125,
"loss": 0.7682,
"margin_dpo/margin_mean": 143.03546142578125,
"margin_dpo/margin_std": 132.19140625,
"step": 340
},
{
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.007092430256307125,
"fcm_dpo/delta": -0.05085266754031181,
"fcm_dpo/margin": 147.26475524902344,
"fcm_dpo/q_t": 0.29424718022346497,
"grad_norm": 47.6697998046875,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.01976841688156128,
"logits/rejected": -0.010042941197752953,
"logps/chosen": -253.1570587158203,
"logps/ref_chosen": -57.9904899597168,
"logps/ref_rejected": -99.11092376708984,
"logps/rejected": -441.542236328125,
"loss": 0.7665,
"margin_dpo/margin_mean": 147.26475524902344,
"margin_dpo/margin_std": 133.5169219970703,
"step": 345
},
{
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.007030009292066097,
"fcm_dpo/delta": 0.02872460149228573,
"fcm_dpo/margin": 138.34315490722656,
"fcm_dpo/q_t": 0.3119054436683655,
"grad_norm": 56.26321792602539,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": 0.1253545731306076,
"logits/rejected": 0.13581940531730652,
"logps/chosen": -304.1668701171875,
"logps/ref_chosen": -58.29923629760742,
"logps/ref_rejected": -89.25711822509766,
"logps/rejected": -473.4678649902344,
"loss": 0.8404,
"margin_dpo/margin_mean": 138.34315490722656,
"margin_dpo/margin_std": 145.17849731445312,
"step": 350
},
{
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.007113814353942871,
"fcm_dpo/delta": -0.0113839628174901,
"fcm_dpo/margin": 141.8028106689453,
"fcm_dpo/q_t": 0.3037022054195404,
"grad_norm": 66.82807922363281,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": 0.024208009243011475,
"logits/rejected": 0.04891639202833176,
"logps/chosen": -290.4024658203125,
"logps/ref_chosen": -60.788482666015625,
"logps/ref_rejected": -85.94129943847656,
"logps/rejected": -457.3580627441406,
"loss": 0.8063,
"margin_dpo/margin_mean": 141.8028106689453,
"margin_dpo/margin_std": 138.77706909179688,
"step": 355
},
{
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.007339042611420155,
"fcm_dpo/delta": 0.027517419308423996,
"fcm_dpo/margin": 132.4971160888672,
"fcm_dpo/q_t": 0.3145308792591095,
"grad_norm": 56.25462341308594,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": 0.04696974158287048,
"logits/rejected": 0.06719812005758286,
"logps/chosen": -284.9296875,
"logps/ref_chosen": -57.6871337890625,
"logps/ref_rejected": -80.62527465820312,
"logps/rejected": -440.364990234375,
"loss": 0.8454,
"margin_dpo/margin_mean": 132.4971160888672,
"margin_dpo/margin_std": 141.62705993652344,
"step": 360
},
{
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.00699001457542181,
"fcm_dpo/delta": -0.04927719756960869,
"fcm_dpo/margin": 149.19345092773438,
"fcm_dpo/q_t": 0.2966029942035675,
"grad_norm": 49.64384460449219,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": 0.04224974662065506,
"logits/rejected": 0.04576627165079117,
"logps/chosen": -241.86990356445312,
"logps/ref_chosen": -51.490867614746094,
"logps/ref_rejected": -91.02871704101562,
"logps/rejected": -430.6012268066406,
"loss": 0.7741,
"margin_dpo/margin_mean": 149.19345092773438,
"margin_dpo/margin_std": 138.5751495361328,
"step": 365
},
{
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.007350992411375046,
"fcm_dpo/delta": 0.1297951340675354,
"fcm_dpo/margin": 120.12471771240234,
"fcm_dpo/q_t": 0.324402391910553,
"grad_norm": 54.650917053222656,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": 0.00014039436064194888,
"logits/rejected": 0.010272528044879436,
"logps/chosen": -229.25048828125,
"logps/ref_chosen": -58.113502502441406,
"logps/ref_rejected": -89.43451690673828,
"logps/rejected": -380.6962890625,
"loss": 0.855,
"margin_dpo/margin_mean": 120.1247329711914,
"margin_dpo/margin_std": 122.92936706542969,
"step": 370
},
{
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.007453347556293011,
"fcm_dpo/delta": -0.0778079703450203,
"fcm_dpo/margin": 143.21746826171875,
"fcm_dpo/q_t": 0.30160221457481384,
"grad_norm": 71.94976043701172,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": 0.05768465995788574,
"logits/rejected": 0.08099120855331421,
"logps/chosen": -273.6061706542969,
"logps/ref_chosen": -65.23600769042969,
"logps/ref_rejected": -89.24995422363281,
"logps/rejected": -440.83758544921875,
"loss": 0.8239,
"margin_dpo/margin_mean": 143.21746826171875,
"margin_dpo/margin_std": 150.7921600341797,
"step": 375
},
{
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.006744793616235256,
"fcm_dpo/delta": -0.134987011551857,
"fcm_dpo/margin": 166.04324340820312,
"fcm_dpo/q_t": 0.28603968024253845,
"grad_norm": 37.941612243652344,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": 0.21572455763816833,
"logits/rejected": 0.21860823035240173,
"logps/chosen": -317.9999694824219,
"logps/ref_chosen": -53.33893966674805,
"logps/ref_rejected": -102.15375518798828,
"logps/rejected": -532.8580322265625,
"loss": 0.7655,
"margin_dpo/margin_mean": 166.04324340820312,
"margin_dpo/margin_std": 156.43357849121094,
"step": 380
},
{
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.0064668795093894005,
"fcm_dpo/delta": 0.0269775390625,
"fcm_dpo/margin": 150.83981323242188,
"fcm_dpo/q_t": 0.30734530091285706,
"grad_norm": 49.370338439941406,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": 0.18446096777915955,
"logits/rejected": 0.1971583068370819,
"logps/chosen": -347.1589660644531,
"logps/ref_chosen": -58.36262130737305,
"logps/ref_rejected": -89.44685363769531,
"logps/rejected": -529.0830078125,
"loss": 0.8122,
"margin_dpo/margin_mean": 150.83981323242188,
"margin_dpo/margin_std": 148.15174865722656,
"step": 385
},
{
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.00651018600910902,
"fcm_dpo/delta": 0.006901745684444904,
"fcm_dpo/margin": 152.48130798339844,
"fcm_dpo/q_t": 0.30407440662384033,
"grad_norm": 77.2577896118164,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": 0.12399164587259293,
"logits/rejected": 0.1388627588748932,
"logps/chosen": -327.7897033691406,
"logps/ref_chosen": -56.380653381347656,
"logps/ref_rejected": -90.47447204589844,
"logps/rejected": -514.3648071289062,
"loss": 0.8056,
"margin_dpo/margin_mean": 152.48129272460938,
"margin_dpo/margin_std": 147.57839965820312,
"step": 390
},
{
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.0068787201307713985,
"fcm_dpo/delta": 0.04187867045402527,
"fcm_dpo/margin": 139.57540893554688,
"fcm_dpo/q_t": 0.31576859951019287,
"grad_norm": 77.93980407714844,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": 0.05469496175646782,
"logits/rejected": 0.06271407008171082,
"logps/chosen": -278.23992919921875,
"logps/ref_chosen": -55.95304489135742,
"logps/ref_rejected": -87.13162994384766,
"logps/rejected": -448.993896484375,
"loss": 0.8367,
"margin_dpo/margin_mean": 139.57540893554688,
"margin_dpo/margin_std": 147.34275817871094,
"step": 395
},
{
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.006794418208301067,
"fcm_dpo/delta": -0.05268029496073723,
"fcm_dpo/margin": 154.1851806640625,
"fcm_dpo/q_t": 0.2933744788169861,
"grad_norm": 71.72708129882812,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.016925137490034103,
"logits/rejected": 0.0033867184538394213,
"logps/chosen": -246.555908203125,
"logps/ref_chosen": -64.59160614013672,
"logps/ref_rejected": -96.700927734375,
"logps/rejected": -432.85040283203125,
"loss": 0.767,
"margin_dpo/margin_mean": 154.1851806640625,
"margin_dpo/margin_std": 139.50790405273438,
"step": 400
},
{
"epoch": 0.5873715124816447,
"eval_fcm_dpo/beta": 0.009887248277664185,
"eval_fcm_dpo/delta": 0.0760737881064415,
"eval_fcm_dpo/margin": 94.04965209960938,
"eval_fcm_dpo/q_t": 0.34363260865211487,
"eval_logits/chosen": 0.006849322468042374,
"eval_logits/rejected": 0.028473298996686935,
"eval_logps/chosen": -305.7621765136719,
"eval_logps/ref_chosen": -79.05104064941406,
"eval_logps/ref_rejected": -86.79793548583984,
"eval_logps/rejected": -407.5587158203125,
"eval_loss": 0.5232132077217102,
"eval_margin_dpo/margin_mean": 94.04964447021484,
"eval_margin_dpo/margin_std": 145.6568603515625,
"eval_runtime": 39.5117,
"eval_samples_per_second": 59.198,
"eval_steps_per_second": 1.873,
"step": 400
},
{
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.00876363180577755,
"fcm_dpo/delta": -0.39350950717926025,
"fcm_dpo/margin": 152.52651977539062,
"fcm_dpo/q_t": 0.2732911705970764,
"grad_norm": 80.5892333984375,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": 0.08245402574539185,
"logits/rejected": 0.09595631062984467,
"logps/chosen": -250.5602569580078,
"logps/ref_chosen": -53.61777877807617,
"logps/ref_rejected": -81.28938293457031,
"logps/rejected": -430.75836181640625,
"loss": 0.8047,
"margin_dpo/margin_mean": 152.5264892578125,
"margin_dpo/margin_std": 158.84365844726562,
"step": 405
},
{
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.007610198110342026,
"fcm_dpo/delta": -0.005237111356109381,
"fcm_dpo/margin": 131.73397827148438,
"fcm_dpo/q_t": 0.30888423323631287,
"grad_norm": 37.665645599365234,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": 0.03756345808506012,
"logits/rejected": 0.06168060749769211,
"logps/chosen": -268.45806884765625,
"logps/ref_chosen": -58.9287223815918,
"logps/ref_rejected": -85.55818176269531,
"logps/rejected": -426.82147216796875,
"loss": 0.8424,
"margin_dpo/margin_mean": 131.73397827148438,
"margin_dpo/margin_std": 140.30471801757812,
"step": 410
},
{
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.0075039928779006,
"fcm_dpo/delta": -0.041446395218372345,
"fcm_dpo/margin": 137.97164916992188,
"fcm_dpo/q_t": 0.30245256423950195,
"grad_norm": 57.69986343383789,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": 0.09034428745508194,
"logits/rejected": 0.1039205938577652,
"logps/chosen": -316.216064453125,
"logps/ref_chosen": -58.45662307739258,
"logps/ref_rejected": -93.67063903808594,
"logps/rejected": -489.40167236328125,
"loss": 0.8107,
"margin_dpo/margin_mean": 137.97164916992188,
"margin_dpo/margin_std": 139.51095581054688,
"step": 415
},
{
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.006540152244269848,
"fcm_dpo/delta": -0.1377246230840683,
"fcm_dpo/margin": 170.90652465820312,
"fcm_dpo/q_t": 0.287302702665329,
"grad_norm": 74.29497528076172,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": 0.19877588748931885,
"logits/rejected": 0.22694358229637146,
"logps/chosen": -374.59149169921875,
"logps/ref_chosen": -64.2349853515625,
"logps/ref_rejected": -84.86299133300781,
"logps/rejected": -566.1260986328125,
"loss": 0.7632,
"margin_dpo/margin_mean": 170.9065399169922,
"margin_dpo/margin_std": 162.65200805664062,
"step": 420
},
{
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.006726304534822702,
"fcm_dpo/delta": 0.09040592610836029,
"fcm_dpo/margin": 136.3487091064453,
"fcm_dpo/q_t": 0.32028037309646606,
"grad_norm": 45.669525146484375,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": 0.08228816837072372,
"logits/rejected": 0.09816490858793259,
"logps/chosen": -280.62823486328125,
"logps/ref_chosen": -56.054161071777344,
"logps/ref_rejected": -85.17119598388672,
"logps/rejected": -446.09393310546875,
"loss": 0.8659,
"margin_dpo/margin_mean": 136.3487091064453,
"margin_dpo/margin_std": 148.36148071289062,
"step": 425
},
{
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.007214270532131195,
"fcm_dpo/delta": 0.05701718479394913,
"fcm_dpo/margin": 131.32492065429688,
"fcm_dpo/q_t": 0.3152838349342346,
"grad_norm": 55.725528717041016,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.05021911859512329,
"logits/rejected": -0.02161559835076332,
"logps/chosen": -260.875,
"logps/ref_chosen": -69.24568176269531,
"logps/ref_rejected": -91.8664321899414,
"logps/rejected": -414.8206481933594,
"loss": 0.843,
"margin_dpo/margin_mean": 131.32492065429688,
"margin_dpo/margin_std": 136.3804473876953,
"step": 430
},
{
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.007029211614280939,
"fcm_dpo/delta": -0.056614942848682404,
"fcm_dpo/margin": 149.05471801757812,
"fcm_dpo/q_t": 0.295867919921875,
"grad_norm": 33.727622985839844,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": 0.04893326759338379,
"logits/rejected": 0.06856132298707962,
"logps/chosen": -266.8647155761719,
"logps/ref_chosen": -60.03449630737305,
"logps/ref_rejected": -90.6872329711914,
"logps/rejected": -446.5721740722656,
"loss": 0.7875,
"margin_dpo/margin_mean": 149.05471801757812,
"margin_dpo/margin_std": 141.10134887695312,
"step": 435
},
{
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.006822484079748392,
"fcm_dpo/delta": 0.013211876153945923,
"fcm_dpo/margin": 144.6233673095703,
"fcm_dpo/q_t": 0.30818983912467957,
"grad_norm": 47.22258758544922,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": 0.08384235203266144,
"logits/rejected": 0.1115858182311058,
"logps/chosen": -308.7546691894531,
"logps/ref_chosen": -65.50349426269531,
"logps/ref_rejected": -85.66627502441406,
"logps/rejected": -473.54083251953125,
"loss": 0.812,
"margin_dpo/margin_mean": 144.6233673095703,
"margin_dpo/margin_std": 141.59542846679688,
"step": 440
},
{
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.006704818457365036,
"fcm_dpo/delta": -0.0901242271065712,
"fcm_dpo/margin": 160.96018981933594,
"fcm_dpo/q_t": 0.2981015145778656,
"grad_norm": 49.451751708984375,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": 0.13737985491752625,
"logits/rejected": 0.1538257598876953,
"logps/chosen": -342.56353759765625,
"logps/ref_chosen": -60.72443389892578,
"logps/ref_rejected": -89.9255142211914,
"logps/rejected": -532.724853515625,
"loss": 0.8005,
"margin_dpo/margin_mean": 160.96018981933594,
"margin_dpo/margin_std": 165.419189453125,
"step": 445
},
{
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.006186770740896463,
"fcm_dpo/delta": -0.08101926743984222,
"fcm_dpo/margin": 173.30300903320312,
"fcm_dpo/q_t": 0.2926211357116699,
"grad_norm": 54.31269073486328,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": 0.17226827144622803,
"logits/rejected": 0.18348528444766998,
"logps/chosen": -358.87921142578125,
"logps/ref_chosen": -59.96248245239258,
"logps/ref_rejected": -98.71509552001953,
"logps/rejected": -570.9348754882812,
"loss": 0.77,
"margin_dpo/margin_mean": 173.30300903320312,
"margin_dpo/margin_std": 166.47640991210938,
"step": 450
},
{
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.005956509616225958,
"fcm_dpo/delta": -0.042925093322992325,
"fcm_dpo/margin": 174.29293823242188,
"fcm_dpo/q_t": 0.2964968979358673,
"grad_norm": 46.66022491455078,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": 0.21964845061302185,
"logits/rejected": 0.239833801984787,
"logps/chosen": -354.7815246582031,
"logps/ref_chosen": -54.48131561279297,
"logps/ref_rejected": -90.6321029663086,
"logps/rejected": -565.2252197265625,
"loss": 0.778,
"margin_dpo/margin_mean": 174.29290771484375,
"margin_dpo/margin_std": 163.45718383789062,
"step": 455
},
{
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.005550770089030266,
"fcm_dpo/delta": -0.038787856698036194,
"fcm_dpo/margin": 186.31370544433594,
"fcm_dpo/q_t": 0.29349032044410706,
"grad_norm": 44.3757438659668,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": 0.18749204277992249,
"logits/rejected": 0.1999995857477188,
"logps/chosen": -343.68719482421875,
"logps/ref_chosen": -52.97611618041992,
"logps/ref_rejected": -95.65971374511719,
"logps/rejected": -572.6843872070312,
"loss": 0.7581,
"margin_dpo/margin_mean": 186.313720703125,
"margin_dpo/margin_std": 163.4239044189453,
"step": 460
},
{
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.005766182206571102,
"fcm_dpo/delta": 0.007664884440600872,
"fcm_dpo/margin": 171.88845825195312,
"fcm_dpo/q_t": 0.30810102820396423,
"grad_norm": 41.693294525146484,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": 0.21744711697101593,
"logits/rejected": 0.22756743431091309,
"logps/chosen": -379.27606201171875,
"logps/ref_chosen": -58.2827033996582,
"logps/ref_rejected": -94.15567779541016,
"logps/rejected": -587.0375366210938,
"loss": 0.8147,
"margin_dpo/margin_mean": 171.88845825195312,
"margin_dpo/margin_std": 170.59255981445312,
"step": 465
},
{
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.00537510309368372,
"fcm_dpo/delta": -0.05350829288363457,
"fcm_dpo/margin": 194.76451110839844,
"fcm_dpo/q_t": 0.29564154148101807,
"grad_norm": 34.60992431640625,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": 0.20110762119293213,
"logits/rejected": 0.22162282466888428,
"logps/chosen": -405.76971435546875,
"logps/ref_chosen": -62.69774627685547,
"logps/ref_rejected": -96.33873748779297,
"logps/rejected": -634.1751708984375,
"loss": 0.7856,
"margin_dpo/margin_mean": 194.7645263671875,
"margin_dpo/margin_std": 185.07159423828125,
"step": 470
},
{
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.005407982971519232,
"fcm_dpo/delta": 0.04896858334541321,
"fcm_dpo/margin": 176.5228271484375,
"fcm_dpo/q_t": 0.31317138671875,
"grad_norm": 59.90799331665039,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": 0.13768652081489563,
"logits/rejected": 0.15576650202274323,
"logps/chosen": -395.1552734375,
"logps/ref_chosen": -61.12194061279297,
"logps/ref_rejected": -92.42192077636719,
"logps/rejected": -602.97802734375,
"loss": 0.8334,
"margin_dpo/margin_mean": 176.52281188964844,
"margin_dpo/margin_std": 181.145263671875,
"step": 475
},
{
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.005679582245647907,
"fcm_dpo/delta": 0.06495720893144608,
"fcm_dpo/margin": 165.47451782226562,
"fcm_dpo/q_t": 0.3155754506587982,
"grad_norm": 74.00138854980469,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": 0.15134042501449585,
"logits/rejected": 0.1730061173439026,
"logps/chosen": -374.6919250488281,
"logps/ref_chosen": -53.550628662109375,
"logps/ref_rejected": -82.65167999267578,
"logps/rejected": -569.2674560546875,
"loss": 0.8451,
"margin_dpo/margin_mean": 165.47451782226562,
"margin_dpo/margin_std": 170.61659240722656,
"step": 480
},
{
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.005770521704107523,
"fcm_dpo/delta": -0.054512571543455124,
"fcm_dpo/margin": 181.7024688720703,
"fcm_dpo/q_t": 0.2955150008201599,
"grad_norm": 38.076080322265625,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": 0.11906009912490845,
"logits/rejected": 0.14106717705726624,
"logps/chosen": -374.17266845703125,
"logps/ref_chosen": -60.76704788208008,
"logps/ref_rejected": -92.70649719238281,
"logps/rejected": -587.8145141601562,
"loss": 0.7769,
"margin_dpo/margin_mean": 181.7024688720703,
"margin_dpo/margin_std": 169.85366821289062,
"step": 485
},
{
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.005647027865052223,
"fcm_dpo/delta": 0.04806803911924362,
"fcm_dpo/margin": 169.46475219726562,
"fcm_dpo/q_t": 0.3089732229709625,
"grad_norm": 46.15083312988281,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": 0.08161328732967377,
"logits/rejected": 0.10188315063714981,
"logps/chosen": -354.3033447265625,
"logps/ref_chosen": -53.78142547607422,
"logps/ref_rejected": -91.67438507080078,
"logps/rejected": -561.6610717773438,
"loss": 0.808,
"margin_dpo/margin_mean": 169.46475219726562,
"margin_dpo/margin_std": 158.5917510986328,
"step": 490
},
{
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.006240219343453646,
"fcm_dpo/delta": 0.039695724844932556,
"fcm_dpo/margin": 153.99688720703125,
"fcm_dpo/q_t": 0.3135624825954437,
"grad_norm": 53.868595123291016,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": 0.12279490381479263,
"logits/rejected": 0.14274553954601288,
"logps/chosen": -352.9208679199219,
"logps/ref_chosen": -58.9004020690918,
"logps/ref_rejected": -82.15424346923828,
"logps/rejected": -530.171630859375,
"loss": 0.8564,
"margin_dpo/margin_mean": 153.99688720703125,
"margin_dpo/margin_std": 165.22085571289062,
"step": 495
},
{
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.0063318656757473946,
"fcm_dpo/delta": 0.0376526415348053,
"fcm_dpo/margin": 152.19857788085938,
"fcm_dpo/q_t": 0.3127899467945099,
"grad_norm": 49.95294189453125,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": 0.11964716762304306,
"logits/rejected": 0.13759367167949677,
"logps/chosen": -378.79486083984375,
"logps/ref_chosen": -62.13483810424805,
"logps/ref_rejected": -87.91773223876953,
"logps/rejected": -556.7762451171875,
"loss": 0.8433,
"margin_dpo/margin_mean": 152.19857788085938,
"margin_dpo/margin_std": 158.50917053222656,
"step": 500
},
{
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.005962314549833536,
"fcm_dpo/delta": -0.04856229946017265,
"fcm_dpo/margin": 174.53497314453125,
"fcm_dpo/q_t": 0.29466599225997925,
"grad_norm": 97.13284301757812,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": 0.1509219855070114,
"logits/rejected": 0.1782134473323822,
"logps/chosen": -379.09417724609375,
"logps/ref_chosen": -62.631813049316406,
"logps/ref_rejected": -87.59168243408203,
"logps/rejected": -578.5890502929688,
"loss": 0.7789,
"margin_dpo/margin_mean": 174.5349884033203,
"margin_dpo/margin_std": 157.0460205078125,
"step": 505
},
{
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.005955955013632774,
"fcm_dpo/delta": -0.02068263851106167,
"fcm_dpo/margin": 170.916748046875,
"fcm_dpo/q_t": 0.30395492911338806,
"grad_norm": 51.482826232910156,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": 0.1670006364583969,
"logits/rejected": 0.18064869940280914,
"logps/chosen": -393.43707275390625,
"logps/ref_chosen": -60.552574157714844,
"logps/ref_rejected": -91.0874252319336,
"logps/rejected": -594.8886108398438,
"loss": 0.7986,
"margin_dpo/margin_mean": 170.916748046875,
"margin_dpo/margin_std": 168.501953125,
"step": 510
},
{
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.005851385649293661,
"fcm_dpo/delta": 0.049037761986255646,
"fcm_dpo/margin": 163.44979858398438,
"fcm_dpo/q_t": 0.3152884542942047,
"grad_norm": 42.39580535888672,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": 0.17646808922290802,
"logits/rejected": 0.19573770463466644,
"logps/chosen": -373.1913757324219,
"logps/ref_chosen": -60.9382438659668,
"logps/ref_rejected": -83.36767578125,
"logps/rejected": -559.0706787109375,
"loss": 0.8517,
"margin_dpo/margin_mean": 163.44979858398438,
"margin_dpo/margin_std": 174.04823303222656,
"step": 515
},
{
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.005991171579807997,
"fcm_dpo/delta": -0.0007207244634628296,
"fcm_dpo/margin": 166.78158569335938,
"fcm_dpo/q_t": 0.30548095703125,
"grad_norm": 38.39859390258789,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": 0.09400401264429092,
"logits/rejected": 0.10684072971343994,
"logps/chosen": -342.5378112792969,
"logps/ref_chosen": -65.47642517089844,
"logps/ref_rejected": -98.10872650146484,
"logps/rejected": -541.9517211914062,
"loss": 0.8064,
"margin_dpo/margin_mean": 166.78158569335938,
"margin_dpo/margin_std": 163.54275512695312,
"step": 520
},
{
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.006436540279537439,
"fcm_dpo/delta": 0.037472836673259735,
"fcm_dpo/margin": 149.70407104492188,
"fcm_dpo/q_t": 0.3091326355934143,
"grad_norm": 40.531497955322266,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": 0.10942535102367401,
"logits/rejected": 0.13167142868041992,
"logps/chosen": -327.15350341796875,
"logps/ref_chosen": -59.072021484375,
"logps/ref_rejected": -82.8821792602539,
"logps/rejected": -500.66766357421875,
"loss": 0.8249,
"margin_dpo/margin_mean": 149.70407104492188,
"margin_dpo/margin_std": 147.34591674804688,
"step": 525
},
{
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.00635374104604125,
"fcm_dpo/delta": -0.006808423902839422,
"fcm_dpo/margin": 158.07611083984375,
"fcm_dpo/q_t": 0.30375149846076965,
"grad_norm": 47.252288818359375,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": 0.11862097680568695,
"logits/rejected": 0.14111658930778503,
"logps/chosen": -342.89794921875,
"logps/ref_chosen": -61.11234664916992,
"logps/ref_rejected": -87.01112365722656,
"logps/rejected": -526.872802734375,
"loss": 0.7995,
"margin_dpo/margin_mean": 158.07608032226562,
"margin_dpo/margin_std": 149.02304077148438,
"step": 530
},
{
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.006171065382659435,
"fcm_dpo/delta": -0.05086851119995117,
"fcm_dpo/margin": 169.0679168701172,
"fcm_dpo/q_t": 0.2947761118412018,
"grad_norm": 43.50486755371094,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": 0.13510656356811523,
"logits/rejected": 0.16101349890232086,
"logps/chosen": -341.5918884277344,
"logps/ref_chosen": -60.96736526489258,
"logps/ref_rejected": -81.727294921875,
"logps/rejected": -531.4197998046875,
"loss": 0.775,
"margin_dpo/margin_mean": 169.0679168701172,
"margin_dpo/margin_std": 153.8284912109375,
"step": 535
},
{
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.005736497696489096,
"fcm_dpo/delta": -0.06654427945613861,
"fcm_dpo/margin": 184.36936950683594,
"fcm_dpo/q_t": 0.2930430471897125,
"grad_norm": 64.07243347167969,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": 0.12933237850666046,
"logits/rejected": 0.1510297656059265,
"logps/chosen": -348.2706604003906,
"logps/ref_chosen": -58.64385986328125,
"logps/ref_rejected": -90.40965270996094,
"logps/rejected": -564.4058837890625,
"loss": 0.7675,
"margin_dpo/margin_mean": 184.36936950683594,
"margin_dpo/margin_std": 169.51486206054688,
"step": 540
},
{
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.005900930613279343,
"fcm_dpo/delta": 0.03893137350678444,
"fcm_dpo/margin": 163.19003295898438,
"fcm_dpo/q_t": 0.3134654462337494,
"grad_norm": 48.44090270996094,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": 0.14699451625347137,
"logits/rejected": 0.16024695336818695,
"logps/chosen": -394.77392578125,
"logps/ref_chosen": -64.73474884033203,
"logps/ref_rejected": -100.44208526611328,
"logps/rejected": -593.6712646484375,
"loss": 0.8348,
"margin_dpo/margin_mean": 163.19003295898438,
"margin_dpo/margin_std": 170.00575256347656,
"step": 545
},
{
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.00598874781280756,
"fcm_dpo/delta": 0.04262319952249527,
"fcm_dpo/margin": 160.43338012695312,
"fcm_dpo/q_t": 0.30985692143440247,
"grad_norm": 48.28050994873047,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": 0.1386023908853531,
"logits/rejected": 0.1567370593547821,
"logps/chosen": -380.16876220703125,
"logps/ref_chosen": -59.13951873779297,
"logps/ref_rejected": -87.15635681152344,
"logps/rejected": -568.6190185546875,
"loss": 0.8232,
"margin_dpo/margin_mean": 160.43338012695312,
"margin_dpo/margin_std": 160.04835510253906,
"step": 550
},
{
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.005587672349065542,
"fcm_dpo/delta": -0.13118276000022888,
"fcm_dpo/margin": 199.08775329589844,
"fcm_dpo/q_t": 0.2846675515174866,
"grad_norm": 65.35738372802734,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": 0.12665621936321259,
"logits/rejected": 0.147970512509346,
"logps/chosen": -381.3880615234375,
"logps/ref_chosen": -62.1995849609375,
"logps/ref_rejected": -102.51883697509766,
"logps/rejected": -620.7949829101562,
"loss": 0.7534,
"margin_dpo/margin_mean": 199.08773803710938,
"margin_dpo/margin_std": 179.85818481445312,
"step": 555
},
{
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.005411994643509388,
"fcm_dpo/delta": 0.03394172340631485,
"fcm_dpo/margin": 178.8895721435547,
"fcm_dpo/q_t": 0.30980902910232544,
"grad_norm": 38.78322219848633,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": 0.17429454624652863,
"logits/rejected": 0.19978392124176025,
"logps/chosen": -396.4398498535156,
"logps/ref_chosen": -61.541908264160156,
"logps/ref_rejected": -97.37491607666016,
"logps/rejected": -611.1624755859375,
"loss": 0.8088,
"margin_dpo/margin_mean": 178.88958740234375,
"margin_dpo/margin_std": 170.66427612304688,
"step": 560
},
{
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.0056685106828808784,
"fcm_dpo/delta": 3.868490603053942e-05,
"fcm_dpo/margin": 176.0917205810547,
"fcm_dpo/q_t": 0.3043767809867859,
"grad_norm": 44.58524703979492,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": 0.2256505936384201,
"logits/rejected": 0.2481400966644287,
"logps/chosen": -416.82061767578125,
"logps/ref_chosen": -62.85475540161133,
"logps/ref_rejected": -93.8392105102539,
"logps/rejected": -623.896728515625,
"loss": 0.7993,
"margin_dpo/margin_mean": 176.09170532226562,
"margin_dpo/margin_std": 167.57518005371094,
"step": 565
},
{
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.005463146138936281,
"fcm_dpo/delta": -0.029030922800302505,
"fcm_dpo/margin": 187.56187438964844,
"fcm_dpo/q_t": 0.30074894428253174,
"grad_norm": 44.8382682800293,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": 0.24869520962238312,
"logits/rejected": 0.2637079358100891,
"logps/chosen": -408.54278564453125,
"logps/ref_chosen": -57.98622512817383,
"logps/ref_rejected": -87.91555786132812,
"logps/rejected": -626.0340576171875,
"loss": 0.8067,
"margin_dpo/margin_mean": 187.56187438964844,
"margin_dpo/margin_std": 182.2978515625,
"step": 570
},
{
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.00532135646790266,
"fcm_dpo/delta": -0.016356075182557106,
"fcm_dpo/margin": 190.32020568847656,
"fcm_dpo/q_t": 0.3059103190898895,
"grad_norm": 91.26822662353516,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": 0.2546055018901825,
"logits/rejected": 0.2656412720680237,
"logps/chosen": -414.08648681640625,
"logps/ref_chosen": -55.194114685058594,
"logps/ref_rejected": -89.68229675292969,
"logps/rejected": -638.8948364257812,
"loss": 0.8292,
"margin_dpo/margin_mean": 190.32022094726562,
"margin_dpo/margin_std": 196.94891357421875,
"step": 575
},
{
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.005366818513721228,
"fcm_dpo/delta": 0.01412141602486372,
"fcm_dpo/margin": 183.69808959960938,
"fcm_dpo/q_t": 0.30419597029685974,
"grad_norm": 35.38365936279297,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": 0.18337729573249817,
"logits/rejected": 0.19277450442314148,
"logps/chosen": -378.4786682128906,
"logps/ref_chosen": -54.605796813964844,
"logps/ref_rejected": -88.9614486694336,
"logps/rejected": -596.5324096679688,
"loss": 0.8049,
"margin_dpo/margin_mean": 183.69808959960938,
"margin_dpo/margin_std": 174.64346313476562,
"step": 580
},
{
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.005487887188792229,
"fcm_dpo/delta": -0.006089937873184681,
"fcm_dpo/margin": 182.86058044433594,
"fcm_dpo/q_t": 0.30417701601982117,
"grad_norm": 45.972862243652344,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": 0.19554153084754944,
"logits/rejected": 0.21006889641284943,
"logps/chosen": -387.3260803222656,
"logps/ref_chosen": -59.03770065307617,
"logps/ref_rejected": -94.4029541015625,
"logps/rejected": -605.5518798828125,
"loss": 0.8099,
"margin_dpo/margin_mean": 182.860595703125,
"margin_dpo/margin_std": 176.96749877929688,
"step": 585
},
{
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.005434809252619743,
"fcm_dpo/delta": -0.017457369714975357,
"fcm_dpo/margin": 186.83787536621094,
"fcm_dpo/q_t": 0.302236407995224,
"grad_norm": 31.776384353637695,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": 0.1919538974761963,
"logits/rejected": 0.19437086582183838,
"logps/chosen": -378.33697509765625,
"logps/ref_chosen": -53.5163688659668,
"logps/ref_rejected": -99.90290832519531,
"logps/rejected": -611.5613403320312,
"loss": 0.7985,
"margin_dpo/margin_mean": 186.83786010742188,
"margin_dpo/margin_std": 180.22854614257812,
"step": 590
},
{
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.005393174476921558,
"fcm_dpo/delta": 0.045437589287757874,
"fcm_dpo/margin": 177.54502868652344,
"fcm_dpo/q_t": 0.31307360529899597,
"grad_norm": 40.15277099609375,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": 0.20439474284648895,
"logits/rejected": 0.22585907578468323,
"logps/chosen": -381.0555419921875,
"logps/ref_chosen": -51.44538497924805,
"logps/ref_rejected": -77.43083190917969,
"logps/rejected": -584.5860595703125,
"loss": 0.8451,
"margin_dpo/margin_mean": 177.5450439453125,
"margin_dpo/margin_std": 184.60360717773438,
"step": 595
},
{
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.005620202049612999,
"fcm_dpo/delta": 0.03544490784406662,
"fcm_dpo/margin": 171.91168212890625,
"fcm_dpo/q_t": 0.30969831347465515,
"grad_norm": 65.1519775390625,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": 0.1829279512166977,
"logits/rejected": 0.19534242153167725,
"logps/chosen": -395.7161560058594,
"logps/ref_chosen": -57.161705017089844,
"logps/ref_rejected": -87.73274230957031,
"logps/rejected": -598.1988525390625,
"loss": 0.8367,
"margin_dpo/margin_mean": 171.91168212890625,
"margin_dpo/margin_std": 174.3584747314453,
"step": 600
},
{
"epoch": 0.8810572687224669,
"eval_fcm_dpo/beta": 0.007840755395591259,
"eval_fcm_dpo/delta": 0.05926959961652756,
"eval_fcm_dpo/margin": 120.67913818359375,
"eval_fcm_dpo/q_t": 0.33643776178359985,
"eval_logits/chosen": 0.15810680389404297,
"eval_logits/rejected": 0.17932601273059845,
"eval_logps/chosen": -446.58221435546875,
"eval_logps/ref_chosen": -79.05104064941406,
"eval_logps/ref_rejected": -86.79793548583984,
"eval_logps/rejected": -575.0082397460938,
"eval_loss": 0.4979900121688843,
"eval_margin_dpo/margin_mean": 120.67913818359375,
"eval_margin_dpo/margin_std": 173.73777770996094,
"eval_runtime": 39.5266,
"eval_samples_per_second": 59.175,
"eval_steps_per_second": 1.872,
"step": 600
},
{
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.006911611650139093,
"fcm_dpo/delta": -0.3668617904186249,
"fcm_dpo/margin": 190.3062286376953,
"fcm_dpo/q_t": 0.26509329676628113,
"grad_norm": 42.82274627685547,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": 0.15809786319732666,
"logits/rejected": 0.1699742078781128,
"logps/chosen": -384.10858154296875,
"logps/ref_chosen": -58.169830322265625,
"logps/ref_rejected": -95.36891174316406,
"logps/rejected": -611.6138916015625,
"loss": 0.7306,
"margin_dpo/margin_mean": 190.3062286376953,
"margin_dpo/margin_std": 184.19656372070312,
"step": 605
},
{
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.005817721597850323,
"fcm_dpo/delta": -0.072767473757267,
"fcm_dpo/margin": 182.8505401611328,
"fcm_dpo/q_t": 0.28939151763916016,
"grad_norm": 43.43704605102539,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": 0.1247420683503151,
"logits/rejected": 0.14462068676948547,
"logps/chosen": -379.78411865234375,
"logps/ref_chosen": -58.97087860107422,
"logps/ref_rejected": -89.0286865234375,
"logps/rejected": -592.6925048828125,
"loss": 0.7518,
"margin_dpo/margin_mean": 182.8505401611328,
"margin_dpo/margin_std": 159.82778930664062,
"step": 610
},
{
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.0054991478100419044,
"fcm_dpo/delta": -0.02074270322918892,
"fcm_dpo/margin": 185.0585174560547,
"fcm_dpo/q_t": 0.2946159839630127,
"grad_norm": 36.950138092041016,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": 0.16577747464179993,
"logits/rejected": 0.17569129168987274,
"logps/chosen": -386.177490234375,
"logps/ref_chosen": -58.081878662109375,
"logps/ref_rejected": -95.92155456542969,
"logps/rejected": -609.0756225585938,
"loss": 0.7557,
"margin_dpo/margin_mean": 185.0585174560547,
"margin_dpo/margin_std": 154.9835205078125,
"step": 615
},
{
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.005858595483005047,
"fcm_dpo/delta": 0.04625372961163521,
"fcm_dpo/margin": 163.02841186523438,
"fcm_dpo/q_t": 0.31683021783828735,
"grad_norm": 54.52793884277344,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": 0.1606747955083847,
"logits/rejected": 0.18675543367862701,
"logps/chosen": -407.07562255859375,
"logps/ref_chosen": -62.203094482421875,
"logps/ref_rejected": -80.53683471679688,
"logps/rejected": -588.4378051757812,
"loss": 0.845,
"margin_dpo/margin_mean": 163.02841186523438,
"margin_dpo/margin_std": 172.63064575195312,
"step": 620
},
{
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.0056380960159003735,
"fcm_dpo/delta": -0.03593330457806587,
"fcm_dpo/margin": 182.74427795410156,
"fcm_dpo/q_t": 0.299526184797287,
"grad_norm": 51.273494720458984,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": 0.17915502190589905,
"logits/rejected": 0.20225441455841064,
"logps/chosen": -396.8714599609375,
"logps/ref_chosen": -61.727455139160156,
"logps/ref_rejected": -88.4387435913086,
"logps/rejected": -606.3270263671875,
"loss": 0.7929,
"margin_dpo/margin_mean": 182.74429321289062,
"margin_dpo/margin_std": 176.95751953125,
"step": 625
},
{
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.0057447524741292,
"fcm_dpo/delta": 0.04814302176237106,
"fcm_dpo/margin": 166.31198120117188,
"fcm_dpo/q_t": 0.3164430558681488,
"grad_norm": 65.81224822998047,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": 0.12732776999473572,
"logits/rejected": 0.14292660355567932,
"logps/chosen": -390.0010070800781,
"logps/ref_chosen": -61.30865478515625,
"logps/ref_rejected": -96.54997253417969,
"logps/rejected": -591.5543212890625,
"loss": 0.8342,
"margin_dpo/margin_mean": 166.31198120117188,
"margin_dpo/margin_std": 171.20651245117188,
"step": 630
},
{
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.0062016574665904045,
"fcm_dpo/delta": 0.09587004035711288,
"fcm_dpo/margin": 147.2493896484375,
"fcm_dpo/q_t": 0.3260436952114105,
"grad_norm": 54.04371643066406,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": 0.1764940768480301,
"logits/rejected": 0.1979142725467682,
"logps/chosen": -417.3326721191406,
"logps/ref_chosen": -63.7315673828125,
"logps/ref_rejected": -89.66435241699219,
"logps/rejected": -590.5148315429688,
"loss": 0.9008,
"margin_dpo/margin_mean": 147.2493896484375,
"margin_dpo/margin_std": 174.3343963623047,
"step": 635
},
{
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.006202409043908119,
"fcm_dpo/delta": -0.1008889302611351,
"fcm_dpo/margin": 175.59921264648438,
"fcm_dpo/q_t": 0.28956547379493713,
"grad_norm": 41.55816650390625,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": 0.1386783868074417,
"logits/rejected": 0.16345861554145813,
"logps/chosen": -397.62249755859375,
"logps/ref_chosen": -59.17915725708008,
"logps/ref_rejected": -88.51210021972656,
"logps/rejected": -602.5546875,
"loss": 0.7647,
"margin_dpo/margin_mean": 175.59921264648438,
"margin_dpo/margin_std": 161.22811889648438,
"step": 640
},
{
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.006019303109496832,
"fcm_dpo/delta": 0.06824000179767609,
"fcm_dpo/margin": 155.67410278320312,
"fcm_dpo/q_t": 0.3227546215057373,
"grad_norm": 61.321075439453125,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": 0.1542472094297409,
"logits/rejected": 0.164066344499588,
"logps/chosen": -400.74029541015625,
"logps/ref_chosen": -59.50596237182617,
"logps/ref_rejected": -93.92404174804688,
"logps/rejected": -590.8323974609375,
"loss": 0.8788,
"margin_dpo/margin_mean": 155.67410278320312,
"margin_dpo/margin_std": 177.89688110351562,
"step": 645
},
{
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.006101357284933329,
"fcm_dpo/delta": -0.057907313108444214,
"fcm_dpo/margin": 172.21653747558594,
"fcm_dpo/q_t": 0.30463922023773193,
"grad_norm": 51.276302337646484,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": 0.19726604223251343,
"logits/rejected": 0.21199996769428253,
"logps/chosen": -395.1019592285156,
"logps/ref_chosen": -57.774566650390625,
"logps/ref_rejected": -89.61600494384766,
"logps/rejected": -599.1598510742188,
"loss": 0.8238,
"margin_dpo/margin_mean": 172.21653747558594,
"margin_dpo/margin_std": 183.01187133789062,
"step": 650
},
{
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.005755548365414143,
"fcm_dpo/delta": -0.010833797045052052,
"fcm_dpo/margin": 175.18466186523438,
"fcm_dpo/q_t": 0.31061989068984985,
"grad_norm": 52.280006408691406,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": 0.1898464411497116,
"logits/rejected": 0.194228395819664,
"logps/chosen": -388.0377502441406,
"logps/ref_chosen": -55.68403244018555,
"logps/ref_rejected": -102.4081802368164,
"logps/rejected": -609.9465942382812,
"loss": 0.8261,
"margin_dpo/margin_mean": 175.18466186523438,
"margin_dpo/margin_std": 185.63772583007812,
"step": 655
},
{
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.0053447578102350235,
"fcm_dpo/delta": -0.11341585963964462,
"fcm_dpo/margin": 205.26394653320312,
"fcm_dpo/q_t": 0.2793430685997009,
"grad_norm": 40.472389221191406,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": 0.19943654537200928,
"logits/rejected": 0.2189749777317047,
"logps/chosen": -392.583740234375,
"logps/ref_chosen": -59.19981002807617,
"logps/ref_rejected": -94.19200134277344,
"logps/rejected": -632.83984375,
"loss": 0.7141,
"margin_dpo/margin_mean": 205.26394653320312,
"margin_dpo/margin_std": 164.3708038330078,
"step": 660
},
{
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.005307146348059177,
"fcm_dpo/delta": 0.015874262899160385,
"fcm_dpo/margin": 185.51779174804688,
"fcm_dpo/q_t": 0.30432578921318054,
"grad_norm": 51.16931915283203,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": 0.1688624769449234,
"logits/rejected": 0.19106578826904297,
"logps/chosen": -390.56341552734375,
"logps/ref_chosen": -61.2533073425293,
"logps/ref_rejected": -95.86351013183594,
"logps/rejected": -610.6913452148438,
"loss": 0.799,
"margin_dpo/margin_mean": 185.51779174804688,
"margin_dpo/margin_std": 175.6309051513672,
"step": 665
},
{
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.005404268391430378,
"fcm_dpo/delta": -0.018390949815511703,
"fcm_dpo/margin": 187.98519897460938,
"fcm_dpo/q_t": 0.2963741421699524,
"grad_norm": 42.112640380859375,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": 0.143265038728714,
"logits/rejected": 0.16587090492248535,
"logps/chosen": -393.91961669921875,
"logps/ref_chosen": -62.95263671875,
"logps/ref_rejected": -92.4662094116211,
"logps/rejected": -611.4183959960938,
"loss": 0.7762,
"margin_dpo/margin_mean": 187.98519897460938,
"margin_dpo/margin_std": 167.12362670898438,
"step": 670
},
{
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.005253970623016357,
"fcm_dpo/delta": -0.041876502335071564,
"fcm_dpo/margin": 197.55996704101562,
"fcm_dpo/q_t": 0.2921511232852936,
"grad_norm": 46.28888702392578,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": 0.21673651039600372,
"logits/rejected": 0.23396320641040802,
"logps/chosen": -382.89788818359375,
"logps/ref_chosen": -48.5856819152832,
"logps/ref_rejected": -81.27871704101562,
"logps/rejected": -613.1509399414062,
"loss": 0.759,
"margin_dpo/margin_mean": 197.55996704101562,
"margin_dpo/margin_std": 170.1479949951172,
"step": 675
},
{
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.0050786687061190605,
"fcm_dpo/delta": 0.0362386479973793,
"fcm_dpo/margin": 190.45541381835938,
"fcm_dpo/q_t": 0.30515041947364807,
"grad_norm": 44.39472961425781,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": 0.14481225609779358,
"logits/rejected": 0.16777487099170685,
"logps/chosen": -393.9770202636719,
"logps/ref_chosen": -60.25421905517578,
"logps/ref_rejected": -87.23457336425781,
"logps/rejected": -611.4127807617188,
"loss": 0.7908,
"margin_dpo/margin_mean": 190.45541381835938,
"margin_dpo/margin_std": 171.58912658691406,
"step": 680
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 0.846195821282447,
"train_runtime": 1845.654,
"train_samples_per_second": 23.622,
"train_steps_per_second": 0.369
}
],
"logging_steps": 5,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink