W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.45-20260428-045924
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC 52da0a5b5d 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.45-20260428-045924
Source: Original Platform
2026-05-22 20:24:35 +08:00

15247 lines
571 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.999244142101285,
"eval_steps": 200,
"global_step": 661,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.02867889404296875,
"KL/mean": 0.029354453086853027,
"KL/rejected_KL_mean": 0.030029296875,
"KL/std": 0.2071000635623932,
"epoch": 0.0015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0013532638549804688,
"fcm_dpo/q_t": 0.5000336766242981,
"grad_norm": 28.22066879272461,
"learning_rate": 0.0,
"logits/chosen": 0.13337239623069763,
"logits/rejected": 0.12492949515581131,
"logps/chosen": -64.5841293334961,
"logps/ref_chosen": -64.61280822753906,
"logps/ref_rejected": -64.17195129394531,
"logps/rejected": -64.14192199707031,
"loss": 1.3866,
"margin_dpo/margin_mean": -0.0013527870178222656,
"margin_dpo/margin_std": 0.2561596930027008,
"step": 1
},
{
"KL/chosen_KL_mean": -0.00289154052734375,
"KL/mean": -0.021616414189338684,
"KL/rejected_KL_mean": -0.04033660888671875,
"KL/std": 0.19624735414981842,
"epoch": 0.0030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.037450045347213745,
"fcm_dpo/q_t": 0.4990639090538025,
"grad_norm": 27.826658248901367,
"learning_rate": 7.462686567164179e-09,
"logits/chosen": 0.09414851665496826,
"logits/rejected": 0.07363267242908478,
"logps/chosen": -56.101890563964844,
"logps/ref_chosen": -56.0989990234375,
"logps/ref_rejected": -66.59971618652344,
"logps/rejected": -66.64006042480469,
"loss": 1.3828,
"margin_dpo/margin_mean": 0.03744968771934509,
"margin_dpo/margin_std": 0.27811938524246216,
"step": 2
},
{
"KL/chosen_KL_mean": 0.04264068603515625,
"KL/mean": 0.053322210907936096,
"KL/rejected_KL_mean": 0.06399917602539062,
"KL/std": 0.2611844837665558,
"epoch": 0.0045351473922902496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.021352916955947876,
"fcm_dpo/q_t": 0.5005335211753845,
"grad_norm": 31.28985023498535,
"learning_rate": 1.4925373134328357e-08,
"logits/chosen": 0.09941151738166809,
"logits/rejected": 0.061459895223379135,
"logps/chosen": -65.41461944580078,
"logps/ref_chosen": -65.45726013183594,
"logps/ref_rejected": -90.82853698730469,
"logps/rejected": -90.76453399658203,
"loss": 1.3888,
"margin_dpo/margin_mean": -0.021352916955947876,
"margin_dpo/margin_std": 0.36793452501296997,
"step": 3
},
{
"KL/chosen_KL_mean": 0.02747344970703125,
"KL/mean": 0.011432260274887085,
"KL/rejected_KL_mean": -0.00461578369140625,
"KL/std": 0.216526597738266,
"epoch": 0.006046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.032090187072753906,
"fcm_dpo/q_t": 0.4991982579231262,
"grad_norm": 34.291229248046875,
"learning_rate": 2.2388059701492534e-08,
"logits/chosen": 0.1126030832529068,
"logits/rejected": 0.09631362557411194,
"logps/chosen": -76.83271026611328,
"logps/ref_chosen": -76.86018371582031,
"logps/ref_rejected": -79.91523742675781,
"logps/rejected": -79.91985321044922,
"loss": 1.3833,
"margin_dpo/margin_mean": 0.03208950161933899,
"margin_dpo/margin_std": 0.3081591725349426,
"step": 4
},
{
"KL/chosen_KL_mean": -0.019184112548828125,
"KL/mean": 0.0005896538496017456,
"KL/rejected_KL_mean": 0.020366668701171875,
"KL/std": 0.24410542845726013,
"epoch": 0.007558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03955063223838806,
"fcm_dpo/q_t": 0.5009881854057312,
"grad_norm": 29.674455642700195,
"learning_rate": 2.9850746268656714e-08,
"logits/chosen": 0.07852072268724442,
"logits/rejected": 0.039799898862838745,
"logps/chosen": -62.99052429199219,
"logps/ref_chosen": -62.97134017944336,
"logps/ref_rejected": -79.9192123413086,
"logps/rejected": -79.89884948730469,
"loss": 1.3905,
"margin_dpo/margin_mean": -0.03954946994781494,
"margin_dpo/margin_std": 0.3197075128555298,
"step": 5
},
{
"KL/chosen_KL_mean": -0.0473175048828125,
"KL/mean": -0.04120214283466339,
"KL/rejected_KL_mean": -0.03508567810058594,
"KL/std": 0.24611763656139374,
"epoch": 0.009070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.012232571840286255,
"fcm_dpo/q_t": 0.5003060698509216,
"grad_norm": 29.636795043945312,
"learning_rate": 3.731343283582089e-08,
"logits/chosen": 0.16185824573040009,
"logits/rejected": 0.12125864624977112,
"logps/chosen": -51.35468292236328,
"logps/ref_chosen": -51.30736541748047,
"logps/ref_rejected": -82.77239227294922,
"logps/rejected": -82.80747985839844,
"loss": 1.3878,
"margin_dpo/margin_mean": -0.012232691049575806,
"margin_dpo/margin_std": 0.35705843567848206,
"step": 6
},
{
"KL/chosen_KL_mean": 0.027551651000976562,
"KL/mean": 0.02225738763809204,
"KL/rejected_KL_mean": 0.016962051391601562,
"KL/std": 0.2204289585351944,
"epoch": 0.010582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01059180498123169,
"fcm_dpo/q_t": 0.49973538517951965,
"grad_norm": 27.118959426879883,
"learning_rate": 4.477611940298507e-08,
"logits/chosen": 0.01888515241444111,
"logits/rejected": -0.025128159672021866,
"logps/chosen": -51.431861877441406,
"logps/ref_chosen": -51.45941162109375,
"logps/ref_rejected": -66.3828125,
"logps/rejected": -66.3658447265625,
"loss": 1.3855,
"margin_dpo/margin_mean": 0.0105915367603302,
"margin_dpo/margin_std": 0.29007700085639954,
"step": 7
},
{
"KL/chosen_KL_mean": -0.00579833984375,
"KL/mean": -0.006937950849533081,
"KL/rejected_KL_mean": -0.008075714111328125,
"KL/std": 0.2304605394601822,
"epoch": 0.012093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0022736191749572754,
"fcm_dpo/q_t": 0.4999435245990753,
"grad_norm": 28.647260665893555,
"learning_rate": 5.223880597014925e-08,
"logits/chosen": 0.06418946385383606,
"logits/rejected": 0.042302586138248444,
"logps/chosen": -62.203346252441406,
"logps/ref_chosen": -62.197547912597656,
"logps/ref_rejected": -74.66180419921875,
"logps/rejected": -74.66988372802734,
"loss": 1.3863,
"margin_dpo/margin_mean": 0.002272665500640869,
"margin_dpo/margin_std": 0.29995080828666687,
"step": 8
},
{
"KL/chosen_KL_mean": -0.039676666259765625,
"KL/mean": -0.03592316806316376,
"KL/rejected_KL_mean": -0.03217315673828125,
"KL/std": 0.22302508354187012,
"epoch": 0.013605442176870748,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.007506042718887329,
"fcm_dpo/q_t": 0.5001875162124634,
"grad_norm": 31.57317352294922,
"learning_rate": 5.970149253731343e-08,
"logits/chosen": 0.17316150665283203,
"logits/rejected": 0.11361009627580643,
"logps/chosen": -55.669395446777344,
"logps/ref_chosen": -55.629722595214844,
"logps/ref_rejected": -86.21221923828125,
"logps/rejected": -86.24440002441406,
"loss": 1.3873,
"margin_dpo/margin_mean": -0.007505506277084351,
"margin_dpo/margin_std": 0.32757535576820374,
"step": 9
},
{
"KL/chosen_KL_mean": -0.015594482421875,
"KL/mean": 0.0028478801250457764,
"KL/rejected_KL_mean": 0.021289825439453125,
"KL/std": 0.2507847547531128,
"epoch": 0.015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03688693046569824,
"fcm_dpo/q_t": 0.5009225010871887,
"grad_norm": 29.66307830810547,
"learning_rate": 6.71641791044776e-08,
"logits/chosen": 0.15237724781036377,
"logits/rejected": 0.12025925517082214,
"logps/chosen": -62.70619583129883,
"logps/ref_chosen": -62.69060134887695,
"logps/ref_rejected": -90.610107421875,
"logps/rejected": -90.58882141113281,
"loss": 1.3903,
"margin_dpo/margin_mean": -0.03688687086105347,
"margin_dpo/margin_std": 0.34781166911125183,
"step": 10
},
{
"KL/chosen_KL_mean": -0.04038047790527344,
"KL/mean": -0.027389228343963623,
"KL/rejected_KL_mean": -0.014400482177734375,
"KL/std": 0.18189536035060883,
"epoch": 0.016628873771730914,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.025982022285461426,
"fcm_dpo/q_t": 0.5006492733955383,
"grad_norm": 29.398035049438477,
"learning_rate": 7.462686567164178e-08,
"logits/chosen": 0.12077412009239197,
"logits/rejected": 0.11373384296894073,
"logps/chosen": -65.8074951171875,
"logps/ref_chosen": -65.76712036132812,
"logps/ref_rejected": -72.4764633178711,
"logps/rejected": -72.49085998535156,
"loss": 1.3891,
"margin_dpo/margin_mean": -0.025981128215789795,
"margin_dpo/margin_std": 0.24610239267349243,
"step": 11
},
{
"KL/chosen_KL_mean": -0.018270492553710938,
"KL/mean": -0.028908535838127136,
"KL/rejected_KL_mean": -0.03954887390136719,
"KL/std": 0.2320721447467804,
"epoch": 0.018140589569160998,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02127799391746521,
"fcm_dpo/q_t": 0.49946796894073486,
"grad_norm": 28.293529510498047,
"learning_rate": 8.208955223880596e-08,
"logits/chosen": 0.00675106979906559,
"logits/rejected": -0.008964652195572853,
"logps/chosen": -60.72315979003906,
"logps/ref_chosen": -60.704891204833984,
"logps/ref_rejected": -69.41564178466797,
"logps/rejected": -69.45519256591797,
"loss": 1.3844,
"margin_dpo/margin_mean": 0.021277964115142822,
"margin_dpo/margin_std": 0.2855193614959717,
"step": 12
},
{
"KL/chosen_KL_mean": -0.014070510864257812,
"KL/mean": 0.011509418487548828,
"KL/rejected_KL_mean": 0.0370941162109375,
"KL/std": 0.2110683023929596,
"epoch": 0.019652305366591082,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.051162004470825195,
"fcm_dpo/q_t": 0.5012787580490112,
"grad_norm": 29.074474334716797,
"learning_rate": 8.955223880597014e-08,
"logits/chosen": 0.11439329385757446,
"logits/rejected": 0.05134594440460205,
"logps/chosen": -49.92332458496094,
"logps/ref_chosen": -49.90925598144531,
"logps/ref_rejected": -92.37818145751953,
"logps/rejected": -92.3410873413086,
"loss": 1.3916,
"margin_dpo/margin_mean": -0.051162540912628174,
"margin_dpo/margin_std": 0.29525551199913025,
"step": 13
},
{
"KL/chosen_KL_mean": 0.0183868408203125,
"KL/mean": -0.0022123605012893677,
"KL/rejected_KL_mean": -0.022808074951171875,
"KL/std": 0.20953845977783203,
"epoch": 0.021164021164021163,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04119846224784851,
"fcm_dpo/q_t": 0.49897050857543945,
"grad_norm": 29.494983673095703,
"learning_rate": 9.701492537313432e-08,
"logits/chosen": 0.09920904040336609,
"logits/rejected": 0.0811566635966301,
"logps/chosen": -60.600406646728516,
"logps/ref_chosen": -60.61879348754883,
"logps/ref_rejected": -71.79306030273438,
"logps/rejected": -71.81587219238281,
"loss": 1.3824,
"margin_dpo/margin_mean": 0.041198402643203735,
"margin_dpo/margin_std": 0.2794230878353119,
"step": 14
},
{
"KL/chosen_KL_mean": -0.027385711669921875,
"KL/mean": -0.029895320534706116,
"KL/rejected_KL_mean": -0.03240966796875,
"KL/std": 0.22818677127361298,
"epoch": 0.022675736961451247,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.005019158124923706,
"fcm_dpo/q_t": 0.49987471103668213,
"grad_norm": 33.24843215942383,
"learning_rate": 1.044776119402985e-07,
"logits/chosen": 0.08923092484474182,
"logits/rejected": 0.04480152577161789,
"logps/chosen": -63.496925354003906,
"logps/ref_chosen": -63.46953582763672,
"logps/ref_rejected": -88.88951110839844,
"logps/rejected": -88.92192077636719,
"loss": 1.3861,
"margin_dpo/margin_mean": 0.005019336938858032,
"margin_dpo/margin_std": 0.3223441541194916,
"step": 15
},
{
"KL/chosen_KL_mean": -0.00457763671875,
"KL/mean": -0.01490524411201477,
"KL/rejected_KL_mean": -0.025234222412109375,
"KL/std": 0.20558756589889526,
"epoch": 0.02418745275888133,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.020651936531066895,
"fcm_dpo/q_t": 0.4994843006134033,
"grad_norm": 26.84224510192871,
"learning_rate": 1.1194029850746268e-07,
"logits/chosen": 0.121390700340271,
"logits/rejected": 0.08392910659313202,
"logps/chosen": -46.5368766784668,
"logps/ref_chosen": -46.53229904174805,
"logps/ref_rejected": -74.27533721923828,
"logps/rejected": -74.30056762695312,
"loss": 1.3845,
"margin_dpo/margin_mean": 0.020652025938034058,
"margin_dpo/margin_std": 0.29893872141838074,
"step": 16
},
{
"KL/chosen_KL_mean": -0.013319015502929688,
"KL/mean": -0.0209941565990448,
"KL/rejected_KL_mean": -0.028667449951171875,
"KL/std": 0.22113090753555298,
"epoch": 0.025699168556311415,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.015350341796875,
"fcm_dpo/q_t": 0.4996163547039032,
"grad_norm": 32.962730407714844,
"learning_rate": 1.1940298507462686e-07,
"logits/chosen": 0.06988856196403503,
"logits/rejected": 0.05055886507034302,
"logps/chosen": -64.09115600585938,
"logps/ref_chosen": -64.07783508300781,
"logps/ref_rejected": -86.40876770019531,
"logps/rejected": -86.43743896484375,
"loss": 1.385,
"margin_dpo/margin_mean": 0.015350043773651123,
"margin_dpo/margin_std": 0.3078957796096802,
"step": 17
},
{
"KL/chosen_KL_mean": 0.03248405456542969,
"KL/mean": 0.027805477380752563,
"KL/rejected_KL_mean": 0.02312469482421875,
"KL/std": 0.17294234037399292,
"epoch": 0.027210884353741496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.009357690811157227,
"fcm_dpo/q_t": 0.4997663199901581,
"grad_norm": 27.813247680664062,
"learning_rate": 1.2686567164179106e-07,
"logits/chosen": 0.0905240923166275,
"logits/rejected": 0.04524059593677521,
"logps/chosen": -44.84185028076172,
"logps/ref_chosen": -44.87433624267578,
"logps/ref_rejected": -70.97604370117188,
"logps/rejected": -70.95292663574219,
"loss": 1.3855,
"margin_dpo/margin_mean": 0.009358137845993042,
"margin_dpo/margin_std": 0.2632210850715637,
"step": 18
},
{
"KL/chosen_KL_mean": 0.03167915344238281,
"KL/mean": 0.002976052463054657,
"KL/rejected_KL_mean": -0.025730133056640625,
"KL/std": 0.2518116235733032,
"epoch": 0.02872260015117158,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.057412102818489075,
"fcm_dpo/q_t": 0.498563677072525,
"grad_norm": 30.920751571655273,
"learning_rate": 1.343283582089552e-07,
"logits/chosen": 0.06809468567371368,
"logits/rejected": 0.05450137332081795,
"logps/chosen": -68.12812805175781,
"logps/ref_chosen": -68.1598129272461,
"logps/ref_rejected": -81.17138671875,
"logps/rejected": -81.1971206665039,
"loss": 1.3809,
"margin_dpo/margin_mean": 0.057412490248680115,
"margin_dpo/margin_std": 0.3499699831008911,
"step": 19
},
{
"KL/chosen_KL_mean": 0.024566650390625,
"KL/mean": 0.012526951730251312,
"KL/rejected_KL_mean": 0.000492095947265625,
"KL/std": 0.18837958574295044,
"epoch": 0.030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.024075254797935486,
"fcm_dpo/q_t": 0.49939823150634766,
"grad_norm": 29.136823654174805,
"learning_rate": 1.4179104477611938e-07,
"logits/chosen": 0.1479886770248413,
"logits/rejected": 0.12440957129001617,
"logps/chosen": -53.653995513916016,
"logps/ref_chosen": -53.67856216430664,
"logps/ref_rejected": -74.16911315917969,
"logps/rejected": -74.16862487792969,
"loss": 1.384,
"margin_dpo/margin_mean": 0.02407585084438324,
"margin_dpo/margin_std": 0.24943657219409943,
"step": 20
},
{
"KL/chosen_KL_mean": 0.007982254028320312,
"KL/mean": 0.0034023597836494446,
"KL/rejected_KL_mean": -0.001178741455078125,
"KL/std": 0.23807811737060547,
"epoch": 0.031746031746031744,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.009163036942481995,
"fcm_dpo/q_t": 0.4997709393501282,
"grad_norm": 29.345401763916016,
"learning_rate": 1.4925373134328355e-07,
"logits/chosen": 0.1197953000664711,
"logits/rejected": 0.09367487579584122,
"logps/chosen": -64.69357299804688,
"logps/ref_chosen": -64.70155334472656,
"logps/ref_rejected": -81.02095031738281,
"logps/rejected": -81.02212524414062,
"loss": 1.3856,
"margin_dpo/margin_mean": 0.00916224718093872,
"margin_dpo/margin_std": 0.2760716676712036,
"step": 21
},
{
"KL/chosen_KL_mean": -0.021139144897460938,
"KL/mean": -0.024111106991767883,
"KL/rejected_KL_mean": -0.027080535888671875,
"KL/std": 0.21007117629051208,
"epoch": 0.03325774754346183,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.005945473909378052,
"fcm_dpo/q_t": 0.4998513460159302,
"grad_norm": 29.009286880493164,
"learning_rate": 1.5671641791044775e-07,
"logits/chosen": 0.00022973224986344576,
"logits/rejected": -0.020442631095647812,
"logps/chosen": -58.05712890625,
"logps/ref_chosen": -58.03599166870117,
"logps/ref_rejected": -80.72721862792969,
"logps/rejected": -80.75430297851562,
"loss": 1.3859,
"margin_dpo/margin_mean": 0.005945086479187012,
"margin_dpo/margin_std": 0.2577439546585083,
"step": 22
},
{
"KL/chosen_KL_mean": -0.01317596435546875,
"KL/mean": -0.014801204204559326,
"KL/rejected_KL_mean": -0.016429901123046875,
"KL/std": 0.24061861634254456,
"epoch": 0.03476946334089191,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.003259122371673584,
"fcm_dpo/q_t": 0.49991869926452637,
"grad_norm": 33.08818054199219,
"learning_rate": 1.6417910447761193e-07,
"logits/chosen": 0.12761254608631134,
"logits/rejected": 0.10276039689779282,
"logps/chosen": -66.3692626953125,
"logps/ref_chosen": -66.35608673095703,
"logps/ref_rejected": -93.02769470214844,
"logps/rejected": -93.04412078857422,
"loss": 1.3863,
"margin_dpo/margin_mean": 0.0032591521739959717,
"margin_dpo/margin_std": 0.3656204640865326,
"step": 23
},
{
"KL/chosen_KL_mean": 0.0018863677978515625,
"KL/mean": -0.009600698947906494,
"KL/rejected_KL_mean": -0.021089553833007812,
"KL/std": 0.19692258536815643,
"epoch": 0.036281179138321996,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.022979021072387695,
"fcm_dpo/q_t": 0.49942547082901,
"grad_norm": 26.260557174682617,
"learning_rate": 1.716417910447761e-07,
"logits/chosen": 0.1274389773607254,
"logits/rejected": 0.09479451179504395,
"logps/chosen": -54.4593505859375,
"logps/ref_chosen": -54.461238861083984,
"logps/ref_rejected": -68.33817291259766,
"logps/rejected": -68.35926818847656,
"loss": 1.3842,
"margin_dpo/margin_mean": 0.02297845482826233,
"margin_dpo/margin_std": 0.25866520404815674,
"step": 24
},
{
"KL/chosen_KL_mean": -0.04363059997558594,
"KL/mean": -0.03292663395404816,
"KL/rejected_KL_mean": -0.02222442626953125,
"KL/std": 0.22737306356430054,
"epoch": 0.03779289493575208,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.021410435438156128,
"fcm_dpo/q_t": 0.5005350112915039,
"grad_norm": 29.813453674316406,
"learning_rate": 1.7910447761194027e-07,
"logits/chosen": 0.09910149872303009,
"logits/rejected": 0.04798390716314316,
"logps/chosen": -60.04783630371094,
"logps/ref_chosen": -60.00420379638672,
"logps/ref_rejected": -90.47376251220703,
"logps/rejected": -90.49598693847656,
"loss": 1.3886,
"margin_dpo/margin_mean": -0.021410197019577026,
"margin_dpo/margin_std": 0.2856305241584778,
"step": 25
},
{
"KL/chosen_KL_mean": -0.03212165832519531,
"KL/mean": -0.010377638041973114,
"KL/rejected_KL_mean": 0.0113677978515625,
"KL/std": 0.23722830414772034,
"epoch": 0.039304610733182165,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04349200427532196,
"fcm_dpo/q_t": 0.5010876655578613,
"grad_norm": 29.885282516479492,
"learning_rate": 1.8656716417910447e-07,
"logits/chosen": 0.10826554894447327,
"logits/rejected": 0.08984041213989258,
"logps/chosen": -56.85127639770508,
"logps/ref_chosen": -56.81915283203125,
"logps/ref_rejected": -77.84333038330078,
"logps/rejected": -77.83196258544922,
"loss": 1.3909,
"margin_dpo/margin_mean": -0.043492019176483154,
"margin_dpo/margin_std": 0.3361806273460388,
"step": 26
},
{
"KL/chosen_KL_mean": -0.0045871734619140625,
"KL/mean": -0.008165434002876282,
"KL/rejected_KL_mean": -0.011743545532226562,
"KL/std": 0.19913233816623688,
"epoch": 0.04081632653061224,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.00715222954750061,
"fcm_dpo/q_t": 0.49982118606567383,
"grad_norm": 28.897428512573242,
"learning_rate": 1.9402985074626865e-07,
"logits/chosen": 0.08622437715530396,
"logits/rejected": 0.062031831592321396,
"logps/chosen": -62.881614685058594,
"logps/ref_chosen": -62.87702560424805,
"logps/ref_rejected": -71.34437561035156,
"logps/rejected": -71.35611724853516,
"loss": 1.3858,
"margin_dpo/margin_mean": 0.007152438163757324,
"margin_dpo/margin_std": 0.29086369276046753,
"step": 27
},
{
"KL/chosen_KL_mean": -0.029260635375976562,
"KL/mean": -0.03680327534675598,
"KL/rejected_KL_mean": -0.044345855712890625,
"KL/std": 0.19967246055603027,
"epoch": 0.042328042328042326,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01508396863937378,
"fcm_dpo/q_t": 0.49962306022644043,
"grad_norm": 27.587074279785156,
"learning_rate": 2.0149253731343282e-07,
"logits/chosen": 0.04726283252239227,
"logits/rejected": 0.03866342082619667,
"logps/chosen": -59.86263656616211,
"logps/ref_chosen": -59.8333740234375,
"logps/ref_rejected": -70.39804077148438,
"logps/rejected": -70.4423828125,
"loss": 1.385,
"margin_dpo/margin_mean": 0.015084385871887207,
"margin_dpo/margin_std": 0.2913114130496979,
"step": 28
},
{
"KL/chosen_KL_mean": -0.01448822021484375,
"KL/mean": -0.02990015596151352,
"KL/rejected_KL_mean": -0.045307159423828125,
"KL/std": 0.2235536277294159,
"epoch": 0.04383975812547241,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03082270920276642,
"fcm_dpo/q_t": 0.4992297887802124,
"grad_norm": 32.74290084838867,
"learning_rate": 2.08955223880597e-07,
"logits/chosen": 0.11370386928319931,
"logits/rejected": 0.09676564484834671,
"logps/chosen": -74.13468933105469,
"logps/ref_chosen": -74.12020111083984,
"logps/ref_rejected": -83.33099365234375,
"logps/rejected": -83.37629699707031,
"loss": 1.3835,
"margin_dpo/margin_mean": 0.03082290291786194,
"margin_dpo/margin_std": 0.3283058702945709,
"step": 29
},
{
"KL/chosen_KL_mean": -0.05242156982421875,
"KL/mean": -0.056987062096595764,
"KL/rejected_KL_mean": -0.061553955078125,
"KL/std": 0.22280657291412354,
"epoch": 0.045351473922902494,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.009135037660598755,
"fcm_dpo/q_t": 0.4997720718383789,
"grad_norm": 30.316654205322266,
"learning_rate": 2.1641791044776117e-07,
"logits/chosen": 0.11641520261764526,
"logits/rejected": 0.06267019361257553,
"logps/chosen": -50.8037109375,
"logps/ref_chosen": -50.75128936767578,
"logps/ref_rejected": -89.29063415527344,
"logps/rejected": -89.35218811035156,
"loss": 1.3856,
"margin_dpo/margin_mean": 0.009134829044342041,
"margin_dpo/margin_std": 0.3106568455696106,
"step": 30
},
{
"KL/chosen_KL_mean": -0.05117988586425781,
"KL/mean": -0.05755278468132019,
"KL/rejected_KL_mean": -0.06392669677734375,
"KL/std": 0.2555512487888336,
"epoch": 0.04686318972033258,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.012740731239318848,
"fcm_dpo/q_t": 0.49968260526657104,
"grad_norm": 34.35220718383789,
"learning_rate": 2.2388059701492537e-07,
"logits/chosen": 0.12674255669116974,
"logits/rejected": 0.07923215627670288,
"logps/chosen": -65.38793182373047,
"logps/ref_chosen": -65.33675384521484,
"logps/ref_rejected": -100.76666259765625,
"logps/rejected": -100.8305892944336,
"loss": 1.3854,
"margin_dpo/margin_mean": 0.01274147629737854,
"margin_dpo/margin_std": 0.3628019690513611,
"step": 31
},
{
"KL/chosen_KL_mean": -0.0181884765625,
"KL/mean": -0.0361301451921463,
"KL/rejected_KL_mean": -0.054073333740234375,
"KL/std": 0.23413747549057007,
"epoch": 0.04837490551776266,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03588786721229553,
"fcm_dpo/q_t": 0.49910300970077515,
"grad_norm": 30.009159088134766,
"learning_rate": 2.3134328358208954e-07,
"logits/chosen": 0.08153019845485687,
"logits/rejected": 0.0736636072397232,
"logps/chosen": -67.20152282714844,
"logps/ref_chosen": -67.18333435058594,
"logps/ref_rejected": -82.80763244628906,
"logps/rejected": -82.86170959472656,
"loss": 1.383,
"margin_dpo/margin_mean": 0.0358881950378418,
"margin_dpo/margin_std": 0.35985904932022095,
"step": 32
},
{
"KL/chosen_KL_mean": -0.034152984619140625,
"KL/mean": -0.05544004589319229,
"KL/rejected_KL_mean": -0.07672500610351562,
"KL/std": 0.24409998953342438,
"epoch": 0.049886621315192746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04257179796695709,
"fcm_dpo/q_t": 0.498935729265213,
"grad_norm": 30.951351165771484,
"learning_rate": 2.388059701492537e-07,
"logits/chosen": 0.027346061542630196,
"logits/rejected": 0.0015142709016799927,
"logps/chosen": -64.0736312866211,
"logps/ref_chosen": -64.03948211669922,
"logps/ref_rejected": -75.68357849121094,
"logps/rejected": -75.76030731201172,
"loss": 1.3823,
"margin_dpo/margin_mean": 0.04257209599018097,
"margin_dpo/margin_std": 0.30183154344558716,
"step": 33
},
{
"KL/chosen_KL_mean": -0.048999786376953125,
"KL/mean": -0.06053346395492554,
"KL/rejected_KL_mean": -0.07206916809082031,
"KL/std": 0.19695493578910828,
"epoch": 0.05139833711262283,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.023070037364959717,
"fcm_dpo/q_t": 0.49942296743392944,
"grad_norm": 28.55535125732422,
"learning_rate": 2.4626865671641786e-07,
"logits/chosen": 0.08082367479801178,
"logits/rejected": 0.05176318809390068,
"logps/chosen": -53.713294982910156,
"logps/ref_chosen": -53.6642951965332,
"logps/ref_rejected": -65.77989959716797,
"logps/rejected": -65.85196685791016,
"loss": 1.3842,
"margin_dpo/margin_mean": 0.023069947957992554,
"margin_dpo/margin_std": 0.2683805227279663,
"step": 34
},
{
"KL/chosen_KL_mean": -0.08650398254394531,
"KL/mean": -0.08371259272098541,
"KL/rejected_KL_mean": -0.0809173583984375,
"KL/std": 0.2462875247001648,
"epoch": 0.05291005291005291,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.005585581064224243,
"fcm_dpo/q_t": 0.500140905380249,
"grad_norm": 27.881855010986328,
"learning_rate": 2.537313432835821e-07,
"logits/chosen": 0.04871156066656113,
"logits/rejected": 0.02646300010383129,
"logps/chosen": -61.10336685180664,
"logps/ref_chosen": -61.01686096191406,
"logps/ref_rejected": -72.78598022460938,
"logps/rejected": -72.86689758300781,
"loss": 1.3872,
"margin_dpo/margin_mean": -0.005585342645645142,
"margin_dpo/margin_std": 0.3802899122238159,
"step": 35
},
{
"KL/chosen_KL_mean": -0.08986663818359375,
"KL/mean": -0.08186689019203186,
"KL/rejected_KL_mean": -0.0738677978515625,
"KL/std": 0.25683659315109253,
"epoch": 0.05442176870748299,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.01600050926208496,
"fcm_dpo/q_t": 0.5004006624221802,
"grad_norm": 28.70569610595703,
"learning_rate": 2.611940298507462e-07,
"logits/chosen": 0.08580096811056137,
"logits/rejected": 0.03410476818680763,
"logps/chosen": -50.627227783203125,
"logps/ref_chosen": -50.53736114501953,
"logps/ref_rejected": -78.11678314208984,
"logps/rejected": -78.1906509399414,
"loss": 1.3883,
"margin_dpo/margin_mean": -0.01600012183189392,
"margin_dpo/margin_std": 0.3842473030090332,
"step": 36
},
{
"KL/chosen_KL_mean": -0.028474807739257812,
"KL/mean": -0.09227629005908966,
"KL/rejected_KL_mean": -0.1560821533203125,
"KL/std": 0.26934385299682617,
"epoch": 0.055933484504913075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12760671973228455,
"fcm_dpo/q_t": 0.49681222438812256,
"grad_norm": 37.727169036865234,
"learning_rate": 2.686567164179104e-07,
"logits/chosen": 0.12197470664978027,
"logits/rejected": 0.03995547443628311,
"logps/chosen": -59.582420349121094,
"logps/ref_chosen": -59.55394744873047,
"logps/ref_rejected": -108.27702331542969,
"logps/rejected": -108.43310546875,
"loss": 1.3739,
"margin_dpo/margin_mean": 0.12760648131370544,
"margin_dpo/margin_std": 0.37096428871154785,
"step": 37
},
{
"KL/chosen_KL_mean": -0.0984039306640625,
"KL/mean": -0.10117024183273315,
"KL/rejected_KL_mean": -0.10393905639648438,
"KL/std": 0.2519524097442627,
"epoch": 0.05744520030234316,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.005530297756195068,
"fcm_dpo/q_t": 0.4998621940612793,
"grad_norm": 29.83066177368164,
"learning_rate": 2.761194029850746e-07,
"logits/chosen": 0.0732765644788742,
"logits/rejected": 0.05932066589593887,
"logps/chosen": -65.88676452636719,
"logps/ref_chosen": -65.78836059570312,
"logps/ref_rejected": -76.1619873046875,
"logps/rejected": -76.26593017578125,
"loss": 1.3861,
"margin_dpo/margin_mean": 0.005530297756195068,
"margin_dpo/margin_std": 0.3729862570762634,
"step": 38
},
{
"KL/chosen_KL_mean": -0.09945297241210938,
"KL/mean": -0.11698070168495178,
"KL/rejected_KL_mean": -0.13450241088867188,
"KL/std": 0.25591549277305603,
"epoch": 0.05895691609977324,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03504788875579834,
"fcm_dpo/q_t": 0.4991235136985779,
"grad_norm": 29.30242347717285,
"learning_rate": 2.8358208955223876e-07,
"logits/chosen": 0.2040865123271942,
"logits/rejected": 0.1760084629058838,
"logps/chosen": -57.27626037597656,
"logps/ref_chosen": -57.17681121826172,
"logps/ref_rejected": -79.486328125,
"logps/rejected": -79.62083435058594,
"loss": 1.3831,
"margin_dpo/margin_mean": 0.03504854440689087,
"margin_dpo/margin_std": 0.3559015691280365,
"step": 39
},
{
"KL/chosen_KL_mean": -0.08912086486816406,
"KL/mean": -0.10761454701423645,
"KL/rejected_KL_mean": -0.12611007690429688,
"KL/std": 0.22716698050498962,
"epoch": 0.06046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.036985158920288086,
"fcm_dpo/q_t": 0.49907582998275757,
"grad_norm": 31.3107967376709,
"learning_rate": 2.9104477611940296e-07,
"logits/chosen": 0.10985475778579712,
"logits/rejected": 0.0602511391043663,
"logps/chosen": -61.42328643798828,
"logps/ref_chosen": -61.33416748046875,
"logps/ref_rejected": -79.10697174072266,
"logps/rejected": -79.23307800292969,
"loss": 1.3829,
"margin_dpo/margin_mean": 0.03698521852493286,
"margin_dpo/margin_std": 0.31080809235572815,
"step": 40
},
{
"KL/chosen_KL_mean": -0.1392059326171875,
"KL/mean": -0.1555485725402832,
"KL/rejected_KL_mean": -0.17188453674316406,
"KL/std": 0.27494168281555176,
"epoch": 0.06198034769463341,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.032675087451934814,
"fcm_dpo/q_t": 0.4991832971572876,
"grad_norm": 30.506553649902344,
"learning_rate": 2.985074626865671e-07,
"logits/chosen": 0.04268798977136612,
"logits/rejected": 0.022809015586972237,
"logps/chosen": -67.6859359741211,
"logps/ref_chosen": -67.5467300415039,
"logps/ref_rejected": -83.87788391113281,
"logps/rejected": -84.04977416992188,
"loss": 1.3833,
"margin_dpo/margin_mean": 0.03267523646354675,
"margin_dpo/margin_std": 0.3265340030193329,
"step": 41
},
{
"KL/chosen_KL_mean": -0.11741065979003906,
"KL/mean": -0.15568827092647552,
"KL/rejected_KL_mean": -0.19396591186523438,
"KL/std": 0.30766791105270386,
"epoch": 0.06349206349206349,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07655629515647888,
"fcm_dpo/q_t": 0.4980867803096771,
"grad_norm": 29.174734115600586,
"learning_rate": 3.059701492537313e-07,
"logits/chosen": 0.05423213541507721,
"logits/rejected": 0.032556917518377304,
"logps/chosen": -61.38227081298828,
"logps/ref_chosen": -61.26485824584961,
"logps/ref_rejected": -76.3629150390625,
"logps/rejected": -76.556884765625,
"loss": 1.3791,
"margin_dpo/margin_mean": 0.07655629515647888,
"margin_dpo/margin_std": 0.42006781697273254,
"step": 42
},
{
"KL/chosen_KL_mean": -0.09850692749023438,
"KL/mean": -0.1341993361711502,
"KL/rejected_KL_mean": -0.169891357421875,
"KL/std": 0.29504144191741943,
"epoch": 0.06500377928949358,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07138493657112122,
"fcm_dpo/q_t": 0.49821680784225464,
"grad_norm": 34.448570251464844,
"learning_rate": 3.134328358208955e-07,
"logits/chosen": 0.08338065445423126,
"logits/rejected": 0.07241372764110565,
"logps/chosen": -71.90753173828125,
"logps/ref_chosen": -71.80902862548828,
"logps/ref_rejected": -81.12464141845703,
"logps/rejected": -81.2945327758789,
"loss": 1.3796,
"margin_dpo/margin_mean": 0.07138559222221375,
"margin_dpo/margin_std": 0.41163527965545654,
"step": 43
},
{
"KL/chosen_KL_mean": -0.13344192504882812,
"KL/mean": -0.16079673171043396,
"KL/rejected_KL_mean": -0.18815231323242188,
"KL/std": 0.26067036390304565,
"epoch": 0.06651549508692366,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05471104383468628,
"fcm_dpo/q_t": 0.49863290786743164,
"grad_norm": 32.42856216430664,
"learning_rate": 3.2089552238805965e-07,
"logits/chosen": 0.05968674272298813,
"logits/rejected": 0.028688661754131317,
"logps/chosen": -66.68386840820312,
"logps/ref_chosen": -66.55043029785156,
"logps/ref_rejected": -85.06198120117188,
"logps/rejected": -85.25013732910156,
"loss": 1.3812,
"margin_dpo/margin_mean": 0.054710716009140015,
"margin_dpo/margin_std": 0.40378284454345703,
"step": 44
},
{
"KL/chosen_KL_mean": -0.13708114624023438,
"KL/mean": -0.18379130959510803,
"KL/rejected_KL_mean": -0.23050308227539062,
"KL/std": 0.2648080289363861,
"epoch": 0.06802721088435375,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09342862665653229,
"fcm_dpo/q_t": 0.49766480922698975,
"grad_norm": 31.722583770751953,
"learning_rate": 3.2835820895522385e-07,
"logits/chosen": 0.130683034658432,
"logits/rejected": 0.07715224474668503,
"logps/chosen": -62.38093566894531,
"logps/ref_chosen": -62.24385452270508,
"logps/ref_rejected": -92.96665954589844,
"logps/rejected": -93.1971664428711,
"loss": 1.3773,
"margin_dpo/margin_mean": 0.09342983365058899,
"margin_dpo/margin_std": 0.346953809261322,
"step": 45
},
{
"KL/chosen_KL_mean": -0.10871696472167969,
"KL/mean": -0.19501355290412903,
"KL/rejected_KL_mean": -0.2813072204589844,
"KL/std": 0.3256569504737854,
"epoch": 0.06953892668178382,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17258453369140625,
"fcm_dpo/q_t": 0.4956880807876587,
"grad_norm": 30.854923248291016,
"learning_rate": 3.3582089552238805e-07,
"logits/chosen": 0.12547335028648376,
"logits/rejected": 0.0794464647769928,
"logps/chosen": -61.60762023925781,
"logps/ref_chosen": -61.498905181884766,
"logps/ref_rejected": -78.91172790527344,
"logps/rejected": -79.19303894042969,
"loss": 1.3696,
"margin_dpo/margin_mean": 0.17258423566818237,
"margin_dpo/margin_std": 0.4200759530067444,
"step": 46
},
{
"KL/chosen_KL_mean": -0.13623046875,
"KL/mean": -0.2177979201078415,
"KL/rejected_KL_mean": -0.2993621826171875,
"KL/std": 0.27024394273757935,
"epoch": 0.0710506424792139,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.16313031315803528,
"fcm_dpo/q_t": 0.4959232807159424,
"grad_norm": 28.427127838134766,
"learning_rate": 3.432835820895522e-07,
"logits/chosen": 0.03357366845011711,
"logits/rejected": -0.009209276176989079,
"logps/chosen": -51.714576721191406,
"logps/ref_chosen": -51.578346252441406,
"logps/ref_rejected": -68.2215576171875,
"logps/rejected": -68.52091979980469,
"loss": 1.3704,
"margin_dpo/margin_mean": 0.16313022375106812,
"margin_dpo/margin_std": 0.3930588364601135,
"step": 47
},
{
"KL/chosen_KL_mean": -0.22270774841308594,
"KL/mean": -0.24841207265853882,
"KL/rejected_KL_mean": -0.27411460876464844,
"KL/std": 0.31731051206588745,
"epoch": 0.07256235827664399,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05140644311904907,
"fcm_dpo/q_t": 0.498715877532959,
"grad_norm": 26.81363296508789,
"learning_rate": 3.507462686567164e-07,
"logits/chosen": 0.14631465077400208,
"logits/rejected": 0.11654843389987946,
"logps/chosen": -52.016357421875,
"logps/ref_chosen": -51.79365158081055,
"logps/ref_rejected": -64.22503662109375,
"logps/rejected": -64.49915313720703,
"loss": 1.3817,
"margin_dpo/margin_mean": 0.0514070987701416,
"margin_dpo/margin_std": 0.4587850868701935,
"step": 48
},
{
"KL/chosen_KL_mean": -0.20112228393554688,
"KL/mean": -0.2560715079307556,
"KL/rejected_KL_mean": -0.3110179901123047,
"KL/std": 0.3602464199066162,
"epoch": 0.07407407407407407,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10989488661289215,
"fcm_dpo/q_t": 0.4972541630268097,
"grad_norm": 27.207046508789062,
"learning_rate": 3.5820895522388055e-07,
"logits/chosen": 0.05348002910614014,
"logits/rejected": 0.03150768578052521,
"logps/chosen": -58.335723876953125,
"logps/ref_chosen": -58.13460159301758,
"logps/ref_rejected": -64.63206481933594,
"logps/rejected": -64.94308471679688,
"loss": 1.3759,
"margin_dpo/margin_mean": 0.10989552736282349,
"margin_dpo/margin_std": 0.48010891675949097,
"step": 49
},
{
"KL/chosen_KL_mean": -0.3021965026855469,
"KL/mean": -0.3455643653869629,
"KL/rejected_KL_mean": -0.3889350891113281,
"KL/std": 0.34557676315307617,
"epoch": 0.07558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0867350697517395,
"fcm_dpo/q_t": 0.4978324770927429,
"grad_norm": 28.065460205078125,
"learning_rate": 3.6567164179104475e-07,
"logits/chosen": 0.11386396735906601,
"logits/rejected": 0.08404330164194107,
"logps/chosen": -53.15863800048828,
"logps/ref_chosen": -52.85643768310547,
"logps/ref_rejected": -72.17460632324219,
"logps/rejected": -72.56353759765625,
"loss": 1.3781,
"margin_dpo/margin_mean": 0.08673503994941711,
"margin_dpo/margin_std": 0.4149363040924072,
"step": 50
},
{
"KL/chosen_KL_mean": -0.31117820739746094,
"KL/mean": -0.42082780599594116,
"KL/rejected_KL_mean": -0.5304794311523438,
"KL/std": 0.4126695394515991,
"epoch": 0.07709750566893424,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.21929967403411865,
"fcm_dpo/q_t": 0.4945247769355774,
"grad_norm": 30.938974380493164,
"learning_rate": 3.7313432835820895e-07,
"logits/chosen": 0.07800806313753128,
"logits/rejected": 0.05045904591679573,
"logps/chosen": -63.967620849609375,
"logps/ref_chosen": -63.65644073486328,
"logps/ref_rejected": -86.13229370117188,
"logps/rejected": -86.66278076171875,
"loss": 1.3652,
"margin_dpo/margin_mean": 0.2192995846271515,
"margin_dpo/margin_std": 0.5256927609443665,
"step": 51
},
{
"KL/chosen_KL_mean": -0.3040885925292969,
"KL/mean": -0.39921391010284424,
"KL/rejected_KL_mean": -0.4943389892578125,
"KL/std": 0.4276493191719055,
"epoch": 0.07860922146636433,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19024460017681122,
"fcm_dpo/q_t": 0.49524950981140137,
"grad_norm": 32.57892990112305,
"learning_rate": 3.805970149253731e-07,
"logits/chosen": 0.06836412101984024,
"logits/rejected": 0.020395169034600258,
"logps/chosen": -68.14430236816406,
"logps/ref_chosen": -67.8402099609375,
"logps/ref_rejected": -96.97090911865234,
"logps/rejected": -97.46524810791016,
"loss": 1.3681,
"margin_dpo/margin_mean": 0.19024433195590973,
"margin_dpo/margin_std": 0.5287501811981201,
"step": 52
},
{
"KL/chosen_KL_mean": -0.322906494140625,
"KL/mean": -0.42890429496765137,
"KL/rejected_KL_mean": -0.5349006652832031,
"KL/std": 0.40694308280944824,
"epoch": 0.0801209372637944,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.21199661493301392,
"fcm_dpo/q_t": 0.49470412731170654,
"grad_norm": 27.37774658203125,
"learning_rate": 3.880597014925373e-07,
"logits/chosen": 0.09767497330904007,
"logits/rejected": 0.08682269603013992,
"logps/chosen": -57.201045989990234,
"logps/ref_chosen": -56.87813949584961,
"logps/ref_rejected": -60.75569152832031,
"logps/rejected": -61.29059600830078,
"loss": 1.3659,
"margin_dpo/margin_mean": 0.2119971513748169,
"margin_dpo/margin_std": 0.5070023536682129,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3383960723876953,
"KL/mean": -0.4334535002708435,
"KL/rejected_KL_mean": -0.5285091400146484,
"KL/std": 0.39724022150039673,
"epoch": 0.08163265306122448,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19011196494102478,
"fcm_dpo/q_t": 0.49525079131126404,
"grad_norm": 26.38343048095703,
"learning_rate": 3.9552238805970144e-07,
"logits/chosen": 0.05778650939464569,
"logits/rejected": 0.04246218502521515,
"logps/chosen": -47.605316162109375,
"logps/ref_chosen": -47.26692199707031,
"logps/ref_rejected": -62.19426727294922,
"logps/rejected": -62.722774505615234,
"loss": 1.3679,
"margin_dpo/margin_mean": 0.19011151790618896,
"margin_dpo/margin_std": 0.46603357791900635,
"step": 54
},
{
"KL/chosen_KL_mean": -0.36114501953125,
"KL/mean": -0.502538800239563,
"KL/rejected_KL_mean": -0.6439323425292969,
"KL/std": 0.4991796016693115,
"epoch": 0.08314436885865457,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.28278931975364685,
"fcm_dpo/q_t": 0.4929451048374176,
"grad_norm": 31.28098487854004,
"learning_rate": 4.0298507462686564e-07,
"logits/chosen": 0.047343235462903976,
"logits/rejected": -0.030307874083518982,
"logps/chosen": -50.68733596801758,
"logps/ref_chosen": -50.32619094848633,
"logps/ref_rejected": -92.44389343261719,
"logps/rejected": -93.08782196044922,
"loss": 1.3593,
"margin_dpo/margin_mean": 0.2827894985675812,
"margin_dpo/margin_std": 0.6369022727012634,
"step": 55
},
{
"KL/chosen_KL_mean": -0.3630104064941406,
"KL/mean": -0.45767152309417725,
"KL/rejected_KL_mean": -0.5523262023925781,
"KL/std": 0.4814542829990387,
"epoch": 0.08465608465608465,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.18931812047958374,
"fcm_dpo/q_t": 0.49526914954185486,
"grad_norm": 27.53534698486328,
"learning_rate": 4.1044776119402984e-07,
"logits/chosen": 0.1598096489906311,
"logits/rejected": 0.13690778613090515,
"logps/chosen": -57.129981994628906,
"logps/ref_chosen": -56.766971588134766,
"logps/ref_rejected": -66.30504608154297,
"logps/rejected": -66.85737609863281,
"loss": 1.3684,
"margin_dpo/margin_mean": 0.18931809067726135,
"margin_dpo/margin_std": 0.6055810451507568,
"step": 56
},
{
"KL/chosen_KL_mean": -0.47999000549316406,
"KL/mean": -0.6331826448440552,
"KL/rejected_KL_mean": -0.7863693237304688,
"KL/std": 0.5945008993148804,
"epoch": 0.08616780045351474,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.30638235807418823,
"fcm_dpo/q_t": 0.49235400557518005,
"grad_norm": 30.548871994018555,
"learning_rate": 4.17910447761194e-07,
"logits/chosen": 0.09187550097703934,
"logits/rejected": 0.028308037668466568,
"logps/chosen": -58.24773406982422,
"logps/ref_chosen": -57.76774597167969,
"logps/ref_rejected": -82.75698852539062,
"logps/rejected": -83.5433578491211,
"loss": 1.3572,
"margin_dpo/margin_mean": 0.30638229846954346,
"margin_dpo/margin_std": 0.6942629814147949,
"step": 57
},
{
"KL/chosen_KL_mean": -0.5946998596191406,
"KL/mean": -0.6939823031425476,
"KL/rejected_KL_mean": -0.7932624816894531,
"KL/std": 0.6381080746650696,
"epoch": 0.08767951625094482,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19855886697769165,
"fcm_dpo/q_t": 0.49508431553840637,
"grad_norm": 30.27342987060547,
"learning_rate": 4.253731343283582e-07,
"logits/chosen": 0.05209419131278992,
"logits/rejected": 0.03686722368001938,
"logps/chosen": -73.3587875366211,
"logps/ref_chosen": -72.76408386230469,
"logps/ref_rejected": -84.49275207519531,
"logps/rejected": -85.2860107421875,
"loss": 1.3689,
"margin_dpo/margin_mean": 0.19855844974517822,
"margin_dpo/margin_std": 0.9104675054550171,
"step": 58
},
{
"KL/chosen_KL_mean": -0.5438365936279297,
"KL/mean": -0.6774130463600159,
"KL/rejected_KL_mean": -0.8109931945800781,
"KL/std": 0.5832891464233398,
"epoch": 0.08919123204837491,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.26715219020843506,
"fcm_dpo/q_t": 0.493341326713562,
"grad_norm": 26.679256439208984,
"learning_rate": 4.3283582089552234e-07,
"logits/chosen": 0.1515531837940216,
"logits/rejected": 0.08378548175096512,
"logps/chosen": -50.3646125793457,
"logps/ref_chosen": -49.820777893066406,
"logps/ref_rejected": -77.14368438720703,
"logps/rejected": -77.95468139648438,
"loss": 1.3614,
"margin_dpo/margin_mean": 0.26715266704559326,
"margin_dpo/margin_std": 0.7313680052757263,
"step": 59
},
{
"KL/chosen_KL_mean": -0.6181087493896484,
"KL/mean": -0.6719140410423279,
"KL/rejected_KL_mean": -0.7257213592529297,
"KL/std": 0.618611216545105,
"epoch": 0.09070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10761374235153198,
"fcm_dpo/q_t": 0.49730825424194336,
"grad_norm": 29.533267974853516,
"learning_rate": 4.4029850746268654e-07,
"logits/chosen": 0.09845545142889023,
"logits/rejected": 0.09702516347169876,
"logps/chosen": -63.84288024902344,
"logps/ref_chosen": -63.22477340698242,
"logps/ref_rejected": -61.360477447509766,
"logps/rejected": -62.08619689941406,
"loss": 1.3771,
"margin_dpo/margin_mean": 0.10761353373527527,
"margin_dpo/margin_std": 0.7893075346946716,
"step": 60
},
{
"KL/chosen_KL_mean": -0.7608661651611328,
"KL/mean": -0.786099374294281,
"KL/rejected_KL_mean": -0.8113327026367188,
"KL/std": 0.7570456862449646,
"epoch": 0.09221466364323508,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05046442151069641,
"fcm_dpo/q_t": 0.49873921275138855,
"grad_norm": 28.125015258789062,
"learning_rate": 4.4776119402985074e-07,
"logits/chosen": 0.15135915577411652,
"logits/rejected": 0.11800111830234528,
"logps/chosen": -49.77766418457031,
"logps/ref_chosen": -49.01679992675781,
"logps/ref_rejected": -74.90817260742188,
"logps/rejected": -75.71949768066406,
"loss": 1.384,
"margin_dpo/margin_mean": 0.05046433210372925,
"margin_dpo/margin_std": 1.048170804977417,
"step": 61
},
{
"KL/chosen_KL_mean": -0.7540721893310547,
"KL/mean": -0.8673438429832458,
"KL/rejected_KL_mean": -0.9806175231933594,
"KL/std": 0.6291642189025879,
"epoch": 0.09372637944066516,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2265465259552002,
"fcm_dpo/q_t": 0.4943479299545288,
"grad_norm": 28.553312301635742,
"learning_rate": 4.552238805970149e-07,
"logits/chosen": 0.11329515278339386,
"logits/rejected": 0.07428479939699173,
"logps/chosen": -63.505943298339844,
"logps/ref_chosen": -62.751869201660156,
"logps/ref_rejected": -78.93360900878906,
"logps/rejected": -79.91423034667969,
"loss": 1.3657,
"margin_dpo/margin_mean": 0.22654661536216736,
"margin_dpo/margin_std": 0.8588752150535583,
"step": 62
},
{
"KL/chosen_KL_mean": -0.5554847717285156,
"KL/mean": -0.8419216871261597,
"KL/rejected_KL_mean": -1.1283645629882812,
"KL/std": 0.7228051424026489,
"epoch": 0.09523809523809523,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5728802680969238,
"fcm_dpo/q_t": 0.4857173562049866,
"grad_norm": 31.28562355041504,
"learning_rate": 4.626865671641791e-07,
"logits/chosen": 0.18709728121757507,
"logits/rejected": 0.1622365117073059,
"logps/chosen": -61.07073211669922,
"logps/ref_chosen": -60.51525115966797,
"logps/ref_rejected": -85.11021423339844,
"logps/rejected": -86.23857879638672,
"loss": 1.3318,
"margin_dpo/margin_mean": 0.5728808641433716,
"margin_dpo/margin_std": 0.8949563503265381,
"step": 63
},
{
"KL/chosen_KL_mean": -0.7727451324462891,
"KL/mean": -0.830580472946167,
"KL/rejected_KL_mean": -0.8884201049804688,
"KL/std": 0.6900042295455933,
"epoch": 0.09674981103552532,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11567190289497375,
"fcm_dpo/q_t": 0.4971042275428772,
"grad_norm": 26.15057373046875,
"learning_rate": 4.701492537313433e-07,
"logits/chosen": 0.1170622780919075,
"logits/rejected": 0.09124461561441422,
"logps/chosen": -51.979591369628906,
"logps/ref_chosen": -51.20684814453125,
"logps/ref_rejected": -66.93081665039062,
"logps/rejected": -67.81924438476562,
"loss": 1.3766,
"margin_dpo/margin_mean": 0.11567208170890808,
"margin_dpo/margin_std": 0.8656256794929504,
"step": 64
},
{
"KL/chosen_KL_mean": -0.8728599548339844,
"KL/mean": -1.1317873001098633,
"KL/rejected_KL_mean": -1.3907203674316406,
"KL/std": 0.8693119287490845,
"epoch": 0.0982615268329554,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5178593397140503,
"fcm_dpo/q_t": 0.48711270093917847,
"grad_norm": 30.82215118408203,
"learning_rate": 4.776119402985074e-07,
"logits/chosen": 0.20676767826080322,
"logits/rejected": 0.17682614922523499,
"logps/chosen": -68.16155242919922,
"logps/ref_chosen": -67.2886962890625,
"logps/ref_rejected": -74.44281005859375,
"logps/rejected": -75.83352661132812,
"loss": 1.3385,
"margin_dpo/margin_mean": 0.5178589224815369,
"margin_dpo/margin_std": 1.1341583728790283,
"step": 65
},
{
"KL/chosen_KL_mean": -0.9256954193115234,
"KL/mean": -1.107551097869873,
"KL/rejected_KL_mean": -1.2894096374511719,
"KL/std": 0.8482298851013184,
"epoch": 0.09977324263038549,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3637177050113678,
"fcm_dpo/q_t": 0.4909314811229706,
"grad_norm": 29.322872161865234,
"learning_rate": 4.850746268656717e-07,
"logits/chosen": 0.10713790357112885,
"logits/rejected": 0.08299659192562103,
"logps/chosen": -71.66910552978516,
"logps/ref_chosen": -70.743408203125,
"logps/ref_rejected": -77.26499938964844,
"logps/rejected": -78.55441284179688,
"loss": 1.3536,
"margin_dpo/margin_mean": 0.363717257976532,
"margin_dpo/margin_std": 1.1283318996429443,
"step": 66
},
{
"KL/chosen_KL_mean": -0.7712230682373047,
"KL/mean": -0.9370452165603638,
"KL/rejected_KL_mean": -1.1028671264648438,
"KL/std": 0.7597904205322266,
"epoch": 0.10128495842781557,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.33164212107658386,
"fcm_dpo/q_t": 0.49173396825790405,
"grad_norm": 29.111217498779297,
"learning_rate": 4.925373134328357e-07,
"logits/chosen": 0.08660893142223358,
"logits/rejected": 0.031195688992738724,
"logps/chosen": -61.37382507324219,
"logps/ref_chosen": -60.60260009765625,
"logps/ref_rejected": -75.22235870361328,
"logps/rejected": -76.32522583007812,
"loss": 1.3555,
"margin_dpo/margin_mean": 0.3316424787044525,
"margin_dpo/margin_std": 0.921684980392456,
"step": 67
},
{
"KL/chosen_KL_mean": -1.242034912109375,
"KL/mean": -1.413798451423645,
"KL/rejected_KL_mean": -1.5855636596679688,
"KL/std": 0.9572592377662659,
"epoch": 0.10279667422524566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.34353235363960266,
"fcm_dpo/q_t": 0.49143853783607483,
"grad_norm": 31.30362892150879,
"learning_rate": 5e-07,
"logits/chosen": 0.0596737265586853,
"logits/rejected": 0.03012073040008545,
"logps/chosen": -78.77040100097656,
"logps/ref_chosen": -77.52836608886719,
"logps/ref_rejected": -93.17778015136719,
"logps/rejected": -94.76333618164062,
"loss": 1.3562,
"margin_dpo/margin_mean": 0.3435323238372803,
"margin_dpo/margin_std": 1.236339807510376,
"step": 68
},
{
"KL/chosen_KL_mean": -1.1912288665771484,
"KL/mean": -1.477524995803833,
"KL/rejected_KL_mean": -1.7638206481933594,
"KL/std": 1.001812219619751,
"epoch": 0.10430839002267574,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5725945234298706,
"fcm_dpo/q_t": 0.4857825040817261,
"grad_norm": 31.007518768310547,
"learning_rate": 4.999965034812934e-07,
"logits/chosen": 0.09101719409227371,
"logits/rejected": 0.04851025342941284,
"logps/chosen": -67.13427734375,
"logps/ref_chosen": -65.94305419921875,
"logps/ref_rejected": -89.7735595703125,
"logps/rejected": -91.53738403320312,
"loss": 1.3337,
"margin_dpo/margin_mean": 0.5725944638252258,
"margin_dpo/margin_std": 1.2039427757263184,
"step": 69
},
{
"KL/chosen_KL_mean": -1.2000751495361328,
"KL/mean": -1.3958642482757568,
"KL/rejected_KL_mean": -1.5916500091552734,
"KL/std": 0.91285240650177,
"epoch": 0.10582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.39157044887542725,
"fcm_dpo/q_t": 0.49025198817253113,
"grad_norm": 28.630355834960938,
"learning_rate": 4.999860140229787e-07,
"logits/chosen": 0.14131593704223633,
"logits/rejected": 0.11839009821414948,
"logps/chosen": -63.157989501953125,
"logps/ref_chosen": -61.95791244506836,
"logps/ref_rejected": -75.80945587158203,
"logps/rejected": -77.40110778808594,
"loss": 1.3515,
"margin_dpo/margin_mean": 0.3915710151195526,
"margin_dpo/margin_std": 1.252969741821289,
"step": 70
},
{
"KL/chosen_KL_mean": -1.4675350189208984,
"KL/mean": -1.5703108310699463,
"KL/rejected_KL_mean": -1.6730918884277344,
"KL/std": 1.0348397493362427,
"epoch": 0.1073318216175359,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20555351674556732,
"fcm_dpo/q_t": 0.4948495030403137,
"grad_norm": 28.58488655090332,
"learning_rate": 4.999685319184688e-07,
"logits/chosen": 0.09156721830368042,
"logits/rejected": 0.07582792639732361,
"logps/chosen": -64.81510925292969,
"logps/ref_chosen": -63.34757995605469,
"logps/ref_rejected": -67.49658203125,
"logps/rejected": -69.16967010498047,
"loss": 1.3713,
"margin_dpo/margin_mean": 0.20555375516414642,
"margin_dpo/margin_std": 1.4667941331863403,
"step": 71
},
{
"KL/chosen_KL_mean": -1.237548828125,
"KL/mean": -1.5851967334747314,
"KL/rejected_KL_mean": -1.9328422546386719,
"KL/std": 1.1038551330566406,
"epoch": 0.10884353741496598,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6952927112579346,
"fcm_dpo/q_t": 0.4827180504798889,
"grad_norm": 30.270689010620117,
"learning_rate": 4.999440576567755e-07,
"logits/chosen": 0.10874535143375397,
"logits/rejected": 0.04621289297938347,
"logps/chosen": -57.09684753417969,
"logps/ref_chosen": -55.85929870605469,
"logps/ref_rejected": -68.45423889160156,
"logps/rejected": -70.3870849609375,
"loss": 1.3231,
"margin_dpo/margin_mean": 0.6952927112579346,
"margin_dpo/margin_std": 1.4010509252548218,
"step": 72
},
{
"KL/chosen_KL_mean": -1.8302345275878906,
"KL/mean": -1.8590284585952759,
"KL/rejected_KL_mean": -1.887826919555664,
"KL/std": 1.2061991691589355,
"epoch": 0.11035525321239607,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.057596683502197266,
"fcm_dpo/q_t": 0.49866846203804016,
"grad_norm": 32.27928924560547,
"learning_rate": 4.999125919224965e-07,
"logits/chosen": 0.10941021144390106,
"logits/rejected": 0.0953536182641983,
"logps/chosen": -70.96903991699219,
"logps/ref_chosen": -69.13880920410156,
"logps/ref_rejected": -79.04586791992188,
"logps/rejected": -80.93370056152344,
"loss": 1.3869,
"margin_dpo/margin_mean": 0.05759665369987488,
"margin_dpo/margin_std": 1.5423662662506104,
"step": 73
},
{
"KL/chosen_KL_mean": -1.2537212371826172,
"KL/mean": -1.6008504629135132,
"KL/rejected_KL_mean": -1.9479789733886719,
"KL/std": 1.0375707149505615,
"epoch": 0.11186696900982615,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6942564249038696,
"fcm_dpo/q_t": 0.48278287053108215,
"grad_norm": 27.944766998291016,
"learning_rate": 4.998741355957963e-07,
"logits/chosen": 0.15747080743312836,
"logits/rejected": 0.10257548838853836,
"logps/chosen": -51.177459716796875,
"logps/ref_chosen": -49.923736572265625,
"logps/ref_rejected": -81.73213958740234,
"logps/rejected": -83.68011474609375,
"loss": 1.323,
"margin_dpo/margin_mean": 0.6942560076713562,
"margin_dpo/margin_std": 1.3548740148544312,
"step": 74
},
{
"KL/chosen_KL_mean": -1.3986186981201172,
"KL/mean": -1.7732503414154053,
"KL/rejected_KL_mean": -2.147878646850586,
"KL/std": 1.1023242473602295,
"epoch": 0.11337868480725624,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7492611408233643,
"fcm_dpo/q_t": 0.4813886284828186,
"grad_norm": 26.229400634765625,
"learning_rate": 4.998286897523808e-07,
"logits/chosen": 0.1198577880859375,
"logits/rejected": 0.08726748824119568,
"logps/chosen": -47.467369079589844,
"logps/ref_chosen": -46.06875228881836,
"logps/ref_rejected": -66.1181411743164,
"logps/rejected": -68.26602172851562,
"loss": 1.3174,
"margin_dpo/margin_mean": 0.7492610812187195,
"margin_dpo/margin_std": 1.3630497455596924,
"step": 75
},
{
"KL/chosen_KL_mean": -1.5863704681396484,
"KL/mean": -1.7453091144561768,
"KL/rejected_KL_mean": -1.9042491912841797,
"KL/std": 1.1934852600097656,
"epoch": 0.11489040060468632,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3178822100162506,
"fcm_dpo/q_t": 0.49208852648735046,
"grad_norm": 29.335805892944336,
"learning_rate": 4.997762556634679e-07,
"logits/chosen": 0.1410784125328064,
"logits/rejected": 0.09613093733787537,
"logps/chosen": -55.64912033081055,
"logps/ref_chosen": -54.06275177001953,
"logps/ref_rejected": -74.87464141845703,
"logps/rejected": -76.77889251708984,
"loss": 1.3609,
"margin_dpo/margin_mean": 0.31788232922554016,
"margin_dpo/margin_std": 1.566576600074768,
"step": 76
},
{
"KL/chosen_KL_mean": -1.64678955078125,
"KL/mean": -2.0163772106170654,
"KL/rejected_KL_mean": -2.3859634399414062,
"KL/std": 1.2249342203140259,
"epoch": 0.1164021164021164,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7391700744628906,
"fcm_dpo/q_t": 0.48164504766464233,
"grad_norm": 29.228374481201172,
"learning_rate": 4.99716834795752e-07,
"logits/chosen": 0.14172329008579254,
"logits/rejected": 0.10119108855724335,
"logps/chosen": -54.72288131713867,
"logps/ref_chosen": -53.07609176635742,
"logps/ref_rejected": -74.45601654052734,
"logps/rejected": -76.84197998046875,
"loss": 1.3191,
"margin_dpo/margin_mean": 0.7391701936721802,
"margin_dpo/margin_std": 1.426438331604004,
"step": 77
},
{
"KL/chosen_KL_mean": -1.7951850891113281,
"KL/mean": -2.0447793006896973,
"KL/rejected_KL_mean": -2.29437255859375,
"KL/std": 1.2816565036773682,
"epoch": 0.11791383219954649,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4991890788078308,
"fcm_dpo/q_t": 0.4875733256340027,
"grad_norm": 29.23097038269043,
"learning_rate": 4.996504288113623e-07,
"logits/chosen": 0.0910310223698616,
"logits/rejected": 0.07102985680103302,
"logps/chosen": -69.5206069946289,
"logps/ref_chosen": -67.72541809082031,
"logps/ref_rejected": -79.03926849365234,
"logps/rejected": -81.3336410522461,
"loss": 1.3421,
"margin_dpo/margin_mean": 0.49918922781944275,
"margin_dpo/margin_std": 1.4283866882324219,
"step": 78
},
{
"KL/chosen_KL_mean": -1.7279739379882812,
"KL/mean": -2.2947750091552734,
"KL/rejected_KL_mean": -2.8615760803222656,
"KL/std": 1.5286821126937866,
"epoch": 0.11942554799697656,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1336019039154053,
"fcm_dpo/q_t": 0.472137987613678,
"grad_norm": 30.839723587036133,
"learning_rate": 4.995770395678171e-07,
"logits/chosen": 0.17144104838371277,
"logits/rejected": 0.11132954061031342,
"logps/chosen": -53.88861846923828,
"logps/ref_chosen": -52.16064453125,
"logps/ref_rejected": -83.31062316894531,
"logps/rejected": -86.17219543457031,
"loss": 1.2855,
"margin_dpo/margin_mean": 1.1336017847061157,
"margin_dpo/margin_std": 1.944055199623108,
"step": 79
},
{
"KL/chosen_KL_mean": -2.089902877807617,
"KL/mean": -2.3355894088745117,
"KL/rejected_KL_mean": -2.5812759399414062,
"KL/std": 1.3842788934707642,
"epoch": 0.12093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4913783669471741,
"fcm_dpo/q_t": 0.4878585636615753,
"grad_norm": 28.682737350463867,
"learning_rate": 4.994966691179711e-07,
"logits/chosen": 0.14792883396148682,
"logits/rejected": 0.08779671043157578,
"logps/chosen": -63.500465393066406,
"logps/ref_chosen": -61.410560607910156,
"logps/ref_rejected": -78.66004943847656,
"logps/rejected": -81.24132537841797,
"loss": 1.3476,
"margin_dpo/margin_mean": 0.4913783073425293,
"margin_dpo/margin_std": 1.9447224140167236,
"step": 80
},
{
"KL/chosen_KL_mean": -2.1452388763427734,
"KL/mean": -2.5972180366516113,
"KL/rejected_KL_mean": -3.0491905212402344,
"KL/std": 1.4756536483764648,
"epoch": 0.12244897959183673,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.903956413269043,
"fcm_dpo/q_t": 0.4777379631996155,
"grad_norm": 28.927122116088867,
"learning_rate": 4.994093197099587e-07,
"logits/chosen": 0.13018305599689484,
"logits/rejected": 0.09627491235733032,
"logps/chosen": -65.9496078491211,
"logps/ref_chosen": -63.80437088012695,
"logps/ref_rejected": -79.3484115600586,
"logps/rejected": -82.39759826660156,
"loss": 1.3065,
"margin_dpo/margin_mean": 0.903956413269043,
"margin_dpo/margin_std": 1.7683978080749512,
"step": 81
},
{
"KL/chosen_KL_mean": -1.7878913879394531,
"KL/mean": -2.3377861976623535,
"KL/rejected_KL_mean": -2.8876800537109375,
"KL/std": 1.384407877922058,
"epoch": 0.12396069538926682,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0997854471206665,
"fcm_dpo/q_t": 0.47270792722702026,
"grad_norm": 26.443857192993164,
"learning_rate": 4.993149937871306e-07,
"logits/chosen": 0.10511986166238785,
"logits/rejected": 0.04272346943616867,
"logps/chosen": -50.60578155517578,
"logps/ref_chosen": -48.817893981933594,
"logps/ref_rejected": -70.31497955322266,
"logps/rejected": -73.2026596069336,
"loss": 1.2857,
"margin_dpo/margin_mean": 1.099785566329956,
"margin_dpo/margin_std": 1.5717735290527344,
"step": 82
},
{
"KL/chosen_KL_mean": -2.0469188690185547,
"KL/mean": -2.6421515941619873,
"KL/rejected_KL_mean": -3.2373886108398438,
"KL/std": 1.444145679473877,
"epoch": 0.1254724111866969,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1904712915420532,
"fcm_dpo/q_t": 0.4705899953842163,
"grad_norm": 29.20498275756836,
"learning_rate": 4.992136939879856e-07,
"logits/chosen": 0.2080458104610443,
"logits/rejected": 0.15732426941394806,
"logps/chosen": -59.19769287109375,
"logps/ref_chosen": -57.15077209472656,
"logps/ref_rejected": -75.1710205078125,
"logps/rejected": -78.40841674804688,
"loss": 1.2784,
"margin_dpo/margin_mean": 1.1904711723327637,
"margin_dpo/margin_std": 1.746569037437439,
"step": 83
},
{
"KL/chosen_KL_mean": -2.537485122680664,
"KL/mean": -2.9616825580596924,
"KL/rejected_KL_mean": -3.385883331298828,
"KL/std": 1.8511030673980713,
"epoch": 0.12698412698412698,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8484005928039551,
"fcm_dpo/q_t": 0.47899675369262695,
"grad_norm": 30.805755615234375,
"learning_rate": 4.991054231460969e-07,
"logits/chosen": 0.13806471228599548,
"logits/rejected": 0.0984412282705307,
"logps/chosen": -67.31478881835938,
"logps/ref_chosen": -64.77729797363281,
"logps/ref_rejected": -84.71949768066406,
"logps/rejected": -88.10537719726562,
"loss": 1.3126,
"margin_dpo/margin_mean": 0.848400354385376,
"margin_dpo/margin_std": 1.9342408180236816,
"step": 84
},
{
"KL/chosen_KL_mean": -2.259206771850586,
"KL/mean": -2.916848659515381,
"KL/rejected_KL_mean": -3.5744857788085938,
"KL/std": 1.7699023485183716,
"epoch": 0.12849584278155707,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3152821063995361,
"fcm_dpo/q_t": 0.4675264358520508,
"grad_norm": 26.661396026611328,
"learning_rate": 4.989901842900325e-07,
"logits/chosen": 0.16218605637550354,
"logits/rejected": 0.11834333837032318,
"logps/chosen": -52.510902404785156,
"logps/ref_chosen": -50.25169372558594,
"logps/ref_rejected": -66.55439758300781,
"logps/rejected": -70.12887573242188,
"loss": 1.2695,
"margin_dpo/margin_mean": 1.3152821063995361,
"margin_dpo/margin_std": 1.9742646217346191,
"step": 85
},
{
"KL/chosen_KL_mean": -2.825664520263672,
"KL/mean": -3.325305700302124,
"KL/rejected_KL_mean": -3.8249435424804688,
"KL/std": 1.9348211288452148,
"epoch": 0.13000755857898716,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9992844462394714,
"fcm_dpo/q_t": 0.4753116965293884,
"grad_norm": 27.131023406982422,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": 0.23128435015678406,
"logits/rejected": 0.21146416664123535,
"logps/chosen": -63.55484390258789,
"logps/ref_chosen": -60.72917938232422,
"logps/ref_rejected": -72.30961608886719,
"logps/rejected": -76.13455200195312,
"loss": 1.2995,
"margin_dpo/margin_mean": 0.9992850422859192,
"margin_dpo/margin_std": 2.070314884185791,
"step": 86
},
{
"KL/chosen_KL_mean": -2.9881515502929688,
"KL/mean": -3.519070625305176,
"KL/rejected_KL_mean": -4.04998779296875,
"KL/std": 2.076875925064087,
"epoch": 0.13151927437641722,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.061842441558838,
"fcm_dpo/q_t": 0.473818302154541,
"grad_norm": 30.4871883392334,
"learning_rate": 4.987388156241114e-07,
"logits/chosen": 0.13899703323841095,
"logits/rejected": 0.08255505561828613,
"logps/chosen": -68.74612426757812,
"logps/ref_chosen": -65.75796508789062,
"logps/ref_rejected": -84.81159973144531,
"logps/rejected": -88.86158752441406,
"loss": 1.2989,
"margin_dpo/margin_mean": 1.061842441558838,
"margin_dpo/margin_std": 2.512241840362549,
"step": 87
},
{
"KL/chosen_KL_mean": -3.012208938598633,
"KL/mean": -3.4354000091552734,
"KL/rejected_KL_mean": -3.8585853576660156,
"KL/std": 2.2582128047943115,
"epoch": 0.1330309901738473,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8463754057884216,
"fcm_dpo/q_t": 0.47936731576919556,
"grad_norm": 30.181737899780273,
"learning_rate": 4.986026928455767e-07,
"logits/chosen": 0.2531355917453766,
"logits/rejected": 0.22557546198368073,
"logps/chosen": -65.83623504638672,
"logps/ref_chosen": -62.82402801513672,
"logps/ref_rejected": -74.9607162475586,
"logps/rejected": -78.81930541992188,
"loss": 1.3251,
"margin_dpo/margin_mean": 0.846375584602356,
"margin_dpo/margin_std": 2.8723127841949463,
"step": 88
},
{
"KL/chosen_KL_mean": -2.5241851806640625,
"KL/mean": -3.2427046298980713,
"KL/rejected_KL_mean": -3.961223602294922,
"KL/std": 2.128784418106079,
"epoch": 0.1345427059712774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4370373487472534,
"fcm_dpo/q_t": 0.4648668169975281,
"grad_norm": 29.15933609008789,
"learning_rate": 4.984596161153135e-07,
"logits/chosen": 0.273225873708725,
"logits/rejected": 0.18827831745147705,
"logps/chosen": -43.71562194824219,
"logps/ref_chosen": -41.191436767578125,
"logps/ref_rejected": -85.44769287109375,
"logps/rejected": -89.40892028808594,
"loss": 1.2629,
"margin_dpo/margin_mean": 1.4370378255844116,
"margin_dpo/margin_std": 2.407123565673828,
"step": 89
},
{
"KL/chosen_KL_mean": -3.1085948944091797,
"KL/mean": -3.753403663635254,
"KL/rejected_KL_mean": -4.398212432861328,
"KL/std": 2.442763566970825,
"epoch": 0.1360544217687075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2896150350570679,
"fcm_dpo/q_t": 0.4683343172073364,
"grad_norm": 29.212316513061523,
"learning_rate": 4.983095894354857e-07,
"logits/chosen": 0.14692363142967224,
"logits/rejected": 0.09251993149518967,
"logps/chosen": -59.6925048828125,
"logps/ref_chosen": -56.58390808105469,
"logps/ref_rejected": -86.86978149414062,
"logps/rejected": -91.26799011230469,
"loss": 1.277,
"margin_dpo/margin_mean": 1.2896147966384888,
"margin_dpo/margin_std": 2.4425294399261475,
"step": 90
},
{
"KL/chosen_KL_mean": -3.105762481689453,
"KL/mean": -3.701371192932129,
"KL/rejected_KL_mean": -4.296977996826172,
"KL/std": 2.3072173595428467,
"epoch": 0.13756613756613756,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1912150382995605,
"fcm_dpo/q_t": 0.4711948037147522,
"grad_norm": 25.63971710205078,
"learning_rate": 4.98152617002662e-07,
"logits/chosen": 0.15437094867229462,
"logits/rejected": 0.1109311506152153,
"logps/chosen": -55.48810577392578,
"logps/ref_chosen": -52.38234329223633,
"logps/ref_rejected": -72.17642211914062,
"logps/rejected": -76.47340393066406,
"loss": 1.2899,
"margin_dpo/margin_mean": 1.191214680671692,
"margin_dpo/margin_std": 2.780487060546875,
"step": 91
},
{
"KL/chosen_KL_mean": -3.237283706665039,
"KL/mean": -4.0282511711120605,
"KL/rejected_KL_mean": -4.819221496582031,
"KL/std": 2.670938014984131,
"epoch": 0.13907785336356765,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.581930160522461,
"fcm_dpo/q_t": 0.4618622064590454,
"grad_norm": 27.040842056274414,
"learning_rate": 4.979887032076988e-07,
"logits/chosen": 0.21197925508022308,
"logits/rejected": 0.17208608984947205,
"logps/chosen": -56.24598693847656,
"logps/ref_chosen": -53.00870132446289,
"logps/ref_rejected": -79.77812957763672,
"logps/rejected": -84.59735107421875,
"loss": 1.2589,
"margin_dpo/margin_mean": 1.5819299221038818,
"margin_dpo/margin_std": 3.144960403442383,
"step": 92
},
{
"KL/chosen_KL_mean": -3.615896224975586,
"KL/mean": -4.108669757843018,
"KL/rejected_KL_mean": -4.601442337036133,
"KL/std": 2.6333351135253906,
"epoch": 0.14058956916099774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9855465888977051,
"fcm_dpo/q_t": 0.4764956831932068,
"grad_norm": 24.519123077392578,
"learning_rate": 4.978178526356172e-07,
"logits/chosen": 0.19642697274684906,
"logits/rejected": 0.1676049828529358,
"logps/chosen": -48.52294921875,
"logps/ref_chosen": -44.90705108642578,
"logps/ref_rejected": -58.7879524230957,
"logps/rejected": -63.38939666748047,
"loss": 1.3158,
"margin_dpo/margin_mean": 0.9855464696884155,
"margin_dpo/margin_std": 3.2435593605041504,
"step": 93
},
{
"KL/chosen_KL_mean": -3.2522754669189453,
"KL/mean": -4.209831237792969,
"KL/rejected_KL_mean": -5.167385101318359,
"KL/std": 3.2133913040161133,
"epoch": 0.1421012849584278,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9151129722595215,
"fcm_dpo/q_t": 0.4539569616317749,
"grad_norm": 27.537464141845703,
"learning_rate": 4.976400700654751e-07,
"logits/chosen": 0.2658318877220154,
"logits/rejected": 0.22487923502922058,
"logps/chosen": -63.19004821777344,
"logps/ref_chosen": -59.93777084350586,
"logps/ref_rejected": -79.3138427734375,
"logps/rejected": -84.4812240600586,
"loss": 1.2425,
"margin_dpo/margin_mean": 1.9151129722595215,
"margin_dpo/margin_std": 3.9685182571411133,
"step": 94
},
{
"KL/chosen_KL_mean": -4.282629013061523,
"KL/mean": -5.0815019607543945,
"KL/rejected_KL_mean": -5.880374908447266,
"KL/std": 2.7092788219451904,
"epoch": 0.1436130007558579,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5977485179901123,
"fcm_dpo/q_t": 0.46140381693840027,
"grad_norm": 29.245241165161133,
"learning_rate": 4.974553604702332e-07,
"logits/chosen": 0.18501371145248413,
"logits/rejected": 0.11809428036212921,
"logps/chosen": -64.45111846923828,
"logps/ref_chosen": -60.168487548828125,
"logps/ref_rejected": -90.73665618896484,
"logps/rejected": -96.61703491210938,
"loss": 1.2618,
"margin_dpo/margin_mean": 1.597748041152954,
"margin_dpo/margin_std": 3.4290518760681152,
"step": 95
},
{
"KL/chosen_KL_mean": -4.180746078491211,
"KL/mean": -5.110323905944824,
"KL/rejected_KL_mean": -6.0399017333984375,
"KL/std": 3.4257893562316895,
"epoch": 0.14512471655328799,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8591567277908325,
"fcm_dpo/q_t": 0.4567599296569824,
"grad_norm": 27.415103912353516,
"learning_rate": 4.972637290166157e-07,
"logits/chosen": 0.19027361273765564,
"logits/rejected": 0.14786377549171448,
"logps/chosen": -64.84952545166016,
"logps/ref_chosen": -60.66877746582031,
"logps/ref_rejected": -88.30673217773438,
"logps/rejected": -94.34663391113281,
"loss": 1.2588,
"margin_dpo/margin_mean": 1.8591563701629639,
"margin_dpo/margin_std": 4.5050578117370605,
"step": 96
},
{
"KL/chosen_KL_mean": -4.967893600463867,
"KL/mean": -5.493868827819824,
"KL/rejected_KL_mean": -6.019844055175781,
"KL/std": 3.6154160499572754,
"epoch": 0.14663643235071808,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0519614219665527,
"fcm_dpo/q_t": 0.4750325083732605,
"grad_norm": 36.224552154541016,
"learning_rate": 4.970651810649666e-07,
"logits/chosen": 0.1339898705482483,
"logits/rejected": 0.09054124355316162,
"logps/chosen": -70.01201629638672,
"logps/ref_chosen": -65.04412078857422,
"logps/ref_rejected": -78.42092895507812,
"logps/rejected": -84.4407730102539,
"loss": 1.3439,
"margin_dpo/margin_mean": 1.0519613027572632,
"margin_dpo/margin_std": 4.982733726501465,
"step": 97
},
{
"KL/chosen_KL_mean": -4.659694671630859,
"KL/mean": -5.020186901092529,
"KL/rejected_KL_mean": -5.380680084228516,
"KL/std": 3.311429738998413,
"epoch": 0.14814814814814814,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7209818363189697,
"fcm_dpo/q_t": 0.4820060729980469,
"grad_norm": 30.29183578491211,
"learning_rate": 4.968597221690985e-07,
"logits/chosen": 0.21896392107009888,
"logits/rejected": 0.19200772047042847,
"logps/chosen": -60.162925720214844,
"logps/ref_chosen": -55.503231048583984,
"logps/ref_rejected": -72.81553649902344,
"logps/rejected": -78.19622039794922,
"loss": 1.3425,
"margin_dpo/margin_mean": 0.720981240272522,
"margin_dpo/margin_std": 3.2679855823516846,
"step": 98
},
{
"KL/chosen_KL_mean": -4.7128143310546875,
"KL/mean": -5.198019981384277,
"KL/rejected_KL_mean": -5.683235168457031,
"KL/std": 3.49711012840271,
"epoch": 0.14965986394557823,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9704172015190125,
"fcm_dpo/q_t": 0.476929634809494,
"grad_norm": 32.677608489990234,
"learning_rate": 4.966473580761389e-07,
"logits/chosen": 0.25504788756370544,
"logits/rejected": 0.21502363681793213,
"logps/chosen": -63.2884521484375,
"logps/ref_chosen": -58.57563781738281,
"logps/ref_rejected": -78.693603515625,
"logps/rejected": -84.37684631347656,
"loss": 1.3429,
"margin_dpo/margin_mean": 0.9704174399375916,
"margin_dpo/margin_std": 4.577587604522705,
"step": 99
},
{
"KL/chosen_KL_mean": -5.356479644775391,
"KL/mean": -6.064589500427246,
"KL/rejected_KL_mean": -6.772693634033203,
"KL/std": 4.213325023651123,
"epoch": 0.15117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4162156581878662,
"fcm_dpo/q_t": 0.46854132413864136,
"grad_norm": 33.35466766357422,
"learning_rate": 4.964280947263676e-07,
"logits/chosen": 0.2241612672805786,
"logits/rejected": 0.21632839739322662,
"logps/chosen": -84.93991088867188,
"logps/ref_chosen": -79.58343505859375,
"logps/ref_rejected": -92.152587890625,
"logps/rejected": -98.92527770996094,
"loss": 1.3376,
"margin_dpo/margin_mean": 1.416215419769287,
"margin_dpo/margin_std": 5.909281253814697,
"step": 100
},
{
"KL/chosen_KL_mean": -4.311086654663086,
"KL/mean": -5.3412017822265625,
"KL/rejected_KL_mean": -6.371318817138672,
"KL/std": 3.681835174560547,
"epoch": 0.15268329554043839,
"fcm_dpo/beta": 0.10105337202548981,
"fcm_dpo/delta": 0.10424260795116425,
"fcm_dpo/margin": 2.0602338314056396,
"fcm_dpo/q_t": 0.45020347833633423,
"grad_norm": 26.12833595275879,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": 0.22986358404159546,
"logits/rejected": 0.19068977236747742,
"logps/chosen": -56.64387512207031,
"logps/ref_chosen": -52.332786560058594,
"logps/ref_rejected": -69.55589294433594,
"logps/rejected": -75.92721557617188,
"loss": 1.233,
"margin_dpo/margin_mean": 2.0602331161499023,
"margin_dpo/margin_std": 4.143397331237793,
"step": 101
},
{
"KL/chosen_KL_mean": -5.332441329956055,
"KL/mean": -5.749258995056152,
"KL/rejected_KL_mean": -6.166080474853516,
"KL/std": 3.662102699279785,
"epoch": 0.15419501133786848,
"fcm_dpo/beta": 0.1021067351102829,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.833631157875061,
"fcm_dpo/q_t": 0.48263412714004517,
"grad_norm": 33.488887786865234,
"learning_rate": 4.959688949822748e-07,
"logits/chosen": 0.1631137728691101,
"logits/rejected": 0.12416748702526093,
"logps/chosen": -70.075927734375,
"logps/ref_chosen": -64.74348449707031,
"logps/ref_rejected": -69.06132507324219,
"logps/rejected": -75.22740936279297,
"loss": 1.3757,
"margin_dpo/margin_mean": 0.833631694316864,
"margin_dpo/margin_std": 5.453669548034668,
"step": 102
},
{
"KL/chosen_KL_mean": -4.815326690673828,
"KL/mean": -5.606250762939453,
"KL/rejected_KL_mean": -6.397182464599609,
"KL/std": 3.9247231483459473,
"epoch": 0.15570672713529857,
"fcm_dpo/beta": 0.10332540422677994,
"fcm_dpo/delta": 0.11795026063919067,
"fcm_dpo/margin": 1.5818533897399902,
"fcm_dpo/q_t": 0.4614016115665436,
"grad_norm": 30.380908966064453,
"learning_rate": 4.957289714327572e-07,
"logits/chosen": 0.2731260359287262,
"logits/rejected": 0.23873889446258545,
"logps/chosen": -68.65196990966797,
"logps/ref_chosen": -63.83664321899414,
"logps/ref_rejected": -79.32362365722656,
"logps/rejected": -85.72080993652344,
"loss": 1.2652,
"margin_dpo/margin_mean": 1.5818531513214111,
"margin_dpo/margin_std": 3.693053960800171,
"step": 103
},
{
"KL/chosen_KL_mean": -4.689603805541992,
"KL/mean": -5.855137825012207,
"KL/rejected_KL_mean": -7.020671844482422,
"KL/std": 5.013128280639648,
"epoch": 0.15721844293272866,
"fcm_dpo/beta": 0.10536164045333862,
"fcm_dpo/delta": 0.07759787887334824,
"fcm_dpo/margin": 2.3310675621032715,
"fcm_dpo/q_t": 0.44842541217803955,
"grad_norm": 32.38325500488281,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": 0.27137792110443115,
"logits/rejected": 0.1822788119316101,
"logps/chosen": -65.68881225585938,
"logps/ref_chosen": -60.99920654296875,
"logps/ref_rejected": -98.84645080566406,
"logps/rejected": -105.86712646484375,
"loss": 1.2606,
"margin_dpo/margin_mean": 2.331068754196167,
"margin_dpo/margin_std": 6.3999152183532715,
"step": 104
},
{
"KL/chosen_KL_mean": -5.333820343017578,
"KL/mean": -6.264369010925293,
"KL/rejected_KL_mean": -7.194919586181641,
"KL/std": 3.99896240234375,
"epoch": 0.15873015873015872,
"fcm_dpo/beta": 0.10617920756340027,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8611037731170654,
"fcm_dpo/q_t": 0.4552581012248993,
"grad_norm": 32.53849792480469,
"learning_rate": 4.952285105344791e-07,
"logits/chosen": 0.18711794912815094,
"logits/rejected": 0.13286931812763214,
"logps/chosen": -76.28408813476562,
"logps/ref_chosen": -70.95027160644531,
"logps/ref_rejected": -87.88340759277344,
"logps/rejected": -95.07833099365234,
"loss": 1.2644,
"margin_dpo/margin_mean": 1.8611032962799072,
"margin_dpo/margin_std": 4.930830955505371,
"step": 105
},
{
"KL/chosen_KL_mean": -5.013969421386719,
"KL/mean": -5.812124252319336,
"KL/rejected_KL_mean": -6.6102752685546875,
"KL/std": 3.922760486602783,
"epoch": 0.1602418745275888,
"fcm_dpo/beta": 0.10617920756340027,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5963062047958374,
"fcm_dpo/q_t": 0.45990967750549316,
"grad_norm": 33.27928924560547,
"learning_rate": 4.949679871846857e-07,
"logits/chosen": 0.22435937821865082,
"logits/rejected": 0.21065881848335266,
"logps/chosen": -67.47329711914062,
"logps/ref_chosen": -62.45933151245117,
"logps/ref_rejected": -67.00595092773438,
"logps/rejected": -73.61622619628906,
"loss": 1.3007,
"margin_dpo/margin_mean": 1.5963062047958374,
"margin_dpo/margin_std": 5.327633380889893,
"step": 106
},
{
"KL/chosen_KL_mean": -6.503017425537109,
"KL/mean": -6.911505699157715,
"KL/rejected_KL_mean": -7.3199920654296875,
"KL/std": 4.925832748413086,
"epoch": 0.1617535903250189,
"fcm_dpo/beta": 0.10617920756340027,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8169774413108826,
"fcm_dpo/q_t": 0.4772740602493286,
"grad_norm": 45.139347076416016,
"learning_rate": 4.947006115536947e-07,
"logits/chosen": 0.1692843735218048,
"logits/rejected": 0.1457558125257492,
"logps/chosen": -82.34098815917969,
"logps/ref_chosen": -75.83796691894531,
"logps/ref_rejected": -87.74038696289062,
"logps/rejected": -95.06037902832031,
"loss": 1.4032,
"margin_dpo/margin_mean": 0.8169773817062378,
"margin_dpo/margin_std": 6.143199920654297,
"step": 107
},
{
"KL/chosen_KL_mean": -5.4889373779296875,
"KL/mean": -6.39979362487793,
"KL/rejected_KL_mean": -7.310649871826172,
"KL/std": 4.071392059326172,
"epoch": 0.16326530612244897,
"fcm_dpo/beta": 0.10617920756340027,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8217132091522217,
"fcm_dpo/q_t": 0.4561387300491333,
"grad_norm": 30.513721466064453,
"learning_rate": 4.944263911205772e-07,
"logits/chosen": 0.19126787781715393,
"logits/rejected": 0.15949223935604095,
"logps/chosen": -73.88217163085938,
"logps/ref_chosen": -68.39323425292969,
"logps/ref_rejected": -83.24267578125,
"logps/rejected": -90.5533218383789,
"loss": 1.2797,
"margin_dpo/margin_mean": 1.8217132091522217,
"margin_dpo/margin_std": 5.339081287384033,
"step": 108
},
{
"KL/chosen_KL_mean": -4.884004592895508,
"KL/mean": -6.358839988708496,
"KL/rejected_KL_mean": -7.833675384521484,
"KL/std": 4.998663425445557,
"epoch": 0.16477702191987906,
"fcm_dpo/beta": 0.10841409116983414,
"fcm_dpo/delta": 0.13440188765525818,
"fcm_dpo/margin": 2.949669361114502,
"fcm_dpo/q_t": 0.43165239691734314,
"grad_norm": 30.32049560546875,
"learning_rate": 4.941453335558681e-07,
"logits/chosen": 0.21224114298820496,
"logits/rejected": 0.15585273504257202,
"logps/chosen": -60.41148376464844,
"logps/ref_chosen": -55.52748107910156,
"logps/ref_rejected": -83.55218505859375,
"logps/rejected": -91.3858642578125,
"loss": 1.168,
"margin_dpo/margin_mean": 2.949669361114502,
"margin_dpo/margin_std": 5.457679271697998,
"step": 109
},
{
"KL/chosen_KL_mean": -6.235057830810547,
"KL/mean": -6.359735488891602,
"KL/rejected_KL_mean": -6.484409332275391,
"KL/std": 4.0676727294921875,
"epoch": 0.16628873771730915,
"fcm_dpo/beta": 0.10907205194234848,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24935531616210938,
"fcm_dpo/q_t": 0.4908309876918793,
"grad_norm": 49.313865661621094,
"learning_rate": 4.938574467213517e-07,
"logits/chosen": 0.1845201551914215,
"logits/rejected": 0.19402091205120087,
"logps/chosen": -87.393798828125,
"logps/ref_chosen": -81.15874481201172,
"logps/ref_rejected": -72.56021118164062,
"logps/rejected": -79.04461669921875,
"loss": 1.4428,
"margin_dpo/margin_mean": 0.24935469031333923,
"margin_dpo/margin_std": 5.4945878982543945,
"step": 110
},
{
"KL/chosen_KL_mean": -5.652936935424805,
"KL/mean": -6.612331390380859,
"KL/rejected_KL_mean": -7.571727752685547,
"KL/std": 4.178367614746094,
"epoch": 0.16780045351473924,
"fcm_dpo/beta": 0.11027145385742188,
"fcm_dpo/delta": 0.10877230763435364,
"fcm_dpo/margin": 1.918793797492981,
"fcm_dpo/q_t": 0.45316487550735474,
"grad_norm": 28.390825271606445,
"learning_rate": 4.935627386698418e-07,
"logits/chosen": 0.26181134581565857,
"logits/rejected": 0.22537365555763245,
"logps/chosen": -58.01192092895508,
"logps/ref_chosen": -52.358985900878906,
"logps/ref_rejected": -77.06150817871094,
"logps/rejected": -84.63323211669922,
"loss": 1.2593,
"margin_dpo/margin_mean": 1.91879403591156,
"margin_dpo/margin_std": 5.053813934326172,
"step": 111
},
{
"KL/chosen_KL_mean": -5.305566787719727,
"KL/mean": -6.52008581161499,
"KL/rejected_KL_mean": -7.7346038818359375,
"KL/std": 4.221207141876221,
"epoch": 0.1693121693121693,
"fcm_dpo/beta": 0.11457160115242004,
"fcm_dpo/delta": 0.17640449106693268,
"fcm_dpo/margin": 2.429037570953369,
"fcm_dpo/q_t": 0.4356866478919983,
"grad_norm": 36.389381408691406,
"learning_rate": 4.932612176449559e-07,
"logits/chosen": 0.1818389594554901,
"logits/rejected": 0.11763059347867966,
"logps/chosen": -68.32563018798828,
"logps/ref_chosen": -63.02006530761719,
"logps/ref_rejected": -111.36941528320312,
"logps/rejected": -119.10401916503906,
"loss": 1.1964,
"margin_dpo/margin_mean": 2.429037570953369,
"margin_dpo/margin_std": 4.7098541259765625,
"step": 112
},
{
"KL/chosen_KL_mean": -5.34063720703125,
"KL/mean": -6.032615661621094,
"KL/rejected_KL_mean": -6.7245941162109375,
"KL/std": 3.539608955383301,
"epoch": 0.1708238851095994,
"fcm_dpo/beta": 0.1167660802602768,
"fcm_dpo/delta": 0.11067280918359756,
"fcm_dpo/margin": 1.3839517831802368,
"fcm_dpo/q_t": 0.4618106484413147,
"grad_norm": 41.924644470214844,
"learning_rate": 4.929528920808854e-07,
"logits/chosen": 0.20727001130580902,
"logits/rejected": 0.1677381694316864,
"logps/chosen": -61.14830017089844,
"logps/ref_chosen": -55.80766296386719,
"logps/ref_rejected": -69.84014129638672,
"logps/rejected": -76.56473541259766,
"loss": 1.3001,
"margin_dpo/margin_mean": 1.3839521408081055,
"margin_dpo/margin_std": 4.557351112365723,
"step": 113
},
{
"KL/chosen_KL_mean": -5.004667282104492,
"KL/mean": -6.154642105102539,
"KL/rejected_KL_mean": -7.304622650146484,
"KL/std": 3.8118882179260254,
"epoch": 0.17233560090702948,
"fcm_dpo/beta": 0.12111715972423553,
"fcm_dpo/delta": 0.1764792948961258,
"fcm_dpo/margin": 2.2999496459960938,
"fcm_dpo/q_t": 0.43808525800704956,
"grad_norm": 35.51799392700195,
"learning_rate": 4.92637770602159e-07,
"logits/chosen": 0.2466518133878708,
"logits/rejected": 0.18633544445037842,
"logps/chosen": -71.33744049072266,
"logps/ref_chosen": -66.33277130126953,
"logps/ref_rejected": -71.61489868164062,
"logps/rejected": -78.91951751708984,
"loss": 1.1996,
"margin_dpo/margin_mean": 2.2999496459960938,
"margin_dpo/margin_std": 4.60300350189209,
"step": 114
},
{
"KL/chosen_KL_mean": -5.393440246582031,
"KL/mean": -6.3272271156311035,
"KL/rejected_KL_mean": -7.261011123657227,
"KL/std": 3.5061001777648926,
"epoch": 0.17384731670445955,
"fcm_dpo/beta": 0.12467402219772339,
"fcm_dpo/delta": 0.096140056848526,
"fcm_dpo/margin": 1.8675713539123535,
"fcm_dpo/q_t": 0.4473262131214142,
"grad_norm": 35.43161392211914,
"learning_rate": 4.923158620234019e-07,
"logits/chosen": 0.24343647062778473,
"logits/rejected": 0.17968247830867767,
"logps/chosen": -61.142478942871094,
"logps/ref_chosen": -55.74903869628906,
"logps/ref_rejected": -79.59849548339844,
"logps/rejected": -86.85951232910156,
"loss": 1.2327,
"margin_dpo/margin_mean": 1.8675715923309326,
"margin_dpo/margin_std": 4.200431823730469,
"step": 115
},
{
"KL/chosen_KL_mean": -5.084573745727539,
"KL/mean": -6.464913845062256,
"KL/rejected_KL_mean": -7.845256805419922,
"KL/std": 3.6170547008514404,
"epoch": 0.17535903250188964,
"fcm_dpo/beta": 0.1270686388015747,
"fcm_dpo/delta": 0.10212881863117218,
"fcm_dpo/margin": 2.760686159133911,
"fcm_dpo/q_t": 0.4189320206642151,
"grad_norm": 31.042070388793945,
"learning_rate": 4.91987175349089e-07,
"logits/chosen": 0.2448294758796692,
"logits/rejected": 0.17530521750450134,
"logps/chosen": -54.449737548828125,
"logps/ref_chosen": -49.36516571044922,
"logps/ref_rejected": -72.84671020507812,
"logps/rejected": -80.69196319580078,
"loss": 1.1271,
"margin_dpo/margin_mean": 2.7606863975524902,
"margin_dpo/margin_std": 3.9437413215637207,
"step": 116
},
{
"KL/chosen_KL_mean": -5.042119979858398,
"KL/mean": -6.001665115356445,
"KL/rejected_KL_mean": -6.961215972900391,
"KL/std": 3.4388794898986816,
"epoch": 0.17687074829931973,
"fcm_dpo/beta": 0.1273379921913147,
"fcm_dpo/delta": 0.0035832971334457397,
"fcm_dpo/margin": 1.9190866947174072,
"fcm_dpo/q_t": 0.4429383873939514,
"grad_norm": 36.0718879699707,
"learning_rate": 4.916517197732933e-07,
"logits/chosen": 0.21628046035766602,
"logits/rejected": 0.18022674322128296,
"logps/chosen": -62.75301742553711,
"logps/ref_chosen": -57.710899353027344,
"logps/ref_rejected": -69.77253723144531,
"logps/rejected": -76.73374938964844,
"loss": 1.235,
"margin_dpo/margin_mean": 1.9190870523452759,
"margin_dpo/margin_std": 4.254258155822754,
"step": 117
},
{
"KL/chosen_KL_mean": -4.869361877441406,
"KL/mean": -6.127657413482666,
"KL/rejected_KL_mean": -7.385948181152344,
"KL/std": 3.6793594360351562,
"epoch": 0.17838246409674982,
"fcm_dpo/beta": 0.12952576577663422,
"fcm_dpo/delta": 0.1283983290195465,
"fcm_dpo/margin": 2.516590118408203,
"fcm_dpo/q_t": 0.4246635138988495,
"grad_norm": 34.91251754760742,
"learning_rate": 4.913095046794281e-07,
"logits/chosen": 0.29132065176963806,
"logits/rejected": 0.24994787573814392,
"logps/chosen": -57.34925842285156,
"logps/ref_chosen": -52.479896545410156,
"logps/ref_rejected": -81.359130859375,
"logps/rejected": -88.74507141113281,
"loss": 1.1443,
"margin_dpo/margin_mean": 2.516590118408203,
"margin_dpo/margin_std": 3.754570484161377,
"step": 118
},
{
"KL/chosen_KL_mean": -6.241527557373047,
"KL/mean": -7.111940860748291,
"KL/rejected_KL_mean": -7.982353210449219,
"KL/std": 4.2465362548828125,
"epoch": 0.17989417989417988,
"fcm_dpo/beta": 0.131746307015419,
"fcm_dpo/delta": 0.08318955451250076,
"fcm_dpo/margin": 1.7408270835876465,
"fcm_dpo/q_t": 0.4494980275630951,
"grad_norm": 39.3913688659668,
"learning_rate": 4.909605396399855e-07,
"logits/chosen": 0.21818503737449646,
"logits/rejected": 0.17873303592205048,
"logps/chosen": -67.59919738769531,
"logps/ref_chosen": -61.35767364501953,
"logps/ref_rejected": -75.71510314941406,
"logps/rejected": -83.69744873046875,
"loss": 1.278,
"margin_dpo/margin_mean": 1.7408275604248047,
"margin_dpo/margin_std": 5.219765663146973,
"step": 119
},
{
"KL/chosen_KL_mean": -5.114025115966797,
"KL/mean": -6.457116603851318,
"KL/rejected_KL_mean": -7.800205230712891,
"KL/std": 3.6140894889831543,
"epoch": 0.18140589569160998,
"fcm_dpo/beta": 0.1348700374364853,
"fcm_dpo/delta": 0.09081363677978516,
"fcm_dpo/margin": 2.6861793994903564,
"fcm_dpo/q_t": 0.41728341579437256,
"grad_norm": 36.61309051513672,
"learning_rate": 4.906048344162676e-07,
"logits/chosen": 0.23546835780143738,
"logits/rejected": 0.17328256368637085,
"logps/chosen": -65.02159118652344,
"logps/ref_chosen": -59.907569885253906,
"logps/ref_rejected": -79.6910629272461,
"logps/rejected": -87.49127197265625,
"loss": 1.1157,
"margin_dpo/margin_mean": 2.6861801147460938,
"margin_dpo/margin_std": 3.675182342529297,
"step": 120
},
{
"KL/chosen_KL_mean": -5.602396011352539,
"KL/mean": -7.010382652282715,
"KL/rejected_KL_mean": -8.41836929321289,
"KL/std": 4.133810043334961,
"epoch": 0.18291761148904007,
"fcm_dpo/beta": 0.13647235929965973,
"fcm_dpo/delta": 0.06837910413742065,
"fcm_dpo/margin": 2.815971851348877,
"fcm_dpo/q_t": 0.41775327920913696,
"grad_norm": 34.387046813964844,
"learning_rate": 4.902423989581143e-07,
"logits/chosen": 0.3467385172843933,
"logits/rejected": 0.2564208209514618,
"logps/chosen": -61.26844024658203,
"logps/ref_chosen": -55.66604232788086,
"logps/ref_rejected": -101.56233978271484,
"logps/rejected": -109.980712890625,
"loss": 1.1232,
"margin_dpo/margin_mean": 2.815971851348877,
"margin_dpo/margin_std": 4.585853576660156,
"step": 121
},
{
"KL/chosen_KL_mean": -5.983097076416016,
"KL/mean": -7.31648063659668,
"KL/rejected_KL_mean": -8.649864196777344,
"KL/std": 4.461297512054443,
"epoch": 0.18442932728647016,
"fcm_dpo/beta": 0.13917195796966553,
"fcm_dpo/delta": 0.0815592035651207,
"fcm_dpo/margin": 2.6667704582214355,
"fcm_dpo/q_t": 0.4239156246185303,
"grad_norm": 42.97147750854492,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": 0.22524026036262512,
"logits/rejected": 0.1896163374185562,
"logps/chosen": -69.31747436523438,
"logps/ref_chosen": -63.334373474121094,
"logps/ref_rejected": -73.67523193359375,
"logps/rejected": -82.32510375976562,
"loss": 1.168,
"margin_dpo/margin_mean": 2.6667704582214355,
"margin_dpo/margin_std": 5.3650221824646,
"step": 122
},
{
"KL/chosen_KL_mean": -5.680210113525391,
"KL/mean": -6.947849273681641,
"KL/rejected_KL_mean": -8.215492248535156,
"KL/std": 3.915778636932373,
"epoch": 0.18594104308390022,
"fcm_dpo/beta": 0.14164075255393982,
"fcm_dpo/delta": 0.09402373433113098,
"fcm_dpo/margin": 2.5352840423583984,
"fcm_dpo/q_t": 0.4203948676586151,
"grad_norm": 36.68033981323242,
"learning_rate": 4.894973780788722e-07,
"logits/chosen": 0.23389171063899994,
"logits/rejected": 0.19273313879966736,
"logps/chosen": -62.57895278930664,
"logps/ref_chosen": -56.89874267578125,
"logps/ref_rejected": -78.97028350830078,
"logps/rejected": -87.18577575683594,
"loss": 1.1451,
"margin_dpo/margin_mean": 2.5352840423583984,
"margin_dpo/margin_std": 4.32481575012207,
"step": 123
},
{
"KL/chosen_KL_mean": -6.0447540283203125,
"KL/mean": -7.777586460113525,
"KL/rejected_KL_mean": -9.510417938232422,
"KL/std": 4.207749366760254,
"epoch": 0.1874527588813303,
"fcm_dpo/beta": 0.1409628838300705,
"fcm_dpo/delta": -0.040624044835567474,
"fcm_dpo/margin": 3.465663194656372,
"fcm_dpo/q_t": 0.3909822106361389,
"grad_norm": 36.187339782714844,
"learning_rate": 4.89114813497619e-07,
"logits/chosen": 0.28463006019592285,
"logits/rejected": 0.22122982144355774,
"logps/chosen": -63.16083908081055,
"logps/ref_chosen": -57.116085052490234,
"logps/ref_rejected": -87.93074035644531,
"logps/rejected": -97.441162109375,
"loss": 1.0433,
"margin_dpo/margin_mean": 3.465662717819214,
"margin_dpo/margin_std": 4.411318778991699,
"step": 124
},
{
"KL/chosen_KL_mean": -6.541475296020508,
"KL/mean": -8.008241653442383,
"KL/rejected_KL_mean": -9.47500228881836,
"KL/std": 4.520659923553467,
"epoch": 0.1889644746787604,
"fcm_dpo/beta": 0.14088676869869232,
"fcm_dpo/delta": 0.0378384031355381,
"fcm_dpo/margin": 2.933529853820801,
"fcm_dpo/q_t": 0.4101593494415283,
"grad_norm": 39.248573303222656,
"learning_rate": 4.887255603610184e-07,
"logits/chosen": 0.2627220153808594,
"logits/rejected": 0.20320525765419006,
"logps/chosen": -72.24765014648438,
"logps/ref_chosen": -65.7061767578125,
"logps/ref_rejected": -91.72711944580078,
"logps/rejected": -101.20211791992188,
"loss": 1.1161,
"margin_dpo/margin_mean": 2.933529853820801,
"margin_dpo/margin_std": 4.710192680358887,
"step": 125
},
{
"KL/chosen_KL_mean": -6.104684829711914,
"KL/mean": -6.989722728729248,
"KL/rejected_KL_mean": -7.874761581420898,
"KL/std": 4.20753288269043,
"epoch": 0.19047619047619047,
"fcm_dpo/beta": 0.14634212851524353,
"fcm_dpo/delta": 0.19579944014549255,
"fcm_dpo/margin": 1.7700788974761963,
"fcm_dpo/q_t": 0.44312962889671326,
"grad_norm": 41.04850769042969,
"learning_rate": 4.883296295573176e-07,
"logits/chosen": 0.11527390033006668,
"logits/rejected": 0.10950449109077454,
"logps/chosen": -74.28077697753906,
"logps/ref_chosen": -68.17608642578125,
"logps/ref_rejected": -65.1175537109375,
"logps/rejected": -72.99231719970703,
"loss": 1.2469,
"margin_dpo/margin_mean": 1.7700791358947754,
"margin_dpo/margin_std": 4.583360195159912,
"step": 126
},
{
"KL/chosen_KL_mean": -6.241485595703125,
"KL/mean": -7.84971809387207,
"KL/rejected_KL_mean": -9.457956314086914,
"KL/std": 3.7725934982299805,
"epoch": 0.19198790627362056,
"fcm_dpo/beta": 0.14626064896583557,
"fcm_dpo/delta": -0.022328916937112808,
"fcm_dpo/margin": 3.2164688110351562,
"fcm_dpo/q_t": 0.39330053329467773,
"grad_norm": 36.990318298339844,
"learning_rate": 4.87927032161552e-07,
"logits/chosen": 0.1813475489616394,
"logits/rejected": 0.15173006057739258,
"logps/chosen": -68.12171936035156,
"logps/ref_chosen": -61.88023376464844,
"logps/ref_rejected": -68.46012878417969,
"logps/rejected": -77.91808319091797,
"loss": 1.0528,
"margin_dpo/margin_mean": 3.216468334197998,
"margin_dpo/margin_std": 3.9844183921813965,
"step": 127
},
{
"KL/chosen_KL_mean": -6.738676071166992,
"KL/mean": -8.41079330444336,
"KL/rejected_KL_mean": -10.08291244506836,
"KL/std": 4.916608810424805,
"epoch": 0.19349962207105065,
"fcm_dpo/beta": 0.14551366865634918,
"fcm_dpo/delta": -0.039026957005262375,
"fcm_dpo/margin": 3.344231605529785,
"fcm_dpo/q_t": 0.398078978061676,
"grad_norm": 37.41531753540039,
"learning_rate": 4.875177794352363e-07,
"logits/chosen": 0.2204674333333969,
"logits/rejected": 0.16373318433761597,
"logps/chosen": -73.44766235351562,
"logps/ref_chosen": -66.708984375,
"logps/ref_rejected": -94.97969055175781,
"logps/rejected": -105.06260681152344,
"loss": 1.118,
"margin_dpo/margin_mean": 3.344231367111206,
"margin_dpo/margin_std": 5.8806328773498535,
"step": 128
},
{
"KL/chosen_KL_mean": -7.556180953979492,
"KL/mean": -8.538894653320312,
"KL/rejected_KL_mean": -9.5216064453125,
"KL/std": 4.42452335357666,
"epoch": 0.19501133786848074,
"fcm_dpo/beta": 0.1494940221309662,
"fcm_dpo/delta": 0.16076956689357758,
"fcm_dpo/margin": 1.9654262065887451,
"fcm_dpo/q_t": 0.43633008003234863,
"grad_norm": 52.058353424072266,
"learning_rate": 4.871018828260491e-07,
"logits/chosen": 0.21647898852825165,
"logits/rejected": 0.2104889303445816,
"logps/chosen": -72.89500427246094,
"logps/ref_chosen": -65.33882904052734,
"logps/ref_rejected": -68.06109619140625,
"logps/rejected": -77.58270263671875,
"loss": 1.2569,
"margin_dpo/margin_mean": 1.9654266834259033,
"margin_dpo/margin_std": 5.319171905517578,
"step": 129
},
{
"KL/chosen_KL_mean": -7.091085433959961,
"KL/mean": -8.634103775024414,
"KL/rejected_KL_mean": -10.1771240234375,
"KL/std": 4.412557125091553,
"epoch": 0.1965230536659108,
"fcm_dpo/beta": 0.14928355813026428,
"fcm_dpo/delta": -0.01228487491607666,
"fcm_dpo/margin": 3.0860366821289062,
"fcm_dpo/q_t": 0.39678919315338135,
"grad_norm": 43.6436882019043,
"learning_rate": 4.866793539675126e-07,
"logits/chosen": 0.17643359303474426,
"logits/rejected": 0.12553930282592773,
"logps/chosen": -65.7518310546875,
"logps/ref_chosen": -58.660743713378906,
"logps/ref_rejected": -79.24510192871094,
"logps/rejected": -89.42222595214844,
"loss": 1.0744,
"margin_dpo/margin_mean": 3.0860366821289062,
"margin_dpo/margin_std": 4.253152847290039,
"step": 130
},
{
"KL/chosen_KL_mean": -6.78155517578125,
"KL/mean": -8.716371536254883,
"KL/rejected_KL_mean": -10.651187896728516,
"KL/std": 5.087396621704102,
"epoch": 0.1980347694633409,
"fcm_dpo/beta": 0.14708811044692993,
"fcm_dpo/delta": -0.12676924467086792,
"fcm_dpo/margin": 3.8696370124816895,
"fcm_dpo/q_t": 0.38106900453567505,
"grad_norm": 36.27388381958008,
"learning_rate": 4.86250204678667e-07,
"logits/chosen": 0.19298642873764038,
"logits/rejected": 0.12822577357292175,
"logps/chosen": -59.29608917236328,
"logps/ref_chosen": -52.51453399658203,
"logps/ref_rejected": -85.18299865722656,
"logps/rejected": -95.83418273925781,
"loss": 1.0539,
"margin_dpo/margin_mean": 3.8696374893188477,
"margin_dpo/margin_std": 5.73309326171875,
"step": 131
},
{
"KL/chosen_KL_mean": -7.4026336669921875,
"KL/mean": -9.053910255432129,
"KL/rejected_KL_mean": -10.705190658569336,
"KL/std": 4.851495742797852,
"epoch": 0.19954648526077098,
"fcm_dpo/beta": 0.14569681882858276,
"fcm_dpo/delta": -0.032763972878456116,
"fcm_dpo/margin": 3.3025588989257812,
"fcm_dpo/q_t": 0.3902255892753601,
"grad_norm": 38.206172943115234,
"learning_rate": 4.858144469637408e-07,
"logits/chosen": 0.2730848789215088,
"logits/rejected": 0.24042408168315887,
"logps/chosen": -73.0877685546875,
"logps/ref_chosen": -65.68513488769531,
"logps/ref_rejected": -69.54120635986328,
"logps/rejected": -80.24639892578125,
"loss": 1.0744,
"margin_dpo/margin_mean": 3.3025588989257812,
"margin_dpo/margin_std": 4.778876304626465,
"step": 132
},
{
"KL/chosen_KL_mean": -7.787448883056641,
"KL/mean": -9.092580795288086,
"KL/rejected_KL_mean": -10.397708892822266,
"KL/std": 4.733419418334961,
"epoch": 0.20105820105820105,
"fcm_dpo/beta": 0.1470513641834259,
"fcm_dpo/delta": 0.06877677142620087,
"fcm_dpo/margin": 2.6102614402770996,
"fcm_dpo/q_t": 0.41596511006355286,
"grad_norm": 45.837135314941406,
"learning_rate": 4.853720930118138e-07,
"logits/chosen": 0.183448925614357,
"logits/rejected": 0.1737731397151947,
"logps/chosen": -71.38555908203125,
"logps/ref_chosen": -63.598114013671875,
"logps/ref_rejected": -73.72798156738281,
"logps/rejected": -84.12568664550781,
"loss": 1.1678,
"margin_dpo/margin_mean": 2.6102609634399414,
"margin_dpo/margin_std": 5.107699394226074,
"step": 133
},
{
"KL/chosen_KL_mean": -7.144420623779297,
"KL/mean": -9.359694480895996,
"KL/rejected_KL_mean": -11.574962615966797,
"KL/std": 5.464275360107422,
"epoch": 0.20256991685563114,
"fcm_dpo/beta": 0.1433096081018448,
"fcm_dpo/delta": -0.1974298059940338,
"fcm_dpo/margin": 4.430543422698975,
"fcm_dpo/q_t": 0.36382484436035156,
"grad_norm": 33.925498962402344,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": 0.29932132363319397,
"logits/rejected": 0.24174457788467407,
"logps/chosen": -60.93899154663086,
"logps/ref_chosen": -53.79457092285156,
"logps/ref_rejected": -74.16741943359375,
"logps/rejected": -85.74238586425781,
"loss": 0.9776,
"margin_dpo/margin_mean": 4.430543899536133,
"margin_dpo/margin_std": 5.441107749938965,
"step": 134
},
{
"KL/chosen_KL_mean": -7.194000244140625,
"KL/mean": -8.616719245910645,
"KL/rejected_KL_mean": -10.039440155029297,
"KL/std": 5.098479270935059,
"epoch": 0.20408163265306123,
"fcm_dpo/beta": 0.1429201066493988,
"fcm_dpo/delta": 0.04509545862674713,
"fcm_dpo/margin": 2.84544038772583,
"fcm_dpo/q_t": 0.41076767444610596,
"grad_norm": 34.94743347167969,
"learning_rate": 4.844676460754862e-07,
"logits/chosen": 0.21902255713939667,
"logits/rejected": 0.18694475293159485,
"logps/chosen": -56.63507843017578,
"logps/ref_chosen": -49.441078186035156,
"logps/ref_rejected": -65.96878051757812,
"logps/rejected": -76.00822448730469,
"loss": 1.1456,
"margin_dpo/margin_mean": 2.845439910888672,
"margin_dpo/margin_std": 5.2115278244018555,
"step": 135
},
{
"KL/chosen_KL_mean": -8.81231689453125,
"KL/mean": -10.606294631958008,
"KL/rejected_KL_mean": -12.400272369384766,
"KL/std": 5.5866851806640625,
"epoch": 0.20559334845049132,
"fcm_dpo/beta": 0.14106975495815277,
"fcm_dpo/delta": -0.06005358695983887,
"fcm_dpo/margin": 3.5879569053649902,
"fcm_dpo/q_t": 0.39729416370391846,
"grad_norm": 43.33884048461914,
"learning_rate": 4.840055783904106e-07,
"logits/chosen": 0.21669332683086395,
"logits/rejected": 0.14369165897369385,
"logps/chosen": -75.57157897949219,
"logps/ref_chosen": -66.75926208496094,
"logps/ref_rejected": -94.61787414550781,
"logps/rejected": -107.01814270019531,
"loss": 1.1576,
"margin_dpo/margin_mean": 3.5879576206207275,
"margin_dpo/margin_std": 7.040832996368408,
"step": 136
},
{
"KL/chosen_KL_mean": -8.229362487792969,
"KL/mean": -10.075550079345703,
"KL/rejected_KL_mean": -11.921737670898438,
"KL/std": 5.315681457519531,
"epoch": 0.20710506424792138,
"fcm_dpo/beta": 0.14049124717712402,
"fcm_dpo/delta": -0.07258465886116028,
"fcm_dpo/margin": 3.692375659942627,
"fcm_dpo/q_t": 0.3897295892238617,
"grad_norm": 34.87916564941406,
"learning_rate": 4.835369650662767e-07,
"logits/chosen": 0.25548824667930603,
"logits/rejected": 0.22579577565193176,
"logps/chosen": -65.0131607055664,
"logps/ref_chosen": -56.78379821777344,
"logps/ref_rejected": -69.89952087402344,
"logps/rejected": -81.82125854492188,
"loss": 1.0721,
"margin_dpo/margin_mean": 3.692375659942627,
"margin_dpo/margin_std": 5.650632858276367,
"step": 137
},
{
"KL/chosen_KL_mean": -9.61297607421875,
"KL/mean": -11.213253021240234,
"KL/rejected_KL_mean": -12.813533782958984,
"KL/std": 5.492915630340576,
"epoch": 0.20861678004535147,
"fcm_dpo/beta": 0.13947460055351257,
"fcm_dpo/delta": 0.003768614027649164,
"fcm_dpo/margin": 3.200556993484497,
"fcm_dpo/q_t": 0.4026796221733093,
"grad_norm": 37.49756622314453,
"learning_rate": 4.830618192112065e-07,
"logits/chosen": 0.2518809735774994,
"logits/rejected": 0.21509861946105957,
"logps/chosen": -68.37899017333984,
"logps/ref_chosen": -58.766014099121094,
"logps/ref_rejected": -68.12371826171875,
"logps/rejected": -80.937255859375,
"loss": 1.1513,
"margin_dpo/margin_mean": 3.200556755065918,
"margin_dpo/margin_std": 6.079831123352051,
"step": 138
},
{
"KL/chosen_KL_mean": -10.039913177490234,
"KL/mean": -11.939605712890625,
"KL/rejected_KL_mean": -13.83929443359375,
"KL/std": 5.361264228820801,
"epoch": 0.21012849584278157,
"fcm_dpo/beta": 0.13811233639717102,
"fcm_dpo/delta": -0.0788048505783081,
"fcm_dpo/margin": 3.7993788719177246,
"fcm_dpo/q_t": 0.38475942611694336,
"grad_norm": 40.01036071777344,
"learning_rate": 4.825801541160509e-07,
"logits/chosen": 0.1889549195766449,
"logits/rejected": 0.15961438417434692,
"logps/chosen": -81.2655029296875,
"logps/ref_chosen": -71.2255859375,
"logps/ref_rejected": -82.1834716796875,
"logps/rejected": -96.02276611328125,
"loss": 1.0565,
"margin_dpo/margin_mean": 3.7993783950805664,
"margin_dpo/margin_std": 5.455878734588623,
"step": 139
},
{
"KL/chosen_KL_mean": -9.19911003112793,
"KL/mean": -11.774042129516602,
"KL/rejected_KL_mean": -14.348968505859375,
"KL/std": 7.151411056518555,
"epoch": 0.21164021164021163,
"fcm_dpo/beta": 0.13198350369930267,
"fcm_dpo/delta": -0.24726887047290802,
"fcm_dpo/margin": 5.149857044219971,
"fcm_dpo/q_t": 0.35907793045043945,
"grad_norm": 40.80632781982422,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": 0.21919316053390503,
"logits/rejected": 0.17110946774482727,
"logps/chosen": -72.47677612304688,
"logps/ref_chosen": -63.27766418457031,
"logps/ref_rejected": -83.30647277832031,
"logps/rejected": -97.65544128417969,
"loss": 1.0468,
"margin_dpo/margin_mean": 5.149857044219971,
"margin_dpo/margin_std": 7.859900951385498,
"step": 140
},
{
"KL/chosen_KL_mean": -10.046802520751953,
"KL/mean": -12.450661659240723,
"KL/rejected_KL_mean": -14.854522705078125,
"KL/std": 5.955142974853516,
"epoch": 0.21315192743764172,
"fcm_dpo/beta": 0.12623918056488037,
"fcm_dpo/delta": -0.17030926048755646,
"fcm_dpo/margin": 4.8077168464660645,
"fcm_dpo/q_t": 0.37628281116485596,
"grad_norm": 35.90546417236328,
"learning_rate": 4.815973202802966e-07,
"logits/chosen": 0.25997358560562134,
"logits/rejected": 0.21349510550498962,
"logps/chosen": -71.81356811523438,
"logps/ref_chosen": -61.76676940917969,
"logps/ref_rejected": -88.60601806640625,
"logps/rejected": -103.46054077148438,
"loss": 1.0647,
"margin_dpo/margin_mean": 4.807717800140381,
"margin_dpo/margin_std": 7.479946613311768,
"step": 141
},
{
"KL/chosen_KL_mean": -9.879066467285156,
"KL/mean": -11.788759231567383,
"KL/rejected_KL_mean": -13.698455810546875,
"KL/std": 5.767460823059082,
"epoch": 0.2146636432350718,
"fcm_dpo/beta": 0.12553216516971588,
"fcm_dpo/delta": -0.031014427542686462,
"fcm_dpo/margin": 3.819387435913086,
"fcm_dpo/q_t": 0.3950141668319702,
"grad_norm": 33.704952239990234,
"learning_rate": 4.810961790316729e-07,
"logits/chosen": 0.252850204706192,
"logits/rejected": 0.22739627957344055,
"logps/chosen": -75.15383911132812,
"logps/ref_chosen": -65.2747802734375,
"logps/ref_rejected": -81.1378173828125,
"logps/rejected": -94.83627319335938,
"loss": 1.0841,
"margin_dpo/margin_mean": 3.819387197494507,
"margin_dpo/margin_std": 5.84207010269165,
"step": 142
},
{
"KL/chosen_KL_mean": -10.517995834350586,
"KL/mean": -12.156927108764648,
"KL/rejected_KL_mean": -13.795856475830078,
"KL/std": 6.709386825561523,
"epoch": 0.2161753590325019,
"fcm_dpo/beta": 0.12634995579719543,
"fcm_dpo/delta": 0.037314772605895996,
"fcm_dpo/margin": 3.2778611183166504,
"fcm_dpo/q_t": 0.40795421600341797,
"grad_norm": 46.63773727416992,
"learning_rate": 4.805885735261454e-07,
"logits/chosen": 0.27107226848602295,
"logits/rejected": 0.25340479612350464,
"logps/chosen": -73.13581848144531,
"logps/ref_chosen": -62.617828369140625,
"logps/ref_rejected": -70.39239501953125,
"logps/rejected": -84.18824768066406,
"loss": 1.2333,
"margin_dpo/margin_mean": 3.2778611183166504,
"margin_dpo/margin_std": 7.917219161987305,
"step": 143
},
{
"KL/chosen_KL_mean": -10.777961730957031,
"KL/mean": -12.884668350219727,
"KL/rejected_KL_mean": -14.991374969482422,
"KL/std": 6.686635971069336,
"epoch": 0.21768707482993196,
"fcm_dpo/beta": 0.12440179288387299,
"fcm_dpo/delta": -0.07902223616838455,
"fcm_dpo/margin": 4.21340799331665,
"fcm_dpo/q_t": 0.39374351501464844,
"grad_norm": 37.01242446899414,
"learning_rate": 4.800745179625307e-07,
"logits/chosen": 0.2540838122367859,
"logits/rejected": 0.2218245565891266,
"logps/chosen": -71.58065032958984,
"logps/ref_chosen": -60.80268859863281,
"logps/ref_rejected": -79.07284545898438,
"logps/rejected": -94.06421661376953,
"loss": 1.1251,
"margin_dpo/margin_mean": 4.21340799331665,
"margin_dpo/margin_std": 7.684472560882568,
"step": 144
},
{
"KL/chosen_KL_mean": -9.943706512451172,
"KL/mean": -12.170425415039062,
"KL/rejected_KL_mean": -14.397136688232422,
"KL/std": 6.7663373947143555,
"epoch": 0.21919879062736206,
"fcm_dpo/beta": 0.12235713750123978,
"fcm_dpo/delta": -0.10066936910152435,
"fcm_dpo/margin": 4.453435897827148,
"fcm_dpo/q_t": 0.3854507803916931,
"grad_norm": 39.809226989746094,
"learning_rate": 4.795540267200686e-07,
"logits/chosen": 0.20279008150100708,
"logits/rejected": 0.21891656517982483,
"logps/chosen": -84.55516815185547,
"logps/ref_chosen": -74.61146545410156,
"logps/ref_rejected": -83.24461364746094,
"logps/rejected": -97.64175415039062,
"loss": 1.1188,
"margin_dpo/margin_mean": 4.453435897827148,
"margin_dpo/margin_std": 8.038009643554688,
"step": 145
},
{
"KL/chosen_KL_mean": -9.593629837036133,
"KL/mean": -11.913301467895508,
"KL/rejected_KL_mean": -14.232978820800781,
"KL/std": 6.6814470291137695,
"epoch": 0.22071050642479215,
"fcm_dpo/beta": 0.12027982622385025,
"fcm_dpo/delta": -0.11424677819013596,
"fcm_dpo/margin": 4.6393537521362305,
"fcm_dpo/q_t": 0.38348451256752014,
"grad_norm": 34.77975845336914,
"learning_rate": 4.790271143580173e-07,
"logits/chosen": 0.19675567746162415,
"logits/rejected": 0.18253400921821594,
"logps/chosen": -67.43461608886719,
"logps/ref_chosen": -57.84098434448242,
"logps/ref_rejected": -67.47422790527344,
"logps/rejected": -81.70720672607422,
"loss": 1.0785,
"margin_dpo/margin_mean": 4.6393537521362305,
"margin_dpo/margin_std": 7.411983489990234,
"step": 146
},
{
"KL/chosen_KL_mean": -11.185792922973633,
"KL/mean": -13.020978927612305,
"KL/rejected_KL_mean": -14.85616683959961,
"KL/std": 7.158926963806152,
"epoch": 0.2222222222222222,
"fcm_dpo/beta": 0.11947156488895416,
"fcm_dpo/delta": 0.012014790438115597,
"fcm_dpo/margin": 3.6703786849975586,
"fcm_dpo/q_t": 0.4066588580608368,
"grad_norm": 42.58670425415039,
"learning_rate": 4.784937956152489e-07,
"logits/chosen": 0.22439086437225342,
"logits/rejected": 0.18046727776527405,
"logps/chosen": -77.999267578125,
"logps/ref_chosen": -66.81346893310547,
"logps/ref_rejected": -81.1796875,
"logps/rejected": -96.03585815429688,
"loss": 1.1787,
"margin_dpo/margin_mean": 3.6703782081604004,
"margin_dpo/margin_std": 7.668435096740723,
"step": 147
},
{
"KL/chosen_KL_mean": -10.262016296386719,
"KL/mean": -12.560417175292969,
"KL/rejected_KL_mean": -14.858814239501953,
"KL/std": 6.569252967834473,
"epoch": 0.2237339380196523,
"fcm_dpo/beta": 0.11826850473880768,
"fcm_dpo/delta": -0.09899365901947021,
"fcm_dpo/margin": 4.596801280975342,
"fcm_dpo/q_t": 0.3836548328399658,
"grad_norm": 28.631181716918945,
"learning_rate": 4.779540854098347e-07,
"logits/chosen": 0.3569459319114685,
"logits/rejected": 0.2824467420578003,
"logps/chosen": -58.949771881103516,
"logps/ref_chosen": -48.6877555847168,
"logps/ref_rejected": -67.50503540039062,
"logps/rejected": -82.36384582519531,
"loss": 1.0861,
"margin_dpo/margin_mean": 4.596801280975342,
"margin_dpo/margin_std": 7.328686237335205,
"step": 148
},
{
"KL/chosen_KL_mean": -12.381685256958008,
"KL/mean": -14.980354309082031,
"KL/rejected_KL_mean": -17.579021453857422,
"KL/std": 7.072737693786621,
"epoch": 0.2252456538170824,
"fcm_dpo/beta": 0.11462083458900452,
"fcm_dpo/delta": -0.1548178791999817,
"fcm_dpo/margin": 5.197340965270996,
"fcm_dpo/q_t": 0.3695276379585266,
"grad_norm": 29.586246490478516,
"learning_rate": 4.774079988386296e-07,
"logits/chosen": 0.21398887038230896,
"logits/rejected": 0.16681325435638428,
"logps/chosen": -67.52546691894531,
"logps/ref_chosen": -55.143775939941406,
"logps/ref_rejected": -64.79888916015625,
"logps/rejected": -82.37791442871094,
"loss": 1.0049,
"margin_dpo/margin_mean": 5.19734001159668,
"margin_dpo/margin_std": 6.64594841003418,
"step": 149
},
{
"KL/chosen_KL_mean": -10.293771743774414,
"KL/mean": -13.691690444946289,
"KL/rejected_KL_mean": -17.089603424072266,
"KL/std": 7.609349250793457,
"epoch": 0.22675736961451248,
"fcm_dpo/beta": 0.10836917161941528,
"fcm_dpo/delta": -0.3096678853034973,
"fcm_dpo/margin": 6.79583740234375,
"fcm_dpo/q_t": 0.34543052315711975,
"grad_norm": 27.946935653686523,
"learning_rate": 4.768555511768486e-07,
"logits/chosen": 0.24622797966003418,
"logits/rejected": 0.20110848546028137,
"logps/chosen": -77.76451873779297,
"logps/ref_chosen": -67.47074890136719,
"logps/ref_rejected": -89.21170806884766,
"logps/rejected": -106.30131530761719,
"loss": 0.933,
"margin_dpo/margin_mean": 6.795836925506592,
"margin_dpo/margin_std": 7.71773624420166,
"step": 150
},
{
"KL/chosen_KL_mean": -9.049201965332031,
"KL/mean": -12.828123092651367,
"KL/rejected_KL_mean": -16.607044219970703,
"KL/std": 8.092508316040039,
"epoch": 0.22826908541194255,
"fcm_dpo/beta": 0.10158209502696991,
"fcm_dpo/delta": -0.3438608646392822,
"fcm_dpo/margin": 7.557843208312988,
"fcm_dpo/q_t": 0.33743754029273987,
"grad_norm": 24.914169311523438,
"learning_rate": 4.762967578776406e-07,
"logits/chosen": 0.2584247887134552,
"logits/rejected": 0.20215699076652527,
"logps/chosen": -61.50874328613281,
"logps/ref_chosen": -52.45954132080078,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -95.67005920410156,
"loss": 0.901,
"margin_dpo/margin_mean": 7.55784273147583,
"margin_dpo/margin_std": 8.19579029083252,
"step": 151
},
{
"KL/chosen_KL_mean": -11.547626495361328,
"KL/mean": -14.018619537353516,
"KL/rejected_KL_mean": -16.489608764648438,
"KL/std": 7.6490159034729,
"epoch": 0.22978080120937264,
"fcm_dpo/beta": 0.09971933811903,
"fcm_dpo/delta": -0.045774128288030624,
"fcm_dpo/margin": 4.941987037658691,
"fcm_dpo/q_t": 0.3965153694152832,
"grad_norm": 29.130266189575195,
"learning_rate": 4.757316345716553e-07,
"logits/chosen": 0.2998914420604706,
"logits/rejected": 0.24673616886138916,
"logps/chosen": -68.10145568847656,
"logps/ref_chosen": -56.5538330078125,
"logps/ref_rejected": -76.55074310302734,
"logps/rejected": -93.04035186767578,
"loss": 1.1252,
"margin_dpo/margin_mean": 4.941986560821533,
"margin_dpo/margin_std": 8.686779975891113,
"step": 152
},
{
"KL/chosen_KL_mean": -10.736824035644531,
"KL/mean": -13.64626693725586,
"KL/rejected_KL_mean": -16.555709838867188,
"KL/std": 7.962043762207031,
"epoch": 0.23129251700680273,
"fcm_dpo/beta": 0.09694686532020569,
"fcm_dpo/delta": -0.12102583050727844,
"fcm_dpo/margin": 5.818882942199707,
"fcm_dpo/q_t": 0.375736802816391,
"grad_norm": 26.531309127807617,
"learning_rate": 4.751601970666064e-07,
"logits/chosen": 0.22037890553474426,
"logits/rejected": 0.18365280330181122,
"logps/chosen": -78.74371337890625,
"logps/ref_chosen": -68.00689697265625,
"logps/ref_rejected": -74.83482360839844,
"logps/rejected": -91.39053344726562,
"loss": 1.0171,
"margin_dpo/margin_mean": 5.818883895874023,
"margin_dpo/margin_std": 7.457840919494629,
"step": 153
},
{
"KL/chosen_KL_mean": -11.988504409790039,
"KL/mean": -14.088945388793945,
"KL/rejected_KL_mean": -16.189388275146484,
"KL/std": 8.195691108703613,
"epoch": 0.2328042328042328,
"fcm_dpo/beta": 0.09695246070623398,
"fcm_dpo/delta": 0.04455181211233139,
"fcm_dpo/margin": 4.2008795738220215,
"fcm_dpo/q_t": 0.4105023741722107,
"grad_norm": 28.0793399810791,
"learning_rate": 4.745824613468292e-07,
"logits/chosen": 0.31881213188171387,
"logits/rejected": 0.31584903597831726,
"logps/chosen": -71.21104431152344,
"logps/ref_chosen": -59.222537994384766,
"logps/ref_rejected": -64.19131469726562,
"logps/rejected": -80.38070678710938,
"loss": 1.1732,
"margin_dpo/margin_mean": 4.2008795738220215,
"margin_dpo/margin_std": 8.427075386047363,
"step": 154
},
{
"KL/chosen_KL_mean": -12.549442291259766,
"KL/mean": -15.515789031982422,
"KL/rejected_KL_mean": -18.482131958007812,
"KL/std": 8.112979888916016,
"epoch": 0.23431594860166288,
"fcm_dpo/beta": 0.09494648873806,
"fcm_dpo/delta": -0.12154512107372284,
"fcm_dpo/margin": 5.932689666748047,
"fcm_dpo/q_t": 0.3839681148529053,
"grad_norm": 28.116670608520508,
"learning_rate": 4.7399844357283393e-07,
"logits/chosen": 0.30328816175460815,
"logits/rejected": 0.28287360072135925,
"logps/chosen": -81.00413513183594,
"logps/ref_chosen": -68.45469665527344,
"logps/ref_rejected": -77.91763305664062,
"logps/rejected": -96.39976501464844,
"loss": 1.1091,
"margin_dpo/margin_mean": 5.932689666748047,
"margin_dpo/margin_std": 10.310593605041504,
"step": 155
},
{
"KL/chosen_KL_mean": -12.504823684692383,
"KL/mean": -16.008132934570312,
"KL/rejected_KL_mean": -19.51144027709961,
"KL/std": 9.073400497436523,
"epoch": 0.23582766439909297,
"fcm_dpo/beta": 0.09212834388017654,
"fcm_dpo/delta": -0.2088165283203125,
"fcm_dpo/margin": 7.006613731384277,
"fcm_dpo/q_t": 0.3617513179779053,
"grad_norm": 26.31268882751465,
"learning_rate": 4.7340816008085305e-07,
"logits/chosen": 0.2535175681114197,
"logits/rejected": 0.20776960253715515,
"logps/chosen": -79.77442169189453,
"logps/ref_chosen": -67.26959991455078,
"logps/ref_rejected": -86.95914459228516,
"logps/rejected": -106.4705810546875,
"loss": 0.9756,
"margin_dpo/margin_mean": 7.006614685058594,
"margin_dpo/margin_std": 8.517118453979492,
"step": 156
},
{
"KL/chosen_KL_mean": -11.273056030273438,
"KL/mean": -14.099236488342285,
"KL/rejected_KL_mean": -16.925411224365234,
"KL/std": 8.791933059692383,
"epoch": 0.23733938019652306,
"fcm_dpo/beta": 0.08914068341255188,
"fcm_dpo/delta": -0.06050056219100952,
"fcm_dpo/margin": 5.652355194091797,
"fcm_dpo/q_t": 0.3902330994606018,
"grad_norm": 25.38435173034668,
"learning_rate": 4.728116273823847e-07,
"logits/chosen": 0.2619755268096924,
"logits/rejected": 0.24316802620887756,
"logps/chosen": -66.04592895507812,
"logps/ref_chosen": -54.77287292480469,
"logps/ref_rejected": -63.87866973876953,
"logps/rejected": -80.8040771484375,
"loss": 1.0866,
"margin_dpo/margin_mean": 5.652355194091797,
"margin_dpo/margin_std": 8.495977401733398,
"step": 157
},
{
"KL/chosen_KL_mean": -12.521953582763672,
"KL/mean": -15.536789894104004,
"KL/rejected_KL_mean": -18.55162811279297,
"KL/std": 8.959171295166016,
"epoch": 0.23885109599395313,
"fcm_dpo/beta": 0.08881160616874695,
"fcm_dpo/delta": -0.09029243886470795,
"fcm_dpo/margin": 6.029675483703613,
"fcm_dpo/q_t": 0.3862513601779938,
"grad_norm": 26.304529190063477,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": 0.29904159903526306,
"logits/rejected": 0.2606283724308014,
"logps/chosen": -77.44467163085938,
"logps/ref_chosen": -64.92271423339844,
"logps/ref_rejected": -82.23789978027344,
"logps/rejected": -100.78952026367188,
"loss": 1.0768,
"margin_dpo/margin_mean": 6.029675483703613,
"margin_dpo/margin_std": 9.455682754516602,
"step": 158
},
{
"KL/chosen_KL_mean": -12.712642669677734,
"KL/mean": -16.16486358642578,
"KL/rejected_KL_mean": -19.617088317871094,
"KL/std": 9.580606460571289,
"epoch": 0.24036281179138322,
"fcm_dpo/beta": 0.08682575821876526,
"fcm_dpo/delta": -0.1588217169046402,
"fcm_dpo/margin": 6.904443264007568,
"fcm_dpo/q_t": 0.37174341082572937,
"grad_norm": 27.926536560058594,
"learning_rate": 4.715998812855304e-07,
"logits/chosen": 0.3188456892967224,
"logits/rejected": 0.2785356044769287,
"logps/chosen": -69.75963592529297,
"logps/ref_chosen": -57.046993255615234,
"logps/ref_rejected": -73.32441711425781,
"logps/rejected": -92.9415054321289,
"loss": 1.0438,
"margin_dpo/margin_mean": 6.90444278717041,
"margin_dpo/margin_std": 10.084371566772461,
"step": 159
},
{
"KL/chosen_KL_mean": -14.375518798828125,
"KL/mean": -17.316848754882812,
"KL/rejected_KL_mean": -20.258174896240234,
"KL/std": 8.959368705749512,
"epoch": 0.2418745275888133,
"fcm_dpo/beta": 0.0849989801645279,
"fcm_dpo/delta": -0.052619896829128265,
"fcm_dpo/margin": 5.882652759552002,
"fcm_dpo/q_t": 0.3973754644393921,
"grad_norm": 24.18868064880371,
"learning_rate": 4.7098470178228755e-07,
"logits/chosen": 0.16954955458641052,
"logits/rejected": 0.1237107664346695,
"logps/chosen": -64.18243408203125,
"logps/ref_chosen": -49.806915283203125,
"logps/ref_rejected": -68.3370132446289,
"logps/rejected": -88.59518432617188,
"loss": 1.1145,
"margin_dpo/margin_mean": 5.882652282714844,
"margin_dpo/margin_std": 10.348878860473633,
"step": 160
},
{
"KL/chosen_KL_mean": -14.157938003540039,
"KL/mean": -17.39502716064453,
"KL/rejected_KL_mean": -20.63211441040039,
"KL/std": 8.806609153747559,
"epoch": 0.24338624338624337,
"fcm_dpo/beta": 0.08354272693395615,
"fcm_dpo/delta": -0.09600973129272461,
"fcm_dpo/margin": 6.474179267883301,
"fcm_dpo/q_t": 0.3830728530883789,
"grad_norm": 23.179370880126953,
"learning_rate": 4.703633408618955e-07,
"logits/chosen": 0.30207502841949463,
"logits/rejected": 0.2627732753753662,
"logps/chosen": -66.6584243774414,
"logps/ref_chosen": -52.50048828125,
"logps/ref_rejected": -66.04540252685547,
"logps/rejected": -86.67752075195312,
"loss": 1.0708,
"margin_dpo/margin_mean": 6.474179267883301,
"margin_dpo/margin_std": 9.89106559753418,
"step": 161
},
{
"KL/chosen_KL_mean": -15.079242706298828,
"KL/mean": -19.381000518798828,
"KL/rejected_KL_mean": -23.682758331298828,
"KL/std": 9.969080924987793,
"epoch": 0.24489795918367346,
"fcm_dpo/beta": 0.07994553446769714,
"fcm_dpo/delta": -0.2555304765701294,
"fcm_dpo/margin": 8.603517532348633,
"fcm_dpo/q_t": 0.3500090539455414,
"grad_norm": 24.3117733001709,
"learning_rate": 4.697358159051549e-07,
"logits/chosen": 0.34407973289489746,
"logits/rejected": 0.29311051964759827,
"logps/chosen": -84.54843139648438,
"logps/ref_chosen": -69.46919250488281,
"logps/ref_rejected": -92.00952911376953,
"logps/rejected": -115.69229125976562,
"loss": 0.9533,
"margin_dpo/margin_mean": 8.603517532348633,
"margin_dpo/margin_std": 10.078315734863281,
"step": 162
},
{
"KL/chosen_KL_mean": -13.317777633666992,
"KL/mean": -17.590572357177734,
"KL/rejected_KL_mean": -21.86336898803711,
"KL/std": 9.784802436828613,
"epoch": 0.24640967498110355,
"fcm_dpo/beta": 0.0772738978266716,
"fcm_dpo/delta": -0.22547154128551483,
"fcm_dpo/margin": 8.545589447021484,
"fcm_dpo/q_t": 0.35784658789634705,
"grad_norm": 22.079208374023438,
"learning_rate": 4.691021444652876e-07,
"logits/chosen": 0.2510836720466614,
"logits/rejected": 0.20358552038669586,
"logps/chosen": -63.931610107421875,
"logps/ref_chosen": -50.613834381103516,
"logps/ref_rejected": -74.62033081054688,
"logps/rejected": -96.48369598388672,
"loss": 0.9873,
"margin_dpo/margin_mean": 8.5455904006958,
"margin_dpo/margin_std": 10.263286590576172,
"step": 163
},
{
"KL/chosen_KL_mean": -14.311813354492188,
"KL/mean": -18.52021026611328,
"KL/rejected_KL_mean": -22.72860336303711,
"KL/std": 10.371530532836914,
"epoch": 0.24792139077853365,
"fcm_dpo/beta": 0.07386015355587006,
"fcm_dpo/delta": -0.18267156183719635,
"fcm_dpo/margin": 8.416790008544922,
"fcm_dpo/q_t": 0.3670443892478943,
"grad_norm": 20.7839298248291,
"learning_rate": 4.6846234426744624e-07,
"logits/chosen": 0.2554076611995697,
"logits/rejected": 0.1913755238056183,
"logps/chosen": -69.15992736816406,
"logps/ref_chosen": -54.848114013671875,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -101.79161071777344,
"loss": 1.0178,
"margin_dpo/margin_mean": 8.416790008544922,
"margin_dpo/margin_std": 11.19088363647461,
"step": 164
},
{
"KL/chosen_KL_mean": -15.9600830078125,
"KL/mean": -19.80939483642578,
"KL/rejected_KL_mean": -23.658706665039062,
"KL/std": 10.531667709350586,
"epoch": 0.2494331065759637,
"fcm_dpo/beta": 0.071999192237854,
"fcm_dpo/delta": -0.1104959100484848,
"fcm_dpo/margin": 7.698622226715088,
"fcm_dpo/q_t": 0.3786641061306,
"grad_norm": 21.82950782775879,
"learning_rate": 4.678164332082175e-07,
"logits/chosen": 0.33307212591171265,
"logits/rejected": 0.27663278579711914,
"logps/chosen": -67.0492935180664,
"logps/ref_chosen": -51.089210510253906,
"logps/ref_rejected": -71.23370361328125,
"logps/rejected": -94.89241027832031,
"loss": 1.0631,
"margin_dpo/margin_mean": 7.69862174987793,
"margin_dpo/margin_std": 11.216785430908203,
"step": 165
},
{
"KL/chosen_KL_mean": -15.295942306518555,
"KL/mean": -18.283523559570312,
"KL/rejected_KL_mean": -21.271102905273438,
"KL/std": 10.442094802856445,
"epoch": 0.2509448223733938,
"fcm_dpo/beta": 0.0712810531258583,
"fcm_dpo/delta": 0.025124577805399895,
"fcm_dpo/margin": 5.975157737731934,
"fcm_dpo/q_t": 0.40849870443344116,
"grad_norm": 24.805681228637695,
"learning_rate": 4.6716442935512214e-07,
"logits/chosen": 0.27621328830718994,
"logits/rejected": 0.1836838275194168,
"logps/chosen": -78.48675537109375,
"logps/ref_chosen": -63.19081115722656,
"logps/ref_rejected": -93.8402099609375,
"logps/rejected": -115.11131286621094,
"loss": 1.1214,
"margin_dpo/margin_mean": 5.975157260894775,
"margin_dpo/margin_std": 10.045669555664062,
"step": 166
},
{
"KL/chosen_KL_mean": -13.135995864868164,
"KL/mean": -17.774768829345703,
"KL/rejected_KL_mean": -22.41353988647461,
"KL/std": 10.714015007019043,
"epoch": 0.25245653817082386,
"fcm_dpo/beta": 0.06891494989395142,
"fcm_dpo/delta": -0.20458835363388062,
"fcm_dpo/margin": 9.277545928955078,
"fcm_dpo/q_t": 0.3598324656486511,
"grad_norm": 20.088937759399414,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": 0.24189813435077667,
"logits/rejected": 0.20661483705043793,
"logps/chosen": -72.06027221679688,
"logps/ref_chosen": -58.92427062988281,
"logps/ref_rejected": -72.97377014160156,
"logps/rejected": -95.38731384277344,
"loss": 0.9602,
"margin_dpo/margin_mean": 9.277545928955078,
"margin_dpo/margin_std": 10.376180648803711,
"step": 167
},
{
"KL/chosen_KL_mean": -15.331819534301758,
"KL/mean": -18.971294403076172,
"KL/rejected_KL_mean": -22.610767364501953,
"KL/std": 11.420768737792969,
"epoch": 0.25396825396825395,
"fcm_dpo/beta": 0.06804715096950531,
"fcm_dpo/delta": -0.04781053215265274,
"fcm_dpo/margin": 7.278944969177246,
"fcm_dpo/q_t": 0.39334410429000854,
"grad_norm": 22.273143768310547,
"learning_rate": 4.6584221638904767e-07,
"logits/chosen": 0.25680267810821533,
"logits/rejected": 0.2203751504421234,
"logps/chosen": -80.98320007324219,
"logps/ref_chosen": -65.65138244628906,
"logps/ref_rejected": -79.71418762207031,
"logps/rejected": -102.324951171875,
"loss": 1.0675,
"margin_dpo/margin_mean": 7.27894401550293,
"margin_dpo/margin_std": 10.532588958740234,
"step": 168
},
{
"KL/chosen_KL_mean": -12.636564254760742,
"KL/mean": -16.88669776916504,
"KL/rejected_KL_mean": -21.13683319091797,
"KL/std": 11.466968536376953,
"epoch": 0.25547996976568405,
"fcm_dpo/beta": 0.06674225628376007,
"fcm_dpo/delta": -0.12439984083175659,
"fcm_dpo/margin": 8.500267028808594,
"fcm_dpo/q_t": 0.38215646147727966,
"grad_norm": 22.956363677978516,
"learning_rate": 4.651720442612075e-07,
"logits/chosen": 0.319760799407959,
"logits/rejected": 0.28525394201278687,
"logps/chosen": -74.06243133544922,
"logps/ref_chosen": -61.425865173339844,
"logps/ref_rejected": -76.09590148925781,
"logps/rejected": -97.23272705078125,
"loss": 1.0698,
"margin_dpo/margin_mean": 8.500267028808594,
"margin_dpo/margin_std": 13.569889068603516,
"step": 169
},
{
"KL/chosen_KL_mean": -12.162647247314453,
"KL/mean": -16.37779998779297,
"KL/rejected_KL_mean": -20.592947006225586,
"KL/std": 11.154409408569336,
"epoch": 0.25699168556311414,
"fcm_dpo/beta": 0.06543616950511932,
"fcm_dpo/delta": -0.10744147002696991,
"fcm_dpo/margin": 8.43030071258545,
"fcm_dpo/q_t": 0.3819403052330017,
"grad_norm": 18.930606842041016,
"learning_rate": 4.6449585330874425e-07,
"logits/chosen": 0.27050989866256714,
"logits/rejected": 0.26806920766830444,
"logps/chosen": -68.81584167480469,
"logps/ref_chosen": -56.65319061279297,
"logps/ref_rejected": -63.45965576171875,
"logps/rejected": -84.05259704589844,
"loss": 1.0897,
"margin_dpo/margin_mean": 8.43030071258545,
"margin_dpo/margin_std": 13.604389190673828,
"step": 170
},
{
"KL/chosen_KL_mean": -13.745929718017578,
"KL/mean": -18.389129638671875,
"KL/rejected_KL_mean": -23.032333374023438,
"KL/std": 12.820600509643555,
"epoch": 0.2585034013605442,
"fcm_dpo/beta": 0.06286909431219101,
"fcm_dpo/delta": -0.14584212005138397,
"fcm_dpo/margin": 9.286405563354492,
"fcm_dpo/q_t": 0.3753986656665802,
"grad_norm": 20.48279571533203,
"learning_rate": 4.6381366244617224e-07,
"logits/chosen": 0.3456369638442993,
"logits/rejected": 0.2936851382255554,
"logps/chosen": -77.4806900024414,
"logps/ref_chosen": -63.73476028442383,
"logps/ref_rejected": -78.50328063964844,
"logps/rejected": -101.53561401367188,
"loss": 1.0752,
"margin_dpo/margin_mean": 9.286405563354492,
"margin_dpo/margin_std": 14.497032165527344,
"step": 171
},
{
"KL/chosen_KL_mean": -15.124223709106445,
"KL/mean": -19.514453887939453,
"KL/rejected_KL_mean": -23.904678344726562,
"KL/std": 12.908832550048828,
"epoch": 0.2600151171579743,
"fcm_dpo/beta": 0.06263021379709244,
"fcm_dpo/delta": -0.10646422207355499,
"fcm_dpo/margin": 8.780457496643066,
"fcm_dpo/q_t": 0.38289976119995117,
"grad_norm": 20.70548439025879,
"learning_rate": 4.631254907558365e-07,
"logits/chosen": 0.3301047384738922,
"logits/rejected": 0.27112317085266113,
"logps/chosen": -67.32598114013672,
"logps/ref_chosen": -52.201759338378906,
"logps/ref_rejected": -82.85285949707031,
"logps/rejected": -106.75753784179688,
"loss": 1.0691,
"margin_dpo/margin_mean": 8.78045654296875,
"margin_dpo/margin_std": 13.278366088867188,
"step": 172
},
{
"KL/chosen_KL_mean": -14.149677276611328,
"KL/mean": -18.881351470947266,
"KL/rejected_KL_mean": -23.613025665283203,
"KL/std": 13.220224380493164,
"epoch": 0.2615268329554044,
"fcm_dpo/beta": 0.059835657477378845,
"fcm_dpo/delta": -0.12774161994457245,
"fcm_dpo/margin": 9.463345527648926,
"fcm_dpo/q_t": 0.3847862184047699,
"grad_norm": 20.46826171875,
"learning_rate": 4.624313574873786e-07,
"logits/chosen": 0.37014636397361755,
"logits/rejected": 0.27701422572135925,
"logps/chosen": -69.58440399169922,
"logps/ref_chosen": -55.434722900390625,
"logps/ref_rejected": -77.81967163085938,
"logps/rejected": -101.43270111083984,
"loss": 1.1185,
"margin_dpo/margin_mean": 9.463346481323242,
"margin_dpo/margin_std": 16.405101776123047,
"step": 173
},
{
"KL/chosen_KL_mean": -15.740091323852539,
"KL/mean": -20.807640075683594,
"KL/rejected_KL_mean": -25.87519073486328,
"KL/std": 13.291604995727539,
"epoch": 0.26303854875283444,
"fcm_dpo/beta": 0.058659717440605164,
"fcm_dpo/delta": -0.15415163338184357,
"fcm_dpo/margin": 10.135098457336426,
"fcm_dpo/q_t": 0.3727102279663086,
"grad_norm": 20.58899688720703,
"learning_rate": 4.61731282057198e-07,
"logits/chosen": 0.34955164790153503,
"logits/rejected": 0.2790573239326477,
"logps/chosen": -72.91204833984375,
"logps/ref_chosen": -57.17195129394531,
"logps/ref_rejected": -85.47578430175781,
"logps/rejected": -111.35096740722656,
"loss": 1.0354,
"margin_dpo/margin_mean": 10.135099411010742,
"margin_dpo/margin_std": 14.499110221862793,
"step": 174
},
{
"KL/chosen_KL_mean": -13.907922744750977,
"KL/mean": -19.6348819732666,
"KL/rejected_KL_mean": -25.36184310913086,
"KL/std": 13.645920753479004,
"epoch": 0.26455026455026454,
"fcm_dpo/beta": 0.05664439499378204,
"fcm_dpo/delta": -0.21225407719612122,
"fcm_dpo/margin": 11.453914642333984,
"fcm_dpo/q_t": 0.36655163764953613,
"grad_norm": 19.182777404785156,
"learning_rate": 4.6102528404790965e-07,
"logits/chosen": 0.368941068649292,
"logits/rejected": 0.3376131057739258,
"logps/chosen": -81.57354736328125,
"logps/ref_chosen": -67.6656265258789,
"logps/ref_rejected": -84.36766815185547,
"logps/rejected": -109.72950744628906,
"loss": 1.0132,
"margin_dpo/margin_mean": 11.453914642333984,
"margin_dpo/margin_std": 15.843772888183594,
"step": 175
},
{
"KL/chosen_KL_mean": -17.130245208740234,
"KL/mean": -20.843347549438477,
"KL/rejected_KL_mean": -24.556453704833984,
"KL/std": 14.491414070129395,
"epoch": 0.2660619803476946,
"fcm_dpo/beta": 0.05589904636144638,
"fcm_dpo/delta": 0.03595317155122757,
"fcm_dpo/margin": 7.426205158233643,
"fcm_dpo/q_t": 0.4137015640735626,
"grad_norm": 24.373594284057617,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": 0.32078686356544495,
"logits/rejected": 0.29332348704338074,
"logps/chosen": -94.98900604248047,
"logps/ref_chosen": -77.8587646484375,
"logps/ref_rejected": -81.08732604980469,
"logps/rejected": -105.64378356933594,
"loss": 1.1976,
"margin_dpo/margin_mean": 7.426205635070801,
"margin_dpo/margin_std": 16.123355865478516,
"step": 176
},
{
"KL/chosen_KL_mean": -13.158845901489258,
"KL/mean": -20.39859390258789,
"KL/rejected_KL_mean": -27.638347625732422,
"KL/std": 14.79086685180664,
"epoch": 0.2675736961451247,
"fcm_dpo/beta": 0.05322066694498062,
"fcm_dpo/delta": -0.34956711530685425,
"fcm_dpo/margin": 14.479496955871582,
"fcm_dpo/q_t": 0.33395838737487793,
"grad_norm": 23.84845733642578,
"learning_rate": 4.5959559945025183e-07,
"logits/chosen": 0.42620840668678284,
"logits/rejected": 0.3288993239402771,
"logps/chosen": -68.37924194335938,
"logps/ref_chosen": -55.22039794921875,
"logps/ref_rejected": -92.54973602294922,
"logps/rejected": -120.18807983398438,
"loss": 0.8946,
"margin_dpo/margin_mean": 14.479496002197266,
"margin_dpo/margin_std": 15.118330955505371,
"step": 177
},
{
"KL/chosen_KL_mean": -15.544981002807617,
"KL/mean": -19.833009719848633,
"KL/rejected_KL_mean": -24.12103271484375,
"KL/std": 13.912124633789062,
"epoch": 0.2690854119425548,
"fcm_dpo/beta": 0.05207317695021629,
"fcm_dpo/delta": 0.002033442258834839,
"fcm_dpo/margin": 8.576053619384766,
"fcm_dpo/q_t": 0.40217965841293335,
"grad_norm": 21.546463012695312,
"learning_rate": 4.588719528532341e-07,
"logits/chosen": 0.2882809042930603,
"logits/rejected": 0.23839804530143738,
"logps/chosen": -76.35546875,
"logps/ref_chosen": -60.81049346923828,
"logps/ref_rejected": -81.12973022460938,
"logps/rejected": -105.25076293945312,
"loss": 1.1183,
"margin_dpo/margin_mean": 8.576054573059082,
"margin_dpo/margin_std": 14.08738899230957,
"step": 178
},
{
"KL/chosen_KL_mean": -16.099651336669922,
"KL/mean": -21.50237464904785,
"KL/rejected_KL_mean": -26.905094146728516,
"KL/std": 14.630666732788086,
"epoch": 0.2705971277399849,
"fcm_dpo/beta": 0.05162374675273895,
"fcm_dpo/delta": -0.11425016075372696,
"fcm_dpo/margin": 10.805438995361328,
"fcm_dpo/q_t": 0.3815948963165283,
"grad_norm": 21.03357696533203,
"learning_rate": 4.581424636586928e-07,
"logits/chosen": 0.3580857515335083,
"logits/rejected": 0.34163224697113037,
"logps/chosen": -81.7713623046875,
"logps/ref_chosen": -65.67171478271484,
"logps/ref_rejected": -75.32586669921875,
"logps/rejected": -102.23095703125,
"loss": 1.0771,
"margin_dpo/margin_mean": 10.805438995361328,
"margin_dpo/margin_std": 17.263511657714844,
"step": 179
},
{
"KL/chosen_KL_mean": -12.344501495361328,
"KL/mean": -16.842422485351562,
"KL/rejected_KL_mean": -21.3403377532959,
"KL/std": 14.285127639770508,
"epoch": 0.272108843537415,
"fcm_dpo/beta": 0.05138145387172699,
"fcm_dpo/delta": -0.01291588507592678,
"fcm_dpo/margin": 8.995841026306152,
"fcm_dpo/q_t": 0.40272700786590576,
"grad_norm": 17.912363052368164,
"learning_rate": 4.5740715227200897e-07,
"logits/chosen": 0.1993003785610199,
"logits/rejected": 0.1796572208404541,
"logps/chosen": -69.02731323242188,
"logps/ref_chosen": -56.68280792236328,
"logps/ref_rejected": -64.94414520263672,
"logps/rejected": -86.28448486328125,
"loss": 1.1388,
"margin_dpo/margin_mean": 8.995841026306152,
"margin_dpo/margin_std": 16.66576385498047,
"step": 180
},
{
"KL/chosen_KL_mean": -10.670660018920898,
"KL/mean": -17.220378875732422,
"KL/rejected_KL_mean": -23.770092010498047,
"KL/std": 15.256805419921875,
"epoch": 0.273620559334845,
"fcm_dpo/beta": 0.049501724541187286,
"fcm_dpo/delta": -0.21285748481750488,
"fcm_dpo/margin": 13.099435806274414,
"fcm_dpo/q_t": 0.3588384985923767,
"grad_norm": 18.757675170898438,
"learning_rate": 4.566660392614228e-07,
"logits/chosen": 0.3928632140159607,
"logits/rejected": 0.35001248121261597,
"logps/chosen": -71.44670867919922,
"logps/ref_chosen": -60.77604675292969,
"logps/ref_rejected": -83.98361206054688,
"logps/rejected": -107.75370788574219,
"loss": 0.9532,
"margin_dpo/margin_mean": 13.099435806274414,
"margin_dpo/margin_std": 14.686405181884766,
"step": 181
},
{
"KL/chosen_KL_mean": -14.897209167480469,
"KL/mean": -21.812625885009766,
"KL/rejected_KL_mean": -28.728042602539062,
"KL/std": 16.35493278503418,
"epoch": 0.2751322751322751,
"fcm_dpo/beta": 0.04746267572045326,
"fcm_dpo/delta": -0.22113311290740967,
"fcm_dpo/margin": 13.830833435058594,
"fcm_dpo/q_t": 0.36115947365760803,
"grad_norm": 19.716379165649414,
"learning_rate": 4.5591914535745817e-07,
"logits/chosen": 0.3558586835861206,
"logits/rejected": 0.269753634929657,
"logps/chosen": -75.15099334716797,
"logps/ref_chosen": -60.2537841796875,
"logps/ref_rejected": -89.7706298828125,
"logps/rejected": -118.49867248535156,
"loss": 1.0055,
"margin_dpo/margin_mean": 13.830833435058594,
"margin_dpo/margin_std": 18.645549774169922,
"step": 182
},
{
"KL/chosen_KL_mean": -18.52231788635254,
"KL/mean": -21.377840042114258,
"KL/rejected_KL_mean": -24.233356475830078,
"KL/std": 15.700705528259277,
"epoch": 0.2766439909297052,
"fcm_dpo/beta": 0.04812941700220108,
"fcm_dpo/delta": 0.1803903579711914,
"fcm_dpo/margin": 5.7110395431518555,
"fcm_dpo/q_t": 0.43941351771354675,
"grad_norm": 19.41120719909668,
"learning_rate": 4.551664914523433e-07,
"logits/chosen": 0.3424264192581177,
"logits/rejected": 0.31929779052734375,
"logps/chosen": -80.28373718261719,
"logps/ref_chosen": -61.76142120361328,
"logps/ref_rejected": -72.54627990722656,
"logps/rejected": -96.7796401977539,
"loss": 1.2632,
"margin_dpo/margin_mean": 5.7110395431518555,
"margin_dpo/margin_std": 15.732259750366211,
"step": 183
},
{
"KL/chosen_KL_mean": -11.37504768371582,
"KL/mean": -16.473596572875977,
"KL/rejected_KL_mean": -21.5721435546875,
"KL/std": 13.631113052368164,
"epoch": 0.2781557067271353,
"fcm_dpo/beta": 0.04807819426059723,
"fcm_dpo/delta": -0.04334007948637009,
"fcm_dpo/margin": 10.197093963623047,
"fcm_dpo/q_t": 0.39357906579971313,
"grad_norm": 18.226768493652344,
"learning_rate": 4.544080985994258e-07,
"logits/chosen": 0.45362773537635803,
"logits/rejected": 0.3854234218597412,
"logps/chosen": -58.21576690673828,
"logps/ref_chosen": -46.840721130371094,
"logps/ref_rejected": -69.3609390258789,
"logps/rejected": -90.9330825805664,
"loss": 1.0616,
"margin_dpo/margin_mean": 10.197093963623047,
"margin_dpo/margin_std": 14.069075584411621,
"step": 184
},
{
"KL/chosen_KL_mean": -14.680137634277344,
"KL/mean": -20.519908905029297,
"KL/rejected_KL_mean": -26.359676361083984,
"KL/std": 16.397075653076172,
"epoch": 0.2796674225245654,
"fcm_dpo/beta": 0.04713146388530731,
"fcm_dpo/delta": -0.10833575576543808,
"fcm_dpo/margin": 11.679544448852539,
"fcm_dpo/q_t": 0.3850805163383484,
"grad_norm": 17.211332321166992,
"learning_rate": 4.5364398801258394e-07,
"logits/chosen": 0.3641235828399658,
"logits/rejected": 0.3162262439727783,
"logps/chosen": -67.00128173828125,
"logps/ref_chosen": -52.32114028930664,
"logps/ref_rejected": -68.3885726928711,
"logps/rejected": -94.74824523925781,
"loss": 1.1116,
"margin_dpo/margin_mean": 11.679544448852539,
"margin_dpo/margin_std": 20.055240631103516,
"step": 185
},
{
"KL/chosen_KL_mean": -12.910173416137695,
"KL/mean": -19.38536834716797,
"KL/rejected_KL_mean": -25.860557556152344,
"KL/std": 16.375133514404297,
"epoch": 0.2811791383219955,
"fcm_dpo/beta": 0.046077094972133636,
"fcm_dpo/delta": -0.15629881620407104,
"fcm_dpo/margin": 12.950380325317383,
"fcm_dpo/q_t": 0.3747071325778961,
"grad_norm": 21.335391998291016,
"learning_rate": 4.5287418106563354e-07,
"logits/chosen": 0.2941705584526062,
"logits/rejected": 0.25050365924835205,
"logps/chosen": -80.33030700683594,
"logps/ref_chosen": -67.42012786865234,
"logps/ref_rejected": -82.50968933105469,
"logps/rejected": -108.37024688720703,
"loss": 1.0573,
"margin_dpo/margin_mean": 12.950380325317383,
"margin_dpo/margin_std": 19.893539428710938,
"step": 186
},
{
"KL/chosen_KL_mean": -16.437469482421875,
"KL/mean": -22.146421432495117,
"KL/rejected_KL_mean": -27.85538101196289,
"KL/std": 17.256420135498047,
"epoch": 0.28269085411942557,
"fcm_dpo/beta": 0.045174386352300644,
"fcm_dpo/delta": -0.06973277032375336,
"fcm_dpo/margin": 11.417915344238281,
"fcm_dpo/q_t": 0.39325594902038574,
"grad_norm": 20.540571212768555,
"learning_rate": 4.520986992917297e-07,
"logits/chosen": 0.3017156720161438,
"logits/rejected": 0.24813276529312134,
"logps/chosen": -91.96296691894531,
"logps/ref_chosen": -75.52549743652344,
"logps/ref_rejected": -94.76289367675781,
"logps/rejected": -122.61827087402344,
"loss": 1.1177,
"margin_dpo/margin_mean": 11.417915344238281,
"margin_dpo/margin_std": 20.136951446533203,
"step": 187
},
{
"KL/chosen_KL_mean": -14.43658447265625,
"KL/mean": -20.95711898803711,
"KL/rejected_KL_mean": -27.47765350341797,
"KL/std": 17.697364807128906,
"epoch": 0.2842025699168556,
"fcm_dpo/beta": 0.044272612780332565,
"fcm_dpo/delta": -0.1350875049829483,
"fcm_dpo/margin": 13.041068077087402,
"fcm_dpo/q_t": 0.3793148994445801,
"grad_norm": 17.25244903564453,
"learning_rate": 4.5131756438276466e-07,
"logits/chosen": 0.3734307885169983,
"logits/rejected": 0.3284209966659546,
"logps/chosen": -85.95991516113281,
"logps/ref_chosen": -71.52333068847656,
"logps/ref_rejected": -78.29949951171875,
"logps/rejected": -105.77715301513672,
"loss": 1.043,
"margin_dpo/margin_mean": 13.041067123413086,
"margin_dpo/margin_std": 19.054115295410156,
"step": 188
},
{
"KL/chosen_KL_mean": -14.128122329711914,
"KL/mean": -20.039127349853516,
"KL/rejected_KL_mean": -25.950130462646484,
"KL/std": 17.645959854125977,
"epoch": 0.2857142857142857,
"fcm_dpo/beta": 0.04308847337961197,
"fcm_dpo/delta": -0.06519506126642227,
"fcm_dpo/margin": 11.822007179260254,
"fcm_dpo/q_t": 0.3899165391921997,
"grad_norm": 19.810237884521484,
"learning_rate": 4.5053079818876096e-07,
"logits/chosen": 0.39445924758911133,
"logits/rejected": 0.4088705778121948,
"logps/chosen": -86.30438995361328,
"logps/ref_chosen": -72.17626953125,
"logps/ref_rejected": -75.26313781738281,
"logps/rejected": -101.21326446533203,
"loss": 1.0893,
"margin_dpo/margin_mean": 11.82200813293457,
"margin_dpo/margin_std": 18.43564224243164,
"step": 189
},
{
"KL/chosen_KL_mean": -13.363594055175781,
"KL/mean": -21.484352111816406,
"KL/rejected_KL_mean": -29.605113983154297,
"KL/std": 18.145763397216797,
"epoch": 0.2872260015117158,
"fcm_dpo/beta": 0.04177084565162659,
"fcm_dpo/delta": -0.24536725878715515,
"fcm_dpo/margin": 16.241512298583984,
"fcm_dpo/q_t": 0.355669766664505,
"grad_norm": 23.54590606689453,
"learning_rate": 4.4973842271726024e-07,
"logits/chosen": 0.39284735918045044,
"logits/rejected": 0.24727825820446014,
"logps/chosen": -67.98786926269531,
"logps/ref_chosen": -54.624271392822266,
"logps/ref_rejected": -101.47068786621094,
"logps/rejected": -131.0758056640625,
"loss": 0.9601,
"margin_dpo/margin_mean": 16.241514205932617,
"margin_dpo/margin_std": 19.464500427246094,
"step": 190
},
{
"KL/chosen_KL_mean": -19.313480377197266,
"KL/mean": -25.634521484375,
"KL/rejected_KL_mean": -31.95555877685547,
"KL/std": 19.023698806762695,
"epoch": 0.2887377173091459,
"fcm_dpo/beta": 0.04085142910480499,
"fcm_dpo/delta": -0.07008600234985352,
"fcm_dpo/margin": 12.642083168029785,
"fcm_dpo/q_t": 0.3872701823711395,
"grad_norm": 20.005186080932617,
"learning_rate": 4.48940460132708e-07,
"logits/chosen": 0.44901180267333984,
"logits/rejected": 0.4179879426956177,
"logps/chosen": -92.2459945678711,
"logps/ref_chosen": -72.93251037597656,
"logps/ref_rejected": -89.95103454589844,
"logps/rejected": -121.9065933227539,
"loss": 1.0917,
"margin_dpo/margin_mean": 12.642083168029785,
"margin_dpo/margin_std": 20.432785034179688,
"step": 191
},
{
"KL/chosen_KL_mean": -20.968889236450195,
"KL/mean": -25.023563385009766,
"KL/rejected_KL_mean": -29.078235626220703,
"KL/std": 19.83245849609375,
"epoch": 0.29024943310657597,
"fcm_dpo/beta": 0.04154283553361893,
"fcm_dpo/delta": 0.11658979952335358,
"fcm_dpo/margin": 8.109346389770508,
"fcm_dpo/q_t": 0.42591214179992676,
"grad_norm": 16.941612243652344,
"learning_rate": 4.481369327558329e-07,
"logits/chosen": 0.4174021780490875,
"logits/rejected": 0.38975629210472107,
"logps/chosen": -74.97000885009766,
"logps/ref_chosen": -54.001121520996094,
"logps/ref_rejected": -63.531551361083984,
"logps/rejected": -92.60978698730469,
"loss": 1.195,
"margin_dpo/margin_mean": 8.109346389770508,
"margin_dpo/margin_std": 17.20096206665039,
"step": 192
},
{
"KL/chosen_KL_mean": -16.72517967224121,
"KL/mean": -24.319087982177734,
"KL/rejected_KL_mean": -31.912994384765625,
"KL/std": 18.46902084350586,
"epoch": 0.29176114890400606,
"fcm_dpo/beta": 0.04037639498710632,
"fcm_dpo/delta": -0.17553646862506866,
"fcm_dpo/margin": 15.187812805175781,
"fcm_dpo/q_t": 0.367713987827301,
"grad_norm": 15.405920028686523,
"learning_rate": 4.47327863063023e-07,
"logits/chosen": 0.33871471881866455,
"logits/rejected": 0.31610235571861267,
"logps/chosen": -73.47445678710938,
"logps/ref_chosen": -56.74927520751953,
"logps/ref_rejected": -58.80629348754883,
"logps/rejected": -90.71928405761719,
"loss": 0.9951,
"margin_dpo/margin_mean": 15.187812805175781,
"margin_dpo/margin_std": 19.09872817993164,
"step": 193
},
{
"KL/chosen_KL_mean": -17.941669464111328,
"KL/mean": -24.060016632080078,
"KL/rejected_KL_mean": -30.178367614746094,
"KL/std": 18.968008041381836,
"epoch": 0.29327286470143615,
"fcm_dpo/beta": 0.039627574384212494,
"fcm_dpo/delta": -0.038923852145671844,
"fcm_dpo/margin": 12.236695289611816,
"fcm_dpo/q_t": 0.3964529037475586,
"grad_norm": 19.522842407226562,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": 0.4086693525314331,
"logits/rejected": 0.37831252813339233,
"logps/chosen": -74.59111022949219,
"logps/ref_chosen": -56.64944076538086,
"logps/ref_rejected": -69.98954772949219,
"logps/rejected": -100.16790771484375,
"loss": 1.1441,
"margin_dpo/margin_mean": 12.2366943359375,
"margin_dpo/margin_std": 22.420700073242188,
"step": 194
},
{
"KL/chosen_KL_mean": -21.67102813720703,
"KL/mean": -29.539018630981445,
"KL/rejected_KL_mean": -37.40701675415039,
"KL/std": 21.462011337280273,
"epoch": 0.2947845804988662,
"fcm_dpo/beta": 0.038745589554309845,
"fcm_dpo/delta": -0.17129415273666382,
"fcm_dpo/margin": 15.735990524291992,
"fcm_dpo/q_t": 0.3701290488243103,
"grad_norm": 19.54598045349121,
"learning_rate": 4.4569318740967043e-07,
"logits/chosen": 0.3189467787742615,
"logits/rejected": 0.32533496618270874,
"logps/chosen": -92.080810546875,
"logps/ref_chosen": -70.40977478027344,
"logps/ref_rejected": -74.39448547363281,
"logps/rejected": -111.80149841308594,
"loss": 1.0249,
"margin_dpo/margin_mean": 15.735990524291992,
"margin_dpo/margin_std": 21.78451156616211,
"step": 195
},
{
"KL/chosen_KL_mean": -21.09269905090332,
"KL/mean": -27.36833953857422,
"KL/rejected_KL_mean": -33.643978118896484,
"KL/std": 21.385547637939453,
"epoch": 0.2962962962962963,
"fcm_dpo/beta": 0.0385778471827507,
"fcm_dpo/delta": -0.036321524530649185,
"fcm_dpo/margin": 12.551275253295898,
"fcm_dpo/q_t": 0.3921698331832886,
"grad_norm": 19.41595458984375,
"learning_rate": 4.448676271745197e-07,
"logits/chosen": 0.4123719036579132,
"logits/rejected": 0.368742972612381,
"logps/chosen": -80.32027435302734,
"logps/ref_chosen": -59.227577209472656,
"logps/ref_rejected": -83.54757690429688,
"logps/rejected": -117.1915512084961,
"loss": 1.0808,
"margin_dpo/margin_mean": 12.551275253295898,
"margin_dpo/margin_std": 18.66332244873047,
"step": 196
},
{
"KL/chosen_KL_mean": -18.31484031677246,
"KL/mean": -26.058391571044922,
"KL/rejected_KL_mean": -33.80194854736328,
"KL/std": 20.686565399169922,
"epoch": 0.29780801209372637,
"fcm_dpo/beta": 0.03779051452875137,
"fcm_dpo/delta": -0.14380398392677307,
"fcm_dpo/margin": 15.487101554870605,
"fcm_dpo/q_t": 0.3769325911998749,
"grad_norm": 18.967296600341797,
"learning_rate": 4.440366160729392e-07,
"logits/chosen": 0.5108134150505066,
"logits/rejected": 0.456866979598999,
"logps/chosen": -69.8439712524414,
"logps/ref_chosen": -51.52912902832031,
"logps/ref_rejected": -73.70631408691406,
"logps/rejected": -107.50825500488281,
"loss": 1.099,
"margin_dpo/margin_mean": 15.487101554870605,
"margin_dpo/margin_std": 25.65450668334961,
"step": 197
},
{
"KL/chosen_KL_mean": -20.055164337158203,
"KL/mean": -28.295562744140625,
"KL/rejected_KL_mean": -36.53596115112305,
"KL/std": 20.62320327758789,
"epoch": 0.29931972789115646,
"fcm_dpo/beta": 0.0365363247692585,
"fcm_dpo/delta": -0.1616281270980835,
"fcm_dpo/margin": 16.480792999267578,
"fcm_dpo/q_t": 0.36763936281204224,
"grad_norm": 17.987558364868164,
"learning_rate": 4.432001773500957e-07,
"logits/chosen": 0.4926878809928894,
"logits/rejected": 0.4490898847579956,
"logps/chosen": -79.83784484863281,
"logps/ref_chosen": -59.78268051147461,
"logps/ref_rejected": -72.24533081054688,
"logps/rejected": -108.78128814697266,
"loss": 1.0007,
"margin_dpo/margin_mean": 16.48079490661621,
"margin_dpo/margin_std": 20.50601577758789,
"step": 198
},
{
"KL/chosen_KL_mean": -24.607725143432617,
"KL/mean": -31.082088470458984,
"KL/rejected_KL_mean": -37.556453704833984,
"KL/std": 21.534305572509766,
"epoch": 0.30083144368858655,
"fcm_dpo/beta": 0.03629198670387268,
"fcm_dpo/delta": -0.022943876683712006,
"fcm_dpo/margin": 12.948731422424316,
"fcm_dpo/q_t": 0.3984564244747162,
"grad_norm": 16.96387481689453,
"learning_rate": 4.4235833440297856e-07,
"logits/chosen": 0.4463760256767273,
"logits/rejected": 0.3538878560066223,
"logps/chosen": -80.99449157714844,
"logps/ref_chosen": -56.38677215576172,
"logps/ref_rejected": -74.56779479980469,
"logps/rejected": -112.12425231933594,
"loss": 1.1585,
"margin_dpo/margin_mean": 12.948731422424316,
"margin_dpo/margin_std": 24.406784057617188,
"step": 199
},
{
"KL/chosen_KL_mean": -19.250396728515625,
"KL/mean": -28.578182220458984,
"KL/rejected_KL_mean": -37.905967712402344,
"KL/std": 22.57465362548828,
"epoch": 0.30234315948601664,
"fcm_dpo/beta": 0.03459034115076065,
"fcm_dpo/delta": -0.21037332713603973,
"fcm_dpo/margin": 18.655567169189453,
"fcm_dpo/q_t": 0.36464548110961914,
"grad_norm": 16.139589309692383,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": 0.4873886704444885,
"logits/rejected": 0.40935078263282776,
"logps/chosen": -77.07472229003906,
"logps/ref_chosen": -57.82432556152344,
"logps/ref_rejected": -89.28246307373047,
"logps/rejected": -127.18843078613281,
"loss": 1.0284,
"margin_dpo/margin_mean": 18.655567169189453,
"margin_dpo/margin_std": 26.809616088867188,
"step": 200
},
{
"KL/chosen_KL_mean": -27.423805236816406,
"KL/mean": -34.75459289550781,
"KL/rejected_KL_mean": -42.08538055419922,
"KL/std": 23.84355926513672,
"epoch": 0.30385487528344673,
"fcm_dpo/beta": 0.033915258944034576,
"fcm_dpo/delta": -0.0508701354265213,
"fcm_dpo/margin": 14.661575317382812,
"fcm_dpo/q_t": 0.3947141766548157,
"grad_norm": 18.19431495666504,
"learning_rate": 4.4065853017905953e-07,
"logits/chosen": 0.5265877842903137,
"logits/rejected": 0.4765250086784363,
"logps/chosen": -86.42356872558594,
"logps/ref_chosen": -58.999759674072266,
"logps/ref_rejected": -84.67575073242188,
"logps/rejected": -126.76112365722656,
"loss": 1.1112,
"margin_dpo/margin_mean": 14.661576271057129,
"margin_dpo/margin_std": 24.935222625732422,
"step": 201
},
{
"KL/chosen_KL_mean": -22.689090728759766,
"KL/mean": -31.702587127685547,
"KL/rejected_KL_mean": -40.7160758972168,
"KL/std": 22.56459617614746,
"epoch": 0.30536659108087677,
"fcm_dpo/beta": 0.033367373049259186,
"fcm_dpo/delta": -0.1608964502811432,
"fcm_dpo/margin": 18.026988983154297,
"fcm_dpo/q_t": 0.36734747886657715,
"grad_norm": 19.372838973999023,
"learning_rate": 4.3980061644943575e-07,
"logits/chosen": 0.42092597484588623,
"logits/rejected": 0.34300437569618225,
"logps/chosen": -70.34973907470703,
"logps/ref_chosen": -47.660648345947266,
"logps/ref_rejected": -73.63249969482422,
"logps/rejected": -114.34857177734375,
"loss": 1.0262,
"margin_dpo/margin_mean": 18.026988983154297,
"margin_dpo/margin_std": 24.56505012512207,
"step": 202
},
{
"KL/chosen_KL_mean": -28.01001739501953,
"KL/mean": -36.113922119140625,
"KL/rejected_KL_mean": -44.21782684326172,
"KL/std": 23.278791427612305,
"epoch": 0.30687830687830686,
"fcm_dpo/beta": 0.03262447193264961,
"fcm_dpo/delta": -0.08308503031730652,
"fcm_dpo/margin": 16.207809448242188,
"fcm_dpo/q_t": 0.3873659372329712,
"grad_norm": 19.63701820373535,
"learning_rate": 4.3893739358856455e-07,
"logits/chosen": 0.4732247591018677,
"logits/rejected": 0.3917619287967682,
"logps/chosen": -90.33555603027344,
"logps/ref_chosen": -62.32553482055664,
"logps/ref_rejected": -99.37226104736328,
"logps/rejected": -143.590087890625,
"loss": 1.0772,
"margin_dpo/margin_mean": 16.207809448242188,
"margin_dpo/margin_std": 25.38452911376953,
"step": 203
},
{
"KL/chosen_KL_mean": -25.417871475219727,
"KL/mean": -34.3574104309082,
"KL/rejected_KL_mean": -43.29694366455078,
"KL/std": 24.221799850463867,
"epoch": 0.30839002267573695,
"fcm_dpo/beta": 0.031458836048841476,
"fcm_dpo/delta": -0.1234700083732605,
"fcm_dpo/margin": 17.87906837463379,
"fcm_dpo/q_t": 0.37869489192962646,
"grad_norm": 17.177324295043945,
"learning_rate": 4.380688857426449e-07,
"logits/chosen": 0.45654842257499695,
"logits/rejected": 0.3831368684768677,
"logps/chosen": -76.04718780517578,
"logps/ref_chosen": -50.62931823730469,
"logps/ref_rejected": -66.60475158691406,
"logps/rejected": -109.90169525146484,
"loss": 1.0412,
"margin_dpo/margin_mean": 17.87906837463379,
"margin_dpo/margin_std": 24.354957580566406,
"step": 204
},
{
"KL/chosen_KL_mean": -29.891223907470703,
"KL/mean": -38.5206298828125,
"KL/rejected_KL_mean": -47.150020599365234,
"KL/std": 25.599483489990234,
"epoch": 0.30990173847316704,
"fcm_dpo/beta": 0.03112097829580307,
"fcm_dpo/delta": -0.0924113467335701,
"fcm_dpo/margin": 17.258800506591797,
"fcm_dpo/q_t": 0.38659146428108215,
"grad_norm": 20.992780685424805,
"learning_rate": 4.3719511720570814e-07,
"logits/chosen": 0.49310895800590515,
"logits/rejected": 0.4214407801628113,
"logps/chosen": -100.24740600585938,
"logps/ref_chosen": -70.3561782836914,
"logps/ref_rejected": -93.39848327636719,
"logps/rejected": -140.54849243164062,
"loss": 1.0924,
"margin_dpo/margin_mean": 17.258800506591797,
"margin_dpo/margin_std": 28.509033203125,
"step": 205
},
{
"KL/chosen_KL_mean": -32.124385833740234,
"KL/mean": -38.33163070678711,
"KL/rejected_KL_mean": -44.53887176513672,
"KL/std": 24.964656829833984,
"epoch": 0.31141345427059713,
"fcm_dpo/beta": 0.03151599317789078,
"fcm_dpo/delta": 0.0600641593337059,
"fcm_dpo/margin": 12.414488792419434,
"fcm_dpo/q_t": 0.419567734003067,
"grad_norm": 18.595619201660156,
"learning_rate": 4.363161124189387e-07,
"logits/chosen": 0.5298315286636353,
"logits/rejected": 0.5128300189971924,
"logps/chosen": -99.76986694335938,
"logps/ref_chosen": -67.64547729492188,
"logps/ref_rejected": -79.89584350585938,
"logps/rejected": -124.4347152709961,
"loss": 1.2328,
"margin_dpo/margin_mean": 12.414488792419434,
"margin_dpo/margin_std": 29.71973419189453,
"step": 206
},
{
"KL/chosen_KL_mean": -38.06774139404297,
"KL/mean": -46.9461784362793,
"KL/rejected_KL_mean": -55.82461929321289,
"KL/std": 26.254493713378906,
"epoch": 0.3129251700680272,
"fcm_dpo/beta": 0.030902881175279617,
"fcm_dpo/delta": -0.1046326532959938,
"fcm_dpo/margin": 17.756877899169922,
"fcm_dpo/q_t": 0.38568490743637085,
"grad_norm": 16.643341064453125,
"learning_rate": 4.3543189596998986e-07,
"logits/chosen": 0.4498439431190491,
"logits/rejected": 0.3794988989830017,
"logps/chosen": -105.73193359375,
"logps/ref_chosen": -67.66419219970703,
"logps/ref_rejected": -85.10249328613281,
"logps/rejected": -140.92710876464844,
"loss": 1.0656,
"margin_dpo/margin_mean": 17.756879806518555,
"margin_dpo/margin_std": 27.332908630371094,
"step": 207
},
{
"KL/chosen_KL_mean": -30.73422622680664,
"KL/mean": -35.53645706176758,
"KL/rejected_KL_mean": -40.33868408203125,
"KL/std": 24.153095245361328,
"epoch": 0.3144368858654573,
"fcm_dpo/beta": 0.031466640532016754,
"fcm_dpo/delta": 0.152464359998703,
"fcm_dpo/margin": 9.60445499420166,
"fcm_dpo/q_t": 0.4369770884513855,
"grad_norm": 23.85514259338379,
"learning_rate": 4.3454249259229664e-07,
"logits/chosen": 0.4734978973865509,
"logits/rejected": 0.4461069107055664,
"logps/chosen": -88.4659423828125,
"logps/ref_chosen": -57.731712341308594,
"logps/ref_rejected": -74.19276428222656,
"logps/rejected": -114.53144836425781,
"loss": 1.2801,
"margin_dpo/margin_mean": 9.604454040527344,
"margin_dpo/margin_std": 27.349924087524414,
"step": 208
},
{
"KL/chosen_KL_mean": -31.75300407409668,
"KL/mean": -42.757266998291016,
"KL/rejected_KL_mean": -53.761539459228516,
"KL/std": 27.966007232666016,
"epoch": 0.31594860166288735,
"fcm_dpo/beta": 0.03062412329018116,
"fcm_dpo/delta": -0.24015069007873535,
"fcm_dpo/margin": 22.008533477783203,
"fcm_dpo/q_t": 0.3586593270301819,
"grad_norm": 19.86554718017578,
"learning_rate": 4.336479271643833e-07,
"logits/chosen": 0.4041667580604553,
"logits/rejected": 0.34923887252807617,
"logps/chosen": -100.30308532714844,
"logps/ref_chosen": -68.55007934570312,
"logps/ref_rejected": -87.90541076660156,
"logps/rejected": -141.66696166992188,
"loss": 1.0325,
"margin_dpo/margin_mean": 22.008533477783203,
"margin_dpo/margin_std": 32.60284423828125,
"step": 209
},
{
"KL/chosen_KL_mean": -31.51213264465332,
"KL/mean": -42.13514709472656,
"KL/rejected_KL_mean": -52.75815200805664,
"KL/std": 29.606407165527344,
"epoch": 0.31746031746031744,
"fcm_dpo/beta": 0.029575884342193604,
"fcm_dpo/delta": -0.18992537260055542,
"fcm_dpo/margin": 21.24602508544922,
"fcm_dpo/q_t": 0.369351863861084,
"grad_norm": 18.382568359375,
"learning_rate": 4.327482247091679e-07,
"logits/chosen": 0.5498272776603699,
"logits/rejected": 0.443499356508255,
"logps/chosen": -88.78041076660156,
"logps/ref_chosen": -57.268272399902344,
"logps/ref_rejected": -85.72807312011719,
"logps/rejected": -138.48623657226562,
"loss": 1.0245,
"margin_dpo/margin_mean": 21.24602508544922,
"margin_dpo/margin_std": 29.765714645385742,
"step": 210
},
{
"KL/chosen_KL_mean": -28.97467613220215,
"KL/mean": -39.192665100097656,
"KL/rejected_KL_mean": -49.41065216064453,
"KL/std": 26.596416473388672,
"epoch": 0.31897203325774753,
"fcm_dpo/beta": 0.028674710541963577,
"fcm_dpo/delta": -0.14435598254203796,
"fcm_dpo/margin": 20.435976028442383,
"fcm_dpo/q_t": 0.37782740592956543,
"grad_norm": 21.482284545898438,
"learning_rate": 4.3184341039326217e-07,
"logits/chosen": 0.5518479347229004,
"logits/rejected": 0.44849225878715515,
"logps/chosen": -82.61538696289062,
"logps/ref_chosen": -53.640708923339844,
"logps/ref_rejected": -93.0387954711914,
"logps/rejected": -142.44944763183594,
"loss": 1.0265,
"margin_dpo/margin_mean": 20.435976028442383,
"margin_dpo/margin_std": 28.254350662231445,
"step": 211
},
{
"KL/chosen_KL_mean": -33.499977111816406,
"KL/mean": -44.172698974609375,
"KL/rejected_KL_mean": -54.845420837402344,
"KL/std": 29.302959442138672,
"epoch": 0.3204837490551776,
"fcm_dpo/beta": 0.02764958329498768,
"fcm_dpo/delta": -0.14908015727996826,
"fcm_dpo/margin": 21.345449447631836,
"fcm_dpo/q_t": 0.37234535813331604,
"grad_norm": 14.397629737854004,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": 0.5605090856552124,
"logits/rejected": 0.48153620958328247,
"logps/chosen": -90.86671447753906,
"logps/ref_chosen": -57.36674499511719,
"logps/ref_rejected": -79.89643096923828,
"logps/rejected": -134.74185180664062,
"loss": 1.0431,
"margin_dpo/margin_mean": 21.345447540283203,
"margin_dpo/margin_std": 30.819881439208984,
"step": 212
},
{
"KL/chosen_KL_mean": -26.205650329589844,
"KL/mean": -38.57976531982422,
"KL/rejected_KL_mean": -50.953880310058594,
"KL/std": 29.666786193847656,
"epoch": 0.3219954648526077,
"fcm_dpo/beta": 0.026420973241329193,
"fcm_dpo/delta": -0.22006627917289734,
"fcm_dpo/margin": 24.748226165771484,
"fcm_dpo/q_t": 0.3609957993030548,
"grad_norm": 13.625341415405273,
"learning_rate": 4.3001854756006724e-07,
"logits/chosen": 0.49518492817878723,
"logits/rejected": 0.47064149379730225,
"logps/chosen": -91.42676544189453,
"logps/ref_chosen": -65.22111511230469,
"logps/ref_rejected": -80.1810302734375,
"logps/rejected": -131.13490295410156,
"loss": 1.0003,
"margin_dpo/margin_mean": 24.748226165771484,
"margin_dpo/margin_std": 33.10297775268555,
"step": 213
},
{
"KL/chosen_KL_mean": -31.163930892944336,
"KL/mean": -43.70262145996094,
"KL/rejected_KL_mean": -56.24131774902344,
"KL/std": 29.9415340423584,
"epoch": 0.3235071806500378,
"fcm_dpo/beta": 0.025676624849438667,
"fcm_dpo/delta": -0.2068711519241333,
"fcm_dpo/margin": 25.077390670776367,
"fcm_dpo/q_t": 0.3648650646209717,
"grad_norm": 16.90061378479004,
"learning_rate": 4.290985500881143e-07,
"logits/chosen": 0.3979254364967346,
"logits/rejected": 0.3791281282901764,
"logps/chosen": -92.45626068115234,
"logps/ref_chosen": -61.292327880859375,
"logps/ref_rejected": -67.69841003417969,
"logps/rejected": -123.93972778320312,
"loss": 1.0147,
"margin_dpo/margin_mean": 25.077390670776367,
"margin_dpo/margin_std": 33.935569763183594,
"step": 214
},
{
"KL/chosen_KL_mean": -36.773006439208984,
"KL/mean": -49.322052001953125,
"KL/rejected_KL_mean": -61.87109375,
"KL/std": 31.468637466430664,
"epoch": 0.3250188964474679,
"fcm_dpo/beta": 0.024757249280810356,
"fcm_dpo/delta": -0.18273316323757172,
"fcm_dpo/margin": 25.09808349609375,
"fcm_dpo/q_t": 0.3704993426799774,
"grad_norm": 15.784854888916016,
"learning_rate": 4.281735428447157e-07,
"logits/chosen": 0.41309452056884766,
"logits/rejected": 0.29789355397224426,
"logps/chosen": -100.64214324951172,
"logps/ref_chosen": -63.869136810302734,
"logps/ref_rejected": -98.7657241821289,
"logps/rejected": -160.63681030273438,
"loss": 1.0395,
"margin_dpo/margin_mean": 25.09808349609375,
"margin_dpo/margin_std": 35.794097900390625,
"step": 215
},
{
"KL/chosen_KL_mean": -31.85125732421875,
"KL/mean": -45.10109329223633,
"KL/rejected_KL_mean": -58.35092544555664,
"KL/std": 30.88085174560547,
"epoch": 0.32653061224489793,
"fcm_dpo/beta": 0.02361372672021389,
"fcm_dpo/delta": -0.18767762184143066,
"fcm_dpo/margin": 26.49966812133789,
"fcm_dpo/q_t": 0.36168473958969116,
"grad_norm": 23.61011505126953,
"learning_rate": 4.2724355170431247e-07,
"logits/chosen": 0.5278521180152893,
"logits/rejected": 0.4376213550567627,
"logps/chosen": -99.67620849609375,
"logps/ref_chosen": -67.824951171875,
"logps/ref_rejected": -96.40231323242188,
"logps/rejected": -154.75323486328125,
"loss": 0.9764,
"margin_dpo/margin_mean": 26.49966812133789,
"margin_dpo/margin_std": 31.30062484741211,
"step": 216
},
{
"KL/chosen_KL_mean": -35.986595153808594,
"KL/mean": -49.955936431884766,
"KL/rejected_KL_mean": -63.92528533935547,
"KL/std": 32.65003204345703,
"epoch": 0.328042328042328,
"fcm_dpo/beta": 0.022649064660072327,
"fcm_dpo/delta": -0.1961820423603058,
"fcm_dpo/margin": 27.938690185546875,
"fcm_dpo/q_t": 0.3633325695991516,
"grad_norm": 13.823315620422363,
"learning_rate": 4.26308602680756e-07,
"logits/chosen": 0.5091556310653687,
"logits/rejected": 0.4011778235435486,
"logps/chosen": -96.49159240722656,
"logps/ref_chosen": -60.5049934387207,
"logps/ref_rejected": -84.26618194580078,
"logps/rejected": -148.19146728515625,
"loss": 0.9818,
"margin_dpo/margin_mean": 27.938690185546875,
"margin_dpo/margin_std": 34.23504638671875,
"step": 217
},
{
"KL/chosen_KL_mean": -39.407283782958984,
"KL/mean": -47.98320007324219,
"KL/rejected_KL_mean": -56.559120178222656,
"KL/std": 31.762195587158203,
"epoch": 0.3295540438397581,
"fcm_dpo/beta": 0.022890662774443626,
"fcm_dpo/delta": 0.05783740431070328,
"fcm_dpo/margin": 17.151836395263672,
"fcm_dpo/q_t": 0.41548776626586914,
"grad_norm": 16.205947875976562,
"learning_rate": 4.253687219265803e-07,
"logits/chosen": 0.37078291177749634,
"logits/rejected": 0.370650053024292,
"logps/chosen": -110.00160217285156,
"logps/ref_chosen": -70.59431457519531,
"logps/ref_rejected": -73.89038848876953,
"logps/rejected": -130.4495086669922,
"loss": 1.2118,
"margin_dpo/margin_mean": 17.151836395263672,
"margin_dpo/margin_std": 37.95496368408203,
"step": 218
},
{
"KL/chosen_KL_mean": -37.35448455810547,
"KL/mean": -46.43778991699219,
"KL/rejected_KL_mean": -55.521095275878906,
"KL/std": 32.03618621826172,
"epoch": 0.3310657596371882,
"fcm_dpo/beta": 0.022805888205766678,
"fcm_dpo/delta": 0.03707420825958252,
"fcm_dpo/margin": 18.166610717773438,
"fcm_dpo/q_t": 0.40925899147987366,
"grad_norm": 15.538980484008789,
"learning_rate": 4.2442393573227043e-07,
"logits/chosen": 0.4556928873062134,
"logits/rejected": 0.4128846526145935,
"logps/chosen": -97.84542846679688,
"logps/ref_chosen": -60.490943908691406,
"logps/ref_rejected": -75.85001373291016,
"logps/rejected": -131.37110900878906,
"loss": 1.1334,
"margin_dpo/margin_mean": 18.166610717773438,
"margin_dpo/margin_std": 31.397647857666016,
"step": 219
},
{
"KL/chosen_KL_mean": -31.5102596282959,
"KL/mean": -42.39979553222656,
"KL/rejected_KL_mean": -53.28932571411133,
"KL/std": 33.013694763183594,
"epoch": 0.3325774754346183,
"fcm_dpo/beta": 0.022611908614635468,
"fcm_dpo/delta": -0.046144720166921616,
"fcm_dpo/margin": 21.7790584564209,
"fcm_dpo/q_t": 0.3967788815498352,
"grad_norm": 15.905135154724121,
"learning_rate": 4.234742705255272e-07,
"logits/chosen": 0.5297170877456665,
"logits/rejected": 0.4632262587547302,
"logps/chosen": -76.5236587524414,
"logps/ref_chosen": -45.013397216796875,
"logps/ref_rejected": -70.49369812011719,
"logps/rejected": -123.78302764892578,
"loss": 1.1211,
"margin_dpo/margin_mean": 21.77906036376953,
"margin_dpo/margin_std": 37.682254791259766,
"step": 220
},
{
"KL/chosen_KL_mean": -31.15220069885254,
"KL/mean": -43.39739990234375,
"KL/rejected_KL_mean": -55.642601013183594,
"KL/std": 32.60444641113281,
"epoch": 0.3340891912320484,
"fcm_dpo/beta": 0.02241317182779312,
"fcm_dpo/delta": -0.10453681647777557,
"fcm_dpo/margin": 24.490406036376953,
"fcm_dpo/q_t": 0.384391188621521,
"grad_norm": 17.505321502685547,
"learning_rate": 4.22519752870528e-07,
"logits/chosen": 0.49240219593048096,
"logits/rejected": 0.4145415425300598,
"logps/chosen": -90.24803924560547,
"logps/ref_chosen": -59.09584045410156,
"logps/ref_rejected": -88.64388275146484,
"logps/rejected": -144.28648376464844,
"loss": 1.0602,
"margin_dpo/margin_mean": 24.490407943725586,
"margin_dpo/margin_std": 36.9871940612793,
"step": 221
},
{
"KL/chosen_KL_mean": -32.00490951538086,
"KL/mean": -48.416385650634766,
"KL/rejected_KL_mean": -64.8278579711914,
"KL/std": 37.11773681640625,
"epoch": 0.3356009070294785,
"fcm_dpo/beta": 0.02136814594268799,
"fcm_dpo/delta": -0.2705325782299042,
"fcm_dpo/margin": 32.82295227050781,
"fcm_dpo/q_t": 0.35026657581329346,
"grad_norm": 14.998271942138672,
"learning_rate": 4.2156040946718343e-07,
"logits/chosen": 0.5728538036346436,
"logits/rejected": 0.47882080078125,
"logps/chosen": -88.00260162353516,
"logps/ref_chosen": -55.9976921081543,
"logps/ref_rejected": -111.94727325439453,
"logps/rejected": -176.77513122558594,
"loss": 0.9547,
"margin_dpo/margin_mean": 32.82295227050781,
"margin_dpo/margin_std": 39.01953125,
"step": 222
},
{
"KL/chosen_KL_mean": -37.9762077331543,
"KL/mean": -51.26795959472656,
"KL/rejected_KL_mean": -64.55970764160156,
"KL/std": 35.981632232666016,
"epoch": 0.3371126228269085,
"fcm_dpo/beta": 0.020683202892541885,
"fcm_dpo/delta": -0.10624384880065918,
"fcm_dpo/margin": 26.58349609375,
"fcm_dpo/q_t": 0.3813319802284241,
"grad_norm": 15.701749801635742,
"learning_rate": 4.2059626715039065e-07,
"logits/chosen": 0.5513671636581421,
"logits/rejected": 0.49275338649749756,
"logps/chosen": -97.86763000488281,
"logps/ref_chosen": -59.891422271728516,
"logps/ref_rejected": -86.28954315185547,
"logps/rejected": -150.8492431640625,
"loss": 1.0349,
"margin_dpo/margin_mean": 26.58349609375,
"margin_dpo/margin_std": 36.03096008300781,
"step": 223
},
{
"KL/chosen_KL_mean": -43.705589294433594,
"KL/mean": -51.3037109375,
"KL/rejected_KL_mean": -58.901832580566406,
"KL/std": 35.61591339111328,
"epoch": 0.3386243386243386,
"fcm_dpo/beta": 0.02102397009730339,
"fcm_dpo/delta": 0.13499858975410461,
"fcm_dpo/margin": 15.196237564086914,
"fcm_dpo/q_t": 0.43189573287963867,
"grad_norm": 20.207576751708984,
"learning_rate": 4.1962735288928304e-07,
"logits/chosen": 0.5725216269493103,
"logits/rejected": 0.5493526458740234,
"logps/chosen": -107.75022888183594,
"logps/ref_chosen": -64.04463195800781,
"logps/ref_rejected": -75.05450439453125,
"logps/rejected": -133.95632934570312,
"loss": 1.2345,
"margin_dpo/margin_mean": 15.196236610412598,
"margin_dpo/margin_std": 37.5325927734375,
"step": 224
},
{
"KL/chosen_KL_mean": -41.65528869628906,
"KL/mean": -56.287723541259766,
"KL/rejected_KL_mean": -70.92013549804688,
"KL/std": 39.70849609375,
"epoch": 0.3401360544217687,
"fcm_dpo/beta": 0.02054654248058796,
"fcm_dpo/delta": -0.16328230500221252,
"fcm_dpo/margin": 29.264860153198242,
"fcm_dpo/q_t": 0.3704851269721985,
"grad_norm": 15.133810043334961,
"learning_rate": 4.186536937864752e-07,
"logits/chosen": 0.5531373620033264,
"logits/rejected": 0.4349825978279114,
"logps/chosen": -107.75111389160156,
"logps/ref_chosen": -66.0958251953125,
"logps/ref_rejected": -97.68675231933594,
"logps/rejected": -168.6068878173828,
"loss": 1.0044,
"margin_dpo/margin_mean": 29.26485824584961,
"margin_dpo/margin_std": 37.1306266784668,
"step": 225
},
{
"KL/chosen_KL_mean": -38.91139221191406,
"KL/mean": -50.595481872558594,
"KL/rejected_KL_mean": -62.27956771850586,
"KL/std": 36.265830993652344,
"epoch": 0.3416477702191988,
"fcm_dpo/beta": 0.020526161417365074,
"fcm_dpo/delta": -0.03146500885486603,
"fcm_dpo/margin": 23.368175506591797,
"fcm_dpo/q_t": 0.3965521454811096,
"grad_norm": 15.107815742492676,
"learning_rate": 4.176753170773052e-07,
"logits/chosen": 0.6142418384552002,
"logits/rejected": 0.5654819011688232,
"logps/chosen": -90.32826232910156,
"logps/ref_chosen": -51.4168701171875,
"logps/ref_rejected": -66.30068969726562,
"logps/rejected": -128.58026123046875,
"loss": 1.1383,
"margin_dpo/margin_mean": 23.368173599243164,
"margin_dpo/margin_std": 42.59581756591797,
"step": 226
},
{
"KL/chosen_KL_mean": -41.923866271972656,
"KL/mean": -54.000038146972656,
"KL/rejected_KL_mean": -66.07620239257812,
"KL/std": 39.64067840576172,
"epoch": 0.3431594860166289,
"fcm_dpo/beta": 0.02022862620651722,
"fcm_dpo/delta": -0.04073462262749672,
"fcm_dpo/margin": 24.15234375,
"fcm_dpo/q_t": 0.39708375930786133,
"grad_norm": 17.119104385375977,
"learning_rate": 4.166922501290729e-07,
"logits/chosen": 0.62053382396698,
"logits/rejected": 0.5810697078704834,
"logps/chosen": -99.91365051269531,
"logps/ref_chosen": -57.989776611328125,
"logps/ref_rejected": -75.05464172363281,
"logps/rejected": -141.130859375,
"loss": 1.1322,
"margin_dpo/margin_mean": 24.152341842651367,
"margin_dpo/margin_std": 43.917694091796875,
"step": 227
},
{
"KL/chosen_KL_mean": -44.68891525268555,
"KL/mean": -58.21568298339844,
"KL/rejected_KL_mean": -71.7424545288086,
"KL/std": 37.54546356201172,
"epoch": 0.34467120181405897,
"fcm_dpo/beta": 0.020018167793750763,
"fcm_dpo/delta": -0.09673406183719635,
"fcm_dpo/margin": 27.053539276123047,
"fcm_dpo/q_t": 0.38233309984207153,
"grad_norm": 15.651515007019043,
"learning_rate": 4.1570452044027405e-07,
"logits/chosen": 0.6193653345108032,
"logits/rejected": 0.5345925688743591,
"logps/chosen": -100.24828338623047,
"logps/ref_chosen": -55.55936813354492,
"logps/ref_rejected": -77.02364349365234,
"logps/rejected": -148.76609802246094,
"loss": 1.052,
"margin_dpo/margin_mean": 27.053539276123047,
"margin_dpo/margin_std": 38.620765686035156,
"step": 228
},
{
"KL/chosen_KL_mean": -37.900203704833984,
"KL/mean": -49.77465057373047,
"KL/rejected_KL_mean": -61.64910125732422,
"KL/std": 36.71519470214844,
"epoch": 0.34618291761148906,
"fcm_dpo/beta": 0.019772969186306,
"fcm_dpo/delta": -0.020564941689372063,
"fcm_dpo/margin": 23.748897552490234,
"fcm_dpo/q_t": 0.3996862471103668,
"grad_norm": 23.846460342407227,
"learning_rate": 4.147121556398312e-07,
"logits/chosen": 0.6539832353591919,
"logits/rejected": 0.5783480405807495,
"logps/chosen": -88.69486999511719,
"logps/ref_chosen": -50.79466247558594,
"logps/ref_rejected": -78.4474105834961,
"logps/rejected": -140.0965118408203,
"loss": 1.1464,
"margin_dpo/margin_mean": 23.748897552490234,
"margin_dpo/margin_std": 44.864593505859375,
"step": 229
},
{
"KL/chosen_KL_mean": -41.51421356201172,
"KL/mean": -55.32987976074219,
"KL/rejected_KL_mean": -69.14554595947266,
"KL/std": 40.503692626953125,
"epoch": 0.3476946334089191,
"fcm_dpo/beta": 0.019660290330648422,
"fcm_dpo/delta": -0.09952552616596222,
"fcm_dpo/margin": 27.631336212158203,
"fcm_dpo/q_t": 0.3826754689216614,
"grad_norm": 14.859014511108398,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": 0.569249153137207,
"logits/rejected": 0.5702674388885498,
"logps/chosen": -98.24343872070312,
"logps/ref_chosen": -56.729225158691406,
"logps/ref_rejected": -62.99180603027344,
"logps/rejected": -132.13735961914062,
"loss": 1.0567,
"margin_dpo/margin_mean": 27.631336212158203,
"margin_dpo/margin_std": 39.70343017578125,
"step": 230
},
{
"KL/chosen_KL_mean": -47.226043701171875,
"KL/mean": -67.83246612548828,
"KL/rejected_KL_mean": -88.43889617919922,
"KL/std": 39.61866760253906,
"epoch": 0.3492063492063492,
"fcm_dpo/beta": 0.018344787880778313,
"fcm_dpo/delta": -0.33336055278778076,
"fcm_dpo/margin": 41.21284484863281,
"fcm_dpo/q_t": 0.33213451504707336,
"grad_norm": 15.280308723449707,
"learning_rate": 4.1271363186719835e-07,
"logits/chosen": 0.5038709044456482,
"logits/rejected": 0.499245822429657,
"logps/chosen": -119.8231430053711,
"logps/ref_chosen": -72.59709930419922,
"logps/ref_rejected": -86.2322998046875,
"logps/rejected": -174.67120361328125,
"loss": 0.8998,
"margin_dpo/margin_mean": 41.21284484863281,
"margin_dpo/margin_std": 41.88604736328125,
"step": 231
},
{
"KL/chosen_KL_mean": -47.655906677246094,
"KL/mean": -61.479305267333984,
"KL/rejected_KL_mean": -75.30270385742188,
"KL/std": 43.63159942626953,
"epoch": 0.3507180650037793,
"fcm_dpo/beta": 0.017985675483942032,
"fcm_dpo/delta": -0.049727872014045715,
"fcm_dpo/margin": 27.646800994873047,
"fcm_dpo/q_t": 0.39663565158843994,
"grad_norm": 15.373372077941895,
"learning_rate": 4.1170752879801436e-07,
"logits/chosen": 0.49574434757232666,
"logits/rejected": 0.46651044487953186,
"logps/chosen": -115.7744369506836,
"logps/ref_chosen": -68.1185302734375,
"logps/ref_rejected": -83.79415893554688,
"logps/rejected": -159.09686279296875,
"loss": 1.1249,
"margin_dpo/margin_mean": 27.646799087524414,
"margin_dpo/margin_std": 49.709197998046875,
"step": 232
},
{
"KL/chosen_KL_mean": -54.42681121826172,
"KL/mean": -64.93189239501953,
"KL/rejected_KL_mean": -75.43698120117188,
"KL/std": 40.570587158203125,
"epoch": 0.35222978080120937,
"fcm_dpo/beta": 0.017682647332549095,
"fcm_dpo/delta": -0.07306541502475739,
"fcm_dpo/margin": 21.010177612304688,
"fcm_dpo/q_t": 0.4198026657104492,
"grad_norm": 15.090434074401855,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": 0.6033905148506165,
"logits/rejected": 0.5428017377853394,
"logps/chosen": -109.49696350097656,
"logps/ref_chosen": -55.070152282714844,
"logps/ref_rejected": -66.61845397949219,
"logps/rejected": -142.05543518066406,
"loss": 1.1757,
"margin_dpo/margin_mean": 21.010177612304688,
"margin_dpo/margin_std": 40.30328369140625,
"step": 233
},
{
"KL/chosen_KL_mean": -48.80119323730469,
"KL/mean": -59.234046936035156,
"KL/rejected_KL_mean": -69.66690063476562,
"KL/std": 39.58076477050781,
"epoch": 0.35374149659863946,
"fcm_dpo/beta": 0.017503011971712112,
"fcm_dpo/delta": -0.10263454169034958,
"fcm_dpo/margin": 20.865713119506836,
"fcm_dpo/q_t": 0.4213663339614868,
"grad_norm": 20.658870697021484,
"learning_rate": 4.09681781007452e-07,
"logits/chosen": 0.5194283723831177,
"logits/rejected": 0.5139452815055847,
"logps/chosen": -104.72708892822266,
"logps/ref_chosen": -55.92589569091797,
"logps/ref_rejected": -51.11608123779297,
"logps/rejected": -120.7829818725586,
"loss": 1.2047,
"margin_dpo/margin_mean": 20.865711212158203,
"margin_dpo/margin_std": 42.82243347167969,
"step": 234
},
{
"KL/chosen_KL_mean": -43.279945373535156,
"KL/mean": -61.363121032714844,
"KL/rejected_KL_mean": -79.44630432128906,
"KL/std": 42.8857421875,
"epoch": 0.35525321239606955,
"fcm_dpo/beta": 0.016922399401664734,
"fcm_dpo/delta": -0.17231330275535583,
"fcm_dpo/margin": 36.166358947753906,
"fcm_dpo/q_t": 0.36375778913497925,
"grad_norm": 16.162338256835938,
"learning_rate": 4.08662192950594e-07,
"logits/chosen": 0.5850652456283569,
"logits/rejected": 0.571670651435852,
"logps/chosen": -107.81967163085938,
"logps/ref_chosen": -64.53972625732422,
"logps/ref_rejected": -77.69151306152344,
"logps/rejected": -157.1378173828125,
"loss": 0.9736,
"margin_dpo/margin_mean": 36.166358947753906,
"margin_dpo/margin_std": 41.02494812011719,
"step": 235
},
{
"KL/chosen_KL_mean": -64.5616683959961,
"KL/mean": -78.20289611816406,
"KL/rejected_KL_mean": -91.8441390991211,
"KL/std": 41.81065368652344,
"epoch": 0.35676492819349964,
"fcm_dpo/beta": 0.01667531207203865,
"fcm_dpo/delta": -0.005363345146179199,
"fcm_dpo/margin": 27.282466888427734,
"fcm_dpo/q_t": 0.40045133233070374,
"grad_norm": 14.700178146362305,
"learning_rate": 4.076381667711306e-07,
"logits/chosen": 0.5476399660110474,
"logits/rejected": 0.5346208810806274,
"logps/chosen": -135.71640014648438,
"logps/ref_chosen": -71.15473937988281,
"logps/ref_rejected": -84.88541412353516,
"logps/rejected": -176.72955322265625,
"loss": 1.1252,
"margin_dpo/margin_mean": 27.282466888427734,
"margin_dpo/margin_std": 47.215057373046875,
"step": 236
},
{
"KL/chosen_KL_mean": -53.720584869384766,
"KL/mean": -70.24567413330078,
"KL/rejected_KL_mean": -86.77075958251953,
"KL/std": 42.25255584716797,
"epoch": 0.35827664399092973,
"fcm_dpo/beta": 0.016512542963027954,
"fcm_dpo/delta": -0.10114513337612152,
"fcm_dpo/margin": 33.0501708984375,
"fcm_dpo/q_t": 0.3799697458744049,
"grad_norm": 17.659883499145508,
"learning_rate": 4.066097311132753e-07,
"logits/chosen": 0.5982592105865479,
"logits/rejected": 0.5887188911437988,
"logps/chosen": -129.8625946044922,
"logps/ref_chosen": -76.14201354980469,
"logps/ref_rejected": -80.88479614257812,
"logps/rejected": -167.65554809570312,
"loss": 1.0614,
"margin_dpo/margin_mean": 33.0501708984375,
"margin_dpo/margin_std": 48.059844970703125,
"step": 237
},
{
"KL/chosen_KL_mean": -48.632911682128906,
"KL/mean": -64.87059020996094,
"KL/rejected_KL_mean": -81.1082763671875,
"KL/std": 43.56510925292969,
"epoch": 0.35978835978835977,
"fcm_dpo/beta": 0.016101723536849022,
"fcm_dpo/delta": -0.0780644491314888,
"fcm_dpo/margin": 32.475364685058594,
"fcm_dpo/q_t": 0.38529616594314575,
"grad_norm": 19.395490646362305,
"learning_rate": 4.0557691474458414e-07,
"logits/chosen": 0.5503574013710022,
"logits/rejected": 0.5416954755783081,
"logps/chosen": -117.51776123046875,
"logps/ref_chosen": -68.88484954833984,
"logps/ref_rejected": -75.8946304321289,
"logps/rejected": -157.00289916992188,
"loss": 1.0578,
"margin_dpo/margin_mean": 32.475364685058594,
"margin_dpo/margin_std": 45.807884216308594,
"step": 238
},
{
"KL/chosen_KL_mean": -59.37183380126953,
"KL/mean": -75.34651947021484,
"KL/rejected_KL_mean": -91.32120513916016,
"KL/std": 47.33335876464844,
"epoch": 0.36130007558578986,
"fcm_dpo/beta": 0.016072561964392662,
"fcm_dpo/delta": -0.06735628843307495,
"fcm_dpo/margin": 31.949378967285156,
"fcm_dpo/q_t": 0.38884609937667847,
"grad_norm": 18.898448944091797,
"learning_rate": 4.045397465551513e-07,
"logits/chosen": 0.6873798966407776,
"logits/rejected": 0.5454525947570801,
"logps/chosen": -116.14366149902344,
"logps/ref_chosen": -56.771827697753906,
"logps/ref_rejected": -116.23050689697266,
"logps/rejected": -207.5517120361328,
"loss": 1.0838,
"margin_dpo/margin_mean": 31.949382781982422,
"margin_dpo/margin_std": 49.25836944580078,
"step": 239
},
{
"KL/chosen_KL_mean": -56.67302322387695,
"KL/mean": -77.18383026123047,
"KL/rejected_KL_mean": -97.69464111328125,
"KL/std": 47.6438102722168,
"epoch": 0.36281179138321995,
"fcm_dpo/beta": 0.01546771451830864,
"fcm_dpo/delta": -0.19688355922698975,
"fcm_dpo/margin": 41.02162170410156,
"fcm_dpo/q_t": 0.3613903522491455,
"grad_norm": 13.303434371948242,
"learning_rate": 4.0349825555680045e-07,
"logits/chosen": 0.6007834672927856,
"logits/rejected": 0.50172358751297,
"logps/chosen": -110.02713012695312,
"logps/ref_chosen": -53.35411071777344,
"logps/ref_rejected": -80.12019348144531,
"logps/rejected": -177.81484985351562,
"loss": 0.9792,
"margin_dpo/margin_mean": 41.02162170410156,
"margin_dpo/margin_std": 49.39841842651367,
"step": 240
},
{
"KL/chosen_KL_mean": -53.514156341552734,
"KL/mean": -66.90748596191406,
"KL/rejected_KL_mean": -80.30081176757812,
"KL/std": 44.207427978515625,
"epoch": 0.36432350718065004,
"fcm_dpo/beta": 0.015427043661475182,
"fcm_dpo/delta": 0.03810626268386841,
"fcm_dpo/margin": 26.786663055419922,
"fcm_dpo/q_t": 0.4107113480567932,
"grad_norm": 19.00992774963379,
"learning_rate": 4.0245247088227377e-07,
"logits/chosen": 0.5664269924163818,
"logits/rejected": 0.5286130905151367,
"logps/chosen": -125.4095687866211,
"logps/ref_chosen": -71.89541625976562,
"logps/ref_rejected": -83.03492736816406,
"logps/rejected": -163.3357391357422,
"loss": 1.1461,
"margin_dpo/margin_mean": 26.786663055419922,
"margin_dpo/margin_std": 48.930908203125,
"step": 241
},
{
"KL/chosen_KL_mean": -55.324440002441406,
"KL/mean": -72.56484985351562,
"KL/rejected_KL_mean": -89.80526733398438,
"KL/std": 47.05958557128906,
"epoch": 0.36583522297808013,
"fcm_dpo/beta": 0.015138762071728706,
"fcm_dpo/delta": -0.07740029692649841,
"fcm_dpo/margin": 34.4808349609375,
"fcm_dpo/q_t": 0.3867493271827698,
"grad_norm": 13.885424613952637,
"learning_rate": 4.0140242178441665e-07,
"logits/chosen": 0.5691415071487427,
"logits/rejected": 0.5489025115966797,
"logps/chosen": -113.25187683105469,
"logps/ref_chosen": -57.927433013916016,
"logps/ref_rejected": -67.838623046875,
"logps/rejected": -157.64389038085938,
"loss": 1.0625,
"margin_dpo/margin_mean": 34.4808349609375,
"margin_dpo/margin_std": 50.14107894897461,
"step": 242
},
{
"KL/chosen_KL_mean": -53.59979248046875,
"KL/mean": -69.44566345214844,
"KL/rejected_KL_mean": -85.29151153564453,
"KL/std": 46.36688232421875,
"epoch": 0.3673469387755102,
"fcm_dpo/beta": 0.015110740438103676,
"fcm_dpo/delta": -0.030339926481246948,
"fcm_dpo/margin": 31.691720962524414,
"fcm_dpo/q_t": 0.39408785104751587,
"grad_norm": 18.434778213500977,
"learning_rate": 4.003481376353596e-07,
"logits/chosen": 0.541643500328064,
"logits/rejected": 0.5529664754867554,
"logps/chosen": -127.87646484375,
"logps/ref_chosen": -74.27667236328125,
"logps/ref_rejected": -73.24340057373047,
"logps/rejected": -158.534912109375,
"loss": 1.0824,
"margin_dpo/margin_mean": 31.691722869873047,
"margin_dpo/margin_std": 47.64122772216797,
"step": 243
},
{
"KL/chosen_KL_mean": -50.46135711669922,
"KL/mean": -71.55018615722656,
"KL/rejected_KL_mean": -92.63902282714844,
"KL/std": 46.358036041259766,
"epoch": 0.3688586545729403,
"fcm_dpo/beta": 0.014749385416507721,
"fcm_dpo/delta": -0.18310438096523285,
"fcm_dpo/margin": 42.177650451660156,
"fcm_dpo/q_t": 0.3620183765888214,
"grad_norm": 13.311697959899902,
"learning_rate": 3.9928964792569654e-07,
"logits/chosen": 0.6441433429718018,
"logits/rejected": 0.5542799830436707,
"logps/chosen": -103.82526397705078,
"logps/ref_chosen": -53.36390686035156,
"logps/ref_rejected": -71.10276794433594,
"logps/rejected": -163.74179077148438,
"loss": 0.9702,
"margin_dpo/margin_mean": 42.177650451660156,
"margin_dpo/margin_std": 47.35203552246094,
"step": 244
},
{
"KL/chosen_KL_mean": -55.57624816894531,
"KL/mean": -79.77105712890625,
"KL/rejected_KL_mean": -103.96587371826172,
"KL/std": 47.97163391113281,
"epoch": 0.37037037037037035,
"fcm_dpo/beta": 0.014003526419401169,
"fcm_dpo/delta": -0.24466592073440552,
"fcm_dpo/margin": 48.389625549316406,
"fcm_dpo/q_t": 0.3471682667732239,
"grad_norm": 14.802033424377441,
"learning_rate": 3.982269822636601e-07,
"logits/chosen": 0.6564725637435913,
"logits/rejected": 0.6286360025405884,
"logps/chosen": -126.77135467529297,
"logps/ref_chosen": -71.19510650634766,
"logps/ref_rejected": -80.76235961914062,
"logps/rejected": -184.72824096679688,
"loss": 0.9235,
"margin_dpo/margin_mean": 48.389625549316406,
"margin_dpo/margin_std": 48.512237548828125,
"step": 245
},
{
"KL/chosen_KL_mean": -66.88098907470703,
"KL/mean": -84.69398498535156,
"KL/rejected_KL_mean": -102.5069580078125,
"KL/std": 47.89185333251953,
"epoch": 0.37188208616780044,
"fcm_dpo/beta": 0.01382741890847683,
"fcm_dpo/delta": -0.04510826617479324,
"fcm_dpo/margin": 35.62596893310547,
"fcm_dpo/q_t": 0.39292967319488525,
"grad_norm": 15.253338813781738,
"learning_rate": 3.971601703742932e-07,
"logits/chosen": 0.641024112701416,
"logits/rejected": 0.5778420567512512,
"logps/chosen": -138.50204467773438,
"logps/ref_chosen": -71.62104797363281,
"logps/ref_rejected": -94.03392028808594,
"logps/rejected": -196.5408935546875,
"loss": 1.1137,
"margin_dpo/margin_mean": 35.62596893310547,
"margin_dpo/margin_std": 60.41918182373047,
"step": 246
},
{
"KL/chosen_KL_mean": -69.6776123046875,
"KL/mean": -81.42636108398438,
"KL/rejected_KL_mean": -93.17510223388672,
"KL/std": 47.25940704345703,
"epoch": 0.37339380196523053,
"fcm_dpo/beta": 0.014045486226677895,
"fcm_dpo/delta": 0.12305961549282074,
"fcm_dpo/margin": 23.497482299804688,
"fcm_dpo/q_t": 0.4268472194671631,
"grad_norm": 18.581787109375,
"learning_rate": 3.960892420986177e-07,
"logits/chosen": 0.6367757320404053,
"logits/rejected": 0.6267802715301514,
"logps/chosen": -149.70016479492188,
"logps/ref_chosen": -80.02254486083984,
"logps/ref_rejected": -89.22705841064453,
"logps/rejected": -182.40216064453125,
"loss": 1.2146,
"margin_dpo/margin_mean": 23.497482299804688,
"margin_dpo/margin_std": 52.682167053222656,
"step": 247
},
{
"KL/chosen_KL_mean": -59.59392547607422,
"KL/mean": -78.8796615600586,
"KL/rejected_KL_mean": -98.1654052734375,
"KL/std": 51.46784210205078,
"epoch": 0.3749055177626606,
"fcm_dpo/beta": 0.013925114646553993,
"fcm_dpo/delta": -0.0919812023639679,
"fcm_dpo/margin": 38.57147979736328,
"fcm_dpo/q_t": 0.3852105438709259,
"grad_norm": 16.99346351623535,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": 0.5982637405395508,
"logits/rejected": 0.6556388139724731,
"logps/chosen": -124.97187805175781,
"logps/ref_chosen": -65.37796020507812,
"logps/ref_rejected": -61.365787506103516,
"logps/rejected": -159.53118896484375,
"loss": 1.0684,
"margin_dpo/margin_mean": 38.57147979736328,
"margin_dpo/margin_std": 58.039794921875,
"step": 248
},
{
"KL/chosen_KL_mean": -75.0390625,
"KL/mean": -82.37818908691406,
"KL/rejected_KL_mean": -89.71731567382812,
"KL/std": 47.77378845214844,
"epoch": 0.3764172335600907,
"fcm_dpo/beta": 0.013879897072911263,
"fcm_dpo/delta": 0.04884837195277214,
"fcm_dpo/margin": 14.678247451782227,
"fcm_dpo/q_t": 0.453724205493927,
"grad_norm": 19.524457931518555,
"learning_rate": 3.9393515632731094e-07,
"logits/chosen": 0.623253345489502,
"logits/rejected": 0.6645709872245789,
"logps/chosen": -149.64053344726562,
"logps/ref_chosen": -74.60145568847656,
"logps/ref_rejected": -63.79338455200195,
"logps/rejected": -153.51071166992188,
"loss": 1.3402,
"margin_dpo/margin_mean": 14.67824649810791,
"margin_dpo/margin_std": 55.51128387451172,
"step": 249
},
{
"KL/chosen_KL_mean": -65.30694580078125,
"KL/mean": -85.24577331542969,
"KL/rejected_KL_mean": -105.18460845947266,
"KL/std": 52.569244384765625,
"epoch": 0.3779289493575208,
"fcm_dpo/beta": 0.013815690763294697,
"fcm_dpo/delta": -0.10696772485971451,
"fcm_dpo/margin": 39.87767028808594,
"fcm_dpo/q_t": 0.3786598742008209,
"grad_norm": 14.615215301513672,
"learning_rate": 3.9285205908608934e-07,
"logits/chosen": 0.6955363750457764,
"logits/rejected": 0.6532368659973145,
"logps/chosen": -127.24514770507812,
"logps/ref_chosen": -61.938209533691406,
"logps/ref_rejected": -72.21602630615234,
"logps/rejected": -177.400634765625,
"loss": 1.0392,
"margin_dpo/margin_mean": 39.87766647338867,
"margin_dpo/margin_std": 54.281761169433594,
"step": 250
},
{
"KL/chosen_KL_mean": -71.13668823242188,
"KL/mean": -83.47338104248047,
"KL/rejected_KL_mean": -95.81005859375,
"KL/std": 47.63330841064453,
"epoch": 0.3794406651549509,
"fcm_dpo/beta": 0.013858238235116005,
"fcm_dpo/delta": 0.11203131079673767,
"fcm_dpo/margin": 24.67337417602539,
"fcm_dpo/q_t": 0.42451345920562744,
"grad_norm": 25.10679054260254,
"learning_rate": 3.9176496596569265e-07,
"logits/chosen": 0.6791577339172363,
"logits/rejected": 0.6360569000244141,
"logps/chosen": -137.99363708496094,
"logps/ref_chosen": -66.85694885253906,
"logps/ref_rejected": -84.83396911621094,
"logps/rejected": -180.64404296875,
"loss": 1.2046,
"margin_dpo/margin_mean": 24.67337417602539,
"margin_dpo/margin_std": 54.10609436035156,
"step": 251
},
{
"KL/chosen_KL_mean": -67.5489501953125,
"KL/mean": -77.78974914550781,
"KL/rejected_KL_mean": -88.03054809570312,
"KL/std": 50.0221061706543,
"epoch": 0.38095238095238093,
"fcm_dpo/beta": 0.013924909755587578,
"fcm_dpo/delta": -0.02651354670524597,
"fcm_dpo/margin": 20.48160171508789,
"fcm_dpo/q_t": 0.43610525131225586,
"grad_norm": 21.32712173461914,
"learning_rate": 3.9067390737445254e-07,
"logits/chosen": 0.6195981502532959,
"logits/rejected": 0.5631581544876099,
"logps/chosen": -123.77287292480469,
"logps/ref_chosen": -56.22393035888672,
"logps/ref_rejected": -77.1136245727539,
"logps/rejected": -165.1441650390625,
"loss": 1.2734,
"margin_dpo/margin_mean": 20.48160171508789,
"margin_dpo/margin_std": 55.925498962402344,
"step": 252
},
{
"KL/chosen_KL_mean": -65.90925598144531,
"KL/mean": -80.541015625,
"KL/rejected_KL_mean": -95.17279052734375,
"KL/std": 48.117431640625,
"epoch": 0.382464096749811,
"fcm_dpo/beta": 0.013884095475077629,
"fcm_dpo/delta": 0.04486459493637085,
"fcm_dpo/margin": 29.263538360595703,
"fcm_dpo/q_t": 0.410060316324234,
"grad_norm": 15.181997299194336,
"learning_rate": 3.8957891383162304e-07,
"logits/chosen": 0.7083457708358765,
"logits/rejected": 0.6677348017692566,
"logps/chosen": -118.1192626953125,
"logps/ref_chosen": -52.21001434326172,
"logps/ref_rejected": -58.75764846801758,
"logps/rejected": -153.93045043945312,
"loss": 1.1326,
"margin_dpo/margin_mean": 29.263538360595703,
"margin_dpo/margin_std": 48.80119323730469,
"step": 253
},
{
"KL/chosen_KL_mean": -71.01615142822266,
"KL/mean": -86.79163360595703,
"KL/rejected_KL_mean": -102.56710815429688,
"KL/std": 51.534637451171875,
"epoch": 0.3839758125472411,
"fcm_dpo/beta": 0.013972971588373184,
"fcm_dpo/delta": 0.009229157119989395,
"fcm_dpo/margin": 31.55096435546875,
"fcm_dpo/q_t": 0.40487420558929443,
"grad_norm": 15.08626937866211,
"learning_rate": 3.884800159665276e-07,
"logits/chosen": 0.628813624382019,
"logits/rejected": 0.5745677947998047,
"logps/chosen": -136.65248107910156,
"logps/ref_chosen": -65.63632202148438,
"logps/ref_rejected": -82.34425354003906,
"logps/rejected": -184.91136169433594,
"loss": 1.1139,
"margin_dpo/margin_mean": 31.55096435546875,
"margin_dpo/margin_std": 51.70745086669922,
"step": 254
},
{
"KL/chosen_KL_mean": -62.9200325012207,
"KL/mean": -82.491943359375,
"KL/rejected_KL_mean": -102.06385803222656,
"KL/std": 51.66278839111328,
"epoch": 0.3854875283446712,
"fcm_dpo/beta": 0.01380923017859459,
"fcm_dpo/delta": -0.09603819251060486,
"fcm_dpo/margin": 39.14384078979492,
"fcm_dpo/q_t": 0.38224583864212036,
"grad_norm": 17.70724105834961,
"learning_rate": 3.873772445177015e-07,
"logits/chosen": 0.6162057518959045,
"logits/rejected": 0.5852491855621338,
"logps/chosen": -130.83111572265625,
"logps/ref_chosen": -67.91108703613281,
"logps/ref_rejected": -83.89114379882812,
"logps/rejected": -185.95501708984375,
"loss": 1.0525,
"margin_dpo/margin_mean": 39.143836975097656,
"margin_dpo/margin_std": 56.28773880004883,
"step": 255
},
{
"KL/chosen_KL_mean": -77.14630889892578,
"KL/mean": -94.94715881347656,
"KL/rejected_KL_mean": -112.74800872802734,
"KL/std": 48.50966262817383,
"epoch": 0.3869992441421013,
"fcm_dpo/beta": 0.013629905879497528,
"fcm_dpo/delta": -0.03769109398126602,
"fcm_dpo/margin": 35.60170364379883,
"fcm_dpo/q_t": 0.39596813917160034,
"grad_norm": 17.2496337890625,
"learning_rate": 3.862706303320329e-07,
"logits/chosen": 0.6348433494567871,
"logits/rejected": 0.5660309791564941,
"logps/chosen": -140.64630126953125,
"logps/ref_chosen": -63.49998474121094,
"logps/ref_rejected": -90.77104187011719,
"logps/rejected": -203.51904296875,
"loss": 1.1243,
"margin_dpo/margin_mean": 35.60170364379883,
"margin_dpo/margin_std": 62.84724044799805,
"step": 256
},
{
"KL/chosen_KL_mean": -73.90077209472656,
"KL/mean": -95.79080200195312,
"KL/rejected_KL_mean": -117.68083953857422,
"KL/std": 54.22140121459961,
"epoch": 0.3885109599395314,
"fcm_dpo/beta": 0.013345291838049889,
"fcm_dpo/delta": -0.14312303066253662,
"fcm_dpo/margin": 43.780067443847656,
"fcm_dpo/q_t": 0.37517955899238586,
"grad_norm": 16.31106948852539,
"learning_rate": 3.851602043638994e-07,
"logits/chosen": 0.5821058750152588,
"logits/rejected": 0.5140093564987183,
"logps/chosen": -144.50143432617188,
"logps/ref_chosen": -70.60064697265625,
"logps/ref_rejected": -108.58313751220703,
"logps/rejected": -226.26397705078125,
"loss": 1.0304,
"margin_dpo/margin_mean": 43.780067443847656,
"margin_dpo/margin_std": 61.248958587646484,
"step": 257
},
{
"KL/chosen_KL_mean": -73.88896179199219,
"KL/mean": -90.58930969238281,
"KL/rejected_KL_mean": -107.2896728515625,
"KL/std": 47.34673309326172,
"epoch": 0.3900226757369615,
"fcm_dpo/beta": 0.013297569938004017,
"fcm_dpo/delta": 0.006121770944446325,
"fcm_dpo/margin": 33.40070343017578,
"fcm_dpo/q_t": 0.3981979489326477,
"grad_norm": 16.508010864257812,
"learning_rate": 3.840459976743023e-07,
"logits/chosen": 0.654517412185669,
"logits/rejected": 0.6004564762115479,
"logps/chosen": -133.14312744140625,
"logps/ref_chosen": -59.25416564941406,
"logps/ref_rejected": -85.58709716796875,
"logps/rejected": -192.87677001953125,
"loss": 1.0747,
"margin_dpo/margin_mean": 33.40070343017578,
"margin_dpo/margin_std": 45.45872497558594,
"step": 258
},
{
"KL/chosen_KL_mean": -63.81354904174805,
"KL/mean": -88.93872833251953,
"KL/rejected_KL_mean": -114.06390380859375,
"KL/std": 53.24052429199219,
"epoch": 0.3915343915343915,
"fcm_dpo/beta": 0.012812048196792603,
"fcm_dpo/delta": -0.2092743068933487,
"fcm_dpo/margin": 50.25034713745117,
"fcm_dpo/q_t": 0.3581874966621399,
"grad_norm": 15.079253196716309,
"learning_rate": 3.8292804142999796e-07,
"logits/chosen": 0.5811412334442139,
"logits/rejected": 0.47360676527023315,
"logps/chosen": -129.24842834472656,
"logps/ref_chosen": -65.43487548828125,
"logps/ref_rejected": -95.41731262207031,
"logps/rejected": -209.48121643066406,
"loss": 0.9841,
"margin_dpo/margin_mean": 50.25034713745117,
"margin_dpo/margin_std": 61.20269775390625,
"step": 259
},
{
"KL/chosen_KL_mean": -63.07318115234375,
"KL/mean": -83.33232116699219,
"KL/rejected_KL_mean": -103.5914535522461,
"KL/std": 49.821250915527344,
"epoch": 0.3930461073318216,
"fcm_dpo/beta": 0.012570559047162533,
"fcm_dpo/delta": -0.06336915493011475,
"fcm_dpo/margin": 40.51826858520508,
"fcm_dpo/q_t": 0.38906094431877136,
"grad_norm": 17.87150764465332,
"learning_rate": 3.818063669026256e-07,
"logits/chosen": 0.6366351842880249,
"logits/rejected": 0.5401675701141357,
"logps/chosen": -112.16276550292969,
"logps/ref_chosen": -49.08958435058594,
"logps/ref_rejected": -79.01708221435547,
"logps/rejected": -182.60853576660156,
"loss": 1.0794,
"margin_dpo/margin_mean": 40.51826858520508,
"margin_dpo/margin_std": 61.59083557128906,
"step": 260
},
{
"KL/chosen_KL_mean": -62.76991653442383,
"KL/mean": -79.77311706542969,
"KL/rejected_KL_mean": -96.77631378173828,
"KL/std": 48.37738037109375,
"epoch": 0.3945578231292517,
"fcm_dpo/beta": 0.012666585855185986,
"fcm_dpo/delta": 0.01990126632153988,
"fcm_dpo/margin": 34.006385803222656,
"fcm_dpo/q_t": 0.4056266248226166,
"grad_norm": 17.378955841064453,
"learning_rate": 3.806810054678331e-07,
"logits/chosen": 0.5030592679977417,
"logits/rejected": 0.5419148206710815,
"logps/chosen": -133.64230346679688,
"logps/ref_chosen": -70.87239074707031,
"logps/ref_rejected": -65.01522064208984,
"logps/rejected": -161.79153442382812,
"loss": 1.1136,
"margin_dpo/margin_mean": 34.006385803222656,
"margin_dpo/margin_std": 55.04287338256836,
"step": 261
},
{
"KL/chosen_KL_mean": -67.97140502929688,
"KL/mean": -83.72860717773438,
"KL/rejected_KL_mean": -99.48582458496094,
"KL/std": 48.292259216308594,
"epoch": 0.3960695389266818,
"fcm_dpo/beta": 0.012770213186740875,
"fcm_dpo/delta": 0.04927818849682808,
"fcm_dpo/margin": 31.5144100189209,
"fcm_dpo/q_t": 0.4105941355228424,
"grad_norm": 17.072776794433594,
"learning_rate": 3.7955198860439887e-07,
"logits/chosen": 0.6593347787857056,
"logits/rejected": 0.5915542840957642,
"logps/chosen": -135.842041015625,
"logps/ref_chosen": -67.8706283569336,
"logps/ref_rejected": -88.7205810546875,
"logps/rejected": -188.20640563964844,
"loss": 1.1254,
"margin_dpo/margin_mean": 31.514413833618164,
"margin_dpo/margin_std": 51.63859939575195,
"step": 262
},
{
"KL/chosen_KL_mean": -63.095603942871094,
"KL/mean": -78.94845581054688,
"KL/rejected_KL_mean": -94.80131530761719,
"KL/std": 48.40270233154297,
"epoch": 0.3975812547241119,
"fcm_dpo/beta": 0.012869570404291153,
"fcm_dpo/delta": 0.04364916682243347,
"fcm_dpo/margin": 31.705705642700195,
"fcm_dpo/q_t": 0.4082304537296295,
"grad_norm": 15.210543632507324,
"learning_rate": 3.784193478933516e-07,
"logits/chosen": 0.585516095161438,
"logits/rejected": 0.47373443841934204,
"logps/chosen": -118.2901840209961,
"logps/ref_chosen": -55.194583892822266,
"logps/ref_rejected": -80.54048156738281,
"logps/rejected": -175.341796875,
"loss": 1.1253,
"margin_dpo/margin_mean": 31.705705642700195,
"margin_dpo/margin_std": 52.43214416503906,
"step": 263
},
{
"KL/chosen_KL_mean": -66.87875366210938,
"KL/mean": -84.63905334472656,
"KL/rejected_KL_mean": -102.39934539794922,
"KL/std": 52.75183868408203,
"epoch": 0.39909297052154197,
"fcm_dpo/beta": 0.01285006757825613,
"fcm_dpo/delta": -0.006798055022954941,
"fcm_dpo/margin": 35.52058792114258,
"fcm_dpo/q_t": 0.3998854160308838,
"grad_norm": 15.444226264953613,
"learning_rate": 3.7728311501708674e-07,
"logits/chosen": 0.5134918093681335,
"logits/rejected": 0.47192150354385376,
"logps/chosen": -150.0494384765625,
"logps/ref_chosen": -83.17068481445312,
"logps/ref_rejected": -88.33625793457031,
"logps/rejected": -190.735595703125,
"loss": 1.1086,
"margin_dpo/margin_mean": 35.52058792114258,
"margin_dpo/margin_std": 57.95384216308594,
"step": 264
},
{
"KL/chosen_KL_mean": -63.74461364746094,
"KL/mean": -82.62153625488281,
"KL/rejected_KL_mean": -101.49846649169922,
"KL/std": 52.51271057128906,
"epoch": 0.40060468631897206,
"fcm_dpo/beta": 0.01283263973891735,
"fcm_dpo/delta": -0.03631345182657242,
"fcm_dpo/margin": 37.753849029541016,
"fcm_dpo/q_t": 0.39464491605758667,
"grad_norm": 13.600537300109863,
"learning_rate": 3.7614332175848027e-07,
"logits/chosen": 0.6821668148040771,
"logits/rejected": 0.6149172782897949,
"logps/chosen": -115.407470703125,
"logps/ref_chosen": -51.66284942626953,
"logps/ref_rejected": -67.1720962524414,
"logps/rejected": -168.67056274414062,
"loss": 1.1192,
"margin_dpo/margin_mean": 37.75384521484375,
"margin_dpo/margin_std": 64.861572265625,
"step": 265
},
{
"KL/chosen_KL_mean": -61.073211669921875,
"KL/mean": -79.75131225585938,
"KL/rejected_KL_mean": -98.42941284179688,
"KL/std": 52.90031433105469,
"epoch": 0.4021164021164021,
"fcm_dpo/beta": 0.012705964967608452,
"fcm_dpo/delta": -0.025934092700481415,
"fcm_dpo/margin": 37.35620880126953,
"fcm_dpo/q_t": 0.3964086174964905,
"grad_norm": 19.823822021484375,
"learning_rate": 3.75e-07,
"logits/chosen": 0.632943332195282,
"logits/rejected": 0.553703784942627,
"logps/chosen": -118.5237045288086,
"logps/ref_chosen": -57.45049285888672,
"logps/ref_rejected": -77.60826110839844,
"logps/rejected": -176.03765869140625,
"loss": 1.0939,
"margin_dpo/margin_mean": 37.35620880126953,
"margin_dpo/margin_std": 59.012001037597656,
"step": 266
},
{
"KL/chosen_KL_mean": -55.029483795166016,
"KL/mean": -69.81303405761719,
"KL/rejected_KL_mean": -84.59658813476562,
"KL/std": 49.53461837768555,
"epoch": 0.4036281179138322,
"fcm_dpo/beta": 0.012619069777429104,
"fcm_dpo/delta": -0.06738676875829697,
"fcm_dpo/margin": 29.567102432250977,
"fcm_dpo/q_t": 0.4169025719165802,
"grad_norm": 16.014333724975586,
"learning_rate": 3.738531817228131e-07,
"logits/chosen": 0.6566835641860962,
"logits/rejected": 0.6375699043273926,
"logps/chosen": -110.06483459472656,
"logps/ref_chosen": -55.03535079956055,
"logps/ref_rejected": -66.0953369140625,
"logps/rejected": -150.69192504882812,
"loss": 1.1897,
"margin_dpo/margin_mean": 29.567100524902344,
"margin_dpo/margin_std": 59.041709899902344,
"step": 267
},
{
"KL/chosen_KL_mean": -55.67205810546875,
"KL/mean": -72.71066284179688,
"KL/rejected_KL_mean": -89.749267578125,
"KL/std": 52.541404724121094,
"epoch": 0.4051398337112623,
"fcm_dpo/beta": 0.012609283439815044,
"fcm_dpo/delta": 0.020995769649744034,
"fcm_dpo/margin": 34.07720947265625,
"fcm_dpo/q_t": 0.40545758605003357,
"grad_norm": 14.707161903381348,
"learning_rate": 3.7270289900589204e-07,
"logits/chosen": 0.48954570293426514,
"logits/rejected": 0.4764719009399414,
"logps/chosen": -120.74380493164062,
"logps/ref_chosen": -65.07174682617188,
"logps/ref_rejected": -71.42485809326172,
"logps/rejected": -161.17413330078125,
"loss": 1.0974,
"margin_dpo/margin_mean": 34.077205657958984,
"margin_dpo/margin_std": 50.785972595214844,
"step": 268
},
{
"KL/chosen_KL_mean": -62.85627746582031,
"KL/mean": -83.8494873046875,
"KL/rejected_KL_mean": -104.84269714355469,
"KL/std": 55.634765625,
"epoch": 0.40665154950869237,
"fcm_dpo/beta": 0.01252103503793478,
"fcm_dpo/delta": -0.08028771728277206,
"fcm_dpo/margin": 41.98643112182617,
"fcm_dpo/q_t": 0.3840331435203552,
"grad_norm": 12.968265533447266,
"learning_rate": 3.7154918402511714e-07,
"logits/chosen": 0.7296715974807739,
"logits/rejected": 0.6805523633956909,
"logps/chosen": -129.99249267578125,
"logps/ref_chosen": -67.1362075805664,
"logps/ref_rejected": -82.55778503417969,
"logps/rejected": -187.40048217773438,
"loss": 1.0429,
"margin_dpo/margin_mean": 41.98643112182617,
"margin_dpo/margin_std": 55.632362365722656,
"step": 269
},
{
"KL/chosen_KL_mean": -63.16179656982422,
"KL/mean": -80.41263580322266,
"KL/rejected_KL_mean": -97.66346740722656,
"KL/std": 53.77967071533203,
"epoch": 0.40816326530612246,
"fcm_dpo/beta": 0.012402812018990517,
"fcm_dpo/delta": 0.023027736693620682,
"fcm_dpo/margin": 34.50166320800781,
"fcm_dpo/q_t": 0.4045753479003906,
"grad_norm": 15.0111665725708,
"learning_rate": 3.7039206905237656e-07,
"logits/chosen": 0.6593344807624817,
"logits/rejected": 0.5738272070884705,
"logps/chosen": -129.85049438476562,
"logps/ref_chosen": -66.6886978149414,
"logps/ref_rejected": -85.16129302978516,
"logps/rejected": -182.82476806640625,
"loss": 1.117,
"margin_dpo/margin_mean": 34.50166320800781,
"margin_dpo/margin_std": 56.74278259277344,
"step": 270
},
{
"KL/chosen_KL_mean": -64.6517333984375,
"KL/mean": -78.27857971191406,
"KL/rejected_KL_mean": -91.90541076660156,
"KL/std": 57.019386291503906,
"epoch": 0.40967498110355255,
"fcm_dpo/beta": 0.012488750740885735,
"fcm_dpo/delta": 0.01816648617386818,
"fcm_dpo/margin": 27.25368881225586,
"fcm_dpo/q_t": 0.4309072196483612,
"grad_norm": 18.295251846313477,
"learning_rate": 3.692315864546635e-07,
"logits/chosen": 0.6561949253082275,
"logits/rejected": 0.5901546478271484,
"logps/chosen": -137.0592803955078,
"logps/ref_chosen": -72.40754699707031,
"logps/ref_rejected": -92.06311798095703,
"logps/rejected": -183.96853637695312,
"loss": 1.2321,
"margin_dpo/margin_mean": 27.253690719604492,
"margin_dpo/margin_std": 66.5860595703125,
"step": 271
},
{
"KL/chosen_KL_mean": -54.95947265625,
"KL/mean": -80.6039810180664,
"KL/rejected_KL_mean": -106.24848937988281,
"KL/std": 51.89457702636719,
"epoch": 0.41118669690098264,
"fcm_dpo/beta": 0.012187042273581028,
"fcm_dpo/delta": -0.18635287880897522,
"fcm_dpo/margin": 51.28901672363281,
"fcm_dpo/q_t": 0.35789382457733154,
"grad_norm": 14.965229988098145,
"learning_rate": 3.6806776869317067e-07,
"logits/chosen": 0.6322664022445679,
"logits/rejected": 0.653581976890564,
"logps/chosen": -121.56086730957031,
"logps/ref_chosen": -66.60140228271484,
"logps/ref_rejected": -67.74340057373047,
"logps/rejected": -173.99188232421875,
"loss": 0.9413,
"margin_dpo/margin_mean": 51.28901672363281,
"margin_dpo/margin_std": 49.99754333496094,
"step": 272
},
{
"KL/chosen_KL_mean": -73.67694091796875,
"KL/mean": -92.46715545654297,
"KL/rejected_KL_mean": -111.25737762451172,
"KL/std": 55.94935989379883,
"epoch": 0.4126984126984127,
"fcm_dpo/beta": 0.011996024288237095,
"fcm_dpo/delta": -0.00102166086435318,
"fcm_dpo/margin": 37.58043670654297,
"fcm_dpo/q_t": 0.4026259779930115,
"grad_norm": 19.010719299316406,
"learning_rate": 3.669006483223828e-07,
"logits/chosen": 0.6612379550933838,
"logits/rejected": 0.5861459970474243,
"logps/chosen": -131.0318145751953,
"logps/ref_chosen": -57.35487747192383,
"logps/ref_rejected": -84.17168426513672,
"logps/rejected": -195.42906188964844,
"loss": 1.1487,
"margin_dpo/margin_mean": 37.58043670654297,
"margin_dpo/margin_std": 70.39936828613281,
"step": 273
},
{
"KL/chosen_KL_mean": -71.5015869140625,
"KL/mean": -91.9190673828125,
"KL/rejected_KL_mean": -112.33653259277344,
"KL/std": 55.5938835144043,
"epoch": 0.41421012849584277,
"fcm_dpo/beta": 0.011979161761701107,
"fcm_dpo/delta": -0.041180066764354706,
"fcm_dpo/margin": 40.83495330810547,
"fcm_dpo/q_t": 0.3926715850830078,
"grad_norm": 14.555620193481445,
"learning_rate": 3.657302579891656e-07,
"logits/chosen": 0.5605759024620056,
"logits/rejected": 0.5425212979316711,
"logps/chosen": -131.14308166503906,
"logps/ref_chosen": -59.64149475097656,
"logps/ref_rejected": -68.29348754882812,
"logps/rejected": -180.63002014160156,
"loss": 1.0978,
"margin_dpo/margin_mean": 40.834957122802734,
"margin_dpo/margin_std": 65.75580596923828,
"step": 274
},
{
"KL/chosen_KL_mean": -70.40373992919922,
"KL/mean": -90.73006439208984,
"KL/rejected_KL_mean": -111.05638885498047,
"KL/std": 53.968345642089844,
"epoch": 0.41572184429327286,
"fcm_dpo/beta": 0.011903250589966774,
"fcm_dpo/delta": -0.03569941222667694,
"fcm_dpo/margin": 40.65265655517578,
"fcm_dpo/q_t": 0.391641229391098,
"grad_norm": 15.478461265563965,
"learning_rate": 3.645566304318526e-07,
"logits/chosen": 0.6459121704101562,
"logits/rejected": 0.5600987076759338,
"logps/chosen": -123.67037963867188,
"logps/ref_chosen": -53.26664352416992,
"logps/ref_rejected": -73.84062194824219,
"logps/rejected": -184.89700317382812,
"loss": 1.0633,
"margin_dpo/margin_mean": 40.65265655517578,
"margin_dpo/margin_std": 56.11018753051758,
"step": 275
},
{
"KL/chosen_KL_mean": -66.95787048339844,
"KL/mean": -87.26176452636719,
"KL/rejected_KL_mean": -107.56565856933594,
"KL/std": 51.03001022338867,
"epoch": 0.41723356009070295,
"fcm_dpo/beta": 0.01180434413254261,
"fcm_dpo/delta": -0.030816514045000076,
"fcm_dpo/margin": 40.607791900634766,
"fcm_dpo/q_t": 0.39217013120651245,
"grad_norm": 15.905548095703125,
"learning_rate": 3.633797984793294e-07,
"logits/chosen": 0.6082146763801575,
"logits/rejected": 0.5748229026794434,
"logps/chosen": -119.97866821289062,
"logps/ref_chosen": -53.02079772949219,
"logps/ref_rejected": -61.56678771972656,
"logps/rejected": -169.1324462890625,
"loss": 1.0616,
"margin_dpo/margin_mean": 40.607791900634766,
"margin_dpo/margin_std": 55.26260757446289,
"step": 276
},
{
"KL/chosen_KL_mean": -77.37110137939453,
"KL/mean": -88.0814208984375,
"KL/rejected_KL_mean": -98.79173278808594,
"KL/std": 54.884521484375,
"epoch": 0.41874527588813304,
"fcm_dpo/beta": 0.011947010643780231,
"fcm_dpo/delta": 0.07333403825759888,
"fcm_dpo/margin": 21.420637130737305,
"fcm_dpo/q_t": 0.4435683786869049,
"grad_norm": 20.63077163696289,
"learning_rate": 3.6219979505011555e-07,
"logits/chosen": 0.6831735372543335,
"logits/rejected": 0.7109507322311401,
"logps/chosen": -148.80409240722656,
"logps/ref_chosen": -71.43299102783203,
"logps/ref_rejected": -67.65852355957031,
"logps/rejected": -166.45025634765625,
"loss": 1.2787,
"margin_dpo/margin_mean": 21.420637130737305,
"margin_dpo/margin_std": 62.50048065185547,
"step": 277
},
{
"KL/chosen_KL_mean": -79.96455383300781,
"KL/mean": -100.06394958496094,
"KL/rejected_KL_mean": -120.16334533691406,
"KL/std": 58.283294677734375,
"epoch": 0.42025699168556313,
"fcm_dpo/beta": 0.011976407840847969,
"fcm_dpo/delta": -0.03391536697745323,
"fcm_dpo/margin": 40.19879150390625,
"fcm_dpo/q_t": 0.39265549182891846,
"grad_norm": 23.727630615234375,
"learning_rate": 3.6101665315144353e-07,
"logits/chosen": 0.5698814392089844,
"logits/rejected": 0.5110211968421936,
"logps/chosen": -147.0753173828125,
"logps/ref_chosen": -67.11076354980469,
"logps/ref_rejected": -88.74851989746094,
"logps/rejected": -208.911865234375,
"loss": 1.0875,
"margin_dpo/margin_mean": 40.19879150390625,
"margin_dpo/margin_std": 60.55006790161133,
"step": 278
},
{
"KL/chosen_KL_mean": -62.668453216552734,
"KL/mean": -89.84060668945312,
"KL/rejected_KL_mean": -117.01275634765625,
"KL/std": 56.908164978027344,
"epoch": 0.4217687074829932,
"fcm_dpo/beta": 0.011571118608117104,
"fcm_dpo/delta": -0.19043880701065063,
"fcm_dpo/margin": 54.344303131103516,
"fcm_dpo/q_t": 0.35756611824035645,
"grad_norm": 17.530946731567383,
"learning_rate": 3.5983040587833563e-07,
"logits/chosen": 0.609818696975708,
"logits/rejected": 0.571174144744873,
"logps/chosen": -117.16593933105469,
"logps/ref_chosen": -54.49748611450195,
"logps/ref_rejected": -70.42373657226562,
"logps/rejected": -187.43649291992188,
"loss": 0.9465,
"margin_dpo/margin_mean": 54.34429931640625,
"margin_dpo/margin_std": 54.498779296875,
"step": 279
},
{
"KL/chosen_KL_mean": -68.33837890625,
"KL/mean": -96.23875427246094,
"KL/rejected_KL_mean": -124.13914489746094,
"KL/std": 59.32992935180664,
"epoch": 0.42328042328042326,
"fcm_dpo/beta": 0.011118508875370026,
"fcm_dpo/delta": -0.18170782923698425,
"fcm_dpo/margin": 55.8007698059082,
"fcm_dpo/q_t": 0.36028069257736206,
"grad_norm": 13.276687622070312,
"learning_rate": 3.586410864126781e-07,
"logits/chosen": 0.6702148914337158,
"logits/rejected": 0.623775064945221,
"logps/chosen": -128.7711944580078,
"logps/ref_chosen": -60.43281173706055,
"logps/ref_rejected": -78.39051818847656,
"logps/rejected": -202.5296630859375,
"loss": 0.9398,
"margin_dpo/margin_mean": 55.80076599121094,
"margin_dpo/margin_std": 54.328269958496094,
"step": 280
},
{
"KL/chosen_KL_mean": -72.25418853759766,
"KL/mean": -95.67463684082031,
"KL/rejected_KL_mean": -119.09507751464844,
"KL/std": 58.33176803588867,
"epoch": 0.42479213907785335,
"fcm_dpo/beta": 0.010939419269561768,
"fcm_dpo/delta": -0.0657629668712616,
"fcm_dpo/margin": 46.84089279174805,
"fcm_dpo/q_t": 0.38581421971321106,
"grad_norm": 14.414909362792969,
"learning_rate": 3.574487280222929e-07,
"logits/chosen": 0.6432371139526367,
"logits/rejected": 0.6703838109970093,
"logps/chosen": -132.53628540039062,
"logps/ref_chosen": -60.2820930480957,
"logps/ref_rejected": -62.04009246826172,
"logps/rejected": -181.13516235351562,
"loss": 1.0502,
"margin_dpo/margin_mean": 46.84089660644531,
"margin_dpo/margin_std": 63.40924072265625,
"step": 281
},
{
"KL/chosen_KL_mean": -80.641845703125,
"KL/mean": -102.34271240234375,
"KL/rejected_KL_mean": -124.0435791015625,
"KL/std": 58.330078125,
"epoch": 0.42630385487528344,
"fcm_dpo/beta": 0.010934704914689064,
"fcm_dpo/delta": -0.027530424296855927,
"fcm_dpo/margin": 43.40172576904297,
"fcm_dpo/q_t": 0.396485298871994,
"grad_norm": 17.353635787963867,
"learning_rate": 3.562533640600075e-07,
"logits/chosen": 0.583552360534668,
"logits/rejected": 0.537044107913971,
"logps/chosen": -141.26577758789062,
"logps/ref_chosen": -60.623924255371094,
"logps/ref_rejected": -68.67400360107422,
"logps/rejected": -192.71759033203125,
"loss": 1.0975,
"margin_dpo/margin_mean": 43.40172576904297,
"margin_dpo/margin_std": 66.38933563232422,
"step": 282
},
{
"KL/chosen_KL_mean": -80.09376525878906,
"KL/mean": -101.1219711303711,
"KL/rejected_KL_mean": -122.15017700195312,
"KL/std": 60.03446578979492,
"epoch": 0.42781557067271353,
"fcm_dpo/beta": 0.010815637186169624,
"fcm_dpo/delta": -0.005119716748595238,
"fcm_dpo/margin": 42.05641174316406,
"fcm_dpo/q_t": 0.399352490901947,
"grad_norm": 15.772919654846191,
"learning_rate": 3.550550279627215e-07,
"logits/chosen": 0.5872868299484253,
"logits/rejected": 0.4839021861553192,
"logps/chosen": -147.74151611328125,
"logps/ref_chosen": -67.64775085449219,
"logps/ref_rejected": -99.96835327148438,
"logps/rejected": -222.1185302734375,
"loss": 1.104,
"margin_dpo/margin_mean": 42.05641174316406,
"margin_dpo/margin_std": 67.12612915039062,
"step": 283
},
{
"KL/chosen_KL_mean": -76.53446960449219,
"KL/mean": -98.15267181396484,
"KL/rejected_KL_mean": -119.7708740234375,
"KL/std": 61.65085983276367,
"epoch": 0.4293272864701436,
"fcm_dpo/beta": 0.010724857449531555,
"fcm_dpo/delta": -0.014916583895683289,
"fcm_dpo/margin": 43.236419677734375,
"fcm_dpo/q_t": 0.3957221508026123,
"grad_norm": 13.379623413085938,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": 0.6814301609992981,
"logits/rejected": 0.6149337291717529,
"logps/chosen": -133.50189208984375,
"logps/ref_chosen": -56.96742630004883,
"logps/ref_rejected": -86.36236572265625,
"logps/rejected": -206.13323974609375,
"loss": 1.0681,
"margin_dpo/margin_mean": 43.236419677734375,
"margin_dpo/margin_std": 58.351688385009766,
"step": 284
},
{
"KL/chosen_KL_mean": -89.5875244140625,
"KL/mean": -107.78976440429688,
"KL/rejected_KL_mean": -125.99200439453125,
"KL/std": 55.76294708251953,
"epoch": 0.4308390022675737,
"fcm_dpo/beta": 0.010900814086198807,
"fcm_dpo/delta": 0.0547969751060009,
"fcm_dpo/margin": 36.404502868652344,
"fcm_dpo/q_t": 0.41238832473754883,
"grad_norm": 17.87129783630371,
"learning_rate": 3.5264957352549375e-07,
"logits/chosen": 0.6561132073402405,
"logits/rejected": 0.629011869430542,
"logps/chosen": -161.24362182617188,
"logps/ref_chosen": -71.65611267089844,
"logps/ref_rejected": -81.63829803466797,
"logps/rejected": -207.63031005859375,
"loss": 1.1345,
"margin_dpo/margin_mean": 36.404502868652344,
"margin_dpo/margin_std": 61.58406066894531,
"step": 285
},
{
"KL/chosen_KL_mean": -85.88138580322266,
"KL/mean": -114.774169921875,
"KL/rejected_KL_mean": -143.66696166992188,
"KL/std": 59.88359832763672,
"epoch": 0.4323507180650038,
"fcm_dpo/beta": 0.010602448135614395,
"fcm_dpo/delta": -0.17343094944953918,
"fcm_dpo/margin": 57.78559112548828,
"fcm_dpo/q_t": 0.363902747631073,
"grad_norm": 14.141740798950195,
"learning_rate": 3.514425224712835e-07,
"logits/chosen": 0.5710334777832031,
"logits/rejected": 0.4783239960670471,
"logps/chosen": -146.96090698242188,
"logps/ref_chosen": -61.07952117919922,
"logps/ref_rejected": -91.28128051757812,
"logps/rejected": -234.9482421875,
"loss": 0.9628,
"margin_dpo/margin_mean": 57.78559112548828,
"margin_dpo/margin_std": 63.03284454345703,
"step": 286
},
{
"KL/chosen_KL_mean": -70.0899887084961,
"KL/mean": -98.25200653076172,
"KL/rejected_KL_mean": -126.41402435302734,
"KL/std": 61.41065979003906,
"epoch": 0.43386243386243384,
"fcm_dpo/beta": 0.010312942788004875,
"fcm_dpo/delta": -0.1388087272644043,
"fcm_dpo/margin": 56.32403564453125,
"fcm_dpo/q_t": 0.36997073888778687,
"grad_norm": 11.145482063293457,
"learning_rate": 3.502326338516534e-07,
"logits/chosen": 0.7032564282417297,
"logits/rejected": 0.6613306999206543,
"logps/chosen": -116.12577819824219,
"logps/ref_chosen": -46.035789489746094,
"logps/ref_rejected": -59.95293426513672,
"logps/rejected": -186.36695861816406,
"loss": 0.9875,
"margin_dpo/margin_mean": 56.32403564453125,
"margin_dpo/margin_std": 64.42481994628906,
"step": 287
},
{
"KL/chosen_KL_mean": -88.99663543701172,
"KL/mean": -110.02481842041016,
"KL/rejected_KL_mean": -131.05300903320312,
"KL/std": 58.78456115722656,
"epoch": 0.43537414965986393,
"fcm_dpo/beta": 0.010274114087224007,
"fcm_dpo/delta": 0.018622491508722305,
"fcm_dpo/margin": 42.05636978149414,
"fcm_dpo/q_t": 0.40322625637054443,
"grad_norm": 15.26879596710205,
"learning_rate": 3.490199415097892e-07,
"logits/chosen": 0.5211523771286011,
"logits/rejected": 0.4631347060203552,
"logps/chosen": -154.38748168945312,
"logps/ref_chosen": -65.3908462524414,
"logps/ref_rejected": -88.53607940673828,
"logps/rejected": -219.58908081054688,
"loss": 1.0982,
"margin_dpo/margin_mean": 42.05636978149414,
"margin_dpo/margin_std": 63.13880920410156,
"step": 288
},
{
"KL/chosen_KL_mean": -90.59356689453125,
"KL/mean": -110.605712890625,
"KL/rejected_KL_mean": -130.61785888671875,
"KL/std": 59.32279968261719,
"epoch": 0.436885865457294,
"fcm_dpo/beta": 0.010357742197811604,
"fcm_dpo/delta": 0.03660314157605171,
"fcm_dpo/margin": 40.02429962158203,
"fcm_dpo/q_t": 0.41001832485198975,
"grad_norm": 15.136282920837402,
"learning_rate": 3.4780447936730247e-07,
"logits/chosen": 0.7384834289550781,
"logits/rejected": 0.6993527412414551,
"logps/chosen": -145.18719482421875,
"logps/ref_chosen": -54.5936279296875,
"logps/ref_rejected": -67.20855712890625,
"logps/rejected": -197.826416015625,
"loss": 1.1311,
"margin_dpo/margin_mean": 40.02430725097656,
"margin_dpo/margin_std": 68.23431396484375,
"step": 289
},
{
"KL/chosen_KL_mean": -103.62715911865234,
"KL/mean": -125.84639739990234,
"KL/rejected_KL_mean": -148.06561279296875,
"KL/std": 57.64268493652344,
"epoch": 0.4383975812547241,
"fcm_dpo/beta": 0.010276639834046364,
"fcm_dpo/delta": -0.007583964616060257,
"fcm_dpo/margin": 44.43846130371094,
"fcm_dpo/q_t": 0.3985353708267212,
"grad_norm": 16.58408546447754,
"learning_rate": 3.465862814232821e-07,
"logits/chosen": 0.7248564958572388,
"logits/rejected": 0.6485689878463745,
"logps/chosen": -165.01173400878906,
"logps/ref_chosen": -61.38457489013672,
"logps/ref_rejected": -91.92778015136719,
"logps/rejected": -239.993408203125,
"loss": 1.0911,
"margin_dpo/margin_mean": 44.43846130371094,
"margin_dpo/margin_std": 66.8131103515625,
"step": 290
},
{
"KL/chosen_KL_mean": -96.03206634521484,
"KL/mean": -122.14167785644531,
"KL/rejected_KL_mean": -148.2512969970703,
"KL/std": 58.933162689208984,
"epoch": 0.4399092970521542,
"fcm_dpo/beta": 0.010280381888151169,
"fcm_dpo/delta": -0.09266703575849533,
"fcm_dpo/margin": 52.21923065185547,
"fcm_dpo/q_t": 0.38349100947380066,
"grad_norm": 14.520237922668457,
"learning_rate": 3.4536538175334343e-07,
"logits/chosen": 0.7851830720901489,
"logits/rejected": 0.7087798118591309,
"logps/chosen": -146.89511108398438,
"logps/ref_chosen": -50.863037109375,
"logps/ref_rejected": -82.20868682861328,
"logps/rejected": -230.45997619628906,
"loss": 1.037,
"margin_dpo/margin_mean": 52.21923065185547,
"margin_dpo/margin_std": 68.57379150390625,
"step": 291
},
{
"KL/chosen_KL_mean": -96.4028091430664,
"KL/mean": -115.39314270019531,
"KL/rejected_KL_mean": -134.38348388671875,
"KL/std": 62.22578430175781,
"epoch": 0.4414210128495843,
"fcm_dpo/beta": 0.010227020829916,
"fcm_dpo/delta": 0.06409165263175964,
"fcm_dpo/margin": 37.98066329956055,
"fcm_dpo/q_t": 0.41515278816223145,
"grad_norm": 17.732135772705078,
"learning_rate": 3.4414181450867465e-07,
"logits/chosen": 0.693466067314148,
"logits/rejected": 0.6415978670120239,
"logps/chosen": -160.75169372558594,
"logps/ref_chosen": -64.34888458251953,
"logps/ref_rejected": -72.86434173583984,
"logps/rejected": -207.24781799316406,
"loss": 1.1693,
"margin_dpo/margin_mean": 37.98065948486328,
"margin_dpo/margin_std": 74.58848571777344,
"step": 292
},
{
"KL/chosen_KL_mean": -93.11393737792969,
"KL/mean": -122.25959777832031,
"KL/rejected_KL_mean": -151.40525817871094,
"KL/std": 62.89670944213867,
"epoch": 0.4429327286470144,
"fcm_dpo/beta": 0.010097621008753777,
"fcm_dpo/delta": -0.1470598578453064,
"fcm_dpo/margin": 58.29131317138672,
"fcm_dpo/q_t": 0.37115803360939026,
"grad_norm": 12.535250663757324,
"learning_rate": 3.4291561391508185e-07,
"logits/chosen": 0.7780508995056152,
"logits/rejected": 0.6862406730651855,
"logps/chosen": -147.9833984375,
"logps/ref_chosen": -54.869468688964844,
"logps/ref_rejected": -81.858642578125,
"logps/rejected": -233.263916015625,
"loss": 1.0148,
"margin_dpo/margin_mean": 58.29132080078125,
"margin_dpo/margin_std": 75.01055908203125,
"step": 293
},
{
"KL/chosen_KL_mean": -97.99851989746094,
"KL/mean": -117.81070709228516,
"KL/rejected_KL_mean": -137.62289428710938,
"KL/std": 65.31426239013672,
"epoch": 0.4444444444444444,
"fcm_dpo/beta": 0.010007752105593681,
"fcm_dpo/delta": 0.0554986447095871,
"fcm_dpo/margin": 39.62438201904297,
"fcm_dpo/q_t": 0.4120730757713318,
"grad_norm": 12.451395988464355,
"learning_rate": 3.4168681427203153e-07,
"logits/chosen": 0.6983363032341003,
"logits/rejected": 0.6498584747314453,
"logps/chosen": -154.66941833496094,
"logps/ref_chosen": -56.670902252197266,
"logps/ref_rejected": -70.32819366455078,
"logps/rejected": -207.95108032226562,
"loss": 1.1098,
"margin_dpo/margin_mean": 39.6243782043457,
"margin_dpo/margin_std": 59.10851287841797,
"step": 294
},
{
"KL/chosen_KL_mean": -103.488525390625,
"KL/mean": -122.23589324951172,
"KL/rejected_KL_mean": -140.98326110839844,
"KL/std": 61.330848693847656,
"epoch": 0.4459561602418745,
"fcm_dpo/beta": 0.010175291448831558,
"fcm_dpo/delta": 0.07125428318977356,
"fcm_dpo/margin": 37.49473571777344,
"fcm_dpo/q_t": 0.41606712341308594,
"grad_norm": 17.44603729248047,
"learning_rate": 3.4045544995169125e-07,
"logits/chosen": 0.6700437664985657,
"logits/rejected": 0.5628273487091064,
"logps/chosen": -153.889404296875,
"logps/ref_chosen": -50.40088653564453,
"logps/ref_rejected": -83.43521881103516,
"logps/rejected": -224.41848754882812,
"loss": 1.1453,
"margin_dpo/margin_mean": 37.49473571777344,
"margin_dpo/margin_std": 66.1406021118164,
"step": 295
},
{
"KL/chosen_KL_mean": -105.82876586914062,
"KL/mean": -129.0733642578125,
"KL/rejected_KL_mean": -152.31796264648438,
"KL/std": 65.7169418334961,
"epoch": 0.4474678760393046,
"fcm_dpo/beta": 0.010146657936275005,
"fcm_dpo/delta": -0.023256970569491386,
"fcm_dpo/margin": 46.48919677734375,
"fcm_dpo/q_t": 0.39643797278404236,
"grad_norm": 14.188161849975586,
"learning_rate": 3.392215553979679e-07,
"logits/chosen": 0.6409674882888794,
"logits/rejected": 0.5902814865112305,
"logps/chosen": -174.9791259765625,
"logps/ref_chosen": -69.15034484863281,
"logps/ref_rejected": -89.60166931152344,
"logps/rejected": -241.9196319580078,
"loss": 1.0983,
"margin_dpo/margin_mean": 46.48919677734375,
"margin_dpo/margin_std": 73.70729064941406,
"step": 296
},
{
"KL/chosen_KL_mean": -110.43616485595703,
"KL/mean": -133.87158203125,
"KL/rejected_KL_mean": -157.3070068359375,
"KL/std": 57.288177490234375,
"epoch": 0.4489795918367347,
"fcm_dpo/beta": 0.010175475850701332,
"fcm_dpo/delta": -0.028494687750935555,
"fcm_dpo/margin": 46.87084197998047,
"fcm_dpo/q_t": 0.39212262630462646,
"grad_norm": 16.023086547851562,
"learning_rate": 3.3798516512554485e-07,
"logits/chosen": 0.6705678701400757,
"logits/rejected": 0.6146547794342041,
"logps/chosen": -168.4524688720703,
"logps/ref_chosen": -58.01630401611328,
"logps/ref_rejected": -69.95780944824219,
"logps/rejected": -227.26483154296875,
"loss": 1.0562,
"margin_dpo/margin_mean": 46.87084197998047,
"margin_dpo/margin_std": 61.05129623413086,
"step": 297
},
{
"KL/chosen_KL_mean": -106.61897277832031,
"KL/mean": -124.87419891357422,
"KL/rejected_KL_mean": -143.12940979003906,
"KL/std": 61.50054168701172,
"epoch": 0.4504913076341648,
"fcm_dpo/beta": 0.010227935388684273,
"fcm_dpo/delta": 0.07961219549179077,
"fcm_dpo/margin": 36.51044464111328,
"fcm_dpo/q_t": 0.41826608777046204,
"grad_norm": 13.112458229064941,
"learning_rate": 3.367463137189156e-07,
"logits/chosen": 0.7707704305648804,
"logits/rejected": 0.7134817838668823,
"logps/chosen": -162.7882843017578,
"logps/ref_chosen": -56.1693115234375,
"logps/ref_rejected": -68.55052185058594,
"logps/rejected": -211.679931640625,
"loss": 1.1702,
"margin_dpo/margin_mean": 36.51044464111328,
"margin_dpo/margin_std": 70.75037384033203,
"step": 298
},
{
"KL/chosen_KL_mean": -111.99178314208984,
"KL/mean": -126.85576629638672,
"KL/rejected_KL_mean": -141.71974182128906,
"KL/std": 59.19386291503906,
"epoch": 0.4520030234315949,
"fcm_dpo/beta": 0.010346837341785431,
"fcm_dpo/delta": 0.03121235966682434,
"fcm_dpo/margin": 29.727954864501953,
"fcm_dpo/q_t": 0.4333716630935669,
"grad_norm": 19.00541114807129,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": 0.5875415205955505,
"logits/rejected": 0.554801881313324,
"logps/chosen": -174.30958557128906,
"logps/ref_chosen": -62.31780242919922,
"logps/ref_rejected": -72.60028839111328,
"logps/rejected": -214.32003784179688,
"loss": 1.243,
"margin_dpo/margin_mean": 29.727954864501953,
"margin_dpo/margin_std": 73.96747589111328,
"step": 299
},
{
"KL/chosen_KL_mean": -109.86944580078125,
"KL/mean": -130.2449951171875,
"KL/rejected_KL_mean": -150.62054443359375,
"KL/std": 60.240020751953125,
"epoch": 0.45351473922902497,
"fcm_dpo/beta": 0.010411659255623817,
"fcm_dpo/delta": 0.026686938479542732,
"fcm_dpo/margin": 40.75108337402344,
"fcm_dpo/q_t": 0.4073053002357483,
"grad_norm": 15.532927513122559,
"learning_rate": 3.3426136618426043e-07,
"logits/chosen": 0.6936973333358765,
"logits/rejected": 0.6232248544692993,
"logps/chosen": -170.25103759765625,
"logps/ref_chosen": -60.38157653808594,
"logps/ref_rejected": -75.45442199707031,
"logps/rejected": -226.074951171875,
"loss": 1.1313,
"margin_dpo/margin_mean": 40.75108337402344,
"margin_dpo/margin_std": 70.04428100585938,
"step": 300
},
{
"KL/chosen_KL_mean": -110.76192474365234,
"KL/mean": -127.73435974121094,
"KL/rejected_KL_mean": -144.706787109375,
"KL/std": 59.79651641845703,
"epoch": 0.455026455026455,
"fcm_dpo/beta": 0.010507562197744846,
"fcm_dpo/delta": 0.09680550545454025,
"fcm_dpo/margin": 33.94485855102539,
"fcm_dpo/q_t": 0.4223489761352539,
"grad_norm": 13.857847213745117,
"learning_rate": 3.3301533956555885e-07,
"logits/chosen": 0.7236491441726685,
"logits/rejected": 0.693558931350708,
"logps/chosen": -163.61282348632812,
"logps/ref_chosen": -52.85089111328125,
"logps/ref_rejected": -69.97584533691406,
"logps/rejected": -214.68263244628906,
"loss": 1.1847,
"margin_dpo/margin_mean": 33.94485855102539,
"margin_dpo/margin_std": 69.35813903808594,
"step": 301
},
{
"KL/chosen_KL_mean": -117.26535034179688,
"KL/mean": -130.64083862304688,
"KL/rejected_KL_mean": -144.016357421875,
"KL/std": 61.98484802246094,
"epoch": 0.4565381708238851,
"fcm_dpo/beta": 0.01086408831179142,
"fcm_dpo/delta": 0.16421742737293243,
"fcm_dpo/margin": 26.750995635986328,
"fcm_dpo/q_t": 0.4370330572128296,
"grad_norm": 18.773527145385742,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": 0.5519767999649048,
"logits/rejected": 0.4936150908470154,
"logps/chosen": -184.23187255859375,
"logps/ref_chosen": -66.96650695800781,
"logps/ref_rejected": -88.09510803222656,
"logps/rejected": -232.1114501953125,
"loss": 1.2423,
"margin_dpo/margin_mean": 26.750995635986328,
"margin_dpo/margin_std": 68.02701568603516,
"step": 302
},
{
"KL/chosen_KL_mean": -102.22068786621094,
"KL/mean": -128.0861053466797,
"KL/rejected_KL_mean": -153.9515380859375,
"KL/std": 63.22712707519531,
"epoch": 0.4580498866213152,
"fcm_dpo/beta": 0.010783663019537926,
"fcm_dpo/delta": -0.11414587497711182,
"fcm_dpo/margin": 51.7308349609375,
"fcm_dpo/q_t": 0.37940526008605957,
"grad_norm": 12.33498764038086,
"learning_rate": 3.3051635489464793e-07,
"logits/chosen": 0.6826484203338623,
"logits/rejected": 0.6071947813034058,
"logps/chosen": -164.34222412109375,
"logps/ref_chosen": -62.12152862548828,
"logps/ref_rejected": -90.31204223632812,
"logps/rejected": -244.26358032226562,
"loss": 1.0652,
"margin_dpo/margin_mean": 51.7308349609375,
"margin_dpo/margin_std": 77.99957275390625,
"step": 303
},
{
"KL/chosen_KL_mean": -94.04334259033203,
"KL/mean": -120.75105285644531,
"KL/rejected_KL_mean": -147.45877075195312,
"KL/std": 66.89263916015625,
"epoch": 0.4595616024187453,
"fcm_dpo/beta": 0.01050855778157711,
"fcm_dpo/delta": -0.118191197514534,
"fcm_dpo/margin": 53.41542053222656,
"fcm_dpo/q_t": 0.3726978898048401,
"grad_norm": 13.780329704284668,
"learning_rate": 3.292634667444117e-07,
"logits/chosen": 0.6737751364707947,
"logits/rejected": 0.6167408227920532,
"logps/chosen": -154.73843383789062,
"logps/ref_chosen": -60.695091247558594,
"logps/ref_rejected": -78.2525405883789,
"logps/rejected": -225.7113037109375,
"loss": 0.9972,
"margin_dpo/margin_mean": 53.41542053222656,
"margin_dpo/margin_std": 61.099151611328125,
"step": 304
},
{
"KL/chosen_KL_mean": -105.29306030273438,
"KL/mean": -124.80269622802734,
"KL/rejected_KL_mean": -144.31231689453125,
"KL/std": 60.25926971435547,
"epoch": 0.46107331821617537,
"fcm_dpo/beta": 0.01046682707965374,
"fcm_dpo/delta": 0.04289165511727333,
"fcm_dpo/margin": 39.01927185058594,
"fcm_dpo/q_t": 0.4094550609588623,
"grad_norm": 15.33985710144043,
"learning_rate": 3.280083614246217e-07,
"logits/chosen": 0.6184134483337402,
"logits/rejected": 0.6530435085296631,
"logps/chosen": -177.9921875,
"logps/ref_chosen": -72.69914245605469,
"logps/ref_rejected": -65.65670776367188,
"logps/rejected": -209.9690399169922,
"loss": 1.1597,
"margin_dpo/margin_mean": 39.0192756652832,
"margin_dpo/margin_std": 73.0045166015625,
"step": 305
},
{
"KL/chosen_KL_mean": -99.73224639892578,
"KL/mean": -119.26398468017578,
"KL/rejected_KL_mean": -138.79571533203125,
"KL/std": 58.238197326660156,
"epoch": 0.46258503401360546,
"fcm_dpo/beta": 0.010540273040533066,
"fcm_dpo/delta": 0.039174970239400864,
"fcm_dpo/margin": 39.063472747802734,
"fcm_dpo/q_t": 0.4074612855911255,
"grad_norm": 14.64108943939209,
"learning_rate": 3.267510740432719e-07,
"logits/chosen": 0.729286789894104,
"logits/rejected": 0.6173923015594482,
"logps/chosen": -153.70277404785156,
"logps/ref_chosen": -53.97052764892578,
"logps/ref_rejected": -71.02423095703125,
"logps/rejected": -209.8199462890625,
"loss": 1.1081,
"margin_dpo/margin_mean": 39.063472747802734,
"margin_dpo/margin_std": 57.965023040771484,
"step": 306
},
{
"KL/chosen_KL_mean": -92.92012786865234,
"KL/mean": -104.03545379638672,
"KL/rejected_KL_mean": -115.15077209472656,
"KL/std": 60.11651611328125,
"epoch": 0.46409674981103555,
"fcm_dpo/beta": 0.010723145678639412,
"fcm_dpo/delta": 0.0820213183760643,
"fcm_dpo/margin": 22.230642318725586,
"fcm_dpo/q_t": 0.44964367151260376,
"grad_norm": 19.061439514160156,
"learning_rate": 3.2549163976939285e-07,
"logits/chosen": 0.6955245733261108,
"logits/rejected": 0.647261381149292,
"logps/chosen": -150.33323669433594,
"logps/ref_chosen": -57.413108825683594,
"logps/ref_rejected": -68.68010711669922,
"logps/rejected": -183.83087158203125,
"loss": 1.3116,
"margin_dpo/margin_mean": 22.230642318725586,
"margin_dpo/margin_std": 74.4197006225586,
"step": 307
},
{
"KL/chosen_KL_mean": -93.56331634521484,
"KL/mean": -113.46038818359375,
"KL/rejected_KL_mean": -133.3574676513672,
"KL/std": 61.609130859375,
"epoch": 0.4656084656084656,
"fcm_dpo/beta": 0.010778220370411873,
"fcm_dpo/delta": 0.021368850022554398,
"fcm_dpo/margin": 39.79413604736328,
"fcm_dpo/q_t": 0.4058694839477539,
"grad_norm": 12.138362884521484,
"learning_rate": 3.2423009383206874e-07,
"logits/chosen": 0.6182932257652283,
"logits/rejected": 0.6067421436309814,
"logps/chosen": -160.162109375,
"logps/ref_chosen": -66.59879302978516,
"logps/ref_rejected": -74.337158203125,
"logps/rejected": -207.69461059570312,
"loss": 1.1387,
"margin_dpo/margin_mean": 39.79413604736328,
"margin_dpo/margin_std": 70.455078125,
"step": 308
},
{
"KL/chosen_KL_mean": -107.09249877929688,
"KL/mean": -126.61676788330078,
"KL/rejected_KL_mean": -146.14102172851562,
"KL/std": 58.26902770996094,
"epoch": 0.4671201814058957,
"fcm_dpo/beta": 0.010832262225449085,
"fcm_dpo/delta": 0.027536533772945404,
"fcm_dpo/margin": 39.04853057861328,
"fcm_dpo/q_t": 0.4072054624557495,
"grad_norm": 13.843729972839355,
"learning_rate": 3.229664715194511e-07,
"logits/chosen": 0.7150523662567139,
"logits/rejected": 0.6556819677352905,
"logps/chosen": -172.48724365234375,
"logps/ref_chosen": -65.39474487304688,
"logps/ref_rejected": -75.70930480957031,
"logps/rejected": -221.850341796875,
"loss": 1.1303,
"margin_dpo/margin_mean": 39.04853057861328,
"margin_dpo/margin_std": 66.06800842285156,
"step": 309
},
{
"KL/chosen_KL_mean": -109.26652526855469,
"KL/mean": -121.12366485595703,
"KL/rejected_KL_mean": -132.98080444335938,
"KL/std": 53.856170654296875,
"epoch": 0.46863189720332576,
"fcm_dpo/beta": 0.011067605577409267,
"fcm_dpo/delta": 0.06834125518798828,
"fcm_dpo/margin": 23.71427345275879,
"fcm_dpo/q_t": 0.44445621967315674,
"grad_norm": 15.862635612487793,
"learning_rate": 3.2170080817777257e-07,
"logits/chosen": 0.6403053998947144,
"logits/rejected": 0.62421053647995,
"logps/chosen": -183.93479919433594,
"logps/ref_chosen": -74.66827392578125,
"logps/ref_rejected": -80.5689697265625,
"logps/rejected": -213.54977416992188,
"loss": 1.2593,
"margin_dpo/margin_mean": 23.714271545410156,
"margin_dpo/margin_std": 64.26668548583984,
"step": 310
},
{
"KL/chosen_KL_mean": -86.25080871582031,
"KL/mean": -107.97197723388672,
"KL/rejected_KL_mean": -129.69314575195312,
"KL/std": 68.0066146850586,
"epoch": 0.47014361300075586,
"fcm_dpo/beta": 0.010956868529319763,
"fcm_dpo/delta": -0.02816399559378624,
"fcm_dpo/margin": 43.44232940673828,
"fcm_dpo/q_t": 0.39939939975738525,
"grad_norm": 14.952717781066895,
"learning_rate": 3.204331392103574e-07,
"logits/chosen": 0.5733008980751038,
"logits/rejected": 0.42495253682136536,
"logps/chosen": -145.9888458251953,
"logps/ref_chosen": -59.738033294677734,
"logps/ref_rejected": -93.60757446289062,
"logps/rejected": -223.30072021484375,
"loss": 1.0981,
"margin_dpo/margin_mean": 43.44232940673828,
"margin_dpo/margin_std": 70.86783599853516,
"step": 311
},
{
"KL/chosen_KL_mean": -88.2499771118164,
"KL/mean": -113.75946044921875,
"KL/rejected_KL_mean": -139.26893615722656,
"KL/std": 60.36577606201172,
"epoch": 0.47165532879818595,
"fcm_dpo/beta": 0.01085340604186058,
"fcm_dpo/delta": -0.10966580361127853,
"fcm_dpo/margin": 51.01895523071289,
"fcm_dpo/q_t": 0.37506595253944397,
"grad_norm": 12.034233093261719,
"learning_rate": 3.1916350007663176e-07,
"logits/chosen": 0.6571813821792603,
"logits/rejected": 0.5639383792877197,
"logps/chosen": -142.06640625,
"logps/ref_chosen": -53.816436767578125,
"logps/ref_rejected": -68.6575698852539,
"logps/rejected": -207.926513671875,
"loss": 0.9963,
"margin_dpo/margin_mean": 51.018951416015625,
"margin_dpo/margin_std": 57.567359924316406,
"step": 312
},
{
"KL/chosen_KL_mean": -91.30401611328125,
"KL/mean": -104.92782592773438,
"KL/rejected_KL_mean": -118.5516357421875,
"KL/std": 58.888465881347656,
"epoch": 0.47316704459561604,
"fcm_dpo/beta": 0.011028505861759186,
"fcm_dpo/delta": 0.15395116806030273,
"fcm_dpo/margin": 27.247615814208984,
"fcm_dpo/q_t": 0.4350872337818146,
"grad_norm": 14.497456550598145,
"learning_rate": 3.178919262911314e-07,
"logits/chosen": 0.6942657232284546,
"logits/rejected": 0.6728841066360474,
"logps/chosen": -151.26138305664062,
"logps/ref_chosen": -59.957359313964844,
"logps/ref_rejected": -69.31729888916016,
"logps/rejected": -187.86892700195312,
"loss": 1.2203,
"margin_dpo/margin_mean": 27.247615814208984,
"margin_dpo/margin_std": 63.275230407714844,
"step": 313
},
{
"KL/chosen_KL_mean": -83.92698669433594,
"KL/mean": -109.30597686767578,
"KL/rejected_KL_mean": -134.68496704101562,
"KL/std": 57.439422607421875,
"epoch": 0.47467876039304613,
"fcm_dpo/beta": 0.010885808616876602,
"fcm_dpo/delta": -0.10918959230184555,
"fcm_dpo/margin": 50.75796890258789,
"fcm_dpo/q_t": 0.3786463439464569,
"grad_norm": 13.35073184967041,
"learning_rate": 3.166184534225087e-07,
"logits/chosen": 0.6736915707588196,
"logits/rejected": 0.70624178647995,
"logps/chosen": -154.1951446533203,
"logps/ref_chosen": -70.26815795898438,
"logps/ref_rejected": -69.23971557617188,
"logps/rejected": -203.9246826171875,
"loss": 1.0157,
"margin_dpo/margin_mean": 50.757972717285156,
"margin_dpo/margin_std": 62.80244445800781,
"step": 314
},
{
"KL/chosen_KL_mean": -87.91320037841797,
"KL/mean": -107.46711730957031,
"KL/rejected_KL_mean": -127.02104187011719,
"KL/std": 57.641319274902344,
"epoch": 0.47619047619047616,
"fcm_dpo/beta": 0.010960625484585762,
"fcm_dpo/delta": 0.02152669057250023,
"fcm_dpo/margin": 39.10783767700195,
"fcm_dpo/q_t": 0.4034097194671631,
"grad_norm": 13.392065048217773,
"learning_rate": 3.1534311709253723e-07,
"logits/chosen": 0.6077339053153992,
"logits/rejected": 0.5718281269073486,
"logps/chosen": -155.7078857421875,
"logps/ref_chosen": -67.79469299316406,
"logps/ref_rejected": -74.55148315429688,
"logps/rejected": -201.572509765625,
"loss": 1.1049,
"margin_dpo/margin_mean": 39.10784149169922,
"margin_dpo/margin_std": 59.44211196899414,
"step": 315
},
{
"KL/chosen_KL_mean": -83.18527221679688,
"KL/mean": -110.18215942382812,
"KL/rejected_KL_mean": -137.1790313720703,
"KL/std": 60.67741012573242,
"epoch": 0.47770219198790626,
"fcm_dpo/beta": 0.010814497247338295,
"fcm_dpo/delta": -0.14357182383537292,
"fcm_dpo/margin": 53.99375915527344,
"fcm_dpo/q_t": 0.37073686718940735,
"grad_norm": 14.253772735595703,
"learning_rate": 3.1406595297511564e-07,
"logits/chosen": 0.5644083023071289,
"logits/rejected": 0.4203554391860962,
"logps/chosen": -138.4737548828125,
"logps/ref_chosen": -55.288482666015625,
"logps/ref_rejected": -96.15723419189453,
"logps/rejected": -233.33627319335938,
"loss": 0.9945,
"margin_dpo/margin_mean": 53.99375915527344,
"margin_dpo/margin_std": 59.711795806884766,
"step": 316
},
{
"KL/chosen_KL_mean": -75.41068267822266,
"KL/mean": -101.35579681396484,
"KL/rejected_KL_mean": -127.30091857910156,
"KL/std": 56.30128860473633,
"epoch": 0.47921390778533635,
"fcm_dpo/beta": 0.010403081774711609,
"fcm_dpo/delta": -0.09583519399166107,
"fcm_dpo/margin": 51.890228271484375,
"fcm_dpo/q_t": 0.37759479880332947,
"grad_norm": 15.830406188964844,
"learning_rate": 3.1278699679526975e-07,
"logits/chosen": 0.6743822693824768,
"logits/rejected": 0.6287072896957397,
"logps/chosen": -129.9920654296875,
"logps/ref_chosen": -54.58137512207031,
"logps/ref_rejected": -72.77232360839844,
"logps/rejected": -200.0732421875,
"loss": 1.0127,
"margin_dpo/margin_mean": 51.890228271484375,
"margin_dpo/margin_std": 61.410675048828125,
"step": 317
},
{
"KL/chosen_KL_mean": -83.8477783203125,
"KL/mean": -102.66434478759766,
"KL/rejected_KL_mean": -121.48091125488281,
"KL/std": 60.022735595703125,
"epoch": 0.48072562358276644,
"fcm_dpo/beta": 0.010466702282428741,
"fcm_dpo/delta": 0.05843011662364006,
"fcm_dpo/margin": 37.633140563964844,
"fcm_dpo/q_t": 0.4175593852996826,
"grad_norm": 13.778341293334961,
"learning_rate": 3.1150628432815336e-07,
"logits/chosen": 0.6658343076705933,
"logits/rejected": 0.5933891534805298,
"logps/chosen": -136.7360076904297,
"logps/ref_chosen": -52.88822937011719,
"logps/ref_rejected": -80.63988494873047,
"logps/rejected": -202.1208038330078,
"loss": 1.1904,
"margin_dpo/margin_mean": 37.633140563964844,
"margin_dpo/margin_std": 80.30189514160156,
"step": 318
},
{
"KL/chosen_KL_mean": -81.49282836914062,
"KL/mean": -108.32252502441406,
"KL/rejected_KL_mean": -135.1522216796875,
"KL/std": 63.14406204223633,
"epoch": 0.48223733938019653,
"fcm_dpo/beta": 0.010345933958888054,
"fcm_dpo/delta": -0.11138413846492767,
"fcm_dpo/margin": 53.65939712524414,
"fcm_dpo/q_t": 0.37948736548423767,
"grad_norm": 13.567173957824707,
"learning_rate": 3.1022385139804707e-07,
"logits/chosen": 0.6083593964576721,
"logits/rejected": 0.5914223194122314,
"logps/chosen": -145.8561553955078,
"logps/ref_chosen": -64.36333465576172,
"logps/ref_rejected": -79.47296142578125,
"logps/rejected": -214.62518310546875,
"loss": 1.0291,
"margin_dpo/margin_mean": 53.65939712524414,
"margin_dpo/margin_std": 72.19100952148438,
"step": 319
},
{
"KL/chosen_KL_mean": -76.05279541015625,
"KL/mean": -97.53375244140625,
"KL/rejected_KL_mean": -119.01470947265625,
"KL/std": 64.29141235351562,
"epoch": 0.4837490551776266,
"fcm_dpo/beta": 0.01016431674361229,
"fcm_dpo/delta": -0.12435748428106308,
"fcm_dpo/margin": 42.96190643310547,
"fcm_dpo/q_t": 0.40559324622154236,
"grad_norm": 14.700331687927246,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": 0.5677880048751831,
"logits/rejected": 0.5255244970321655,
"logps/chosen": -125.61154174804688,
"logps/ref_chosen": -49.558746337890625,
"logps/ref_rejected": -71.23444366455078,
"logps/rejected": -190.2491455078125,
"loss": 1.1179,
"margin_dpo/margin_mean": 42.96190643310547,
"margin_dpo/margin_std": 68.222412109375,
"step": 320
},
{
"KL/chosen_KL_mean": -80.76130676269531,
"KL/mean": -103.32257080078125,
"KL/rejected_KL_mean": -125.88383483886719,
"KL/std": 56.74087905883789,
"epoch": 0.4852607709750567,
"fcm_dpo/beta": 0.009960871189832687,
"fcm_dpo/delta": -0.0006339475512504578,
"fcm_dpo/margin": 45.12252426147461,
"fcm_dpo/q_t": 0.39828598499298096,
"grad_norm": 18.722091674804688,
"learning_rate": 3.0765396768561004e-07,
"logits/chosen": 0.650843620300293,
"logits/rejected": 0.6300288438796997,
"logps/chosen": -132.84657287597656,
"logps/ref_chosen": -52.08526611328125,
"logps/ref_rejected": -55.58674621582031,
"logps/rejected": -181.4705810546875,
"loss": 1.0956,
"margin_dpo/margin_mean": 45.12252426147461,
"margin_dpo/margin_std": 66.43511962890625,
"step": 321
},
{
"KL/chosen_KL_mean": -90.73258209228516,
"KL/mean": -119.62736511230469,
"KL/rejected_KL_mean": -148.5221405029297,
"KL/std": 63.579566955566406,
"epoch": 0.48677248677248675,
"fcm_dpo/beta": 0.009883934631943703,
"fcm_dpo/delta": -0.1283361166715622,
"fcm_dpo/margin": 57.78955841064453,
"fcm_dpo/q_t": 0.370322048664093,
"grad_norm": 11.981712341308594,
"learning_rate": 3.063665887884511e-07,
"logits/chosen": 0.7016029357910156,
"logits/rejected": 0.6129163503646851,
"logps/chosen": -138.13668823242188,
"logps/ref_chosen": -47.404109954833984,
"logps/ref_rejected": -73.4260025024414,
"logps/rejected": -221.94815063476562,
"loss": 0.9847,
"margin_dpo/margin_mean": 57.78955841064453,
"margin_dpo/margin_std": 62.76979446411133,
"step": 322
},
{
"KL/chosen_KL_mean": -92.734619140625,
"KL/mean": -112.15545654296875,
"KL/rejected_KL_mean": -131.57632446289062,
"KL/std": 61.908042907714844,
"epoch": 0.48828420256991684,
"fcm_dpo/beta": 0.009892433881759644,
"fcm_dpo/delta": 0.06829862296581268,
"fcm_dpo/margin": 38.841705322265625,
"fcm_dpo/q_t": 0.4185178875923157,
"grad_norm": 14.59123420715332,
"learning_rate": 3.0507763319663517e-07,
"logits/chosen": 0.5701498985290527,
"logits/rejected": 0.49318498373031616,
"logps/chosen": -162.74090576171875,
"logps/ref_chosen": -70.00630187988281,
"logps/ref_rejected": -86.96690368652344,
"logps/rejected": -218.543212890625,
"loss": 1.176,
"margin_dpo/margin_mean": 38.841705322265625,
"margin_dpo/margin_std": 78.66917419433594,
"step": 323
},
{
"KL/chosen_KL_mean": -82.48573303222656,
"KL/mean": -107.55412292480469,
"KL/rejected_KL_mean": -132.62252807617188,
"KL/std": 68.80017852783203,
"epoch": 0.4897959183673469,
"fcm_dpo/beta": 0.009812276810407639,
"fcm_dpo/delta": -0.0446263924241066,
"fcm_dpo/margin": 50.13678741455078,
"fcm_dpo/q_t": 0.3889816403388977,
"grad_norm": 18.301029205322266,
"learning_rate": 3.0378713696502097e-07,
"logits/chosen": 0.6554989814758301,
"logits/rejected": 0.5961704254150391,
"logps/chosen": -138.37454223632812,
"logps/ref_chosen": -55.88882064819336,
"logps/ref_rejected": -75.23088073730469,
"logps/rejected": -207.8533935546875,
"loss": 1.0441,
"margin_dpo/margin_mean": 50.13678741455078,
"margin_dpo/margin_std": 63.38947677612305,
"step": 324
},
{
"KL/chosen_KL_mean": -99.98908996582031,
"KL/mean": -125.22124481201172,
"KL/rejected_KL_mean": -150.45339965820312,
"KL/std": 67.48681640625,
"epoch": 0.491307634164777,
"fcm_dpo/beta": 0.009717625565826893,
"fcm_dpo/delta": -0.04319227859377861,
"fcm_dpo/margin": 50.46430587768555,
"fcm_dpo/q_t": 0.3927181661128998,
"grad_norm": 13.410660743713379,
"learning_rate": 3.0249513619156206e-07,
"logits/chosen": 0.6126409769058228,
"logits/rejected": 0.5459693670272827,
"logps/chosen": -164.1361083984375,
"logps/ref_chosen": -64.14701843261719,
"logps/ref_rejected": -79.91143798828125,
"logps/rejected": -230.36483764648438,
"loss": 1.0903,
"margin_dpo/margin_mean": 50.46430587768555,
"margin_dpo/margin_std": 80.43098449707031,
"step": 325
},
{
"KL/chosen_KL_mean": -118.30567932128906,
"KL/mean": -128.3629150390625,
"KL/rejected_KL_mean": -138.420166015625,
"KL/std": 63.34078598022461,
"epoch": 0.4928193499622071,
"fcm_dpo/beta": 0.009744199924170971,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 20.11448860168457,
"fcm_dpo/q_t": 0.45632901787757874,
"grad_norm": 15.810664176940918,
"learning_rate": 3.012016670162977e-07,
"logits/chosen": 0.5627347230911255,
"logits/rejected": 0.5708246827125549,
"logps/chosen": -193.8369903564453,
"logps/ref_chosen": -75.53131103515625,
"logps/ref_rejected": -76.5898666381836,
"logps/rejected": -215.01004028320312,
"loss": 1.3113,
"margin_dpo/margin_mean": 20.114490509033203,
"margin_dpo/margin_std": 70.28466796875,
"step": 326
},
{
"KL/chosen_KL_mean": -105.60182189941406,
"KL/mean": -125.68409729003906,
"KL/rejected_KL_mean": -145.76638793945312,
"KL/std": 67.13113403320312,
"epoch": 0.4943310657596372,
"fcm_dpo/beta": 0.009856300428509712,
"fcm_dpo/delta": 0.055987969040870667,
"fcm_dpo/margin": 40.16455078125,
"fcm_dpo/q_t": 0.4134991765022278,
"grad_norm": 14.7315092086792,
"learning_rate": 2.99906765620341e-07,
"logits/chosen": 0.5232765674591064,
"logits/rejected": 0.4972417950630188,
"logps/chosen": -174.93899536132812,
"logps/ref_chosen": -69.33717346191406,
"logps/ref_rejected": -73.37751770019531,
"logps/rejected": -219.14389038085938,
"loss": 1.1596,
"margin_dpo/margin_mean": 40.16455078125,
"margin_dpo/margin_std": 76.08860778808594,
"step": 327
},
{
"KL/chosen_KL_mean": -92.72146606445312,
"KL/mean": -117.269775390625,
"KL/rejected_KL_mean": -141.81808471679688,
"KL/std": 67.56961059570312,
"epoch": 0.4958427815570673,
"fcm_dpo/beta": 0.009826852940022945,
"fcm_dpo/delta": -0.03414200246334076,
"fcm_dpo/margin": 49.09660720825195,
"fcm_dpo/q_t": 0.3925701975822449,
"grad_norm": 13.953093528747559,
"learning_rate": 2.9861046822486766e-07,
"logits/chosen": 0.5673099756240845,
"logits/rejected": 0.5271446108818054,
"logps/chosen": -154.42770385742188,
"logps/ref_chosen": -61.70623016357422,
"logps/ref_rejected": -83.73808288574219,
"logps/rejected": -225.55615234375,
"loss": 1.054,
"margin_dpo/margin_mean": 49.09661102294922,
"margin_dpo/margin_std": 64.48529815673828,
"step": 328
},
{
"KL/chosen_KL_mean": -104.82115936279297,
"KL/mean": -129.3842315673828,
"KL/rejected_KL_mean": -153.94729614257812,
"KL/std": 70.55194091796875,
"epoch": 0.4973544973544973,
"fcm_dpo/beta": 0.009733401238918304,
"fcm_dpo/delta": -0.029633961617946625,
"fcm_dpo/margin": 49.12614440917969,
"fcm_dpo/q_t": 0.39622172713279724,
"grad_norm": 15.438725471496582,
"learning_rate": 2.9731281109010253e-07,
"logits/chosen": 0.6905697584152222,
"logits/rejected": 0.628017783164978,
"logps/chosen": -169.319580078125,
"logps/ref_chosen": -64.4984130859375,
"logps/ref_rejected": -83.6591796875,
"logps/rejected": -237.60647583007812,
"loss": 1.077,
"margin_dpo/margin_mean": 49.12614440917969,
"margin_dpo/margin_std": 72.67239379882812,
"step": 329
},
{
"KL/chosen_KL_mean": -90.5877685546875,
"KL/mean": -115.56993103027344,
"KL/rejected_KL_mean": -140.55209350585938,
"KL/std": 66.15676879882812,
"epoch": 0.4988662131519274,
"fcm_dpo/beta": 0.009712828323245049,
"fcm_dpo/delta": -0.0372566320002079,
"fcm_dpo/margin": 49.964324951171875,
"fcm_dpo/q_t": 0.393964946269989,
"grad_norm": 13.776653289794922,
"learning_rate": 2.9601383051430505e-07,
"logits/chosen": 0.6362694501876831,
"logits/rejected": 0.5670721530914307,
"logps/chosen": -145.3924102783203,
"logps/ref_chosen": -54.80464172363281,
"logps/ref_rejected": -75.3194351196289,
"logps/rejected": -215.87152099609375,
"loss": 1.1115,
"margin_dpo/margin_mean": 49.96432113647461,
"margin_dpo/margin_std": 83.51506042480469,
"step": 330
},
{
"KL/chosen_KL_mean": -97.71769714355469,
"KL/mean": -129.8302001953125,
"KL/rejected_KL_mean": -161.9427032470703,
"KL/std": 70.34434509277344,
"epoch": 0.5003779289493575,
"fcm_dpo/beta": 0.00942898727953434,
"fcm_dpo/delta": -0.1655063033103943,
"fcm_dpo/margin": 64.22500610351562,
"fcm_dpo/q_t": 0.36448371410369873,
"grad_norm": 12.452539443969727,
"learning_rate": 2.947135628327544e-07,
"logits/chosen": 0.7674143314361572,
"logits/rejected": 0.7401211261749268,
"logps/chosen": -156.96026611328125,
"logps/ref_chosen": -59.242584228515625,
"logps/ref_rejected": -69.87483215332031,
"logps/rejected": -231.81753540039062,
"loss": 0.9871,
"margin_dpo/margin_mean": 64.22500610351562,
"margin_dpo/margin_std": 76.57498168945312,
"step": 331
},
{
"KL/chosen_KL_mean": -100.70648956298828,
"KL/mean": -127.30098724365234,
"KL/rejected_KL_mean": -153.8954620361328,
"KL/std": 67.10773468017578,
"epoch": 0.5018896447467877,
"fcm_dpo/beta": 0.009348525665700436,
"fcm_dpo/delta": -0.050655677914619446,
"fcm_dpo/margin": 53.18898010253906,
"fcm_dpo/q_t": 0.39014729857444763,
"grad_norm": 12.76447868347168,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": 0.5541119575500488,
"logits/rejected": 0.5114009976387024,
"logps/chosen": -167.81625366210938,
"logps/ref_chosen": -67.10975646972656,
"logps/ref_rejected": -77.11839294433594,
"logps/rejected": -231.01385498046875,
"loss": 1.0501,
"margin_dpo/margin_mean": 53.18898010253906,
"margin_dpo/margin_std": 69.1185073852539,
"step": 332
},
{
"KL/chosen_KL_mean": -107.39823150634766,
"KL/mean": -136.9010467529297,
"KL/rejected_KL_mean": -166.40386962890625,
"KL/std": 67.92105102539062,
"epoch": 0.5034013605442177,
"fcm_dpo/beta": 0.009133165702223778,
"fcm_dpo/delta": -0.09388077259063721,
"fcm_dpo/margin": 59.0056266784668,
"fcm_dpo/q_t": 0.3798203468322754,
"grad_norm": 12.019691467285156,
"learning_rate": 2.921093116725076e-07,
"logits/chosen": 0.6719864010810852,
"logits/rejected": 0.5902762413024902,
"logps/chosen": -165.77935791015625,
"logps/ref_chosen": -58.381134033203125,
"logps/ref_rejected": -85.02839660644531,
"logps/rejected": -251.4322509765625,
"loss": 1.0135,
"margin_dpo/margin_mean": 59.0056266784668,
"margin_dpo/margin_std": 71.16836547851562,
"step": 333
},
{
"KL/chosen_KL_mean": -103.21528625488281,
"KL/mean": -124.10235595703125,
"KL/rejected_KL_mean": -144.98941040039062,
"KL/std": 69.16629028320312,
"epoch": 0.5049130763416477,
"fcm_dpo/beta": 0.009185167029500008,
"fcm_dpo/delta": 0.0687691867351532,
"fcm_dpo/margin": 41.77412414550781,
"fcm_dpo/q_t": 0.41626012325286865,
"grad_norm": 13.901957511901855,
"learning_rate": 2.9080540104031484e-07,
"logits/chosen": 0.6650213599205017,
"logits/rejected": 0.6160895824432373,
"logps/chosen": -170.1072998046875,
"logps/ref_chosen": -66.89199829101562,
"logps/ref_rejected": -91.83695220947266,
"logps/rejected": -236.8263702392578,
"loss": 1.1629,
"margin_dpo/margin_mean": 41.77412414550781,
"margin_dpo/margin_std": 79.92010498046875,
"step": 334
},
{
"KL/chosen_KL_mean": -103.90148162841797,
"KL/mean": -127.40682983398438,
"KL/rejected_KL_mean": -150.9121856689453,
"KL/std": 68.83200073242188,
"epoch": 0.5064247921390779,
"fcm_dpo/beta": 0.009263388812541962,
"fcm_dpo/delta": 0.014709331095218658,
"fcm_dpo/margin": 47.010711669921875,
"fcm_dpo/q_t": 0.4052375257015228,
"grad_norm": 18.1312313079834,
"learning_rate": 2.895003489933375e-07,
"logits/chosen": 0.6490761041641235,
"logits/rejected": 0.6102247834205627,
"logps/chosen": -165.4159393310547,
"logps/ref_chosen": -61.51445770263672,
"logps/ref_rejected": -75.68916320800781,
"logps/rejected": -226.60134887695312,
"loss": 1.1211,
"margin_dpo/margin_mean": 47.010711669921875,
"margin_dpo/margin_std": 78.88675689697266,
"step": 335
},
{
"KL/chosen_KL_mean": -116.6889877319336,
"KL/mean": -141.385498046875,
"KL/rejected_KL_mean": -166.08203125,
"KL/std": 72.22708892822266,
"epoch": 0.5079365079365079,
"fcm_dpo/beta": 0.00915299728512764,
"fcm_dpo/delta": -0.003460422158241272,
"fcm_dpo/margin": 49.393028259277344,
"fcm_dpo/q_t": 0.40451768040657043,
"grad_norm": 13.747593879699707,
"learning_rate": 2.8819419203668675e-07,
"logits/chosen": 0.6120268106460571,
"logits/rejected": 0.5861070156097412,
"logps/chosen": -185.5390625,
"logps/ref_chosen": -68.85006713867188,
"logps/ref_rejected": -92.99603271484375,
"logps/rejected": -259.07806396484375,
"loss": 1.1113,
"margin_dpo/margin_mean": 49.393028259277344,
"margin_dpo/margin_std": 80.9678726196289,
"step": 336
},
{
"KL/chosen_KL_mean": -121.39860534667969,
"KL/mean": -140.76492309570312,
"KL/rejected_KL_mean": -160.13125610351562,
"KL/std": 71.30752563476562,
"epoch": 0.509448223733938,
"fcm_dpo/beta": 0.009357749484479427,
"fcm_dpo/delta": 0.09072916209697723,
"fcm_dpo/margin": 38.732643127441406,
"fcm_dpo/q_t": 0.4197903275489807,
"grad_norm": 12.64812183380127,
"learning_rate": 2.8688696670638053e-07,
"logits/chosen": 0.5565305352210999,
"logits/rejected": 0.5242564082145691,
"logps/chosen": -194.58644104003906,
"logps/ref_chosen": -73.18783569335938,
"logps/ref_rejected": -86.89118957519531,
"logps/rejected": -247.02244567871094,
"loss": 1.169,
"margin_dpo/margin_mean": 38.732643127441406,
"margin_dpo/margin_std": 74.53764343261719,
"step": 337
},
{
"KL/chosen_KL_mean": -118.91609954833984,
"KL/mean": -139.07220458984375,
"KL/rejected_KL_mean": -159.22833251953125,
"KL/std": 67.26274108886719,
"epoch": 0.5109599395313681,
"fcm_dpo/beta": 0.00947779230773449,
"fcm_dpo/delta": 0.07068651914596558,
"fcm_dpo/margin": 40.312225341796875,
"fcm_dpo/q_t": 0.41567832231521606,
"grad_norm": 13.105402946472168,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": 0.6425968408584595,
"logits/rejected": 0.6147984862327576,
"logps/chosen": -182.855712890625,
"logps/ref_chosen": -63.939613342285156,
"logps/ref_rejected": -75.34243774414062,
"logps/rejected": -234.57077026367188,
"loss": 1.1589,
"margin_dpo/margin_mean": 40.312225341796875,
"margin_dpo/margin_std": 75.49674224853516,
"step": 338
},
{
"KL/chosen_KL_mean": -96.85836791992188,
"KL/mean": -120.70976257324219,
"KL/rejected_KL_mean": -144.56114196777344,
"KL/std": 67.41364288330078,
"epoch": 0.5124716553287982,
"fcm_dpo/beta": 0.009497357532382011,
"fcm_dpo/delta": -0.0033246222883462906,
"fcm_dpo/margin": 47.70277404785156,
"fcm_dpo/q_t": 0.39905327558517456,
"grad_norm": 15.153426170349121,
"learning_rate": 2.842694572172736e-07,
"logits/chosen": 0.7378226518630981,
"logits/rejected": 0.6509774923324585,
"logps/chosen": -142.40750122070312,
"logps/ref_chosen": -45.54913330078125,
"logps/ref_rejected": -67.0482177734375,
"logps/rejected": -211.609375,
"loss": 1.093,
"margin_dpo/margin_mean": 47.70277404785156,
"margin_dpo/margin_std": 71.71929931640625,
"step": 339
},
{
"KL/chosen_KL_mean": -109.52645111083984,
"KL/mean": -132.99769592285156,
"KL/rejected_KL_mean": -156.46893310546875,
"KL/std": 73.52427673339844,
"epoch": 0.5139833711262283,
"fcm_dpo/beta": 0.009554904885590076,
"fcm_dpo/delta": 0.0012706257402896881,
"fcm_dpo/margin": 46.94248962402344,
"fcm_dpo/q_t": 0.4044419229030609,
"grad_norm": 13.740220069885254,
"learning_rate": 2.8295924627584004e-07,
"logits/chosen": 0.6307625770568848,
"logits/rejected": 0.6090366840362549,
"logps/chosen": -163.5321044921875,
"logps/ref_chosen": -54.00564956665039,
"logps/ref_rejected": -61.314430236816406,
"logps/rejected": -217.7833709716797,
"loss": 1.1408,
"margin_dpo/margin_mean": 46.94248580932617,
"margin_dpo/margin_std": 85.59992218017578,
"step": 340
},
{
"KL/chosen_KL_mean": -109.53600311279297,
"KL/mean": -136.73626708984375,
"KL/rejected_KL_mean": -163.9365234375,
"KL/std": 72.4736099243164,
"epoch": 0.5154950869236583,
"fcm_dpo/beta": 0.009304164908826351,
"fcm_dpo/delta": -0.06382884830236435,
"fcm_dpo/margin": 54.4005241394043,
"fcm_dpo/q_t": 0.3888283371925354,
"grad_norm": 15.317728996276855,
"learning_rate": 2.816481133934373e-07,
"logits/chosen": 0.6532335877418518,
"logits/rejected": 0.6039453744888306,
"logps/chosen": -172.93109130859375,
"logps/ref_chosen": -63.39509582519531,
"logps/ref_rejected": -76.20973205566406,
"logps/rejected": -240.14625549316406,
"loss": 1.0768,
"margin_dpo/margin_mean": 54.4005241394043,
"margin_dpo/margin_std": 76.9643783569336,
"step": 341
},
{
"KL/chosen_KL_mean": -108.70679473876953,
"KL/mean": -135.81114196777344,
"KL/rejected_KL_mean": -162.91549682617188,
"KL/std": 76.89732360839844,
"epoch": 0.5170068027210885,
"fcm_dpo/beta": 0.009291356429457664,
"fcm_dpo/delta": -0.05691445991396904,
"fcm_dpo/margin": 54.20869445800781,
"fcm_dpo/q_t": 0.39120563864707947,
"grad_norm": 12.805046081542969,
"learning_rate": 2.8033609524527046e-07,
"logits/chosen": 0.6862562894821167,
"logits/rejected": 0.6442948579788208,
"logps/chosen": -161.75460815429688,
"logps/ref_chosen": -53.047813415527344,
"logps/ref_rejected": -68.2854232788086,
"logps/rejected": -231.200927734375,
"loss": 1.0676,
"margin_dpo/margin_mean": 54.20869445800781,
"margin_dpo/margin_std": 78.99603271484375,
"step": 342
},
{
"KL/chosen_KL_mean": -102.84151458740234,
"KL/mean": -122.31297302246094,
"KL/rejected_KL_mean": -141.784423828125,
"KL/std": 71.11018371582031,
"epoch": 0.5185185185185185,
"fcm_dpo/beta": 0.009219045750796795,
"fcm_dpo/delta": -0.040550898760557175,
"fcm_dpo/margin": 38.94290542602539,
"fcm_dpo/q_t": 0.4192054271697998,
"grad_norm": 12.088910102844238,
"learning_rate": 2.7902322853130753e-07,
"logits/chosen": 0.5472891330718994,
"logits/rejected": 0.5373716354370117,
"logps/chosen": -173.4200439453125,
"logps/ref_chosen": -70.57852935791016,
"logps/ref_rejected": -84.73873901367188,
"logps/rejected": -226.52316284179688,
"loss": 1.167,
"margin_dpo/margin_mean": 38.94290542602539,
"margin_dpo/margin_std": 72.6668701171875,
"step": 343
},
{
"KL/chosen_KL_mean": -113.80350494384766,
"KL/mean": -141.92852783203125,
"KL/rejected_KL_mean": -170.05355834960938,
"KL/std": 72.05269622802734,
"epoch": 0.5200302343159486,
"fcm_dpo/beta": 0.009113233536481857,
"fcm_dpo/delta": -0.06617586314678192,
"fcm_dpo/margin": 56.250038146972656,
"fcm_dpo/q_t": 0.3878791332244873,
"grad_norm": 15.124133110046387,
"learning_rate": 2.7770954997525274e-07,
"logits/chosen": 0.6614542007446289,
"logits/rejected": 0.5895565152168274,
"logps/chosen": -169.614501953125,
"logps/ref_chosen": -55.811004638671875,
"logps/ref_rejected": -84.77637481689453,
"logps/rejected": -254.82992553710938,
"loss": 1.0575,
"margin_dpo/margin_mean": 56.250038146972656,
"margin_dpo/margin_std": 79.96159362792969,
"step": 344
},
{
"KL/chosen_KL_mean": -91.85005187988281,
"KL/mean": -116.61634826660156,
"KL/rejected_KL_mean": -141.3826446533203,
"KL/std": 65.72059631347656,
"epoch": 0.5215419501133787,
"fcm_dpo/beta": 0.009115805849432945,
"fcm_dpo/delta": -0.0016993321478366852,
"fcm_dpo/margin": 49.532596588134766,
"fcm_dpo/q_t": 0.4003925323486328,
"grad_norm": 14.183326721191406,
"learning_rate": 2.7639509632351927e-07,
"logits/chosen": 0.7415436506271362,
"logits/rejected": 0.6945260763168335,
"logps/chosen": -149.63613891601562,
"logps/ref_chosen": -57.78609848022461,
"logps/ref_rejected": -78.91847229003906,
"logps/rejected": -220.30111694335938,
"loss": 1.1019,
"margin_dpo/margin_mean": 49.5325927734375,
"margin_dpo/margin_std": 78.25393676757812,
"step": 345
},
{
"KL/chosen_KL_mean": -98.70977783203125,
"KL/mean": -128.33563232421875,
"KL/rejected_KL_mean": -157.96148681640625,
"KL/std": 76.5794677734375,
"epoch": 0.5230536659108088,
"fcm_dpo/beta": 0.009013254195451736,
"fcm_dpo/delta": -0.08877884596586227,
"fcm_dpo/margin": 59.251705169677734,
"fcm_dpo/q_t": 0.38100147247314453,
"grad_norm": 14.048331260681152,
"learning_rate": 2.7507990434420123e-07,
"logits/chosen": 0.6886954307556152,
"logits/rejected": 0.5999356508255005,
"logps/chosen": -154.99490356445312,
"logps/ref_chosen": -56.285125732421875,
"logps/ref_rejected": -91.15303039550781,
"logps/rejected": -249.11451721191406,
"loss": 1.0448,
"margin_dpo/margin_mean": 59.251708984375,
"margin_dpo/margin_std": 79.60140991210938,
"step": 346
},
{
"KL/chosen_KL_mean": -108.50074005126953,
"KL/mean": -131.3124542236328,
"KL/rejected_KL_mean": -154.1241455078125,
"KL/std": 73.10723876953125,
"epoch": 0.5245653817082389,
"fcm_dpo/beta": 0.008980346843600273,
"fcm_dpo/delta": 0.04202822968363762,
"fcm_dpo/margin": 45.6234130859375,
"fcm_dpo/q_t": 0.40984588861465454,
"grad_norm": 15.117777824401855,
"learning_rate": 2.737640108260456e-07,
"logits/chosen": 0.8087772130966187,
"logits/rejected": 0.7555938959121704,
"logps/chosen": -162.00027465820312,
"logps/ref_chosen": -53.499542236328125,
"logps/ref_rejected": -72.52565002441406,
"logps/rejected": -226.64981079101562,
"loss": 1.1206,
"margin_dpo/margin_mean": 45.6234130859375,
"margin_dpo/margin_std": 74.35516357421875,
"step": 347
},
{
"KL/chosen_KL_mean": -97.05572509765625,
"KL/mean": -123.86107635498047,
"KL/rejected_KL_mean": -150.6664276123047,
"KL/std": 69.09683990478516,
"epoch": 0.5260770975056689,
"fcm_dpo/beta": 0.008932996541261673,
"fcm_dpo/delta": -0.030811918899416924,
"fcm_dpo/margin": 53.610694885253906,
"fcm_dpo/q_t": 0.3972897529602051,
"grad_norm": 14.440524101257324,
"learning_rate": 2.724474525774229e-07,
"logits/chosen": 0.7605525255203247,
"logits/rejected": 0.728769063949585,
"logps/chosen": -147.8425750732422,
"logps/ref_chosen": -50.78684997558594,
"logps/ref_rejected": -68.63732147216797,
"logps/rejected": -219.30374145507812,
"loss": 1.0883,
"margin_dpo/margin_mean": 53.61069107055664,
"margin_dpo/margin_std": 83.18544006347656,
"step": 348
},
{
"KL/chosen_KL_mean": -97.84178161621094,
"KL/mean": -125.2623062133789,
"KL/rejected_KL_mean": -152.68283081054688,
"KL/std": 75.20608520507812,
"epoch": 0.527588813303099,
"fcm_dpo/beta": 0.008946020156145096,
"fcm_dpo/delta": -0.04307024925947189,
"fcm_dpo/margin": 54.84104919433594,
"fcm_dpo/q_t": 0.39336341619491577,
"grad_norm": 13.364866256713867,
"learning_rate": 2.711302664252973e-07,
"logits/chosen": 0.702577531337738,
"logits/rejected": 0.6050245761871338,
"logps/chosen": -151.16677856445312,
"logps/ref_chosen": -53.325008392333984,
"logps/ref_rejected": -83.21236419677734,
"logps/rejected": -235.8951873779297,
"loss": 1.0727,
"margin_dpo/margin_mean": 54.84105682373047,
"margin_dpo/margin_std": 79.70642852783203,
"step": 349
},
{
"KL/chosen_KL_mean": -102.39527130126953,
"KL/mean": -137.85421752929688,
"KL/rejected_KL_mean": -173.31314086914062,
"KL/std": 80.3461685180664,
"epoch": 0.5291005291005291,
"fcm_dpo/beta": 0.008636381477117538,
"fcm_dpo/delta": -0.17346930503845215,
"fcm_dpo/margin": 70.91788482666016,
"fcm_dpo/q_t": 0.3626285195350647,
"grad_norm": 15.910598754882812,
"learning_rate": 2.698124892141971e-07,
"logits/chosen": 0.6693747639656067,
"logits/rejected": 0.5895169377326965,
"logps/chosen": -164.0210418701172,
"logps/ref_chosen": -61.625770568847656,
"logps/ref_rejected": -87.63627624511719,
"logps/rejected": -260.94940185546875,
"loss": 0.9643,
"margin_dpo/margin_mean": 70.91788482666016,
"margin_dpo/margin_std": 77.03778076171875,
"step": 350
},
{
"KL/chosen_KL_mean": -97.79383850097656,
"KL/mean": -122.77336120605469,
"KL/rejected_KL_mean": -147.7528839111328,
"KL/std": 68.2166748046875,
"epoch": 0.5306122448979592,
"fcm_dpo/beta": 0.008579917252063751,
"fcm_dpo/delta": 0.02221677638590336,
"fcm_dpo/margin": 49.95904541015625,
"fcm_dpo/q_t": 0.4026418626308441,
"grad_norm": 14.179925918579102,
"learning_rate": 2.6849415780518357e-07,
"logits/chosen": 0.6331157088279724,
"logits/rejected": 0.5555412769317627,
"logps/chosen": -154.0501708984375,
"logps/ref_chosen": -56.2563362121582,
"logps/ref_rejected": -79.11589813232422,
"logps/rejected": -226.8687744140625,
"loss": 1.1297,
"margin_dpo/margin_mean": 49.95904541015625,
"margin_dpo/margin_std": 85.19073486328125,
"step": 351
},
{
"KL/chosen_KL_mean": -99.13749694824219,
"KL/mean": -127.44450378417969,
"KL/rejected_KL_mean": -155.75152587890625,
"KL/std": 76.02853393554688,
"epoch": 0.5321239606953893,
"fcm_dpo/beta": 0.008539706468582153,
"fcm_dpo/delta": -0.03550984710454941,
"fcm_dpo/margin": 56.61402893066406,
"fcm_dpo/q_t": 0.3938150107860565,
"grad_norm": 12.919422149658203,
"learning_rate": 2.6717530907482024e-07,
"logits/chosen": 0.677651047706604,
"logits/rejected": 0.6238071918487549,
"logps/chosen": -162.189453125,
"logps/ref_chosen": -63.05195236206055,
"logps/ref_rejected": -85.52035522460938,
"logps/rejected": -241.27188110351562,
"loss": 1.0692,
"margin_dpo/margin_mean": 56.61402893066406,
"margin_dpo/margin_std": 81.5572509765625,
"step": 352
},
{
"KL/chosen_KL_mean": -96.16493225097656,
"KL/mean": -123.23371124267578,
"KL/rejected_KL_mean": -150.302490234375,
"KL/std": 70.93081665039062,
"epoch": 0.5336356764928194,
"fcm_dpo/beta": 0.008550824597477913,
"fcm_dpo/delta": -0.013591569848358631,
"fcm_dpo/margin": 54.137550354003906,
"fcm_dpo/q_t": 0.3958391547203064,
"grad_norm": 12.857582092285156,
"learning_rate": 2.658559799141411e-07,
"logits/chosen": 0.6801486015319824,
"logits/rejected": 0.6855990886688232,
"logps/chosen": -165.1741180419922,
"logps/ref_chosen": -69.00918579101562,
"logps/ref_rejected": -72.65840148925781,
"logps/rejected": -222.96090698242188,
"loss": 1.0738,
"margin_dpo/margin_mean": 54.137550354003906,
"margin_dpo/margin_std": 75.59597778320312,
"step": 353
},
{
"KL/chosen_KL_mean": -97.02384948730469,
"KL/mean": -128.12098693847656,
"KL/rejected_KL_mean": -159.21810913085938,
"KL/std": 68.00271606445312,
"epoch": 0.5351473922902494,
"fcm_dpo/beta": 0.008396446704864502,
"fcm_dpo/delta": -0.0768899917602539,
"fcm_dpo/margin": 62.19426727294922,
"fcm_dpo/q_t": 0.3830644488334656,
"grad_norm": 14.596508026123047,
"learning_rate": 2.6453620722761895e-07,
"logits/chosen": 0.7443443536758423,
"logits/rejected": 0.6121722459793091,
"logps/chosen": -136.81219482421875,
"logps/ref_chosen": -39.78833770751953,
"logps/ref_rejected": -69.56885528564453,
"logps/rejected": -228.7869873046875,
"loss": 1.0408,
"margin_dpo/margin_mean": 62.19426727294922,
"margin_dpo/margin_std": 81.65336608886719,
"step": 354
},
{
"KL/chosen_KL_mean": -102.66900634765625,
"KL/mean": -135.7369842529297,
"KL/rejected_KL_mean": -168.80494689941406,
"KL/std": 78.304443359375,
"epoch": 0.5366591080876795,
"fcm_dpo/beta": 0.008291777223348618,
"fcm_dpo/delta": -0.10396140813827515,
"fcm_dpo/margin": 66.13594055175781,
"fcm_dpo/q_t": 0.378852516412735,
"grad_norm": 16.355134963989258,
"learning_rate": 2.632160279321328e-07,
"logits/chosen": 0.6968790292739868,
"logits/rejected": 0.5650628805160522,
"logps/chosen": -148.9243927001953,
"logps/ref_chosen": -46.25537872314453,
"logps/ref_rejected": -78.20236206054688,
"logps/rejected": -247.00730895996094,
"loss": 1.0354,
"margin_dpo/margin_mean": 66.13594055175781,
"margin_dpo/margin_std": 88.17823791503906,
"step": 355
},
{
"KL/chosen_KL_mean": -96.81419372558594,
"KL/mean": -123.33544921875,
"KL/rejected_KL_mean": -149.85671997070312,
"KL/std": 73.71067810058594,
"epoch": 0.5381708238851096,
"fcm_dpo/beta": 0.008203265257179737,
"fcm_dpo/delta": 0.015107132494449615,
"fcm_dpo/margin": 53.042518615722656,
"fcm_dpo/q_t": 0.4048543870449066,
"grad_norm": 12.561837196350098,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": 0.6723008155822754,
"logits/rejected": 0.5905227065086365,
"logps/chosen": -144.72035217285156,
"logps/ref_chosen": -47.906158447265625,
"logps/ref_rejected": -74.29397583007812,
"logps/rejected": -224.15069580078125,
"loss": 1.1379,
"margin_dpo/margin_mean": 53.042518615722656,
"margin_dpo/margin_std": 94.193359375,
"step": 356
},
{
"KL/chosen_KL_mean": -115.66548919677734,
"KL/mean": -137.5971221923828,
"KL/rejected_KL_mean": -159.52877807617188,
"KL/std": 73.30964660644531,
"epoch": 0.5396825396825397,
"fcm_dpo/beta": 0.00813913345336914,
"fcm_dpo/delta": -0.06833313405513763,
"fcm_dpo/margin": 43.86327362060547,
"fcm_dpo/q_t": 0.418012797832489,
"grad_norm": 13.875882148742676,
"learning_rate": 2.6057459723762076e-07,
"logits/chosen": 0.6871232986450195,
"logits/rejected": 0.6637519001960754,
"logps/chosen": -178.3004913330078,
"logps/ref_chosen": -62.63500213623047,
"logps/ref_rejected": -65.11399841308594,
"logps/rejected": -224.64276123046875,
"loss": 1.1691,
"margin_dpo/margin_mean": 43.86327362060547,
"margin_dpo/margin_std": 79.24894714355469,
"step": 357
},
{
"KL/chosen_KL_mean": -112.24562072753906,
"KL/mean": -142.50857543945312,
"KL/rejected_KL_mean": -172.77151489257812,
"KL/std": 76.78482055664062,
"epoch": 0.5411942554799698,
"fcm_dpo/beta": 0.008158707991242409,
"fcm_dpo/delta": -0.04745243489742279,
"fcm_dpo/margin": 60.52588653564453,
"fcm_dpo/q_t": 0.3917842209339142,
"grad_norm": 18.900074005126953,
"learning_rate": 2.5925341972508954e-07,
"logits/chosen": 0.6384870409965515,
"logits/rejected": 0.6525646448135376,
"logps/chosen": -179.45523071289062,
"logps/ref_chosen": -67.20960998535156,
"logps/ref_rejected": -69.34715270996094,
"logps/rejected": -242.11868286132812,
"loss": 1.0702,
"margin_dpo/margin_mean": 60.52588653564453,
"margin_dpo/margin_std": 85.34688568115234,
"step": 358
},
{
"KL/chosen_KL_mean": -125.95619201660156,
"KL/mean": -141.68975830078125,
"KL/rejected_KL_mean": -157.42330932617188,
"KL/std": 75.13418579101562,
"epoch": 0.5427059712773998,
"fcm_dpo/beta": 0.008086791262030602,
"fcm_dpo/delta": 0.030342087149620056,
"fcm_dpo/margin": 31.467113494873047,
"fcm_dpo/q_t": 0.44308972358703613,
"grad_norm": 14.887833595275879,
"learning_rate": 2.579319833745169e-07,
"logits/chosen": 0.6606814861297607,
"logits/rejected": 0.6267046928405762,
"logps/chosen": -188.4819793701172,
"logps/ref_chosen": -62.52578353881836,
"logps/ref_rejected": -76.63114929199219,
"logps/rejected": -234.05445861816406,
"loss": 1.2485,
"margin_dpo/margin_mean": 31.467111587524414,
"margin_dpo/margin_std": 78.3302993774414,
"step": 359
},
{
"KL/chosen_KL_mean": -121.25907897949219,
"KL/mean": -148.23548889160156,
"KL/rejected_KL_mean": -175.21189880371094,
"KL/std": 79.970947265625,
"epoch": 0.54421768707483,
"fcm_dpo/beta": 0.008114282041788101,
"fcm_dpo/delta": 0.012742086313664913,
"fcm_dpo/margin": 53.95281219482422,
"fcm_dpo/q_t": 0.40300747752189636,
"grad_norm": 13.132290840148926,
"learning_rate": 2.5661032514931834e-07,
"logits/chosen": 0.6002212762832642,
"logits/rejected": 0.5079668164253235,
"logps/chosen": -184.74679565429688,
"logps/ref_chosen": -63.48772048950195,
"logps/ref_rejected": -90.6891098022461,
"logps/rejected": -265.9010009765625,
"loss": 1.0905,
"margin_dpo/margin_mean": 53.95281219482422,
"margin_dpo/margin_std": 78.95816802978516,
"step": 360
},
{
"KL/chosen_KL_mean": -115.78294372558594,
"KL/mean": -146.99960327148438,
"KL/rejected_KL_mean": -178.21627807617188,
"KL/std": 77.42436218261719,
"epoch": 0.54572940287226,
"fcm_dpo/beta": 0.008079019375145435,
"fcm_dpo/delta": -0.05725649744272232,
"fcm_dpo/margin": 62.43333435058594,
"fcm_dpo/q_t": 0.38624024391174316,
"grad_norm": 12.444530487060547,
"learning_rate": 2.552884820191154e-07,
"logits/chosen": 0.7191639542579651,
"logits/rejected": 0.6731234788894653,
"logps/chosen": -173.70008850097656,
"logps/ref_chosen": -57.917144775390625,
"logps/ref_rejected": -72.39089965820312,
"logps/rejected": -250.607177734375,
"loss": 1.0394,
"margin_dpo/margin_mean": 62.43333053588867,
"margin_dpo/margin_std": 79.18809509277344,
"step": 361
},
{
"KL/chosen_KL_mean": -117.07453918457031,
"KL/mean": -151.28367614746094,
"KL/rejected_KL_mean": -185.49281311035156,
"KL/std": 84.75254821777344,
"epoch": 0.54724111866969,
"fcm_dpo/beta": 0.007967658340930939,
"fcm_dpo/delta": -0.10074599087238312,
"fcm_dpo/margin": 68.41827392578125,
"fcm_dpo/q_t": 0.3835322856903076,
"grad_norm": 13.671100616455078,
"learning_rate": 2.53966490958702e-07,
"logits/chosen": 0.7531605958938599,
"logits/rejected": 0.6426513195037842,
"logps/chosen": -180.51800537109375,
"logps/ref_chosen": -63.4434700012207,
"logps/ref_rejected": -103.45516967773438,
"logps/rejected": -288.947998046875,
"loss": 1.0478,
"margin_dpo/margin_mean": 68.41827392578125,
"margin_dpo/margin_std": 98.29641723632812,
"step": 362
},
{
"KL/chosen_KL_mean": -121.70635986328125,
"KL/mean": -153.733642578125,
"KL/rejected_KL_mean": -185.76092529296875,
"KL/std": 75.45001220703125,
"epoch": 0.5487528344671202,
"fcm_dpo/beta": 0.007809008937329054,
"fcm_dpo/delta": -0.05292538180947304,
"fcm_dpo/margin": 64.0545654296875,
"fcm_dpo/q_t": 0.38914868235588074,
"grad_norm": 14.454474449157715,
"learning_rate": 2.526443889470099e-07,
"logits/chosen": 0.771051287651062,
"logits/rejected": 0.630893349647522,
"logps/chosen": -170.35818481445312,
"logps/ref_chosen": -48.65182876586914,
"logps/ref_rejected": -88.65904235839844,
"logps/rejected": -274.41998291015625,
"loss": 1.0513,
"margin_dpo/margin_mean": 64.0545654296875,
"margin_dpo/margin_std": 87.23719787597656,
"step": 363
},
{
"KL/chosen_KL_mean": -107.86466979980469,
"KL/mean": -143.8165740966797,
"KL/rejected_KL_mean": -179.76844787597656,
"KL/std": 83.8006591796875,
"epoch": 0.5502645502645502,
"fcm_dpo/beta": 0.007666376419365406,
"fcm_dpo/delta": -0.10725465416908264,
"fcm_dpo/margin": 71.90379333496094,
"fcm_dpo/q_t": 0.3811180293560028,
"grad_norm": 11.80333423614502,
"learning_rate": 2.513222129660744e-07,
"logits/chosen": 0.6106956005096436,
"logits/rejected": 0.5245239734649658,
"logps/chosen": -165.73574829101562,
"logps/ref_chosen": -57.87107467651367,
"logps/ref_rejected": -80.95503234863281,
"logps/rejected": -260.7234802246094,
"loss": 1.0498,
"margin_dpo/margin_mean": 71.90379333496094,
"margin_dpo/margin_std": 103.74449920654297,
"step": 364
},
{
"KL/chosen_KL_mean": -100.33560180664062,
"KL/mean": -133.744384765625,
"KL/rejected_KL_mean": -167.15313720703125,
"KL/std": 88.45748138427734,
"epoch": 0.5517762660619804,
"fcm_dpo/beta": 0.007543657440692186,
"fcm_dpo/delta": -0.057265233248472214,
"fcm_dpo/margin": 66.81753540039062,
"fcm_dpo/q_t": 0.3860504627227783,
"grad_norm": 11.33962631225586,
"learning_rate": 2.5e-07,
"logits/chosen": 0.6893282532691956,
"logits/rejected": 0.6836937665939331,
"logps/chosen": -165.27777099609375,
"logps/ref_chosen": -64.94217681884766,
"logps/ref_rejected": -74.8599853515625,
"logps/rejected": -242.01312255859375,
"loss": 1.0215,
"margin_dpo/margin_mean": 66.81754302978516,
"margin_dpo/margin_std": 78.03083038330078,
"step": 365
},
{
"KL/chosen_KL_mean": -99.90927124023438,
"KL/mean": -127.51522827148438,
"KL/rejected_KL_mean": -155.1211700439453,
"KL/std": 76.94725799560547,
"epoch": 0.5532879818594104,
"fcm_dpo/beta": 0.007609900087118149,
"fcm_dpo/delta": 0.0306707676500082,
"fcm_dpo/margin": 55.21190643310547,
"fcm_dpo/q_t": 0.4082038402557373,
"grad_norm": 14.167494773864746,
"learning_rate": 2.486777870339255e-07,
"logits/chosen": 0.6601539254188538,
"logits/rejected": 0.6505333185195923,
"logps/chosen": -155.07525634765625,
"logps/ref_chosen": -55.16598129272461,
"logps/ref_rejected": -65.26121520996094,
"logps/rejected": -220.38238525390625,
"loss": 1.1391,
"margin_dpo/margin_mean": 55.21190643310547,
"margin_dpo/margin_std": 98.03353118896484,
"step": 366
},
{
"KL/chosen_KL_mean": -115.7052001953125,
"KL/mean": -143.0133056640625,
"KL/rejected_KL_mean": -170.3214111328125,
"KL/std": 78.01261901855469,
"epoch": 0.5547996976568406,
"fcm_dpo/beta": 0.007616519462317228,
"fcm_dpo/delta": 0.03548750281333923,
"fcm_dpo/margin": 54.61621856689453,
"fcm_dpo/q_t": 0.4044143259525299,
"grad_norm": 13.201713562011719,
"learning_rate": 2.4735561105299014e-07,
"logits/chosen": 0.651557207107544,
"logits/rejected": 0.5488114953041077,
"logps/chosen": -171.71566772460938,
"logps/ref_chosen": -56.01046371459961,
"logps/ref_rejected": -77.31010437011719,
"logps/rejected": -247.6315155029297,
"loss": 1.1061,
"margin_dpo/margin_mean": 54.61621856689453,
"margin_dpo/margin_std": 82.45671844482422,
"step": 367
},
{
"KL/chosen_KL_mean": -127.14485168457031,
"KL/mean": -152.2775421142578,
"KL/rejected_KL_mean": -177.41024780273438,
"KL/std": 77.05934143066406,
"epoch": 0.5563114134542706,
"fcm_dpo/beta": 0.007702920585870743,
"fcm_dpo/delta": 0.06538652628660202,
"fcm_dpo/margin": 50.265403747558594,
"fcm_dpo/q_t": 0.41213297843933105,
"grad_norm": 15.009796142578125,
"learning_rate": 2.46033509041298e-07,
"logits/chosen": 0.5128130316734314,
"logits/rejected": 0.514595091342926,
"logps/chosen": -201.97412109375,
"logps/ref_chosen": -74.82927703857422,
"logps/ref_rejected": -76.11680603027344,
"logps/rejected": -253.5270538330078,
"loss": 1.1298,
"margin_dpo/margin_mean": 50.265403747558594,
"margin_dpo/margin_std": 81.61317443847656,
"step": 368
},
{
"KL/chosen_KL_mean": -118.11318969726562,
"KL/mean": -140.47128295898438,
"KL/rejected_KL_mean": -162.8293914794922,
"KL/std": 77.84164428710938,
"epoch": 0.5578231292517006,
"fcm_dpo/beta": 0.007902087643742561,
"fcm_dpo/delta": 0.09908389300107956,
"fcm_dpo/margin": 44.716209411621094,
"fcm_dpo/q_t": 0.42133742570877075,
"grad_norm": 12.984330177307129,
"learning_rate": 2.447115179808846e-07,
"logits/chosen": 0.7052586078643799,
"logits/rejected": 0.6519788503646851,
"logps/chosen": -176.4394073486328,
"logps/ref_chosen": -58.32621765136719,
"logps/ref_rejected": -80.92183685302734,
"logps/rejected": -243.75123596191406,
"loss": 1.165,
"margin_dpo/margin_mean": 44.716209411621094,
"margin_dpo/margin_std": 82.68061828613281,
"step": 369
},
{
"KL/chosen_KL_mean": -113.09016418457031,
"KL/mean": -145.8719024658203,
"KL/rejected_KL_mean": -178.6536407470703,
"KL/std": 81.12561798095703,
"epoch": 0.5593348450491308,
"fcm_dpo/beta": 0.007803306449204683,
"fcm_dpo/delta": -0.06524214148521423,
"fcm_dpo/margin": 65.5634765625,
"fcm_dpo/q_t": 0.3852328062057495,
"grad_norm": 13.21735668182373,
"learning_rate": 2.4338967485068164e-07,
"logits/chosen": 0.7980337142944336,
"logits/rejected": 0.7273433208465576,
"logps/chosen": -165.973876953125,
"logps/ref_chosen": -52.88372039794922,
"logps/ref_rejected": -79.43692016601562,
"logps/rejected": -258.0905456542969,
"loss": 1.0628,
"margin_dpo/margin_mean": 65.5634765625,
"margin_dpo/margin_std": 94.4046630859375,
"step": 370
},
{
"KL/chosen_KL_mean": -112.4964599609375,
"KL/mean": -140.91160583496094,
"KL/rejected_KL_mean": -169.3267364501953,
"KL/std": 77.54881286621094,
"epoch": 0.5608465608465608,
"fcm_dpo/beta": 0.00784794706851244,
"fcm_dpo/delta": 0.0034468621015548706,
"fcm_dpo/margin": 56.83028030395508,
"fcm_dpo/q_t": 0.4004814028739929,
"grad_norm": 17.490827560424805,
"learning_rate": 2.420680166254831e-07,
"logits/chosen": 0.8194184899330139,
"logits/rejected": 0.7865846753120422,
"logps/chosen": -161.72067260742188,
"logps/ref_chosen": -49.224212646484375,
"logps/ref_rejected": -63.348472595214844,
"logps/rejected": -232.6752166748047,
"loss": 1.092,
"margin_dpo/margin_mean": 56.83028030395508,
"margin_dpo/margin_std": 83.48664855957031,
"step": 371
},
{
"KL/chosen_KL_mean": -119.92819213867188,
"KL/mean": -136.84817504882812,
"KL/rejected_KL_mean": -153.76815795898438,
"KL/std": 77.0000991821289,
"epoch": 0.562358276643991,
"fcm_dpo/beta": 0.007822863757610321,
"fcm_dpo/delta": 0.02560269646346569,
"fcm_dpo/margin": 33.83995819091797,
"fcm_dpo/q_t": 0.44220101833343506,
"grad_norm": 16.425230026245117,
"learning_rate": 2.4074658027491044e-07,
"logits/chosen": 0.7374979257583618,
"logits/rejected": 0.6381915807723999,
"logps/chosen": -172.19775390625,
"logps/ref_chosen": -52.269554138183594,
"logps/ref_rejected": -72.99522399902344,
"logps/rejected": -226.7633819580078,
"loss": 1.2855,
"margin_dpo/margin_mean": 33.8399543762207,
"margin_dpo/margin_std": 99.59162902832031,
"step": 372
},
{
"KL/chosen_KL_mean": -134.54428100585938,
"KL/mean": -158.06080627441406,
"KL/rejected_KL_mean": -181.57733154296875,
"KL/std": 80.29035186767578,
"epoch": 0.563869992441421,
"fcm_dpo/beta": 0.007942916825413704,
"fcm_dpo/delta": 0.07930424809455872,
"fcm_dpo/margin": 47.033042907714844,
"fcm_dpo/q_t": 0.41851696372032166,
"grad_norm": 13.92640209197998,
"learning_rate": 2.394254027623792e-07,
"logits/chosen": 0.6737028956413269,
"logits/rejected": 0.6056632399559021,
"logps/chosen": -195.65728759765625,
"logps/ref_chosen": -61.112998962402344,
"logps/ref_rejected": -76.24851989746094,
"logps/rejected": -257.8258361816406,
"loss": 1.1941,
"margin_dpo/margin_mean": 47.03303909301758,
"margin_dpo/margin_std": 100.17448425292969,
"step": 373
},
{
"KL/chosen_KL_mean": -114.75213623046875,
"KL/mean": -152.3450469970703,
"KL/rejected_KL_mean": -189.9379425048828,
"KL/std": 82.41384887695312,
"epoch": 0.5653817082388511,
"fcm_dpo/beta": 0.0077880797907710075,
"fcm_dpo/delta": -0.14424122869968414,
"fcm_dpo/margin": 75.18580627441406,
"fcm_dpo/q_t": 0.37012046575546265,
"grad_norm": 15.205571174621582,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": 0.6250724196434021,
"logits/rejected": 0.6265436410903931,
"logps/chosen": -187.4213409423828,
"logps/ref_chosen": -72.66920471191406,
"logps/ref_rejected": -76.83158874511719,
"logps/rejected": -266.76953125,
"loss": 1.0123,
"margin_dpo/margin_mean": 75.1858139038086,
"margin_dpo/margin_std": 97.47200012207031,
"step": 374
},
{
"KL/chosen_KL_mean": -113.03237915039062,
"KL/mean": -140.780029296875,
"KL/rejected_KL_mean": -168.5276641845703,
"KL/std": 88.9169921875,
"epoch": 0.5668934240362812,
"fcm_dpo/beta": 0.0077394102700054646,
"fcm_dpo/delta": 0.02124682255089283,
"fcm_dpo/margin": 55.49528121948242,
"fcm_dpo/q_t": 0.40597110986709595,
"grad_norm": 15.411624908447266,
"learning_rate": 2.3678397206786715e-07,
"logits/chosen": 0.7049387693405151,
"logits/rejected": 0.6460511684417725,
"logps/chosen": -170.7156982421875,
"logps/ref_chosen": -57.68330383300781,
"logps/ref_rejected": -79.34097290039062,
"logps/rejected": -247.86863708496094,
"loss": 1.1328,
"margin_dpo/margin_mean": 55.49528121948242,
"margin_dpo/margin_std": 96.86477661132812,
"step": 375
},
{
"KL/chosen_KL_mean": -120.8951416015625,
"KL/mean": -155.48785400390625,
"KL/rejected_KL_mean": -190.08056640625,
"KL/std": 89.38397216796875,
"epoch": 0.5684051398337112,
"fcm_dpo/beta": 0.007656695321202278,
"fcm_dpo/delta": -0.08457353711128235,
"fcm_dpo/margin": 69.18541717529297,
"fcm_dpo/q_t": 0.3872775137424469,
"grad_norm": 12.870294570922852,
"learning_rate": 2.3546379277238103e-07,
"logits/chosen": 0.735195517539978,
"logits/rejected": 0.6646612882614136,
"logps/chosen": -172.5692138671875,
"logps/ref_chosen": -51.674072265625,
"logps/ref_rejected": -75.69713592529297,
"logps/rejected": -265.7777099609375,
"loss": 1.067,
"margin_dpo/margin_mean": 69.18541717529297,
"margin_dpo/margin_std": 104.7180404663086,
"step": 376
},
{
"KL/chosen_KL_mean": -122.86614990234375,
"KL/mean": -146.974853515625,
"KL/rejected_KL_mean": -171.0835418701172,
"KL/std": 75.72106170654297,
"epoch": 0.5699168556311414,
"fcm_dpo/beta": 0.007768765091896057,
"fcm_dpo/delta": 0.07755792140960693,
"fcm_dpo/margin": 48.21738052368164,
"fcm_dpo/q_t": 0.41530439257621765,
"grad_norm": 13.496039390563965,
"learning_rate": 2.3414402008585886e-07,
"logits/chosen": 0.7239181995391846,
"logits/rejected": 0.6990246772766113,
"logps/chosen": -169.044677734375,
"logps/ref_chosen": -46.17853546142578,
"logps/ref_rejected": -57.756500244140625,
"logps/rejected": -228.8400421142578,
"loss": 1.1513,
"margin_dpo/margin_mean": 48.217384338378906,
"margin_dpo/margin_std": 85.44389343261719,
"step": 377
},
{
"KL/chosen_KL_mean": -121.64041137695312,
"KL/mean": -144.5426025390625,
"KL/rejected_KL_mean": -167.44479370117188,
"KL/std": 83.25043487548828,
"epoch": 0.5714285714285714,
"fcm_dpo/beta": 0.007907286286354065,
"fcm_dpo/delta": 0.09018823504447937,
"fcm_dpo/margin": 45.80438232421875,
"fcm_dpo/q_t": 0.4190833270549774,
"grad_norm": 13.746359825134277,
"learning_rate": 2.3282469092517977e-07,
"logits/chosen": 0.7368068695068359,
"logits/rejected": 0.6892712712287903,
"logps/chosen": -180.85928344726562,
"logps/ref_chosen": -59.21887969970703,
"logps/ref_rejected": -71.24818420410156,
"logps/rejected": -238.69297790527344,
"loss": 1.1613,
"margin_dpo/margin_mean": 45.80437469482422,
"margin_dpo/margin_std": 82.95941162109375,
"step": 378
},
{
"KL/chosen_KL_mean": -119.11299133300781,
"KL/mean": -152.1615753173828,
"KL/rejected_KL_mean": -185.21017456054688,
"KL/std": 85.97073364257812,
"epoch": 0.5729402872260015,
"fcm_dpo/beta": 0.007825326174497604,
"fcm_dpo/delta": -0.07087454199790955,
"fcm_dpo/margin": 66.09716796875,
"fcm_dpo/q_t": 0.3876013159751892,
"grad_norm": 13.917744636535645,
"learning_rate": 2.3150584219481643e-07,
"logits/chosen": 0.6688319444656372,
"logits/rejected": 0.5949773788452148,
"logps/chosen": -195.42958068847656,
"logps/ref_chosen": -76.31658935546875,
"logps/ref_rejected": -104.26200103759766,
"logps/rejected": -289.47216796875,
"loss": 1.0541,
"margin_dpo/margin_mean": 66.09716796875,
"margin_dpo/margin_std": 94.03168487548828,
"step": 379
},
{
"KL/chosen_KL_mean": -103.0672378540039,
"KL/mean": -140.85955810546875,
"KL/rejected_KL_mean": -178.65188598632812,
"KL/std": 78.66189575195312,
"epoch": 0.5744520030234316,
"fcm_dpo/beta": 0.007645877078175545,
"fcm_dpo/delta": -0.13564595580101013,
"fcm_dpo/margin": 75.58464813232422,
"fcm_dpo/q_t": 0.37022942304611206,
"grad_norm": 11.984086990356445,
"learning_rate": 2.3018751078580283e-07,
"logits/chosen": 0.7207067012786865,
"logits/rejected": 0.6817103624343872,
"logps/chosen": -164.35040283203125,
"logps/ref_chosen": -61.283164978027344,
"logps/ref_rejected": -72.38892364501953,
"logps/rejected": -251.04080200195312,
"loss": 1.0073,
"margin_dpo/margin_mean": 75.58465576171875,
"margin_dpo/margin_std": 93.482421875,
"step": 380
},
{
"KL/chosen_KL_mean": -126.96179962158203,
"KL/mean": -141.54598999023438,
"KL/rejected_KL_mean": -156.13018798828125,
"KL/std": 81.3851547241211,
"epoch": 0.5759637188208617,
"fcm_dpo/beta": 0.0076386271975934505,
"fcm_dpo/delta": 0.06993352621793747,
"fcm_dpo/margin": 29.168384552001953,
"fcm_dpo/q_t": 0.45122307538986206,
"grad_norm": 13.51224422454834,
"learning_rate": 2.288697335747027e-07,
"logits/chosen": 0.6346052885055542,
"logits/rejected": 0.6169425249099731,
"logps/chosen": -185.17578125,
"logps/ref_chosen": -58.2139892578125,
"logps/ref_rejected": -60.78669357299805,
"logps/rejected": -216.9168701171875,
"loss": 1.3022,
"margin_dpo/margin_mean": 29.168384552001953,
"margin_dpo/margin_std": 96.48456573486328,
"step": 381
},
{
"KL/chosen_KL_mean": -127.53329467773438,
"KL/mean": -153.88670349121094,
"KL/rejected_KL_mean": -180.24008178710938,
"KL/std": 81.65504455566406,
"epoch": 0.5774754346182918,
"fcm_dpo/beta": 0.0077784061431884766,
"fcm_dpo/delta": 0.04100748896598816,
"fcm_dpo/margin": 52.70681381225586,
"fcm_dpo/q_t": 0.4081585109233856,
"grad_norm": 14.638057708740234,
"learning_rate": 2.2755254742257706e-07,
"logits/chosen": 0.7190566062927246,
"logits/rejected": 0.6629340052604675,
"logps/chosen": -189.35861206054688,
"logps/ref_chosen": -61.82532501220703,
"logps/ref_rejected": -83.0452880859375,
"logps/rejected": -263.2853698730469,
"loss": 1.1092,
"margin_dpo/margin_mean": 52.706817626953125,
"margin_dpo/margin_std": 79.73265075683594,
"step": 382
},
{
"KL/chosen_KL_mean": -127.44752502441406,
"KL/mean": -151.69912719726562,
"KL/rejected_KL_mean": -175.9507293701172,
"KL/std": 81.36878204345703,
"epoch": 0.5789871504157218,
"fcm_dpo/beta": 0.007807151880115271,
"fcm_dpo/delta": 0.07406416535377502,
"fcm_dpo/margin": 48.50321960449219,
"fcm_dpo/q_t": 0.41831278800964355,
"grad_norm": 15.773255348205566,
"learning_rate": 2.2623598917395436e-07,
"logits/chosen": 0.5699061751365662,
"logits/rejected": 0.6028285026550293,
"logps/chosen": -208.01077270507812,
"logps/ref_chosen": -80.56326293945312,
"logps/ref_rejected": -74.62922668457031,
"logps/rejected": -250.5799560546875,
"loss": 1.1889,
"margin_dpo/margin_mean": 48.50322341918945,
"margin_dpo/margin_std": 101.3597640991211,
"step": 383
},
{
"KL/chosen_KL_mean": -120.73493957519531,
"KL/mean": -148.515869140625,
"KL/rejected_KL_mean": -176.29678344726562,
"KL/std": 80.29554748535156,
"epoch": 0.5804988662131519,
"fcm_dpo/beta": 0.007890386506915092,
"fcm_dpo/delta": 0.012110977433621883,
"fcm_dpo/margin": 55.561851501464844,
"fcm_dpo/q_t": 0.39945393800735474,
"grad_norm": 16.28131675720215,
"learning_rate": 2.2492009565579875e-07,
"logits/chosen": 0.7039550542831421,
"logits/rejected": 0.6640291810035706,
"logps/chosen": -186.2100830078125,
"logps/ref_chosen": -65.47514343261719,
"logps/ref_rejected": -79.67378234863281,
"logps/rejected": -255.9705810546875,
"loss": 1.0962,
"margin_dpo/margin_mean": 55.561851501464844,
"margin_dpo/margin_std": 83.95750427246094,
"step": 384
},
{
"KL/chosen_KL_mean": -118.50875854492188,
"KL/mean": -154.27658081054688,
"KL/rejected_KL_mean": -190.04440307617188,
"KL/std": 81.92251586914062,
"epoch": 0.582010582010582,
"fcm_dpo/beta": 0.007788371294736862,
"fcm_dpo/delta": -0.11333450675010681,
"fcm_dpo/margin": 71.53565216064453,
"fcm_dpo/q_t": 0.3762691617012024,
"grad_norm": 13.468798637390137,
"learning_rate": 2.2360490367648084e-07,
"logits/chosen": 0.6351601481437683,
"logits/rejected": 0.5921720266342163,
"logps/chosen": -184.56527709960938,
"logps/ref_chosen": -66.0565185546875,
"logps/ref_rejected": -86.68023681640625,
"logps/rejected": -276.7246398925781,
"loss": 1.0081,
"margin_dpo/margin_mean": 71.53565216064453,
"margin_dpo/margin_std": 85.75909423828125,
"step": 385
},
{
"KL/chosen_KL_mean": -138.46185302734375,
"KL/mean": -161.70205688476562,
"KL/rejected_KL_mean": -184.94227600097656,
"KL/std": 86.34553527832031,
"epoch": 0.5835222978080121,
"fcm_dpo/beta": 0.0078004514798521996,
"fcm_dpo/delta": 0.09082190692424774,
"fcm_dpo/margin": 46.48042297363281,
"fcm_dpo/q_t": 0.41729068756103516,
"grad_norm": 13.531447410583496,
"learning_rate": 2.2229045002474724e-07,
"logits/chosen": 0.635225772857666,
"logits/rejected": 0.5782843828201294,
"logps/chosen": -214.08551025390625,
"logps/ref_chosen": -75.6236572265625,
"logps/ref_rejected": -92.62330627441406,
"logps/rejected": -277.5655822753906,
"loss": 1.1624,
"margin_dpo/margin_mean": 46.48042297363281,
"margin_dpo/margin_std": 85.95610046386719,
"step": 386
},
{
"KL/chosen_KL_mean": -121.05953979492188,
"KL/mean": -154.82073974609375,
"KL/rejected_KL_mean": -188.58192443847656,
"KL/std": 78.66697692871094,
"epoch": 0.5850340136054422,
"fcm_dpo/beta": 0.007759010884910822,
"fcm_dpo/delta": -0.07805424928665161,
"fcm_dpo/margin": 67.52236938476562,
"fcm_dpo/q_t": 0.38322460651397705,
"grad_norm": 15.326830863952637,
"learning_rate": 2.209767714686924e-07,
"logits/chosen": 0.6912319660186768,
"logits/rejected": 0.5814871788024902,
"logps/chosen": -168.28125,
"logps/ref_chosen": -47.22170639038086,
"logps/ref_rejected": -87.338134765625,
"logps/rejected": -275.9200439453125,
"loss": 1.0211,
"margin_dpo/margin_mean": 67.52236938476562,
"margin_dpo/margin_std": 82.26397705078125,
"step": 387
},
{
"KL/chosen_KL_mean": -123.56349182128906,
"KL/mean": -145.304443359375,
"KL/rejected_KL_mean": -167.04537963867188,
"KL/std": 83.71977233886719,
"epoch": 0.5865457294028723,
"fcm_dpo/beta": 0.007735843770205975,
"fcm_dpo/delta": 0.0017733016284182668,
"fcm_dpo/margin": 43.481903076171875,
"fcm_dpo/q_t": 0.42785027623176575,
"grad_norm": 14.318989753723145,
"learning_rate": 2.1966390475472954e-07,
"logits/chosen": 0.6570320129394531,
"logits/rejected": 0.6532206535339355,
"logps/chosen": -198.14295959472656,
"logps/ref_chosen": -74.5794677734375,
"logps/ref_rejected": -79.92558288574219,
"logps/rejected": -246.97097778320312,
"loss": 1.2134,
"margin_dpo/margin_mean": 43.481903076171875,
"margin_dpo/margin_std": 98.93060302734375,
"step": 388
},
{
"KL/chosen_KL_mean": -118.34381103515625,
"KL/mean": -151.6417694091797,
"KL/rejected_KL_mean": -184.9397430419922,
"KL/std": 81.11581420898438,
"epoch": 0.5880574452003023,
"fcm_dpo/beta": 0.007675846107304096,
"fcm_dpo/delta": -0.06442630290985107,
"fcm_dpo/margin": 66.59591674804688,
"fcm_dpo/q_t": 0.3868892788887024,
"grad_norm": 26.09174346923828,
"learning_rate": 2.1835188660656265e-07,
"logits/chosen": 0.6963808536529541,
"logits/rejected": 0.6595287322998047,
"logps/chosen": -179.96817016601562,
"logps/ref_chosen": -61.624366760253906,
"logps/ref_rejected": -76.50978088378906,
"logps/rejected": -261.44952392578125,
"loss": 1.0465,
"margin_dpo/margin_mean": 66.59591674804688,
"margin_dpo/margin_std": 89.59396362304688,
"step": 389
},
{
"KL/chosen_KL_mean": -105.8058090209961,
"KL/mean": -132.39251708984375,
"KL/rejected_KL_mean": -158.979248046875,
"KL/std": 78.03878784179688,
"epoch": 0.5895691609977324,
"fcm_dpo/beta": 0.0076671745628118515,
"fcm_dpo/delta": 0.04408419877290726,
"fcm_dpo/margin": 53.173431396484375,
"fcm_dpo/q_t": 0.40860432386398315,
"grad_norm": 12.027225494384766,
"learning_rate": 2.170407537241599e-07,
"logits/chosen": 0.7897613048553467,
"logits/rejected": 0.7145026922225952,
"logps/chosen": -151.67767333984375,
"logps/ref_chosen": -45.871864318847656,
"logps/ref_rejected": -61.305999755859375,
"logps/rejected": -220.28524780273438,
"loss": 1.1127,
"margin_dpo/margin_mean": 53.17342758178711,
"margin_dpo/margin_std": 82.32190704345703,
"step": 390
},
{
"KL/chosen_KL_mean": -117.52503967285156,
"KL/mean": -149.52325439453125,
"KL/rejected_KL_mean": -181.52145385742188,
"KL/std": 79.88783264160156,
"epoch": 0.5910808767951625,
"fcm_dpo/beta": 0.00763491727411747,
"fcm_dpo/delta": -0.040894802659749985,
"fcm_dpo/margin": 63.99640655517578,
"fcm_dpo/q_t": 0.3913359045982361,
"grad_norm": 12.352397918701172,
"learning_rate": 2.1573054278272636e-07,
"logits/chosen": 0.6845928430557251,
"logits/rejected": 0.616992712020874,
"logps/chosen": -175.71205139160156,
"logps/ref_chosen": -58.18701171875,
"logps/ref_rejected": -83.63442993164062,
"logps/rejected": -265.1558837890625,
"loss": 1.0819,
"margin_dpo/margin_mean": 63.99640655517578,
"margin_dpo/margin_std": 96.66595458984375,
"step": 391
},
{
"KL/chosen_KL_mean": -106.76398468017578,
"KL/mean": -140.35391235351562,
"KL/rejected_KL_mean": -173.94381713867188,
"KL/std": 85.97457122802734,
"epoch": 0.5925925925925926,
"fcm_dpo/beta": 0.007620956748723984,
"fcm_dpo/delta": -0.06586946547031403,
"fcm_dpo/margin": 67.17984008789062,
"fcm_dpo/q_t": 0.38879668712615967,
"grad_norm": 13.259527206420898,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": 0.7438907027244568,
"logits/rejected": 0.6849305629730225,
"logps/chosen": -176.50851440429688,
"logps/ref_chosen": -69.7445297241211,
"logps/ref_rejected": -94.05877685546875,
"logps/rejected": -268.0025939941406,
"loss": 1.0735,
"margin_dpo/margin_mean": 67.17984008789062,
"margin_dpo/margin_std": 100.96359252929688,
"step": 392
},
{
"KL/chosen_KL_mean": -120.26972961425781,
"KL/mean": -155.8102264404297,
"KL/rejected_KL_mean": -191.35073852539062,
"KL/std": 83.73542785644531,
"epoch": 0.5941043083900227,
"fcm_dpo/beta": 0.007404985371977091,
"fcm_dpo/delta": -0.08186736702919006,
"fcm_dpo/margin": 71.08099365234375,
"fcm_dpo/q_t": 0.38141682744026184,
"grad_norm": 11.25800609588623,
"learning_rate": 2.131130332936195e-07,
"logits/chosen": 0.7393004894256592,
"logits/rejected": 0.6986830830574036,
"logps/chosen": -172.60462951660156,
"logps/ref_chosen": -52.33489990234375,
"logps/ref_rejected": -74.33809661865234,
"logps/rejected": -265.6888427734375,
"loss": 1.0188,
"margin_dpo/margin_mean": 71.08099365234375,
"margin_dpo/margin_std": 83.78794860839844,
"step": 393
},
{
"KL/chosen_KL_mean": -115.41416931152344,
"KL/mean": -145.79640197753906,
"KL/rejected_KL_mean": -176.17861938476562,
"KL/std": 76.976318359375,
"epoch": 0.5956160241874527,
"fcm_dpo/beta": 0.0074429926462471485,
"fcm_dpo/delta": -0.0026309899985790253,
"fcm_dpo/margin": 60.76445770263672,
"fcm_dpo/q_t": 0.3960072994232178,
"grad_norm": 12.980253219604492,
"learning_rate": 2.1180580796331323e-07,
"logits/chosen": 0.7144701480865479,
"logits/rejected": 0.6844040155410767,
"logps/chosen": -176.09030151367188,
"logps/ref_chosen": -60.6761360168457,
"logps/ref_rejected": -71.36074829101562,
"logps/rejected": -247.53936767578125,
"loss": 1.0635,
"margin_dpo/margin_mean": 60.76445770263672,
"margin_dpo/margin_std": 76.67507934570312,
"step": 394
},
{
"KL/chosen_KL_mean": -118.93069458007812,
"KL/mean": -146.92593383789062,
"KL/rejected_KL_mean": -174.92117309570312,
"KL/std": 80.11039733886719,
"epoch": 0.5971277399848829,
"fcm_dpo/beta": 0.00747601967304945,
"fcm_dpo/delta": 0.0324532687664032,
"fcm_dpo/margin": 55.990455627441406,
"fcm_dpo/q_t": 0.4076528251171112,
"grad_norm": 14.716011047363281,
"learning_rate": 2.104996510066625e-07,
"logits/chosen": 0.7400883436203003,
"logits/rejected": 0.6406676173210144,
"logps/chosen": -169.53501892089844,
"logps/ref_chosen": -50.60432434082031,
"logps/ref_rejected": -77.08731079101562,
"logps/rejected": -252.00848388671875,
"loss": 1.1032,
"margin_dpo/margin_mean": 55.99046325683594,
"margin_dpo/margin_std": 84.89546203613281,
"step": 395
},
{
"KL/chosen_KL_mean": -113.71238708496094,
"KL/mean": -143.36878967285156,
"KL/rejected_KL_mean": -173.02520751953125,
"KL/std": 88.9266128540039,
"epoch": 0.5986394557823129,
"fcm_dpo/beta": 0.007402296643704176,
"fcm_dpo/delta": 0.009609527885913849,
"fcm_dpo/margin": 59.31281280517578,
"fcm_dpo/q_t": 0.4014135003089905,
"grad_norm": 12.139056205749512,
"learning_rate": 2.0919459895968517e-07,
"logits/chosen": 0.7061029672622681,
"logits/rejected": 0.6125322580337524,
"logps/chosen": -165.07199096679688,
"logps/ref_chosen": -51.35961151123047,
"logps/ref_rejected": -79.89360046386719,
"logps/rejected": -252.91879272460938,
"loss": 1.0793,
"margin_dpo/margin_mean": 59.31280517578125,
"margin_dpo/margin_std": 77.40509033203125,
"step": 396
},
{
"KL/chosen_KL_mean": -130.69134521484375,
"KL/mean": -147.45037841796875,
"KL/rejected_KL_mean": -164.2094268798828,
"KL/std": 81.297607421875,
"epoch": 0.600151171579743,
"fcm_dpo/beta": 0.007710670121014118,
"fcm_dpo/delta": 0.19634832441806793,
"fcm_dpo/margin": 33.518104553222656,
"fcm_dpo/q_t": 0.4427967071533203,
"grad_norm": 13.76626968383789,
"learning_rate": 2.078906883274924e-07,
"logits/chosen": 0.6195926070213318,
"logits/rejected": 0.5693718791007996,
"logps/chosen": -197.14755249023438,
"logps/ref_chosen": -66.45622253417969,
"logps/ref_rejected": -85.74736785888672,
"logps/rejected": -249.95680236816406,
"loss": 1.2701,
"margin_dpo/margin_mean": 33.518104553222656,
"margin_dpo/margin_std": 94.12562561035156,
"step": 397
},
{
"KL/chosen_KL_mean": -111.14080810546875,
"KL/mean": -148.89974975585938,
"KL/rejected_KL_mean": -186.65869140625,
"KL/std": 85.36833190917969,
"epoch": 0.6016628873771731,
"fcm_dpo/beta": 0.007586984895169735,
"fcm_dpo/delta": -0.1316794604063034,
"fcm_dpo/margin": 75.51786804199219,
"fcm_dpo/q_t": 0.37503597140312195,
"grad_norm": 11.452016830444336,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": 0.6672321557998657,
"logits/rejected": 0.6032127141952515,
"logps/chosen": -160.38504028320312,
"logps/ref_chosen": -49.244239807128906,
"logps/ref_rejected": -75.18949127197266,
"logps/rejected": -261.8481750488281,
"loss": 0.9958,
"margin_dpo/margin_mean": 75.51786804199219,
"margin_dpo/margin_std": 90.60173797607422,
"step": 398
},
{
"KL/chosen_KL_mean": -130.374267578125,
"KL/mean": -173.41238403320312,
"KL/rejected_KL_mean": -216.45053100585938,
"KL/std": 94.34305572509766,
"epoch": 0.6031746031746031,
"fcm_dpo/beta": 0.0073226685635745525,
"fcm_dpo/delta": -0.19398213922977448,
"fcm_dpo/margin": 86.07627868652344,
"fcm_dpo/q_t": 0.3651480972766876,
"grad_norm": 12.927462577819824,
"learning_rate": 2.052864371672457e-07,
"logits/chosen": 0.6225359439849854,
"logits/rejected": 0.48039311170578003,
"logps/chosen": -198.68104553222656,
"logps/ref_chosen": -68.30679321289062,
"logps/ref_rejected": -113.2708511352539,
"logps/rejected": -329.72137451171875,
"loss": 0.9782,
"margin_dpo/margin_mean": 86.07627868652344,
"margin_dpo/margin_std": 106.19702911376953,
"step": 399
},
{
"KL/chosen_KL_mean": -144.94996643066406,
"KL/mean": -169.4925537109375,
"KL/rejected_KL_mean": -194.03512573242188,
"KL/std": 85.74641418457031,
"epoch": 0.6046863189720333,
"fcm_dpo/beta": 0.007226614747196436,
"fcm_dpo/delta": -0.04161422699689865,
"fcm_dpo/margin": 49.08516311645508,
"fcm_dpo/q_t": 0.41834330558776855,
"grad_norm": 17.31791114807129,
"learning_rate": 2.0398616948569493e-07,
"logits/chosen": 0.6973024606704712,
"logits/rejected": 0.6369521617889404,
"logps/chosen": -216.5764617919922,
"logps/ref_chosen": -71.62649536132812,
"logps/ref_rejected": -90.98765563964844,
"logps/rejected": -285.02276611328125,
"loss": 1.1537,
"margin_dpo/margin_mean": 49.08516311645508,
"margin_dpo/margin_std": 82.55448150634766,
"step": 400
},
{
"KL/chosen_KL_mean": -110.20097351074219,
"KL/mean": -145.64926147460938,
"KL/rejected_KL_mean": -181.0975799560547,
"KL/std": 92.50545501708984,
"epoch": 0.6061980347694633,
"fcm_dpo/beta": 0.0071340445429086685,
"fcm_dpo/delta": -0.059332482516765594,
"fcm_dpo/margin": 70.89659118652344,
"fcm_dpo/q_t": 0.3873620331287384,
"grad_norm": 9.548161506652832,
"learning_rate": 2.0268718890989752e-07,
"logits/chosen": 0.7134464383125305,
"logits/rejected": 0.6242314577102661,
"logps/chosen": -163.92593383789062,
"logps/ref_chosen": -53.72495651245117,
"logps/ref_rejected": -75.06304931640625,
"logps/rejected": -256.1606140136719,
"loss": 1.0338,
"margin_dpo/margin_mean": 70.89659118652344,
"margin_dpo/margin_std": 88.12057495117188,
"step": 401
},
{
"KL/chosen_KL_mean": -120.5625,
"KL/mean": -148.1356201171875,
"KL/rejected_KL_mean": -175.708740234375,
"KL/std": 78.65602111816406,
"epoch": 0.6077097505668935,
"fcm_dpo/beta": 0.007157139014452696,
"fcm_dpo/delta": 0.05712353438138962,
"fcm_dpo/margin": 55.146236419677734,
"fcm_dpo/q_t": 0.4119277596473694,
"grad_norm": 14.758689880371094,
"learning_rate": 2.013895317751323e-07,
"logits/chosen": 0.6909421682357788,
"logits/rejected": 0.6727443933486938,
"logps/chosen": -182.43643188476562,
"logps/ref_chosen": -61.873931884765625,
"logps/ref_rejected": -66.15198516845703,
"logps/rejected": -241.8607177734375,
"loss": 1.1408,
"margin_dpo/margin_mean": 55.146236419677734,
"margin_dpo/margin_std": 93.9888916015625,
"step": 402
},
{
"KL/chosen_KL_mean": -127.60636901855469,
"KL/mean": -163.28607177734375,
"KL/rejected_KL_mean": -198.9657745361328,
"KL/std": 91.14511108398438,
"epoch": 0.6092214663643235,
"fcm_dpo/beta": 0.007140805944800377,
"fcm_dpo/delta": -0.06299025565385818,
"fcm_dpo/margin": 71.35940551757812,
"fcm_dpo/q_t": 0.3879600167274475,
"grad_norm": 10.904901504516602,
"learning_rate": 2.0009323437965898e-07,
"logits/chosen": 0.8188765645027161,
"logits/rejected": 0.730493426322937,
"logps/chosen": -178.92787170410156,
"logps/ref_chosen": -51.321502685546875,
"logps/ref_rejected": -86.54010772705078,
"logps/rejected": -285.5058898925781,
"loss": 1.0464,
"margin_dpo/margin_mean": 71.35940551757812,
"margin_dpo/margin_std": 95.07559204101562,
"step": 403
},
{
"KL/chosen_KL_mean": -119.84092712402344,
"KL/mean": -156.6473388671875,
"KL/rejected_KL_mean": -193.4537353515625,
"KL/std": 92.27593231201172,
"epoch": 0.6107331821617535,
"fcm_dpo/beta": 0.007005881518125534,
"fcm_dpo/delta": -0.07096034288406372,
"fcm_dpo/margin": 73.61279296875,
"fcm_dpo/q_t": 0.38724684715270996,
"grad_norm": 14.632216453552246,
"learning_rate": 1.9879833298370237e-07,
"logits/chosen": 0.6507140398025513,
"logits/rejected": 0.5575799345970154,
"logps/chosen": -182.10382080078125,
"logps/ref_chosen": -62.26288604736328,
"logps/ref_rejected": -95.19029998779297,
"logps/rejected": -288.64404296875,
"loss": 1.0525,
"margin_dpo/margin_mean": 73.61280059814453,
"margin_dpo/margin_std": 100.62483215332031,
"step": 404
},
{
"KL/chosen_KL_mean": -123.88162231445312,
"KL/mean": -152.46707153320312,
"KL/rejected_KL_mean": -181.05255126953125,
"KL/std": 82.95892333984375,
"epoch": 0.6122448979591837,
"fcm_dpo/beta": 0.007027004845440388,
"fcm_dpo/delta": 0.04931598901748657,
"fcm_dpo/margin": 57.17091751098633,
"fcm_dpo/q_t": 0.4105098247528076,
"grad_norm": 12.742623329162598,
"learning_rate": 1.975048638084379e-07,
"logits/chosen": 0.7530736923217773,
"logits/rejected": 0.7074278593063354,
"logps/chosen": -174.46597290039062,
"logps/ref_chosen": -50.5843391418457,
"logps/ref_rejected": -65.43156433105469,
"logps/rejected": -246.48411560058594,
"loss": 1.1115,
"margin_dpo/margin_mean": 57.17091369628906,
"margin_dpo/margin_std": 83.23585510253906,
"step": 405
},
{
"KL/chosen_KL_mean": -117.66008758544922,
"KL/mean": -155.90249633789062,
"KL/rejected_KL_mean": -194.14488220214844,
"KL/std": 89.51800537109375,
"epoch": 0.6137566137566137,
"fcm_dpo/beta": 0.0070230658166110516,
"fcm_dpo/delta": -0.09200313687324524,
"fcm_dpo/margin": 76.48478698730469,
"fcm_dpo/q_t": 0.3798549175262451,
"grad_norm": 12.587457656860352,
"learning_rate": 1.9621286303497914e-07,
"logits/chosen": 0.7463070750236511,
"logits/rejected": 0.5869977474212646,
"logps/chosen": -166.65570068359375,
"logps/ref_chosen": -48.99560546875,
"logps/ref_rejected": -92.47774505615234,
"logps/rejected": -286.62261962890625,
"loss": 1.0331,
"margin_dpo/margin_mean": 76.48478698730469,
"margin_dpo/margin_std": 99.57765197753906,
"step": 406
},
{
"KL/chosen_KL_mean": -150.7066650390625,
"KL/mean": -180.73129272460938,
"KL/rejected_KL_mean": -210.7559356689453,
"KL/std": 102.36092376708984,
"epoch": 0.6152683295540439,
"fcm_dpo/beta": 0.007019747979938984,
"fcm_dpo/delta": 0.029542850330471992,
"fcm_dpo/margin": 60.04927062988281,
"fcm_dpo/q_t": 0.4063884913921356,
"grad_norm": 14.38724136352539,
"learning_rate": 1.9492236680336483e-07,
"logits/chosen": 0.5551047325134277,
"logits/rejected": 0.4944635033607483,
"logps/chosen": -240.1072235107422,
"logps/ref_chosen": -89.40056610107422,
"logps/ref_rejected": -99.28775024414062,
"logps/rejected": -310.043701171875,
"loss": 1.1289,
"margin_dpo/margin_mean": 60.04927062988281,
"margin_dpo/margin_std": 102.39197540283203,
"step": 407
},
{
"KL/chosen_KL_mean": -111.80641174316406,
"KL/mean": -154.49441528320312,
"KL/rejected_KL_mean": -197.18243408203125,
"KL/std": 84.27525329589844,
"epoch": 0.6167800453514739,
"fcm_dpo/beta": 0.006904586683958769,
"fcm_dpo/delta": -0.14809463918209076,
"fcm_dpo/margin": 85.37601470947266,
"fcm_dpo/q_t": 0.3662289083003998,
"grad_norm": 11.406484603881836,
"learning_rate": 1.9363341121154895e-07,
"logits/chosen": 0.7426500916481018,
"logits/rejected": 0.6647111177444458,
"logps/chosen": -166.5103302001953,
"logps/ref_chosen": -54.70391845703125,
"logps/ref_rejected": -73.98648834228516,
"logps/rejected": -271.1689147949219,
"loss": 0.971,
"margin_dpo/margin_mean": 85.37600708007812,
"margin_dpo/margin_std": 88.67381286621094,
"step": 408
},
{
"KL/chosen_KL_mean": -139.26519775390625,
"KL/mean": -159.29217529296875,
"KL/rejected_KL_mean": -179.31918334960938,
"KL/std": 72.91958618164062,
"epoch": 0.618291761148904,
"fcm_dpo/beta": 0.006983055733144283,
"fcm_dpo/delta": 0.17536059021949768,
"fcm_dpo/margin": 40.053977966308594,
"fcm_dpo/q_t": 0.43714118003845215,
"grad_norm": 13.305119514465332,
"learning_rate": 1.9234603231438994e-07,
"logits/chosen": 0.7231974601745605,
"logits/rejected": 0.7324143052101135,
"logps/chosen": -201.3834228515625,
"logps/ref_chosen": -62.11822509765625,
"logps/ref_rejected": -61.933509826660156,
"logps/rejected": -241.252685546875,
"loss": 1.2059,
"margin_dpo/margin_mean": 40.053977966308594,
"margin_dpo/margin_std": 82.67475891113281,
"step": 409
},
{
"KL/chosen_KL_mean": -130.318359375,
"KL/mean": -164.8314208984375,
"KL/rejected_KL_mean": -199.34449768066406,
"KL/std": 82.56102752685547,
"epoch": 0.6198034769463341,
"fcm_dpo/beta": 0.0069589754566550255,
"fcm_dpo/delta": -0.03359462320804596,
"fcm_dpo/margin": 69.02615356445312,
"fcm_dpo/q_t": 0.39046138525009155,
"grad_norm": 11.623514175415039,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": 0.695282518863678,
"logits/rejected": 0.6719921827316284,
"logps/chosen": -192.1210174560547,
"logps/ref_chosen": -61.80266189575195,
"logps/ref_rejected": -76.60002136230469,
"logps/rejected": -275.94451904296875,
"loss": 1.0414,
"margin_dpo/margin_mean": 69.02615356445312,
"margin_dpo/margin_std": 80.98117065429688,
"step": 410
},
{
"KL/chosen_KL_mean": -136.67645263671875,
"KL/mean": -169.91497802734375,
"KL/rejected_KL_mean": -203.15347290039062,
"KL/std": 89.70034790039062,
"epoch": 0.6213151927437641,
"fcm_dpo/beta": 0.006991643458604813,
"fcm_dpo/delta": -0.0155013557523489,
"fcm_dpo/margin": 66.47702026367188,
"fcm_dpo/q_t": 0.39620882272720337,
"grad_norm": 10.967887878417969,
"learning_rate": 1.8977614860195296e-07,
"logits/chosen": 0.7672784328460693,
"logits/rejected": 0.7097710371017456,
"logps/chosen": -191.12184143066406,
"logps/ref_chosen": -54.44539260864258,
"logps/ref_rejected": -74.5650863647461,
"logps/rejected": -277.71856689453125,
"loss": 1.0737,
"margin_dpo/margin_mean": 66.47702026367188,
"margin_dpo/margin_std": 94.61786651611328,
"step": 411
},
{
"KL/chosen_KL_mean": -139.16372680664062,
"KL/mean": -169.79412841796875,
"KL/rejected_KL_mean": -200.42453002929688,
"KL/std": 76.77566528320312,
"epoch": 0.6228269085411943,
"fcm_dpo/beta": 0.006982295773923397,
"fcm_dpo/delta": 0.02307012677192688,
"fcm_dpo/margin": 61.26079559326172,
"fcm_dpo/q_t": 0.4027141332626343,
"grad_norm": 12.258587837219238,
"learning_rate": 1.8849371567184662e-07,
"logits/chosen": 0.7537434101104736,
"logits/rejected": 0.6913366317749023,
"logps/chosen": -194.41180419921875,
"logps/ref_chosen": -55.248085021972656,
"logps/ref_rejected": -68.96623229980469,
"logps/rejected": -269.3907470703125,
"loss": 1.0801,
"margin_dpo/margin_mean": 61.26079177856445,
"margin_dpo/margin_std": 81.34800720214844,
"step": 412
},
{
"KL/chosen_KL_mean": -154.973388671875,
"KL/mean": -183.91961669921875,
"KL/rejected_KL_mean": -212.8658447265625,
"KL/std": 88.90245819091797,
"epoch": 0.6243386243386243,
"fcm_dpo/beta": 0.0070727551355957985,
"fcm_dpo/delta": 0.04212770238518715,
"fcm_dpo/margin": 57.892452239990234,
"fcm_dpo/q_t": 0.41068193316459656,
"grad_norm": 15.375361442565918,
"learning_rate": 1.872130032047302e-07,
"logits/chosen": 0.557374119758606,
"logits/rejected": 0.5279108285903931,
"logps/chosen": -223.69415283203125,
"logps/ref_chosen": -68.72074890136719,
"logps/ref_rejected": -78.76539611816406,
"logps/rejected": -291.6312255859375,
"loss": 1.1519,
"margin_dpo/margin_mean": 57.892452239990234,
"margin_dpo/margin_std": 107.5550537109375,
"step": 413
},
{
"KL/chosen_KL_mean": -137.41403198242188,
"KL/mean": -171.55357360839844,
"KL/rejected_KL_mean": -205.693115234375,
"KL/std": 96.39564514160156,
"epoch": 0.6258503401360545,
"fcm_dpo/beta": 0.0070444131270051,
"fcm_dpo/delta": -0.032540664076805115,
"fcm_dpo/margin": 68.27906799316406,
"fcm_dpo/q_t": 0.39073142409324646,
"grad_norm": 11.638055801391602,
"learning_rate": 1.8593404702488436e-07,
"logits/chosen": 0.7413473129272461,
"logits/rejected": 0.682194709777832,
"logps/chosen": -191.55224609375,
"logps/ref_chosen": -54.138214111328125,
"logps/ref_rejected": -74.65741729736328,
"logps/rejected": -280.35052490234375,
"loss": 1.0561,
"margin_dpo/margin_mean": 68.2790756225586,
"margin_dpo/margin_std": 89.36869812011719,
"step": 414
},
{
"KL/chosen_KL_mean": -135.49588012695312,
"KL/mean": -163.8934326171875,
"KL/rejected_KL_mean": -192.29098510742188,
"KL/std": 89.41297912597656,
"epoch": 0.6273620559334845,
"fcm_dpo/beta": 0.00707012414932251,
"fcm_dpo/delta": 0.050504814833402634,
"fcm_dpo/margin": 56.79509735107422,
"fcm_dpo/q_t": 0.41016697883605957,
"grad_norm": 12.561532974243164,
"learning_rate": 1.846568829074628e-07,
"logits/chosen": 0.749638557434082,
"logits/rejected": 0.7332205772399902,
"logps/chosen": -191.4144287109375,
"logps/ref_chosen": -55.91856002807617,
"logps/ref_rejected": -61.747703552246094,
"logps/rejected": -254.0386962890625,
"loss": 1.1275,
"margin_dpo/margin_mean": 56.79510498046875,
"margin_dpo/margin_std": 94.32406616210938,
"step": 415
},
{
"KL/chosen_KL_mean": -143.9766387939453,
"KL/mean": -168.79827880859375,
"KL/rejected_KL_mean": -193.61993408203125,
"KL/std": 93.01155853271484,
"epoch": 0.6288737717309146,
"fcm_dpo/beta": 0.007032511755824089,
"fcm_dpo/delta": -0.09679044038057327,
"fcm_dpo/margin": 49.64328384399414,
"fcm_dpo/q_t": 0.42265474796295166,
"grad_norm": 14.138065338134766,
"learning_rate": 1.8338154657749128e-07,
"logits/chosen": 0.7095975875854492,
"logits/rejected": 0.6624330282211304,
"logps/chosen": -198.69972229003906,
"logps/ref_chosen": -54.72308349609375,
"logps/ref_rejected": -69.17388916015625,
"logps/rejected": -262.7938232421875,
"loss": 1.179,
"margin_dpo/margin_mean": 49.64328384399414,
"margin_dpo/margin_std": 89.73538208007812,
"step": 416
},
{
"KL/chosen_KL_mean": -149.44570922851562,
"KL/mean": -181.24932861328125,
"KL/rejected_KL_mean": -213.0529327392578,
"KL/std": 86.96218872070312,
"epoch": 0.6303854875283447,
"fcm_dpo/beta": 0.007011952809989452,
"fcm_dpo/delta": 0.003346942365169525,
"fcm_dpo/margin": 63.607215881347656,
"fcm_dpo/q_t": 0.39805179834365845,
"grad_norm": 13.41283893585205,
"learning_rate": 1.8210807370886849e-07,
"logits/chosen": 0.8221265077590942,
"logits/rejected": 0.7597347497940063,
"logps/chosen": -206.23696899414062,
"logps/ref_chosen": -56.791259765625,
"logps/ref_rejected": -68.7791748046875,
"logps/rejected": -281.83209228515625,
"loss": 1.1164,
"margin_dpo/margin_mean": 63.607215881347656,
"margin_dpo/margin_std": 105.42054748535156,
"step": 417
},
{
"KL/chosen_KL_mean": -159.71566772460938,
"KL/mean": -186.35943603515625,
"KL/rejected_KL_mean": -213.00320434570312,
"KL/std": 97.96890258789062,
"epoch": 0.6318972033257747,
"fcm_dpo/beta": 0.006870288867503405,
"fcm_dpo/delta": -0.07140669971704483,
"fcm_dpo/margin": 53.28753662109375,
"fcm_dpo/q_t": 0.4186713695526123,
"grad_norm": 14.105439186096191,
"learning_rate": 1.8083649992336825e-07,
"logits/chosen": 0.7837098240852356,
"logits/rejected": 0.7870790958404541,
"logps/chosen": -228.8236541748047,
"logps/ref_chosen": -69.10798645019531,
"logps/ref_rejected": -75.09132385253906,
"logps/rejected": -288.09454345703125,
"loss": 1.1693,
"margin_dpo/margin_mean": 53.28753662109375,
"margin_dpo/margin_std": 99.01031494140625,
"step": 418
},
{
"KL/chosen_KL_mean": -126.68461608886719,
"KL/mean": -165.0189971923828,
"KL/rejected_KL_mean": -203.35337829589844,
"KL/std": 88.75230407714844,
"epoch": 0.6334089191232048,
"fcm_dpo/beta": 0.006787517108023167,
"fcm_dpo/delta": -0.07435769587755203,
"fcm_dpo/margin": 76.66874694824219,
"fcm_dpo/q_t": 0.38583898544311523,
"grad_norm": 11.909916877746582,
"learning_rate": 1.7956686078964255e-07,
"logits/chosen": 0.6351292729377747,
"logits/rejected": 0.5898942947387695,
"logps/chosen": -184.85638427734375,
"logps/ref_chosen": -58.1717643737793,
"logps/ref_rejected": -71.67066955566406,
"logps/rejected": -275.0240478515625,
"loss": 1.0373,
"margin_dpo/margin_mean": 76.66874694824219,
"margin_dpo/margin_std": 101.29539489746094,
"step": 419
},
{
"KL/chosen_KL_mean": -158.27415466308594,
"KL/mean": -177.7630615234375,
"KL/rejected_KL_mean": -197.251953125,
"KL/std": 92.9329833984375,
"epoch": 0.6349206349206349,
"fcm_dpo/beta": 0.00685450155287981,
"fcm_dpo/delta": 0.06285488605499268,
"fcm_dpo/margin": 38.977783203125,
"fcm_dpo/q_t": 0.4422045350074768,
"grad_norm": 13.392801284790039,
"learning_rate": 1.782991918222275e-07,
"logits/chosen": 0.7376530170440674,
"logits/rejected": 0.694350004196167,
"logps/chosen": -215.32766723632812,
"logps/ref_chosen": -57.05351257324219,
"logps/ref_rejected": -62.670982360839844,
"logps/rejected": -259.9229431152344,
"loss": 1.2604,
"margin_dpo/margin_mean": 38.977783203125,
"margin_dpo/margin_std": 105.80848693847656,
"step": 420
},
{
"KL/chosen_KL_mean": -145.82496643066406,
"KL/mean": -174.51287841796875,
"KL/rejected_KL_mean": -203.20077514648438,
"KL/std": 91.58183288574219,
"epoch": 0.636432350718065,
"fcm_dpo/beta": 0.006877418141812086,
"fcm_dpo/delta": 0.057417359203100204,
"fcm_dpo/margin": 57.37579345703125,
"fcm_dpo/q_t": 0.41433507204055786,
"grad_norm": 14.985981941223145,
"learning_rate": 1.7703352848054887e-07,
"logits/chosen": 0.6757022738456726,
"logits/rejected": 0.6184811592102051,
"logps/chosen": -203.14822387695312,
"logps/ref_chosen": -57.32324981689453,
"logps/ref_rejected": -75.33782958984375,
"logps/rejected": -278.5386047363281,
"loss": 1.1868,
"margin_dpo/margin_mean": 57.375797271728516,
"margin_dpo/margin_std": 119.31109619140625,
"step": 421
},
{
"KL/chosen_KL_mean": -128.066162109375,
"KL/mean": -165.34466552734375,
"KL/rejected_KL_mean": -202.62313842773438,
"KL/std": 90.69136047363281,
"epoch": 0.6379440665154951,
"fcm_dpo/beta": 0.006889001466333866,
"fcm_dpo/delta": -0.0670836940407753,
"fcm_dpo/margin": 74.55697631835938,
"fcm_dpo/q_t": 0.38474151492118835,
"grad_norm": 13.526140213012695,
"learning_rate": 1.7576990616793137e-07,
"logits/chosen": 0.6843360662460327,
"logits/rejected": 0.6799574494361877,
"logps/chosen": -195.12374877929688,
"logps/ref_chosen": -67.05757141113281,
"logps/ref_rejected": -72.12803649902344,
"logps/rejected": -274.75115966796875,
"loss": 1.0293,
"margin_dpo/margin_mean": 74.55697631835938,
"margin_dpo/margin_std": 91.39720153808594,
"step": 422
},
{
"KL/chosen_KL_mean": -130.3348388671875,
"KL/mean": -169.43580627441406,
"KL/rejected_KL_mean": -208.5367431640625,
"KL/std": 92.26570129394531,
"epoch": 0.6394557823129252,
"fcm_dpo/beta": 0.0067449817433953285,
"fcm_dpo/delta": -0.08202774077653885,
"fcm_dpo/margin": 78.201904296875,
"fcm_dpo/q_t": 0.38311636447906494,
"grad_norm": 12.166535377502441,
"learning_rate": 1.745083602306071e-07,
"logits/chosen": 0.7933779954910278,
"logits/rejected": 0.7255642414093018,
"logps/chosen": -184.39651489257812,
"logps/ref_chosen": -54.06167221069336,
"logps/ref_rejected": -76.64092254638672,
"logps/rejected": -285.17767333984375,
"loss": 1.025,
"margin_dpo/margin_mean": 78.201904296875,
"margin_dpo/margin_std": 97.99603271484375,
"step": 423
},
{
"KL/chosen_KL_mean": -138.32078552246094,
"KL/mean": -177.15737915039062,
"KL/rejected_KL_mean": -215.99398803710938,
"KL/std": 86.63241577148438,
"epoch": 0.6409674981103552,
"fcm_dpo/beta": 0.006620635278522968,
"fcm_dpo/delta": -0.0689234584569931,
"fcm_dpo/margin": 77.6731948852539,
"fcm_dpo/q_t": 0.3853021264076233,
"grad_norm": 15.852065086364746,
"learning_rate": 1.7324892595672804e-07,
"logits/chosen": 0.6060911417007446,
"logits/rejected": 0.5659924745559692,
"logps/chosen": -191.92965698242188,
"logps/ref_chosen": -53.60887145996094,
"logps/ref_rejected": -79.2139892578125,
"logps/rejected": -295.2079772949219,
"loss": 1.0347,
"margin_dpo/margin_mean": 77.6731948852539,
"margin_dpo/margin_std": 98.35594940185547,
"step": 424
},
{
"KL/chosen_KL_mean": -139.3746337890625,
"KL/mean": -168.45822143554688,
"KL/rejected_KL_mean": -197.5418243408203,
"KL/std": 84.505615234375,
"epoch": 0.6424792139077853,
"fcm_dpo/beta": 0.006698750890791416,
"fcm_dpo/delta": 0.06281259655952454,
"fcm_dpo/margin": 58.16719055175781,
"fcm_dpo/q_t": 0.41317880153656006,
"grad_norm": 13.068650245666504,
"learning_rate": 1.7199163857537824e-07,
"logits/chosen": 0.7463515996932983,
"logits/rejected": 0.7196699380874634,
"logps/chosen": -197.789306640625,
"logps/ref_chosen": -58.41468048095703,
"logps/ref_rejected": -66.59054565429688,
"logps/rejected": -264.13238525390625,
"loss": 1.1314,
"margin_dpo/margin_mean": 58.16719055175781,
"margin_dpo/margin_std": 96.10154724121094,
"step": 425
},
{
"KL/chosen_KL_mean": -166.42657470703125,
"KL/mean": -184.3075714111328,
"KL/rejected_KL_mean": -202.18856811523438,
"KL/std": 90.15042114257812,
"epoch": 0.6439909297052154,
"fcm_dpo/beta": 0.006936525925993919,
"fcm_dpo/delta": 0.2069658637046814,
"fcm_dpo/margin": 35.762001037597656,
"fcm_dpo/q_t": 0.4449042081832886,
"grad_norm": 18.208776473999023,
"learning_rate": 1.7073653325558828e-07,
"logits/chosen": 0.7006687521934509,
"logits/rejected": 0.7039576768875122,
"logps/chosen": -238.13479614257812,
"logps/ref_chosen": -71.70822143554688,
"logps/ref_rejected": -73.57725524902344,
"logps/rejected": -275.76580810546875,
"loss": 1.2917,
"margin_dpo/margin_mean": 35.762001037597656,
"margin_dpo/margin_std": 112.21145629882812,
"step": 426
},
{
"KL/chosen_KL_mean": -157.49887084960938,
"KL/mean": -188.9153289794922,
"KL/rejected_KL_mean": -220.331787109375,
"KL/std": 99.3335952758789,
"epoch": 0.6455026455026455,
"fcm_dpo/beta": 0.007007491309195757,
"fcm_dpo/delta": 0.010125400498509407,
"fcm_dpo/margin": 62.83293914794922,
"fcm_dpo/q_t": 0.40540921688079834,
"grad_norm": 14.336414337158203,
"learning_rate": 1.6948364510535218e-07,
"logits/chosen": 0.6997349262237549,
"logits/rejected": 0.6372700929641724,
"logps/chosen": -216.14163208007812,
"logps/ref_chosen": -58.64276885986328,
"logps/ref_rejected": -86.25437927246094,
"logps/rejected": -306.586181640625,
"loss": 1.1242,
"margin_dpo/margin_mean": 62.83293914794922,
"margin_dpo/margin_std": 109.12852478027344,
"step": 427
},
{
"KL/chosen_KL_mean": -151.69796752929688,
"KL/mean": -185.02224731445312,
"KL/rejected_KL_mean": -218.3465118408203,
"KL/std": 100.86468505859375,
"epoch": 0.6470143613000756,
"fcm_dpo/beta": 0.007032737135887146,
"fcm_dpo/delta": -0.020013775676488876,
"fcm_dpo/margin": 66.64852905273438,
"fcm_dpo/q_t": 0.3979109227657318,
"grad_norm": 12.575758934020996,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": 0.6532288789749146,
"logits/rejected": 0.6143096685409546,
"logps/chosen": -218.2940216064453,
"logps/ref_chosen": -66.5960464477539,
"logps/ref_rejected": -82.3941650390625,
"logps/rejected": -300.74066162109375,
"loss": 1.1049,
"margin_dpo/margin_mean": 66.64852905273438,
"margin_dpo/margin_std": 107.984130859375,
"step": 428
},
{
"KL/chosen_KL_mean": -154.6398468017578,
"KL/mean": -180.8323974609375,
"KL/rejected_KL_mean": -207.02491760253906,
"KL/std": 84.92485046386719,
"epoch": 0.6485260770975056,
"fcm_dpo/beta": 0.007110248319804668,
"fcm_dpo/delta": 0.07964736223220825,
"fcm_dpo/margin": 52.385066986083984,
"fcm_dpo/q_t": 0.41526907682418823,
"grad_norm": 14.710054397583008,
"learning_rate": 1.669846604344412e-07,
"logits/chosen": 0.6934762001037598,
"logits/rejected": 0.7128642797470093,
"logps/chosen": -211.64955139160156,
"logps/ref_chosen": -57.00970458984375,
"logps/ref_rejected": -59.86549377441406,
"logps/rejected": -266.8904113769531,
"loss": 1.1572,
"margin_dpo/margin_mean": 52.38506317138672,
"margin_dpo/margin_std": 94.00373077392578,
"step": 429
},
{
"KL/chosen_KL_mean": -137.94509887695312,
"KL/mean": -179.18109130859375,
"KL/rejected_KL_mean": -220.41705322265625,
"KL/std": 86.64157104492188,
"epoch": 0.6500377928949358,
"fcm_dpo/beta": 0.006960110738873482,
"fcm_dpo/delta": -0.131709486246109,
"fcm_dpo/margin": 82.47195434570312,
"fcm_dpo/q_t": 0.36921268701553345,
"grad_norm": 13.911884307861328,
"learning_rate": 1.6573863381573954e-07,
"logits/chosen": 0.6297199726104736,
"logits/rejected": 0.6272151470184326,
"logps/chosen": -197.50830078125,
"logps/ref_chosen": -59.563194274902344,
"logps/ref_rejected": -70.52289581298828,
"logps/rejected": -290.93994140625,
"loss": 1.0029,
"margin_dpo/margin_mean": 82.47195434570312,
"margin_dpo/margin_std": 98.84146118164062,
"step": 430
},
{
"KL/chosen_KL_mean": -136.78176879882812,
"KL/mean": -167.47360229492188,
"KL/rejected_KL_mean": -198.16543579101562,
"KL/std": 88.53604125976562,
"epoch": 0.6515495086923658,
"fcm_dpo/beta": 0.006935927085578442,
"fcm_dpo/delta": 0.02532074972987175,
"fcm_dpo/margin": 61.3836669921875,
"fcm_dpo/q_t": 0.405121386051178,
"grad_norm": 13.702001571655273,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": 0.7203608751296997,
"logits/rejected": 0.6643437147140503,
"logps/chosen": -186.9821014404297,
"logps/ref_chosen": -50.20032501220703,
"logps/ref_rejected": -77.81680297851562,
"logps/rejected": -275.98223876953125,
"loss": 1.1097,
"margin_dpo/margin_mean": 61.3836669921875,
"margin_dpo/margin_std": 96.4728775024414,
"step": 431
},
{
"KL/chosen_KL_mean": -142.304443359375,
"KL/mean": -174.9478759765625,
"KL/rejected_KL_mean": -207.59133911132812,
"KL/std": 86.27505493164062,
"epoch": 0.6530612244897959,
"fcm_dpo/beta": 0.006943239830434322,
"fcm_dpo/delta": -0.003489219583570957,
"fcm_dpo/margin": 65.28689575195312,
"fcm_dpo/q_t": 0.3982744514942169,
"grad_norm": 14.40606689453125,
"learning_rate": 1.632536862810844e-07,
"logits/chosen": 0.7262982130050659,
"logits/rejected": 0.6743370294570923,
"logps/chosen": -203.96719360351562,
"logps/ref_chosen": -61.662757873535156,
"logps/ref_rejected": -83.94496154785156,
"logps/rejected": -291.53631591796875,
"loss": 1.0941,
"margin_dpo/margin_mean": 65.28689575195312,
"margin_dpo/margin_std": 99.4925765991211,
"step": 432
},
{
"KL/chosen_KL_mean": -141.8938751220703,
"KL/mean": -179.70875549316406,
"KL/rejected_KL_mean": -217.52365112304688,
"KL/std": 83.667236328125,
"epoch": 0.654572940287226,
"fcm_dpo/beta": 0.006893531884998083,
"fcm_dpo/delta": -0.07524176687002182,
"fcm_dpo/margin": 75.62977600097656,
"fcm_dpo/q_t": 0.3817235231399536,
"grad_norm": 13.342097282409668,
"learning_rate": 1.6201483487445515e-07,
"logits/chosen": 0.8095422983169556,
"logits/rejected": 0.8106013536453247,
"logps/chosen": -205.623046875,
"logps/ref_chosen": -63.72917938232422,
"logps/ref_rejected": -65.8391342163086,
"logps/rejected": -283.36279296875,
"loss": 1.0331,
"margin_dpo/margin_mean": 75.6297607421875,
"margin_dpo/margin_std": 95.17214965820312,
"step": 433
},
{
"KL/chosen_KL_mean": -120.78085327148438,
"KL/mean": -161.9495849609375,
"KL/rejected_KL_mean": -203.11831665039062,
"KL/std": 91.25202941894531,
"epoch": 0.656084656084656,
"fcm_dpo/beta": 0.006684788502752781,
"fcm_dpo/delta": -0.1083269864320755,
"fcm_dpo/margin": 82.33747100830078,
"fcm_dpo/q_t": 0.375964879989624,
"grad_norm": 12.028138160705566,
"learning_rate": 1.6077844460203204e-07,
"logits/chosen": 0.8332556486129761,
"logits/rejected": 0.7700821161270142,
"logps/chosen": -168.75416564941406,
"logps/ref_chosen": -47.97331619262695,
"logps/ref_rejected": -72.51132202148438,
"logps/rejected": -275.629638671875,
"loss": 1.0374,
"margin_dpo/margin_mean": 82.33747100830078,
"margin_dpo/margin_std": 109.64669799804688,
"step": 434
},
{
"KL/chosen_KL_mean": -145.86680603027344,
"KL/mean": -177.239501953125,
"KL/rejected_KL_mean": -208.6121826171875,
"KL/std": 87.94329833984375,
"epoch": 0.6575963718820862,
"fcm_dpo/beta": 0.006757765542715788,
"fcm_dpo/delta": 0.02654809132218361,
"fcm_dpo/margin": 62.74540710449219,
"fcm_dpo/q_t": 0.4058331251144409,
"grad_norm": 15.042875289916992,
"learning_rate": 1.5954455004830878e-07,
"logits/chosen": 0.8290047645568848,
"logits/rejected": 0.7902365922927856,
"logps/chosen": -202.9270477294922,
"logps/ref_chosen": -57.06024932861328,
"logps/ref_rejected": -71.69146728515625,
"logps/rejected": -280.30364990234375,
"loss": 1.1169,
"margin_dpo/margin_mean": 62.745399475097656,
"margin_dpo/margin_std": 101.01614379882812,
"step": 435
},
{
"KL/chosen_KL_mean": -145.9172821044922,
"KL/mean": -172.13966369628906,
"KL/rejected_KL_mean": -198.362060546875,
"KL/std": 86.71946716308594,
"epoch": 0.6591080876795162,
"fcm_dpo/beta": 0.006820957642048597,
"fcm_dpo/delta": 0.09580140560865402,
"fcm_dpo/margin": 52.444766998291016,
"fcm_dpo/q_t": 0.4212391674518585,
"grad_norm": 15.2418794631958,
"learning_rate": 1.5831318572796847e-07,
"logits/chosen": 0.7261216640472412,
"logits/rejected": 0.6715189218521118,
"logps/chosen": -202.07533264160156,
"logps/ref_chosen": -56.158050537109375,
"logps/ref_rejected": -67.63787841796875,
"logps/rejected": -265.99993896484375,
"loss": 1.1837,
"margin_dpo/margin_mean": 52.444766998291016,
"margin_dpo/margin_std": 106.6795654296875,
"step": 436
},
{
"KL/chosen_KL_mean": -152.7239990234375,
"KL/mean": -183.65553283691406,
"KL/rejected_KL_mean": -214.5870819091797,
"KL/std": 94.8659896850586,
"epoch": 0.6606198034769464,
"fcm_dpo/beta": 0.00675150565803051,
"fcm_dpo/delta": -0.08352308720350266,
"fcm_dpo/margin": 61.86308288574219,
"fcm_dpo/q_t": 0.4087408185005188,
"grad_norm": 16.527193069458008,
"learning_rate": 1.5708438608491815e-07,
"logits/chosen": 0.7057990431785583,
"logits/rejected": 0.5852597951889038,
"logps/chosen": -209.70977783203125,
"logps/ref_chosen": -56.98578643798828,
"logps/ref_rejected": -85.61524963378906,
"logps/rejected": -300.20233154296875,
"loss": 1.1707,
"margin_dpo/margin_mean": 61.86308288574219,
"margin_dpo/margin_std": 120.68586730957031,
"step": 437
},
{
"KL/chosen_KL_mean": -131.84364318847656,
"KL/mean": -173.12020874023438,
"KL/rejected_KL_mean": -214.39675903320312,
"KL/std": 98.55461120605469,
"epoch": 0.6621315192743764,
"fcm_dpo/beta": 0.006669066846370697,
"fcm_dpo/delta": -0.1062895730137825,
"fcm_dpo/margin": 82.55313110351562,
"fcm_dpo/q_t": 0.38156580924987793,
"grad_norm": 13.007272720336914,
"learning_rate": 1.558581854913253e-07,
"logits/chosen": 0.805670440196991,
"logits/rejected": 0.7402873039245605,
"logps/chosen": -173.12142944335938,
"logps/ref_chosen": -41.27777862548828,
"logps/ref_rejected": -65.33840942382812,
"logps/rejected": -279.73516845703125,
"loss": 1.0229,
"margin_dpo/margin_mean": 82.55313110351562,
"margin_dpo/margin_std": 109.02909851074219,
"step": 438
},
{
"KL/chosen_KL_mean": -149.43386840820312,
"KL/mean": -186.30145263671875,
"KL/rejected_KL_mean": -223.1690216064453,
"KL/std": 99.49622344970703,
"epoch": 0.6636432350718064,
"fcm_dpo/beta": 0.006624848581850529,
"fcm_dpo/delta": -0.04157322645187378,
"fcm_dpo/margin": 73.73515319824219,
"fcm_dpo/q_t": 0.39052993059158325,
"grad_norm": 13.092687606811523,
"learning_rate": 1.5463461824665658e-07,
"logits/chosen": 0.6147295832633972,
"logits/rejected": 0.5832754373550415,
"logps/chosen": -230.8515167236328,
"logps/ref_chosen": -81.41764831542969,
"logps/ref_rejected": -94.72309875488281,
"logps/rejected": -317.8921203613281,
"loss": 1.0622,
"margin_dpo/margin_mean": 73.73515319824219,
"margin_dpo/margin_std": 99.93533325195312,
"step": 439
},
{
"KL/chosen_KL_mean": -129.64244079589844,
"KL/mean": -165.80941772460938,
"KL/rejected_KL_mean": -201.9763946533203,
"KL/std": 88.39790344238281,
"epoch": 0.6651549508692366,
"fcm_dpo/beta": 0.006558232009410858,
"fcm_dpo/delta": -0.025874171406030655,
"fcm_dpo/margin": 72.33395385742188,
"fcm_dpo/q_t": 0.39480775594711304,
"grad_norm": 24.257293701171875,
"learning_rate": 1.534137185767178e-07,
"logits/chosen": 0.7508381605148315,
"logits/rejected": 0.6510604619979858,
"logps/chosen": -172.18063354492188,
"logps/ref_chosen": -42.538185119628906,
"logps/ref_rejected": -69.78813934326172,
"logps/rejected": -271.7645263671875,
"loss": 1.0767,
"margin_dpo/margin_mean": 72.33395385742188,
"margin_dpo/margin_std": 103.9335708618164,
"step": 440
},
{
"KL/chosen_KL_mean": -131.30650329589844,
"KL/mean": -170.871826171875,
"KL/rejected_KL_mean": -210.43714904785156,
"KL/std": 95.84239196777344,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.006433665752410889,
"fcm_dpo/delta": -0.06287907809019089,
"fcm_dpo/margin": 79.13067626953125,
"fcm_dpo/q_t": 0.3833358883857727,
"grad_norm": 15.967854499816895,
"learning_rate": 1.521955206326976e-07,
"logits/chosen": 0.6979551315307617,
"logits/rejected": 0.6069918870925903,
"logps/chosen": -188.89971923828125,
"logps/ref_chosen": -57.593223571777344,
"logps/ref_rejected": -84.82878875732422,
"logps/rejected": -295.26593017578125,
"loss": 1.0171,
"margin_dpo/margin_mean": 79.13067626953125,
"margin_dpo/margin_std": 87.90113830566406,
"step": 441
},
{
"KL/chosen_KL_mean": -157.26657104492188,
"KL/mean": -197.35610961914062,
"KL/rejected_KL_mean": -237.4456787109375,
"KL/std": 95.72440338134766,
"epoch": 0.6681783824640968,
"fcm_dpo/beta": 0.006357879377901554,
"fcm_dpo/delta": -0.06341610103845596,
"fcm_dpo/margin": 80.17911529541016,
"fcm_dpo/q_t": 0.38497215509414673,
"grad_norm": 14.785154342651367,
"learning_rate": 1.5098005849021078e-07,
"logits/chosen": 0.6711052656173706,
"logits/rejected": 0.628170907497406,
"logps/chosen": -224.727783203125,
"logps/ref_chosen": -67.46121978759766,
"logps/ref_rejected": -89.0693588256836,
"logps/rejected": -326.5150451660156,
"loss": 1.0311,
"margin_dpo/margin_mean": 80.17912292480469,
"margin_dpo/margin_std": 98.379638671875,
"step": 442
},
{
"KL/chosen_KL_mean": -129.59542846679688,
"KL/mean": -178.85983276367188,
"KL/rejected_KL_mean": -228.12423706054688,
"KL/std": 101.99020385742188,
"epoch": 0.6696900982615268,
"fcm_dpo/beta": 0.006190933287143707,
"fcm_dpo/delta": -0.17065072059631348,
"fcm_dpo/margin": 98.52880859375,
"fcm_dpo/q_t": 0.3641754388809204,
"grad_norm": 12.864545822143555,
"learning_rate": 1.4976736614834662e-07,
"logits/chosen": 0.744697093963623,
"logits/rejected": 0.6813009977340698,
"logps/chosen": -184.3915252685547,
"logps/ref_chosen": -54.79610061645508,
"logps/ref_rejected": -77.80781555175781,
"logps/rejected": -305.93206787109375,
"loss": 0.9731,
"margin_dpo/margin_mean": 98.52880859375,
"margin_dpo/margin_std": 113.32872772216797,
"step": 443
},
{
"KL/chosen_KL_mean": -160.04959106445312,
"KL/mean": -181.8095703125,
"KL/rejected_KL_mean": -203.569580078125,
"KL/std": 100.75285339355469,
"epoch": 0.671201814058957,
"fcm_dpo/beta": 0.006196248810738325,
"fcm_dpo/delta": 0.04157605022192001,
"fcm_dpo/margin": 43.519989013671875,
"fcm_dpo/q_t": 0.4417763352394104,
"grad_norm": 16.792774200439453,
"learning_rate": 1.4855747752871654e-07,
"logits/chosen": 0.7347674369812012,
"logits/rejected": 0.6471496224403381,
"logps/chosen": -218.79864501953125,
"logps/ref_chosen": -58.749061584472656,
"logps/ref_rejected": -86.87396240234375,
"logps/rejected": -290.44354248046875,
"loss": 1.247,
"margin_dpo/margin_mean": 43.519989013671875,
"margin_dpo/margin_std": 110.95816040039062,
"step": 444
},
{
"KL/chosen_KL_mean": -150.85191345214844,
"KL/mean": -192.16619873046875,
"KL/rejected_KL_mean": -233.48049926757812,
"KL/std": 93.49421691894531,
"epoch": 0.672713529856387,
"fcm_dpo/beta": 0.006143460050225258,
"fcm_dpo/delta": -0.060683369636535645,
"fcm_dpo/margin": 82.62857818603516,
"fcm_dpo/q_t": 0.38386598229408264,
"grad_norm": 12.43869400024414,
"learning_rate": 1.473504264745062e-07,
"logits/chosen": 0.675220251083374,
"logits/rejected": 0.6696294546127319,
"logps/chosen": -211.76934814453125,
"logps/ref_chosen": -60.91743850708008,
"logps/ref_rejected": -71.5637435913086,
"logps/rejected": -305.04425048828125,
"loss": 1.0258,
"margin_dpo/margin_mean": 82.62857818603516,
"margin_dpo/margin_std": 97.68711853027344,
"step": 445
},
{
"KL/chosen_KL_mean": -136.88546752929688,
"KL/mean": -178.89035034179688,
"KL/rejected_KL_mean": -220.89523315429688,
"KL/std": 95.0992431640625,
"epoch": 0.674225245653817,
"fcm_dpo/beta": 0.006117424461990595,
"fcm_dpo/delta": -0.06857110559940338,
"fcm_dpo/margin": 84.00977325439453,
"fcm_dpo/q_t": 0.38333478569984436,
"grad_norm": 12.378578186035156,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": 0.7436650395393372,
"logits/rejected": 0.6679918766021729,
"logps/chosen": -185.68470764160156,
"logps/ref_chosen": -48.79924774169922,
"logps/ref_rejected": -71.8719482421875,
"logps/rejected": -292.7671813964844,
"loss": 1.0182,
"margin_dpo/margin_mean": 84.00977325439453,
"margin_dpo/margin_std": 91.514404296875,
"step": 446
},
{
"KL/chosen_KL_mean": -132.49256896972656,
"KL/mean": -180.810791015625,
"KL/rejected_KL_mean": -229.1290283203125,
"KL/std": 94.01569366455078,
"epoch": 0.6757369614512472,
"fcm_dpo/beta": 0.005898823030292988,
"fcm_dpo/delta": -0.12822897732257843,
"fcm_dpo/margin": 96.63645935058594,
"fcm_dpo/q_t": 0.36920368671417236,
"grad_norm": 13.93958854675293,
"learning_rate": 1.4494497203727843e-07,
"logits/chosen": 0.6841769814491272,
"logits/rejected": 0.5845425128936768,
"logps/chosen": -186.17529296875,
"logps/ref_chosen": -53.682716369628906,
"logps/ref_rejected": -88.17315673828125,
"logps/rejected": -317.30218505859375,
"loss": 0.9901,
"margin_dpo/margin_mean": 96.63645935058594,
"margin_dpo/margin_std": 106.86634826660156,
"step": 447
},
{
"KL/chosen_KL_mean": -145.4189453125,
"KL/mean": -183.78155517578125,
"KL/rejected_KL_mean": -222.14418029785156,
"KL/std": 96.9927749633789,
"epoch": 0.6772486772486772,
"fcm_dpo/beta": 0.005885988939553499,
"fcm_dpo/delta": -0.0016819881275296211,
"fcm_dpo/margin": 76.72523498535156,
"fcm_dpo/q_t": 0.3971629738807678,
"grad_norm": 9.772438049316406,
"learning_rate": 1.4374663593999256e-07,
"logits/chosen": 0.7255531549453735,
"logits/rejected": 0.6777403950691223,
"logps/chosen": -199.17019653320312,
"logps/ref_chosen": -53.75125503540039,
"logps/ref_rejected": -77.17623901367188,
"logps/rejected": -299.3204345703125,
"loss": 1.0711,
"margin_dpo/margin_mean": 76.72523498535156,
"margin_dpo/margin_std": 103.92894744873047,
"step": 448
},
{
"KL/chosen_KL_mean": -172.9747772216797,
"KL/mean": -192.73361206054688,
"KL/rejected_KL_mean": -212.49246215820312,
"KL/std": 100.64846801757812,
"epoch": 0.6787603930461074,
"fcm_dpo/beta": 0.005969383753836155,
"fcm_dpo/delta": 0.07266371697187424,
"fcm_dpo/margin": 39.5177116394043,
"fcm_dpo/q_t": 0.44623565673828125,
"grad_norm": 19.279781341552734,
"learning_rate": 1.4255127197770707e-07,
"logits/chosen": 0.5781837701797485,
"logits/rejected": 0.5781469345092773,
"logps/chosen": -248.80215454101562,
"logps/ref_chosen": -75.82737731933594,
"logps/ref_rejected": -82.20687866210938,
"logps/rejected": -294.6993408203125,
"loss": 1.2458,
"margin_dpo/margin_mean": 39.5177116394043,
"margin_dpo/margin_std": 97.13715362548828,
"step": 449
},
{
"KL/chosen_KL_mean": -142.44183349609375,
"KL/mean": -173.92239379882812,
"KL/rejected_KL_mean": -205.40293884277344,
"KL/std": 99.94340515136719,
"epoch": 0.6802721088435374,
"fcm_dpo/beta": 0.006016598083078861,
"fcm_dpo/delta": 0.07402040809392929,
"fcm_dpo/margin": 62.96111297607422,
"fcm_dpo/q_t": 0.41852670907974243,
"grad_norm": 12.642474174499512,
"learning_rate": 1.4135891358732205e-07,
"logits/chosen": 0.7799849510192871,
"logits/rejected": 0.6756715774536133,
"logps/chosen": -189.55755615234375,
"logps/ref_chosen": -47.11572265625,
"logps/ref_rejected": -78.7546615600586,
"logps/rejected": -284.1575927734375,
"loss": 1.153,
"margin_dpo/margin_mean": 62.96112060546875,
"margin_dpo/margin_std": 114.33680725097656,
"step": 450
},
{
"KL/chosen_KL_mean": -143.79136657714844,
"KL/mean": -172.44857788085938,
"KL/rejected_KL_mean": -201.1057891845703,
"KL/std": 95.12226104736328,
"epoch": 0.6817838246409675,
"fcm_dpo/beta": 0.006161220371723175,
"fcm_dpo/delta": 0.10015745460987091,
"fcm_dpo/margin": 57.314430236816406,
"fcm_dpo/q_t": 0.42018577456474304,
"grad_norm": 13.891311645507812,
"learning_rate": 1.4016959412166437e-07,
"logits/chosen": 0.6785339117050171,
"logits/rejected": 0.63237464427948,
"logps/chosen": -207.14181518554688,
"logps/ref_chosen": -63.350440979003906,
"logps/ref_rejected": -76.28530883789062,
"logps/rejected": -277.39111328125,
"loss": 1.149,
"margin_dpo/margin_mean": 57.314430236816406,
"margin_dpo/margin_std": 97.8006362915039,
"step": 451
},
{
"KL/chosen_KL_mean": -144.1630859375,
"KL/mean": -176.96450805664062,
"KL/rejected_KL_mean": -209.76593017578125,
"KL/std": 91.98998260498047,
"epoch": 0.6832955404383976,
"fcm_dpo/beta": 0.006219558417797089,
"fcm_dpo/delta": 0.04377629607915878,
"fcm_dpo/margin": 65.60285949707031,
"fcm_dpo/q_t": 0.4090433716773987,
"grad_norm": 14.884453773498535,
"learning_rate": 1.3898334684855645e-07,
"logits/chosen": 0.6643181443214417,
"logits/rejected": 0.5869356393814087,
"logps/chosen": -199.74891662597656,
"logps/ref_chosen": -55.58583450317383,
"logps/ref_rejected": -77.68738555908203,
"logps/rejected": -287.45330810546875,
"loss": 1.132,
"margin_dpo/margin_mean": 65.60285949707031,
"margin_dpo/margin_std": 111.69452667236328,
"step": 452
},
{
"KL/chosen_KL_mean": -144.0391387939453,
"KL/mean": -176.14346313476562,
"KL/rejected_KL_mean": -208.247802734375,
"KL/std": 94.88691711425781,
"epoch": 0.6848072562358276,
"fcm_dpo/beta": 0.006271724589169025,
"fcm_dpo/delta": 0.049313317984342575,
"fcm_dpo/margin": 64.20864868164062,
"fcm_dpo/q_t": 0.4105488657951355,
"grad_norm": 15.54833698272705,
"learning_rate": 1.3780020494988445e-07,
"logits/chosen": 0.680355429649353,
"logits/rejected": 0.6594034433364868,
"logps/chosen": -205.81735229492188,
"logps/ref_chosen": -61.778202056884766,
"logps/ref_rejected": -71.51403045654297,
"logps/rejected": -279.7618408203125,
"loss": 1.1303,
"margin_dpo/margin_mean": 64.20864868164062,
"margin_dpo/margin_std": 107.24003601074219,
"step": 453
},
{
"KL/chosen_KL_mean": -132.9048614501953,
"KL/mean": -169.541748046875,
"KL/rejected_KL_mean": -206.17861938476562,
"KL/std": 95.81768035888672,
"epoch": 0.6863189720332578,
"fcm_dpo/beta": 0.0062804133631289005,
"fcm_dpo/delta": -0.01071581058204174,
"fcm_dpo/margin": 73.27376556396484,
"fcm_dpo/q_t": 0.3961000144481659,
"grad_norm": 13.032012939453125,
"learning_rate": 1.366202015206706e-07,
"logits/chosen": 0.723847508430481,
"logits/rejected": 0.6875928044319153,
"logps/chosen": -184.5,
"logps/ref_chosen": -51.59515380859375,
"logps/ref_rejected": -63.96732711791992,
"logps/rejected": -270.14593505859375,
"loss": 1.0833,
"margin_dpo/margin_mean": 73.27375793457031,
"margin_dpo/margin_std": 108.64644622802734,
"step": 454
},
{
"KL/chosen_KL_mean": -154.15054321289062,
"KL/mean": -188.80511474609375,
"KL/rejected_KL_mean": -223.4596710205078,
"KL/std": 95.76663208007812,
"epoch": 0.6878306878306878,
"fcm_dpo/beta": 0.006290521007031202,
"fcm_dpo/delta": 0.01463963370770216,
"fcm_dpo/margin": 69.30914306640625,
"fcm_dpo/q_t": 0.40427112579345703,
"grad_norm": 13.61314868927002,
"learning_rate": 1.354433695681474e-07,
"logits/chosen": 0.5870033502578735,
"logits/rejected": 0.5615803003311157,
"logps/chosen": -224.80224609375,
"logps/ref_chosen": -70.65170288085938,
"logps/ref_rejected": -77.44276428222656,
"logps/rejected": -300.9024353027344,
"loss": 1.0927,
"margin_dpo/margin_mean": 69.30914306640625,
"margin_dpo/margin_std": 103.27223205566406,
"step": 455
},
{
"KL/chosen_KL_mean": -149.3287353515625,
"KL/mean": -182.22406005859375,
"KL/rejected_KL_mean": -215.119384765625,
"KL/std": 94.61996459960938,
"epoch": 0.6893424036281179,
"fcm_dpo/beta": 0.006323617417365313,
"fcm_dpo/delta": 0.035434018820524216,
"fcm_dpo/margin": 65.79066467285156,
"fcm_dpo/q_t": 0.40913882851600647,
"grad_norm": 17.2503719329834,
"learning_rate": 1.3426974201083439e-07,
"logits/chosen": 0.631847620010376,
"logits/rejected": 0.5720112323760986,
"logps/chosen": -205.72702026367188,
"logps/ref_chosen": -56.398284912109375,
"logps/ref_rejected": -82.61642456054688,
"logps/rejected": -297.7358093261719,
"loss": 1.1165,
"margin_dpo/margin_mean": 65.79066467285156,
"margin_dpo/margin_std": 107.55733489990234,
"step": 456
},
{
"KL/chosen_KL_mean": -147.87754821777344,
"KL/mean": -182.2469482421875,
"KL/rejected_KL_mean": -216.61634826660156,
"KL/std": 97.08610534667969,
"epoch": 0.690854119425548,
"fcm_dpo/beta": 0.0063408189453184605,
"fcm_dpo/delta": 0.01462765596807003,
"fcm_dpo/margin": 68.7387924194336,
"fcm_dpo/q_t": 0.40054911375045776,
"grad_norm": 13.827205657958984,
"learning_rate": 1.3309935167761717e-07,
"logits/chosen": 0.8234766721725464,
"logits/rejected": 0.7447128295898438,
"logps/chosen": -192.59811401367188,
"logps/ref_chosen": -44.72057342529297,
"logps/ref_rejected": -68.1158676147461,
"logps/rejected": -284.7322082519531,
"loss": 1.0766,
"margin_dpo/margin_mean": 68.7387924194336,
"margin_dpo/margin_std": 91.6793212890625,
"step": 457
},
{
"KL/chosen_KL_mean": -142.45022583007812,
"KL/mean": -179.52110290527344,
"KL/rejected_KL_mean": -216.5919952392578,
"KL/std": 99.73497009277344,
"epoch": 0.6923658352229781,
"fcm_dpo/beta": 0.006352492608129978,
"fcm_dpo/delta": -0.022026075050234795,
"fcm_dpo/margin": 74.14176940917969,
"fcm_dpo/q_t": 0.3945360779762268,
"grad_norm": 13.504679679870605,
"learning_rate": 1.3193223130682936e-07,
"logits/chosen": 0.7190458178520203,
"logits/rejected": 0.6072895526885986,
"logps/chosen": -192.4559326171875,
"logps/ref_chosen": -50.00569152832031,
"logps/ref_rejected": -87.50015258789062,
"logps/rejected": -304.0921630859375,
"loss": 1.0825,
"margin_dpo/margin_mean": 74.14176940917969,
"margin_dpo/margin_std": 110.75813293457031,
"step": 458
},
{
"KL/chosen_KL_mean": -136.78713989257812,
"KL/mean": -181.85948181152344,
"KL/rejected_KL_mean": -226.93182373046875,
"KL/std": 114.09321594238281,
"epoch": 0.6938775510204082,
"fcm_dpo/beta": 0.006281477864831686,
"fcm_dpo/delta": -0.12387188524007797,
"fcm_dpo/margin": 90.14466857910156,
"fcm_dpo/q_t": 0.3718501925468445,
"grad_norm": 12.169325828552246,
"learning_rate": 1.3076841354533658e-07,
"logits/chosen": 0.7203759551048279,
"logits/rejected": 0.690513014793396,
"logps/chosen": -202.16510009765625,
"logps/ref_chosen": -65.37794494628906,
"logps/ref_rejected": -88.19244384765625,
"logps/rejected": -315.124267578125,
"loss": 1.0117,
"margin_dpo/margin_mean": 90.14466857910156,
"margin_dpo/margin_std": 107.88766479492188,
"step": 459
},
{
"KL/chosen_KL_mean": -149.89292907714844,
"KL/mean": -193.3321075439453,
"KL/rejected_KL_mean": -236.77130126953125,
"KL/std": 100.56085205078125,
"epoch": 0.6953892668178382,
"fcm_dpo/beta": 0.006080328952521086,
"fcm_dpo/delta": -0.08334297686815262,
"fcm_dpo/margin": 86.87835693359375,
"fcm_dpo/q_t": 0.3830181360244751,
"grad_norm": 12.787505149841309,
"learning_rate": 1.2960793094762345e-07,
"logits/chosen": 0.7253769636154175,
"logits/rejected": 0.6118832230567932,
"logps/chosen": -214.45458984375,
"logps/ref_chosen": -64.5616683959961,
"logps/ref_rejected": -88.67890167236328,
"logps/rejected": -325.4501953125,
"loss": 1.0242,
"margin_dpo/margin_mean": 86.87835693359375,
"margin_dpo/margin_std": 108.14886474609375,
"step": 460
},
{
"KL/chosen_KL_mean": -124.567138671875,
"KL/mean": -168.6573028564453,
"KL/rejected_KL_mean": -212.74746704101562,
"KL/std": 93.67964172363281,
"epoch": 0.6969009826152683,
"fcm_dpo/beta": 0.00594598101451993,
"fcm_dpo/delta": -0.08142174780368805,
"fcm_dpo/margin": 88.1803207397461,
"fcm_dpo/q_t": 0.3814476430416107,
"grad_norm": 11.927419662475586,
"learning_rate": 1.2845081597488286e-07,
"logits/chosen": 0.834221363067627,
"logits/rejected": 0.754862904548645,
"logps/chosen": -174.04505920410156,
"logps/ref_chosen": -49.4779167175293,
"logps/ref_rejected": -72.65262603759766,
"logps/rejected": -285.40008544921875,
"loss": 1.0205,
"margin_dpo/margin_mean": 88.18031311035156,
"margin_dpo/margin_std": 100.82464599609375,
"step": 461
},
{
"KL/chosen_KL_mean": -137.95655822753906,
"KL/mean": -181.53204345703125,
"KL/rejected_KL_mean": -225.10751342773438,
"KL/std": 91.580078125,
"epoch": 0.6984126984126984,
"fcm_dpo/beta": 0.005903769284486771,
"fcm_dpo/delta": -0.06840892881155014,
"fcm_dpo/margin": 87.15096282958984,
"fcm_dpo/q_t": 0.38177424669265747,
"grad_norm": 12.905652046203613,
"learning_rate": 1.27297100994108e-07,
"logits/chosen": 0.7005641460418701,
"logits/rejected": 0.6541897058486938,
"logps/chosen": -198.45167541503906,
"logps/ref_chosen": -60.4951171875,
"logps/ref_rejected": -74.82136535644531,
"logps/rejected": -299.92889404296875,
"loss": 1.0221,
"margin_dpo/margin_mean": 87.15095520019531,
"margin_dpo/margin_std": 102.30754852294922,
"step": 462
},
{
"KL/chosen_KL_mean": -165.4473876953125,
"KL/mean": -191.84483337402344,
"KL/rejected_KL_mean": -218.24227905273438,
"KL/std": 91.26084899902344,
"epoch": 0.6999244142101285,
"fcm_dpo/beta": 0.0059958938509225845,
"fcm_dpo/delta": 0.138011172413826,
"fcm_dpo/margin": 52.794891357421875,
"fcm_dpo/q_t": 0.4275769591331482,
"grad_norm": 18.58067512512207,
"learning_rate": 1.2614681827718695e-07,
"logits/chosen": 0.7361761331558228,
"logits/rejected": 0.7334294319152832,
"logps/chosen": -233.13250732421875,
"logps/ref_chosen": -67.68511962890625,
"logps/ref_rejected": -71.32196044921875,
"logps/rejected": -289.564208984375,
"loss": 1.1823,
"margin_dpo/margin_mean": 52.79489517211914,
"margin_dpo/margin_std": 99.66201782226562,
"step": 463
},
{
"KL/chosen_KL_mean": -150.11727905273438,
"KL/mean": -191.15919494628906,
"KL/rejected_KL_mean": -232.2011260986328,
"KL/std": 103.76582336425781,
"epoch": 0.7014361300075586,
"fcm_dpo/beta": 0.006046008318662643,
"fcm_dpo/delta": -0.04882443696260452,
"fcm_dpo/margin": 82.08385467529297,
"fcm_dpo/q_t": 0.3890731930732727,
"grad_norm": 12.270607948303223,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": 0.7439402937889099,
"logits/rejected": 0.7154823541641235,
"logps/chosen": -209.2829132080078,
"logps/ref_chosen": -59.16564178466797,
"logps/ref_rejected": -69.56146240234375,
"logps/rejected": -301.7625732421875,
"loss": 1.0684,
"margin_dpo/margin_mean": 82.08385467529297,
"margin_dpo/margin_std": 115.9161376953125,
"step": 464
},
{
"KL/chosen_KL_mean": -157.9586181640625,
"KL/mean": -192.47720336914062,
"KL/rejected_KL_mean": -226.9957733154297,
"KL/std": 97.54608154296875,
"epoch": 0.7029478458049887,
"fcm_dpo/beta": 0.006019483786076307,
"fcm_dpo/delta": 0.0358240120112896,
"fcm_dpo/margin": 69.03716278076172,
"fcm_dpo/q_t": 0.4071248471736908,
"grad_norm": 13.349013328552246,
"learning_rate": 1.238566782415197e-07,
"logits/chosen": 0.7654883861541748,
"logits/rejected": 0.7077640295028687,
"logps/chosen": -216.4722900390625,
"logps/ref_chosen": -58.513671875,
"logps/ref_rejected": -84.31745910644531,
"logps/rejected": -311.313232421875,
"loss": 1.1183,
"margin_dpo/margin_mean": 69.03716278076172,
"margin_dpo/margin_std": 110.43283081054688,
"step": 465
},
{
"KL/chosen_KL_mean": -170.02719116210938,
"KL/mean": -190.02499389648438,
"KL/rejected_KL_mean": -210.02281188964844,
"KL/std": 105.02227783203125,
"epoch": 0.7044595616024187,
"fcm_dpo/beta": 0.006132124457508326,
"fcm_dpo/delta": 0.06729494035243988,
"fcm_dpo/margin": 39.99561309814453,
"fcm_dpo/q_t": 0.4432896673679352,
"grad_norm": 20.618934631347656,
"learning_rate": 1.2271688498291334e-07,
"logits/chosen": 0.710273265838623,
"logits/rejected": 0.7162467241287231,
"logps/chosen": -243.2930145263672,
"logps/ref_chosen": -73.26580810546875,
"logps/ref_rejected": -74.83621215820312,
"logps/rejected": -284.8590087890625,
"loss": 1.2579,
"margin_dpo/margin_mean": 39.99561309814453,
"margin_dpo/margin_std": 106.95750427246094,
"step": 466
},
{
"KL/chosen_KL_mean": -150.3333740234375,
"KL/mean": -183.27291870117188,
"KL/rejected_KL_mean": -216.21246337890625,
"KL/std": 102.4985580444336,
"epoch": 0.7059712773998488,
"fcm_dpo/beta": 0.006167709827423096,
"fcm_dpo/delta": 0.04555200785398483,
"fcm_dpo/margin": 65.87907409667969,
"fcm_dpo/q_t": 0.4112234115600586,
"grad_norm": 11.9649658203125,
"learning_rate": 1.2158065210664848e-07,
"logits/chosen": 0.784168004989624,
"logits/rejected": 0.6448608636856079,
"logps/chosen": -197.912841796875,
"logps/ref_chosen": -47.57947540283203,
"logps/ref_rejected": -78.68522644042969,
"logps/rejected": -294.897705078125,
"loss": 1.117,
"margin_dpo/margin_mean": 65.87907409667969,
"margin_dpo/margin_std": 107.44978332519531,
"step": 467
},
{
"KL/chosen_KL_mean": -141.78823852539062,
"KL/mean": -185.31454467773438,
"KL/rejected_KL_mean": -228.84085083007812,
"KL/std": 100.20425415039062,
"epoch": 0.7074829931972789,
"fcm_dpo/beta": 0.006109039299190044,
"fcm_dpo/delta": -0.08640697598457336,
"fcm_dpo/margin": 87.0526123046875,
"fcm_dpo/q_t": 0.37998446822166443,
"grad_norm": 18.61052131652832,
"learning_rate": 1.204480113956011e-07,
"logits/chosen": 0.6854424476623535,
"logits/rejected": 0.6727529168128967,
"logps/chosen": -205.71603393554688,
"logps/ref_chosen": -63.92778778076172,
"logps/ref_rejected": -76.51626586914062,
"logps/rejected": -305.35711669921875,
"loss": 1.0337,
"margin_dpo/margin_mean": 87.0526123046875,
"margin_dpo/margin_std": 113.15974426269531,
"step": 468
},
{
"KL/chosen_KL_mean": -144.42633056640625,
"KL/mean": -183.31387329101562,
"KL/rejected_KL_mean": -222.201416015625,
"KL/std": 97.5907974243164,
"epoch": 0.708994708994709,
"fcm_dpo/beta": 0.006026652175933123,
"fcm_dpo/delta": -0.020541474223136902,
"fcm_dpo/margin": 77.77510070800781,
"fcm_dpo/q_t": 0.39321011304855347,
"grad_norm": 13.9988431930542,
"learning_rate": 1.1931899453216697e-07,
"logits/chosen": 0.7652057409286499,
"logits/rejected": 0.7501240968704224,
"logps/chosen": -203.48451232910156,
"logps/ref_chosen": -59.05818176269531,
"logps/ref_rejected": -75.67672729492188,
"logps/rejected": -297.8781433105469,
"loss": 1.0453,
"margin_dpo/margin_mean": 77.77510070800781,
"margin_dpo/margin_std": 91.79092407226562,
"step": 469
},
{
"KL/chosen_KL_mean": -139.2176055908203,
"KL/mean": -177.15162658691406,
"KL/rejected_KL_mean": -215.08563232421875,
"KL/std": 95.38059997558594,
"epoch": 0.7105064247921391,
"fcm_dpo/beta": 0.006067805923521519,
"fcm_dpo/delta": -0.011047637090086937,
"fcm_dpo/margin": 75.86803436279297,
"fcm_dpo/q_t": 0.396224707365036,
"grad_norm": 12.56069278717041,
"learning_rate": 1.1819363309737438e-07,
"logits/chosen": 0.7496016025543213,
"logits/rejected": 0.6867067217826843,
"logps/chosen": -187.08505249023438,
"logps/ref_chosen": -47.86743927001953,
"logps/ref_rejected": -65.96859741210938,
"logps/rejected": -281.0542297363281,
"loss": 1.0764,
"margin_dpo/margin_mean": 75.86803436279297,
"margin_dpo/margin_std": 106.75106811523438,
"step": 470
},
{
"KL/chosen_KL_mean": -134.6875762939453,
"KL/mean": -176.7728729248047,
"KL/rejected_KL_mean": -218.85816955566406,
"KL/std": 94.34480285644531,
"epoch": 0.7120181405895691,
"fcm_dpo/beta": 0.005986287258565426,
"fcm_dpo/delta": -0.05681845545768738,
"fcm_dpo/margin": 84.17059326171875,
"fcm_dpo/q_t": 0.384713351726532,
"grad_norm": 13.130870819091797,
"learning_rate": 1.1707195857000215e-07,
"logits/chosen": 0.7122259140014648,
"logits/rejected": 0.6619666814804077,
"logps/chosen": -192.46542358398438,
"logps/ref_chosen": -57.777854919433594,
"logps/ref_rejected": -73.81172180175781,
"logps/rejected": -292.6698913574219,
"loss": 1.0365,
"margin_dpo/margin_mean": 84.17059326171875,
"margin_dpo/margin_std": 104.3567123413086,
"step": 471
},
{
"KL/chosen_KL_mean": -134.97317504882812,
"KL/mean": -168.95550537109375,
"KL/rejected_KL_mean": -202.93785095214844,
"KL/std": 98.41152954101562,
"epoch": 0.7135298563869993,
"fcm_dpo/beta": 0.0060343313962221146,
"fcm_dpo/delta": 0.04116275534033775,
"fcm_dpo/margin": 67.96469116210938,
"fcm_dpo/q_t": 0.41018474102020264,
"grad_norm": 13.530449867248535,
"learning_rate": 1.1595400232569768e-07,
"logits/chosen": 0.7044432163238525,
"logits/rejected": 0.6615398526191711,
"logps/chosen": -190.8818359375,
"logps/ref_chosen": -55.908668518066406,
"logps/ref_rejected": -74.70294189453125,
"logps/rejected": -277.64080810546875,
"loss": 1.1442,
"margin_dpo/margin_mean": 67.96469116210938,
"margin_dpo/margin_std": 123.55488586425781,
"step": 472
},
{
"KL/chosen_KL_mean": -138.88629150390625,
"KL/mean": -180.80670166015625,
"KL/rejected_KL_mean": -222.72711181640625,
"KL/std": 114.38705444335938,
"epoch": 0.7150415721844293,
"fcm_dpo/beta": 0.005974816158413887,
"fcm_dpo/delta": -0.05364374816417694,
"fcm_dpo/margin": 83.84082794189453,
"fcm_dpo/q_t": 0.3932644724845886,
"grad_norm": 15.695032119750977,
"learning_rate": 1.1483979563610069e-07,
"logits/chosen": 0.7801591157913208,
"logits/rejected": 0.6740267276763916,
"logps/chosen": -193.04718017578125,
"logps/ref_chosen": -54.16088104248047,
"logps/ref_rejected": -92.76789855957031,
"logps/rejected": -315.4949951171875,
"loss": 1.0901,
"margin_dpo/margin_mean": 83.8408203125,
"margin_dpo/margin_std": 134.4425048828125,
"step": 473
},
{
"KL/chosen_KL_mean": -142.9180908203125,
"KL/mean": -177.65631103515625,
"KL/rejected_KL_mean": -212.39453125,
"KL/std": 101.67129516601562,
"epoch": 0.7165532879818595,
"fcm_dpo/beta": 0.005997130647301674,
"fcm_dpo/delta": 0.034741394221782684,
"fcm_dpo/margin": 69.47645568847656,
"fcm_dpo/q_t": 0.40932339429855347,
"grad_norm": 18.85576629638672,
"learning_rate": 1.1372936966796709e-07,
"logits/chosen": 0.7585621476173401,
"logits/rejected": 0.6906800270080566,
"logps/chosen": -189.60379028320312,
"logps/ref_chosen": -46.685707092285156,
"logps/ref_rejected": -71.44731903076172,
"logps/rejected": -283.84185791015625,
"loss": 1.1316,
"margin_dpo/margin_mean": 69.47645568847656,
"margin_dpo/margin_std": 120.51841735839844,
"step": 474
},
{
"KL/chosen_KL_mean": -136.62750244140625,
"KL/mean": -186.64437866210938,
"KL/rejected_KL_mean": -236.6612548828125,
"KL/std": 102.17213439941406,
"epoch": 0.7180650037792895,
"fcm_dpo/beta": 0.005852770060300827,
"fcm_dpo/delta": -0.14472949504852295,
"fcm_dpo/margin": 100.03376770019531,
"fcm_dpo/q_t": 0.36991116404533386,
"grad_norm": 10.036908149719238,
"learning_rate": 1.126227554822985e-07,
"logits/chosen": 0.7174779176712036,
"logits/rejected": 0.6689811944961548,
"logps/chosen": -195.11480712890625,
"logps/ref_chosen": -58.4873046875,
"logps/ref_rejected": -87.00187683105469,
"logps/rejected": -323.66314697265625,
"loss": 0.9834,
"margin_dpo/margin_mean": 100.03376770019531,
"margin_dpo/margin_std": 113.25033569335938,
"step": 475
},
{
"KL/chosen_KL_mean": -166.2928924560547,
"KL/mean": -197.72381591796875,
"KL/rejected_KL_mean": -229.15475463867188,
"KL/std": 107.59396362304688,
"epoch": 0.7195767195767195,
"fcm_dpo/beta": 0.005927635356783867,
"fcm_dpo/delta": 0.07958348095417023,
"fcm_dpo/margin": 62.861881256103516,
"fcm_dpo/q_t": 0.418168306350708,
"grad_norm": 14.75139045715332,
"learning_rate": 1.1151998403347243e-07,
"logits/chosen": 0.6256033778190613,
"logits/rejected": 0.6278666257858276,
"logps/chosen": -241.6745147705078,
"logps/ref_chosen": -75.38162231445312,
"logps/ref_rejected": -76.99822235107422,
"logps/rejected": -306.1529846191406,
"loss": 1.1569,
"margin_dpo/margin_mean": 62.861881256103516,
"margin_dpo/margin_std": 114.92433166503906,
"step": 476
},
{
"KL/chosen_KL_mean": -164.40463256835938,
"KL/mean": -199.77825927734375,
"KL/rejected_KL_mean": -235.1519012451172,
"KL/std": 107.18777465820312,
"epoch": 0.7210884353741497,
"fcm_dpo/beta": 0.0059497784823179245,
"fcm_dpo/delta": 0.03035357967019081,
"fcm_dpo/margin": 70.74728393554688,
"fcm_dpo/q_t": 0.4081571102142334,
"grad_norm": 15.135383605957031,
"learning_rate": 1.1042108616837692e-07,
"logits/chosen": 0.7753346562385559,
"logits/rejected": 0.726446270942688,
"logps/chosen": -225.47802734375,
"logps/ref_chosen": -61.073387145996094,
"logps/ref_rejected": -81.34375,
"logps/rejected": -316.49566650390625,
"loss": 1.1603,
"margin_dpo/margin_mean": 70.74728393554688,
"margin_dpo/margin_std": 136.30105590820312,
"step": 477
},
{
"KL/chosen_KL_mean": -145.6103973388672,
"KL/mean": -173.1075439453125,
"KL/rejected_KL_mean": -200.6046905517578,
"KL/std": 91.68518829345703,
"epoch": 0.7226001511715797,
"fcm_dpo/beta": 0.006051356438547373,
"fcm_dpo/delta": 0.1214258000254631,
"fcm_dpo/margin": 54.99429702758789,
"fcm_dpo/q_t": 0.4251078963279724,
"grad_norm": 16.03772735595703,
"learning_rate": 1.0932609262554746e-07,
"logits/chosen": 0.6870132088661194,
"logits/rejected": 0.6989500522613525,
"logps/chosen": -202.7777099609375,
"logps/ref_chosen": -57.16731643676758,
"logps/ref_rejected": -53.30917739868164,
"logps/rejected": -253.9138641357422,
"loss": 1.1908,
"margin_dpo/margin_mean": 54.99428939819336,
"margin_dpo/margin_std": 111.90939331054688,
"step": 478
},
{
"KL/chosen_KL_mean": -150.79190063476562,
"KL/mean": -175.4144287109375,
"KL/rejected_KL_mean": -200.03695678710938,
"KL/std": 92.3194580078125,
"epoch": 0.7241118669690099,
"fcm_dpo/beta": 0.006136768963187933,
"fcm_dpo/delta": 0.024153757840394974,
"fcm_dpo/margin": 49.24507522583008,
"fcm_dpo/q_t": 0.4326751232147217,
"grad_norm": 15.933855056762695,
"learning_rate": 1.0823503403430734e-07,
"logits/chosen": 0.7064397931098938,
"logits/rejected": 0.6684491634368896,
"logps/chosen": -209.70521545410156,
"logps/ref_chosen": -58.91331481933594,
"logps/ref_rejected": -63.7403450012207,
"logps/rejected": -263.7773132324219,
"loss": 1.2137,
"margin_dpo/margin_mean": 49.24507141113281,
"margin_dpo/margin_std": 109.21455383300781,
"step": 479
},
{
"KL/chosen_KL_mean": -154.2449951171875,
"KL/mean": -192.55990600585938,
"KL/rejected_KL_mean": -230.87478637695312,
"KL/std": 95.96563720703125,
"epoch": 0.7256235827664399,
"fcm_dpo/beta": 0.0061669060960412025,
"fcm_dpo/delta": -0.02497832477092743,
"fcm_dpo/margin": 76.62979125976562,
"fcm_dpo/q_t": 0.39488065242767334,
"grad_norm": 17.156362533569336,
"learning_rate": 1.0714794091391072e-07,
"logits/chosen": 0.6846721768379211,
"logits/rejected": 0.6770238280296326,
"logps/chosen": -217.04562377929688,
"logps/ref_chosen": -62.80061340332031,
"logps/ref_rejected": -67.58859252929688,
"logps/rejected": -298.46337890625,
"loss": 1.0847,
"margin_dpo/margin_mean": 76.62979125976562,
"margin_dpo/margin_std": 112.43879699707031,
"step": 480
},
{
"KL/chosen_KL_mean": -152.42149353027344,
"KL/mean": -187.66525268554688,
"KL/rejected_KL_mean": -222.90902709960938,
"KL/std": 100.48785400390625,
"epoch": 0.72713529856387,
"fcm_dpo/beta": 0.006108821835368872,
"fcm_dpo/delta": 0.020193161442875862,
"fcm_dpo/margin": 70.48753356933594,
"fcm_dpo/q_t": 0.40493303537368774,
"grad_norm": 14.735018730163574,
"learning_rate": 1.0606484367268906e-07,
"logits/chosen": 0.6829352974891663,
"logits/rejected": 0.6849657297134399,
"logps/chosen": -217.70797729492188,
"logps/ref_chosen": -65.28649139404297,
"logps/ref_rejected": -70.78668212890625,
"logps/rejected": -293.6957092285156,
"loss": 1.1072,
"margin_dpo/margin_mean": 70.48753356933594,
"margin_dpo/margin_std": 112.09474182128906,
"step": 481
},
{
"KL/chosen_KL_mean": -170.580810546875,
"KL/mean": -207.0380401611328,
"KL/rejected_KL_mean": -243.49526977539062,
"KL/std": 107.98014831542969,
"epoch": 0.7286470143613001,
"fcm_dpo/beta": 0.00615047849714756,
"fcm_dpo/delta": 0.0014284886419773102,
"fcm_dpo/margin": 72.91446685791016,
"fcm_dpo/q_t": 0.40593189001083374,
"grad_norm": 16.25693130493164,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": 0.6703163385391235,
"logits/rejected": 0.521394670009613,
"logps/chosen": -231.48699951171875,
"logps/ref_chosen": -60.906185150146484,
"logps/ref_rejected": -103.44656372070312,
"logps/rejected": -346.94183349609375,
"loss": 1.1419,
"margin_dpo/margin_mean": 72.91446685791016,
"margin_dpo/margin_std": 136.18093872070312,
"step": 482
},
{
"KL/chosen_KL_mean": -142.95286560058594,
"KL/mean": -186.32806396484375,
"KL/rejected_KL_mean": -229.70327758789062,
"KL/std": 97.5377426147461,
"epoch": 0.7301587301587301,
"fcm_dpo/beta": 0.0060723647475242615,
"fcm_dpo/delta": -0.0809704065322876,
"fcm_dpo/margin": 86.75039672851562,
"fcm_dpo/q_t": 0.38196927309036255,
"grad_norm": 13.32763957977295,
"learning_rate": 1.0391075790138232e-07,
"logits/chosen": 0.7873438000679016,
"logits/rejected": 0.6818989515304565,
"logps/chosen": -196.14486694335938,
"logps/ref_chosen": -53.192012786865234,
"logps/ref_rejected": -81.83927154541016,
"logps/rejected": -311.54254150390625,
"loss": 1.0308,
"margin_dpo/margin_mean": 86.75039672851562,
"margin_dpo/margin_std": 110.80722045898438,
"step": 483
},
{
"KL/chosen_KL_mean": -149.74163818359375,
"KL/mean": -176.8128662109375,
"KL/rejected_KL_mean": -203.8840789794922,
"KL/std": 95.46682739257812,
"epoch": 0.7316704459561603,
"fcm_dpo/beta": 0.006167872808873653,
"fcm_dpo/delta": 0.11922429502010345,
"fcm_dpo/margin": 54.142452239990234,
"fcm_dpo/q_t": 0.4233402609825134,
"grad_norm": 17.306833267211914,
"learning_rate": 1.0283982962570681e-07,
"logits/chosen": 0.7923409342765808,
"logits/rejected": 0.7621510028839111,
"logps/chosen": -207.5111083984375,
"logps/ref_chosen": -57.76945877075195,
"logps/ref_rejected": -71.6829833984375,
"logps/rejected": -275.56707763671875,
"loss": 1.1463,
"margin_dpo/margin_mean": 54.142452239990234,
"margin_dpo/margin_std": 85.11724853515625,
"step": 484
},
{
"KL/chosen_KL_mean": -152.07223510742188,
"KL/mean": -183.80691528320312,
"KL/rejected_KL_mean": -215.54156494140625,
"KL/std": 97.00895690917969,
"epoch": 0.7331821617535903,
"fcm_dpo/beta": 0.006109254900366068,
"fcm_dpo/delta": -0.057364415377378464,
"fcm_dpo/margin": 63.46931457519531,
"fcm_dpo/q_t": 0.4122996926307678,
"grad_norm": 14.168609619140625,
"learning_rate": 1.0177301773633992e-07,
"logits/chosen": 0.7602566480636597,
"logits/rejected": 0.7402700185775757,
"logps/chosen": -208.70809936523438,
"logps/ref_chosen": -56.63584899902344,
"logps/ref_rejected": -70.85614013671875,
"logps/rejected": -286.397705078125,
"loss": 1.1199,
"margin_dpo/margin_mean": 63.46931457519531,
"margin_dpo/margin_std": 95.38653564453125,
"step": 485
},
{
"KL/chosen_KL_mean": -176.1962127685547,
"KL/mean": -206.28851318359375,
"KL/rejected_KL_mean": -236.38079833984375,
"KL/std": 112.07026672363281,
"epoch": 0.7346938775510204,
"fcm_dpo/beta": 0.006182005628943443,
"fcm_dpo/delta": 0.0809686928987503,
"fcm_dpo/margin": 60.1845817565918,
"fcm_dpo/q_t": 0.4215923547744751,
"grad_norm": 12.775336265563965,
"learning_rate": 1.007103520743035e-07,
"logits/chosen": 0.776502251625061,
"logits/rejected": 0.6627446413040161,
"logps/chosen": -232.54324340820312,
"logps/ref_chosen": -56.347023010253906,
"logps/ref_rejected": -85.97221374511719,
"logps/rejected": -322.35302734375,
"loss": 1.1926,
"margin_dpo/margin_mean": 60.18458557128906,
"margin_dpo/margin_std": 129.93482971191406,
"step": 486
},
{
"KL/chosen_KL_mean": -157.27584838867188,
"KL/mean": -193.54388427734375,
"KL/rejected_KL_mean": -229.81195068359375,
"KL/std": 102.25735473632812,
"epoch": 0.7362055933484505,
"fcm_dpo/beta": 0.0062069520354270935,
"fcm_dpo/delta": -0.0002405631821602583,
"fcm_dpo/margin": 72.53608703613281,
"fcm_dpo/q_t": 0.40433698892593384,
"grad_norm": 14.756111145019531,
"learning_rate": 9.965186236464046e-08,
"logits/chosen": 0.823411762714386,
"logits/rejected": 0.7664943337440491,
"logps/chosen": -217.89306640625,
"logps/ref_chosen": -60.617218017578125,
"logps/ref_rejected": -82.50975036621094,
"logps/rejected": -312.3216857910156,
"loss": 1.1053,
"margin_dpo/margin_mean": 72.53608703613281,
"margin_dpo/margin_std": 120.83760070800781,
"step": 487
},
{
"KL/chosen_KL_mean": -145.82931518554688,
"KL/mean": -185.6131591796875,
"KL/rejected_KL_mean": -225.39697265625,
"KL/std": 97.22247314453125,
"epoch": 0.7377173091458806,
"fcm_dpo/beta": 0.006201518699526787,
"fcm_dpo/delta": -0.046092621982097626,
"fcm_dpo/margin": 79.56764221191406,
"fcm_dpo/q_t": 0.39139896631240845,
"grad_norm": 17.251914978027344,
"learning_rate": 9.859757821558337e-08,
"logits/chosen": 0.7474915981292725,
"logits/rejected": 0.6937694549560547,
"logps/chosen": -208.93836975097656,
"logps/ref_chosen": -63.10905075073242,
"logps/ref_rejected": -82.49348449707031,
"logps/rejected": -307.89044189453125,
"loss": 1.0545,
"margin_dpo/margin_mean": 79.56764221191406,
"margin_dpo/margin_std": 107.11445617675781,
"step": 488
},
{
"KL/chosen_KL_mean": -171.64735412597656,
"KL/mean": -195.64874267578125,
"KL/rejected_KL_mean": -219.65011596679688,
"KL/std": 107.1324462890625,
"epoch": 0.7392290249433107,
"fcm_dpo/beta": 0.006290414370596409,
"fcm_dpo/delta": 0.15272760391235352,
"fcm_dpo/margin": 48.002750396728516,
"fcm_dpo/q_t": 0.43335413932800293,
"grad_norm": 14.13779067993164,
"learning_rate": 9.754752911772615e-08,
"logits/chosen": 0.7004495859146118,
"logits/rejected": 0.656787633895874,
"logps/chosen": -236.63632202148438,
"logps/ref_chosen": -64.98896026611328,
"logps/ref_rejected": -84.39607238769531,
"logps/rejected": -304.04620361328125,
"loss": 1.2294,
"margin_dpo/margin_mean": 48.002750396728516,
"margin_dpo/margin_std": 114.78465270996094,
"step": 489
},
{
"KL/chosen_KL_mean": -154.73526000976562,
"KL/mean": -183.689453125,
"KL/rejected_KL_mean": -212.64366149902344,
"KL/std": 103.60529327392578,
"epoch": 0.7407407407407407,
"fcm_dpo/beta": 0.006394874304533005,
"fcm_dpo/delta": 0.08273956179618835,
"fcm_dpo/margin": 57.908416748046875,
"fcm_dpo/q_t": 0.4194272756576538,
"grad_norm": 13.123128890991211,
"learning_rate": 9.650174444319956e-08,
"logits/chosen": 0.7914144396781921,
"logits/rejected": 0.7736309170722961,
"logps/chosen": -216.6439971923828,
"logps/ref_chosen": -61.90874481201172,
"logps/ref_rejected": -70.58566284179688,
"logps/rejected": -283.22930908203125,
"loss": 1.2074,
"margin_dpo/margin_mean": 57.908416748046875,
"margin_dpo/margin_std": 129.22569274902344,
"step": 490
},
{
"KL/chosen_KL_mean": -150.71292114257812,
"KL/mean": -181.09390258789062,
"KL/rejected_KL_mean": -211.47488403320312,
"KL/std": 94.73994445800781,
"epoch": 0.7422524565381708,
"fcm_dpo/beta": 0.0064573828130960464,
"fcm_dpo/delta": 0.05928955599665642,
"fcm_dpo/margin": 60.761940002441406,
"fcm_dpo/q_t": 0.4131777584552765,
"grad_norm": 13.748710632324219,
"learning_rate": 9.546025344484868e-08,
"logits/chosen": 0.675628125667572,
"logits/rejected": 0.6172371506690979,
"logps/chosen": -206.18862915039062,
"logps/ref_chosen": -55.47570037841797,
"logps/ref_rejected": -78.70318603515625,
"logps/rejected": -290.17803955078125,
"loss": 1.1328,
"margin_dpo/margin_mean": 60.761940002441406,
"margin_dpo/margin_std": 100.07881164550781,
"step": 491
},
{
"KL/chosen_KL_mean": -177.33346557617188,
"KL/mean": -204.11441040039062,
"KL/rejected_KL_mean": -230.89535522460938,
"KL/std": 107.49058532714844,
"epoch": 0.7437641723356009,
"fcm_dpo/beta": 0.006651863921433687,
"fcm_dpo/delta": 0.09622493386268616,
"fcm_dpo/margin": 53.56187438964844,
"fcm_dpo/q_t": 0.42310231924057007,
"grad_norm": 16.768766403198242,
"learning_rate": 9.442308525541589e-08,
"logits/chosen": 0.7349872589111328,
"logits/rejected": 0.6678378582000732,
"logps/chosen": -244.61985778808594,
"logps/ref_chosen": -67.28638458251953,
"logps/ref_rejected": -82.78628540039062,
"logps/rejected": -313.681640625,
"loss": 1.2139,
"margin_dpo/margin_mean": 53.5618782043457,
"margin_dpo/margin_std": 119.93057250976562,
"step": 492
},
{
"KL/chosen_KL_mean": -146.36495971679688,
"KL/mean": -186.2227783203125,
"KL/rejected_KL_mean": -226.0806121826172,
"KL/std": 105.06057739257812,
"epoch": 0.745275888133031,
"fcm_dpo/beta": 0.006630874238908291,
"fcm_dpo/delta": -0.08393573015928268,
"fcm_dpo/margin": 79.71566772460938,
"fcm_dpo/q_t": 0.385233610868454,
"grad_norm": 15.71538257598877,
"learning_rate": 9.339026888672468e-08,
"logits/chosen": 0.692692220211029,
"logits/rejected": 0.621835470199585,
"logps/chosen": -202.29244995117188,
"logps/ref_chosen": -55.92750549316406,
"logps/ref_rejected": -79.12149810791016,
"logps/rejected": -305.20208740234375,
"loss": 1.0568,
"margin_dpo/margin_mean": 79.71566772460938,
"margin_dpo/margin_std": 111.91082763671875,
"step": 493
},
{
"KL/chosen_KL_mean": -150.67694091796875,
"KL/mean": -185.75869750976562,
"KL/rejected_KL_mean": -220.8404998779297,
"KL/std": 104.7548828125,
"epoch": 0.7467876039304611,
"fcm_dpo/beta": 0.0065204668790102005,
"fcm_dpo/delta": -0.007957713678479195,
"fcm_dpo/margin": 70.16357421875,
"fcm_dpo/q_t": 0.401015967130661,
"grad_norm": 15.422093391418457,
"learning_rate": 9.236183322886945e-08,
"logits/chosen": 0.6373249888420105,
"logits/rejected": 0.5916510224342346,
"logps/chosen": -218.63104248046875,
"logps/ref_chosen": -67.95410919189453,
"logps/ref_rejected": -90.50865173339844,
"logps/rejected": -311.3491516113281,
"loss": 1.1375,
"margin_dpo/margin_mean": 70.16357421875,
"margin_dpo/margin_std": 129.2499542236328,
"step": 494
},
{
"KL/chosen_KL_mean": -147.87957763671875,
"KL/mean": -175.8821563720703,
"KL/rejected_KL_mean": -203.88473510742188,
"KL/std": 102.84738159179688,
"epoch": 0.7482993197278912,
"fcm_dpo/beta": 0.00664489483460784,
"fcm_dpo/delta": 0.07983549684286118,
"fcm_dpo/margin": 56.00517272949219,
"fcm_dpo/q_t": 0.42484697699546814,
"grad_norm": 19.291988372802734,
"learning_rate": 9.133780704940594e-08,
"logits/chosen": 0.7651863098144531,
"logits/rejected": 0.7069584131240845,
"logps/chosen": -200.50503540039062,
"logps/ref_chosen": -52.62546157836914,
"logps/ref_rejected": -72.06781005859375,
"logps/rejected": -275.9525451660156,
"loss": 1.1991,
"margin_dpo/margin_mean": 56.005165100097656,
"margin_dpo/margin_std": 125.55036163330078,
"step": 495
},
{
"KL/chosen_KL_mean": -161.83767700195312,
"KL/mean": -200.32595825195312,
"KL/rejected_KL_mean": -238.8142547607422,
"KL/std": 111.56478881835938,
"epoch": 0.7498110355253212,
"fcm_dpo/beta": 0.006527569144964218,
"fcm_dpo/delta": -0.05702162906527519,
"fcm_dpo/margin": 76.9765853881836,
"fcm_dpo/q_t": 0.4012041985988617,
"grad_norm": 13.641803741455078,
"learning_rate": 9.031821899254797e-08,
"logits/chosen": 0.7423851490020752,
"logits/rejected": 0.6301846504211426,
"logps/chosen": -219.43499755859375,
"logps/ref_chosen": -57.597320556640625,
"logps/ref_rejected": -94.36127471923828,
"logps/rejected": -333.175537109375,
"loss": 1.1167,
"margin_dpo/margin_mean": 76.9765853881836,
"margin_dpo/margin_std": 141.50401306152344,
"step": 496
},
{
"KL/chosen_KL_mean": -163.78262329101562,
"KL/mean": -201.80593872070312,
"KL/rejected_KL_mean": -239.82925415039062,
"KL/std": 103.0841064453125,
"epoch": 0.7513227513227513,
"fcm_dpo/beta": 0.006504581309854984,
"fcm_dpo/delta": -0.04710300266742706,
"fcm_dpo/margin": 76.04662322998047,
"fcm_dpo/q_t": 0.38855764269828796,
"grad_norm": 14.237540245056152,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": 0.7395977973937988,
"logits/rejected": 0.7060419321060181,
"logps/chosen": -236.57257080078125,
"logps/ref_chosen": -72.78994750976562,
"logps/ref_rejected": -89.48483276367188,
"logps/rejected": -329.3140869140625,
"loss": 1.0734,
"margin_dpo/margin_mean": 76.04662322998047,
"margin_dpo/margin_std": 111.859619140625,
"step": 497
},
{
"KL/chosen_KL_mean": -146.14315795898438,
"KL/mean": -186.0857696533203,
"KL/rejected_KL_mean": -226.02841186523438,
"KL/std": 97.7952880859375,
"epoch": 0.7528344671201814,
"fcm_dpo/beta": 0.00642948504537344,
"fcm_dpo/delta": -0.06713449209928513,
"fcm_dpo/margin": 79.88525390625,
"fcm_dpo/q_t": 0.38355961441993713,
"grad_norm": 16.2513484954834,
"learning_rate": 8.829247120198563e-08,
"logits/chosen": 0.7451342344284058,
"logits/rejected": 0.7235029935836792,
"logps/chosen": -214.5088653564453,
"logps/ref_chosen": -68.36572265625,
"logps/ref_rejected": -71.28846740722656,
"logps/rejected": -297.31689453125,
"loss": 1.0396,
"margin_dpo/margin_mean": 79.88526153564453,
"margin_dpo/margin_std": 104.2471694946289,
"step": 498
},
{
"KL/chosen_KL_mean": -148.076904296875,
"KL/mean": -188.3824462890625,
"KL/rejected_KL_mean": -228.68801879882812,
"KL/std": 107.20639038085938,
"epoch": 0.7543461829176115,
"fcm_dpo/beta": 0.006349165923893452,
"fcm_dpo/delta": -0.06516575068235397,
"fcm_dpo/margin": 80.61111450195312,
"fcm_dpo/q_t": 0.39157482981681824,
"grad_norm": 18.03789710998535,
"learning_rate": 8.728636813280163e-08,
"logits/chosen": 0.717039942741394,
"logits/rejected": 0.6537841558456421,
"logps/chosen": -209.98573303222656,
"logps/ref_chosen": -61.90882873535156,
"logps/ref_rejected": -91.9411392211914,
"logps/rejected": -320.629150390625,
"loss": 1.1156,
"margin_dpo/margin_mean": 80.6111068725586,
"margin_dpo/margin_std": 139.66329956054688,
"step": 499
},
{
"KL/chosen_KL_mean": -154.04995727539062,
"KL/mean": -187.4798583984375,
"KL/rejected_KL_mean": -220.9097442626953,
"KL/std": 95.2972640991211,
"epoch": 0.7558578987150416,
"fcm_dpo/beta": 0.0063400594517588615,
"fcm_dpo/delta": 0.027207905426621437,
"fcm_dpo/margin": 66.85978698730469,
"fcm_dpo/q_t": 0.40306317806243896,
"grad_norm": 16.55160140991211,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": 0.6679513454437256,
"logits/rejected": 0.6745506525039673,
"logps/chosen": -224.27578735351562,
"logps/ref_chosen": -70.225830078125,
"logps/ref_rejected": -71.72203063964844,
"logps/rejected": -292.63177490234375,
"loss": 1.1462,
"margin_dpo/margin_mean": 66.85978698730469,
"margin_dpo/margin_std": 121.90635681152344,
"step": 500
},
{
"KL/chosen_KL_mean": -154.33560180664062,
"KL/mean": -184.90542602539062,
"KL/rejected_KL_mean": -215.47528076171875,
"KL/std": 102.1161880493164,
"epoch": 0.7573696145124716,
"fcm_dpo/beta": 0.006436236202716827,
"fcm_dpo/delta": 0.05852247402071953,
"fcm_dpo/margin": 61.139686584472656,
"fcm_dpo/q_t": 0.4098885953426361,
"grad_norm": 12.187454223632812,
"learning_rate": 8.528784436016878e-08,
"logits/chosen": 0.725420355796814,
"logits/rejected": 0.7295132279396057,
"logps/chosen": -218.9344024658203,
"logps/ref_chosen": -64.59880828857422,
"logps/ref_rejected": -70.59329223632812,
"logps/rejected": -286.0685729980469,
"loss": 1.0981,
"margin_dpo/margin_mean": 61.139678955078125,
"margin_dpo/margin_std": 82.2115707397461,
"step": 501
},
{
"KL/chosen_KL_mean": -153.7998046875,
"KL/mean": -186.72488403320312,
"KL/rejected_KL_mean": -219.6499481201172,
"KL/std": 107.63894653320312,
"epoch": 0.7588813303099018,
"fcm_dpo/beta": 0.006486368365585804,
"fcm_dpo/delta": 0.023503903299570084,
"fcm_dpo/margin": 65.85012817382812,
"fcm_dpo/q_t": 0.40453580021858215,
"grad_norm": 17.191099166870117,
"learning_rate": 8.4295479559726e-08,
"logits/chosen": 0.7299618721008301,
"logits/rejected": 0.6835644841194153,
"logps/chosen": -219.26644897460938,
"logps/ref_chosen": -65.46662902832031,
"logps/ref_rejected": -90.22233581542969,
"logps/rejected": -309.8722839355469,
"loss": 1.1001,
"margin_dpo/margin_mean": 65.85012817382812,
"margin_dpo/margin_std": 98.904296875,
"step": 502
},
{
"KL/chosen_KL_mean": -139.2997283935547,
"KL/mean": -173.0640869140625,
"KL/rejected_KL_mean": -206.82846069335938,
"KL/std": 94.63929748535156,
"epoch": 0.7603930461073318,
"fcm_dpo/beta": 0.00647312868386507,
"fcm_dpo/delta": 0.013444061391055584,
"fcm_dpo/margin": 67.52874755859375,
"fcm_dpo/q_t": 0.401027113199234,
"grad_norm": 12.830598831176758,
"learning_rate": 8.330774987092712e-08,
"logits/chosen": 0.717086672782898,
"logits/rejected": 0.7222782373428345,
"logps/chosen": -191.13449096679688,
"logps/ref_chosen": -51.83476257324219,
"logps/ref_rejected": -57.62522506713867,
"logps/rejected": -264.4537048339844,
"loss": 1.1121,
"margin_dpo/margin_mean": 67.52874755859375,
"margin_dpo/margin_std": 108.20802307128906,
"step": 503
},
{
"KL/chosen_KL_mean": -143.835693359375,
"KL/mean": -188.02395629882812,
"KL/rejected_KL_mean": -232.21218872070312,
"KL/std": 94.23897552490234,
"epoch": 0.7619047619047619,
"fcm_dpo/beta": 0.006416743155568838,
"fcm_dpo/delta": -0.12435752153396606,
"fcm_dpo/margin": 88.37649536132812,
"fcm_dpo/q_t": 0.371462345123291,
"grad_norm": 14.499225616455078,
"learning_rate": 8.232468292269479e-08,
"logits/chosen": 0.7206200361251831,
"logits/rejected": 0.7034634351730347,
"logps/chosen": -212.48690795898438,
"logps/ref_chosen": -68.65119934082031,
"logps/ref_rejected": -77.91394805908203,
"logps/rejected": -310.12615966796875,
"loss": 0.991,
"margin_dpo/margin_mean": 88.3764877319336,
"margin_dpo/margin_std": 96.57669067382812,
"step": 504
},
{
"KL/chosen_KL_mean": -151.95733642578125,
"KL/mean": -181.8119659423828,
"KL/rejected_KL_mean": -211.66659545898438,
"KL/std": 109.19900512695312,
"epoch": 0.763416477702192,
"fcm_dpo/beta": 0.006348426453769207,
"fcm_dpo/delta": 0.07310114800930023,
"fcm_dpo/margin": 59.709259033203125,
"fcm_dpo/q_t": 0.41927751898765564,
"grad_norm": 15.147652626037598,
"learning_rate": 8.134630621352483e-08,
"logits/chosen": 0.7179839611053467,
"logits/rejected": 0.6813881397247314,
"logps/chosen": -211.9561767578125,
"logps/ref_chosen": -59.99884796142578,
"logps/ref_rejected": -76.88048553466797,
"logps/rejected": -288.5470886230469,
"loss": 1.1806,
"margin_dpo/margin_mean": 59.709259033203125,
"margin_dpo/margin_std": 120.05146789550781,
"step": 505
},
{
"KL/chosen_KL_mean": -151.50389099121094,
"KL/mean": -183.47604370117188,
"KL/rejected_KL_mean": -215.44818115234375,
"KL/std": 96.9446792602539,
"epoch": 0.764928193499622,
"fcm_dpo/beta": 0.006442956626415253,
"fcm_dpo/delta": 0.03961796313524246,
"fcm_dpo/margin": 63.944297790527344,
"fcm_dpo/q_t": 0.40785694122314453,
"grad_norm": 15.846113204956055,
"learning_rate": 8.037264711071698e-08,
"logits/chosen": 0.701289176940918,
"logits/rejected": 0.687470555305481,
"logps/chosen": -221.5751953125,
"logps/ref_chosen": -70.07130432128906,
"logps/ref_rejected": -82.03775024414062,
"logps/rejected": -297.4859313964844,
"loss": 1.1567,
"margin_dpo/margin_mean": 63.944297790527344,
"margin_dpo/margin_std": 120.85015869140625,
"step": 506
},
{
"KL/chosen_KL_mean": -162.0356903076172,
"KL/mean": -198.49484252929688,
"KL/rejected_KL_mean": -234.95399475097656,
"KL/std": 108.42218017578125,
"epoch": 0.7664399092970522,
"fcm_dpo/beta": 0.006417691707611084,
"fcm_dpo/delta": -0.019571736454963684,
"fcm_dpo/margin": 72.91832733154297,
"fcm_dpo/q_t": 0.40374889969825745,
"grad_norm": 15.331804275512695,
"learning_rate": 7.940373284960933e-08,
"logits/chosen": 0.7045880556106567,
"logits/rejected": 0.6585378646850586,
"logps/chosen": -234.042724609375,
"logps/ref_chosen": -72.00703430175781,
"logps/ref_rejected": -93.94987487792969,
"logps/rejected": -328.90386962890625,
"loss": 1.1235,
"margin_dpo/margin_mean": 72.91831970214844,
"margin_dpo/margin_std": 127.28765869140625,
"step": 507
},
{
"KL/chosen_KL_mean": -148.79429626464844,
"KL/mean": -191.38870239257812,
"KL/rejected_KL_mean": -233.9831085205078,
"KL/std": 114.09550476074219,
"epoch": 0.7679516250944822,
"fcm_dpo/beta": 0.006400927901268005,
"fcm_dpo/delta": -0.1012621819972992,
"fcm_dpo/margin": 85.18881225585938,
"fcm_dpo/q_t": 0.3824944794178009,
"grad_norm": 17.070287704467773,
"learning_rate": 7.843959053281663e-08,
"logits/chosen": 0.6818482875823975,
"logits/rejected": 0.5656284093856812,
"logps/chosen": -209.01422119140625,
"logps/ref_chosen": -60.21992492675781,
"logps/ref_rejected": -95.9200668334961,
"logps/rejected": -329.9031982421875,
"loss": 1.0558,
"margin_dpo/margin_mean": 85.18881225585938,
"margin_dpo/margin_std": 121.49737548828125,
"step": 508
},
{
"KL/chosen_KL_mean": -159.0069580078125,
"KL/mean": -190.4010009765625,
"KL/rejected_KL_mean": -221.79501342773438,
"KL/std": 101.37274169921875,
"epoch": 0.7694633408919124,
"fcm_dpo/beta": 0.006350069306790829,
"fcm_dpo/delta": 0.05341218784451485,
"fcm_dpo/margin": 62.78805160522461,
"fcm_dpo/q_t": 0.41115373373031616,
"grad_norm": 17.830284118652344,
"learning_rate": 7.748024712947204e-08,
"logits/chosen": 0.6710245013237,
"logits/rejected": 0.649438738822937,
"logps/chosen": -225.27713012695312,
"logps/ref_chosen": -66.27017211914062,
"logps/ref_rejected": -71.73065185546875,
"logps/rejected": -293.5256652832031,
"loss": 1.1576,
"margin_dpo/margin_mean": 62.78805160522461,
"margin_dpo/margin_std": 117.54412841796875,
"step": 509
},
{
"KL/chosen_KL_mean": -157.57489013671875,
"KL/mean": -197.96450805664062,
"KL/rejected_KL_mean": -238.3541259765625,
"KL/std": 110.64065551757812,
"epoch": 0.7709750566893424,
"fcm_dpo/beta": 0.006295748520642519,
"fcm_dpo/delta": -0.06247474253177643,
"fcm_dpo/margin": 80.77923583984375,
"fcm_dpo/q_t": 0.3951059579849243,
"grad_norm": 14.694042205810547,
"learning_rate": 7.652572947447272e-08,
"logits/chosen": 0.8205969929695129,
"logits/rejected": 0.727057933807373,
"logps/chosen": -211.11976623535156,
"logps/ref_chosen": -53.54487609863281,
"logps/ref_rejected": -91.36648559570312,
"logps/rejected": -329.7206115722656,
"loss": 1.1097,
"margin_dpo/margin_mean": 80.77923583984375,
"margin_dpo/margin_std": 140.24911499023438,
"step": 510
},
{
"KL/chosen_KL_mean": -148.6263427734375,
"KL/mean": -194.43792724609375,
"KL/rejected_KL_mean": -240.24951171875,
"KL/std": 100.10411071777344,
"epoch": 0.7724867724867724,
"fcm_dpo/beta": 0.006196873262524605,
"fcm_dpo/delta": -0.12475550174713135,
"fcm_dpo/margin": 91.62318420410156,
"fcm_dpo/q_t": 0.37250590324401855,
"grad_norm": 16.96703338623047,
"learning_rate": 7.557606426772961e-08,
"logits/chosen": 0.7572273015975952,
"logits/rejected": 0.7002270817756653,
"logps/chosen": -204.47071838378906,
"logps/ref_chosen": -55.844383239746094,
"logps/ref_rejected": -86.49819946289062,
"logps/rejected": -326.7477111816406,
"loss": 1.0031,
"margin_dpo/margin_mean": 91.62318420410156,
"margin_dpo/margin_std": 108.27359008789062,
"step": 511
},
{
"KL/chosen_KL_mean": -153.93524169921875,
"KL/mean": -182.5100555419922,
"KL/rejected_KL_mean": -211.08486938476562,
"KL/std": 93.58463287353516,
"epoch": 0.7739984882842026,
"fcm_dpo/beta": 0.006222460884600878,
"fcm_dpo/delta": 0.09797990322113037,
"fcm_dpo/margin": 57.149635314941406,
"fcm_dpo/q_t": 0.4194071888923645,
"grad_norm": 21.45284080505371,
"learning_rate": 7.463127807341966e-08,
"logits/chosen": 0.6186809539794922,
"logits/rejected": 0.6180380582809448,
"logps/chosen": -215.58827209472656,
"logps/ref_chosen": -61.653038024902344,
"logps/ref_rejected": -72.83148193359375,
"logps/rejected": -283.9163513183594,
"loss": 1.1794,
"margin_dpo/margin_mean": 57.14964294433594,
"margin_dpo/margin_std": 112.34751892089844,
"step": 512
},
{
"KL/chosen_KL_mean": -137.3463592529297,
"KL/mean": -174.01168823242188,
"KL/rejected_KL_mean": -210.67701721191406,
"KL/std": 95.78628540039062,
"epoch": 0.7755102040816326,
"fcm_dpo/beta": 0.006251271814107895,
"fcm_dpo/delta": -0.008913135156035423,
"fcm_dpo/margin": 73.33065032958984,
"fcm_dpo/q_t": 0.39796391129493713,
"grad_norm": 11.89295482635498,
"learning_rate": 7.369139731924401e-08,
"logits/chosen": 0.816609263420105,
"logits/rejected": 0.7631587386131287,
"logps/chosen": -188.1989288330078,
"logps/ref_chosen": -50.85256576538086,
"logps/ref_rejected": -69.21754455566406,
"logps/rejected": -279.8945617675781,
"loss": 1.0705,
"margin_dpo/margin_mean": 73.33065032958984,
"margin_dpo/margin_std": 101.00604248046875,
"step": 513
},
{
"KL/chosen_KL_mean": -153.13296508789062,
"KL/mean": -195.20777893066406,
"KL/rejected_KL_mean": -237.2825927734375,
"KL/std": 104.29148864746094,
"epoch": 0.7770219198790628,
"fcm_dpo/beta": 0.0062185414135456085,
"fcm_dpo/delta": -0.07742288708686829,
"fcm_dpo/margin": 84.14960479736328,
"fcm_dpo/q_t": 0.3840315639972687,
"grad_norm": 15.968329429626465,
"learning_rate": 7.275644829568747e-08,
"logits/chosen": 0.7511647343635559,
"logits/rejected": 0.7200503349304199,
"logps/chosen": -222.51791381835938,
"logps/ref_chosen": -69.38493347167969,
"logps/ref_rejected": -83.32447814941406,
"logps/rejected": -320.6070556640625,
"loss": 1.052,
"margin_dpo/margin_mean": 84.14961242675781,
"margin_dpo/margin_std": 117.60154724121094,
"step": 514
},
{
"KL/chosen_KL_mean": -162.0972442626953,
"KL/mean": -195.05587768554688,
"KL/rejected_KL_mean": -228.0145263671875,
"KL/std": 97.84542846679688,
"epoch": 0.7785336356764928,
"fcm_dpo/beta": 0.006194347515702248,
"fcm_dpo/delta": 0.043475374579429626,
"fcm_dpo/margin": 65.9172592163086,
"fcm_dpo/q_t": 0.4080887734889984,
"grad_norm": 15.475692749023438,
"learning_rate": 7.182645715528435e-08,
"logits/chosen": 0.7449712753295898,
"logits/rejected": 0.6657108068466187,
"logps/chosen": -215.78427124023438,
"logps/ref_chosen": -53.687034606933594,
"logps/ref_rejected": -83.59614562988281,
"logps/rejected": -311.61065673828125,
"loss": 1.1306,
"margin_dpo/margin_mean": 65.9172592163086,
"margin_dpo/margin_std": 111.5773696899414,
"step": 515
},
{
"KL/chosen_KL_mean": -140.24081420898438,
"KL/mean": -171.61367797851562,
"KL/rejected_KL_mean": -202.98654174804688,
"KL/std": 97.13780212402344,
"epoch": 0.780045351473923,
"fcm_dpo/beta": 0.0062939864583313465,
"fcm_dpo/delta": 0.056877270340919495,
"fcm_dpo/margin": 62.74571228027344,
"fcm_dpo/q_t": 0.4098943769931793,
"grad_norm": 17.302248001098633,
"learning_rate": 7.090144991188568e-08,
"logits/chosen": 0.7184008359909058,
"logits/rejected": 0.6824341416358948,
"logps/chosen": -197.14254760742188,
"logps/ref_chosen": -56.9017219543457,
"logps/ref_rejected": -67.83477783203125,
"logps/rejected": -270.8213195800781,
"loss": 1.1527,
"margin_dpo/margin_mean": 62.7457160949707,
"margin_dpo/margin_std": 114.64778900146484,
"step": 516
},
{
"KL/chosen_KL_mean": -167.3245849609375,
"KL/mean": -192.08224487304688,
"KL/rejected_KL_mean": -216.83987426757812,
"KL/std": 102.61844635009766,
"epoch": 0.781557067271353,
"fcm_dpo/beta": 0.006441779434680939,
"fcm_dpo/delta": 0.1345398724079132,
"fcm_dpo/margin": 49.51531219482422,
"fcm_dpo/q_t": 0.43090230226516724,
"grad_norm": 16.201496124267578,
"learning_rate": 6.998145243993284e-08,
"logits/chosen": 0.7385038137435913,
"logits/rejected": 0.7405730485916138,
"logps/chosen": -229.0997314453125,
"logps/ref_chosen": -61.775142669677734,
"logps/ref_rejected": -62.88270950317383,
"logps/rejected": -279.72259521484375,
"loss": 1.202,
"margin_dpo/margin_mean": 49.51531219482422,
"margin_dpo/margin_std": 106.55221557617188,
"step": 517
},
{
"KL/chosen_KL_mean": -143.09214782714844,
"KL/mean": -177.44760131835938,
"KL/rejected_KL_mean": -211.80307006835938,
"KL/std": 100.13240051269531,
"epoch": 0.783068783068783,
"fcm_dpo/beta": 0.006467289291322231,
"fcm_dpo/delta": 0.005887898616492748,
"fcm_dpo/margin": 68.71090698242188,
"fcm_dpo/q_t": 0.40420591831207275,
"grad_norm": 13.878226280212402,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": 0.7254435420036316,
"logits/rejected": 0.6841272115707397,
"logps/chosen": -205.1173858642578,
"logps/ref_chosen": -62.02523422241211,
"logps/ref_rejected": -79.06085205078125,
"logps/rejected": -290.8639221191406,
"loss": 1.109,
"margin_dpo/margin_mean": 68.71090698242188,
"margin_dpo/margin_std": 112.37191009521484,
"step": 518
},
{
"KL/chosen_KL_mean": -171.6400604248047,
"KL/mean": -191.7677764892578,
"KL/rejected_KL_mean": -211.89549255371094,
"KL/std": 102.14537811279297,
"epoch": 0.7845804988662132,
"fcm_dpo/beta": 0.006517443805932999,
"fcm_dpo/delta": 0.0746041014790535,
"fcm_dpo/margin": 40.25545883178711,
"fcm_dpo/q_t": 0.4420524835586548,
"grad_norm": 21.90707778930664,
"learning_rate": 6.815658960673781e-08,
"logits/chosen": 0.745517373085022,
"logits/rejected": 0.6990114450454712,
"logps/chosen": -233.24642944335938,
"logps/ref_chosen": -61.60636901855469,
"logps/ref_rejected": -74.50727844238281,
"logps/rejected": -286.40277099609375,
"loss": 1.3203,
"margin_dpo/margin_mean": 40.25545883178711,
"margin_dpo/margin_std": 135.0486602783203,
"step": 519
},
{
"KL/chosen_KL_mean": -157.34315490722656,
"KL/mean": -187.73007202148438,
"KL/rejected_KL_mean": -218.11697387695312,
"KL/std": 101.43348693847656,
"epoch": 0.7860922146636432,
"fcm_dpo/beta": 0.006574805825948715,
"fcm_dpo/delta": 0.05200670659542084,
"fcm_dpo/margin": 60.773807525634766,
"fcm_dpo/q_t": 0.412746787071228,
"grad_norm": 16.04285430908203,
"learning_rate": 6.725177529083209e-08,
"logits/chosen": 0.7855746746063232,
"logits/rejected": 0.7317807078361511,
"logps/chosen": -220.21658325195312,
"logps/ref_chosen": -62.87343215942383,
"logps/ref_rejected": -76.505615234375,
"logps/rejected": -294.62255859375,
"loss": 1.1375,
"margin_dpo/margin_mean": 60.773807525634766,
"margin_dpo/margin_std": 104.73448181152344,
"step": 520
},
{
"KL/chosen_KL_mean": -154.99295043945312,
"KL/mean": -199.1806640625,
"KL/rejected_KL_mean": -243.36837768554688,
"KL/std": 101.28709411621094,
"epoch": 0.7876039304610734,
"fcm_dpo/beta": 0.00651928223669529,
"fcm_dpo/delta": -0.13365957140922546,
"fcm_dpo/margin": 88.37542724609375,
"fcm_dpo/q_t": 0.3741111755371094,
"grad_norm": 13.415428161621094,
"learning_rate": 6.63520728356167e-08,
"logits/chosen": 0.6261991262435913,
"logits/rejected": 0.5538595914840698,
"logps/chosen": -219.1996307373047,
"logps/ref_chosen": -64.20668029785156,
"logps/ref_rejected": -92.28083038330078,
"logps/rejected": -335.64923095703125,
"loss": 1.0104,
"margin_dpo/margin_mean": 88.37542724609375,
"margin_dpo/margin_std": 113.64947509765625,
"step": 521
},
{
"KL/chosen_KL_mean": -156.89218139648438,
"KL/mean": -183.0986328125,
"KL/rejected_KL_mean": -209.30506896972656,
"KL/std": 101.82064056396484,
"epoch": 0.7891156462585034,
"fcm_dpo/beta": 0.006550894118845463,
"fcm_dpo/delta": 0.11058982461690903,
"fcm_dpo/margin": 52.412906646728516,
"fcm_dpo/q_t": 0.42578643560409546,
"grad_norm": 16.121353149414062,
"learning_rate": 6.545750740770336e-08,
"logits/chosen": 0.7278043627738953,
"logits/rejected": 0.7202616333961487,
"logps/chosen": -215.26190185546875,
"logps/ref_chosen": -58.369720458984375,
"logps/ref_rejected": -68.79248046875,
"logps/rejected": -278.0975341796875,
"loss": 1.2466,
"margin_dpo/margin_mean": 52.41290283203125,
"margin_dpo/margin_std": 133.0802764892578,
"step": 522
},
{
"KL/chosen_KL_mean": -160.9173583984375,
"KL/mean": -192.372314453125,
"KL/rejected_KL_mean": -223.8272705078125,
"KL/std": 102.92727661132812,
"epoch": 0.7906273620559335,
"fcm_dpo/beta": 0.006632317788898945,
"fcm_dpo/delta": 0.034202300012111664,
"fcm_dpo/margin": 62.9099006652832,
"fcm_dpo/q_t": 0.40550702810287476,
"grad_norm": 17.54810333251953,
"learning_rate": 6.456810403001012e-08,
"logits/chosen": 0.7431254386901855,
"logits/rejected": 0.6213644742965698,
"logps/chosen": -226.63058471679688,
"logps/ref_chosen": -65.71324157714844,
"logps/ref_rejected": -91.98896789550781,
"logps/rejected": -315.81622314453125,
"loss": 1.1495,
"margin_dpo/margin_mean": 62.9099006652832,
"margin_dpo/margin_std": 116.45848083496094,
"step": 523
},
{
"KL/chosen_KL_mean": -139.40798950195312,
"KL/mean": -172.82437133789062,
"KL/rejected_KL_mean": -206.24075317382812,
"KL/std": 98.07855987548828,
"epoch": 0.7921390778533636,
"fcm_dpo/beta": 0.006677767261862755,
"fcm_dpo/delta": 0.003585953265428543,
"fcm_dpo/margin": 66.83275604248047,
"fcm_dpo/q_t": 0.3996120095252991,
"grad_norm": 15.2020845413208,
"learning_rate": 6.368388758106134e-08,
"logits/chosen": 0.6536479592323303,
"logits/rejected": 0.6307451725006104,
"logps/chosen": -215.75924682617188,
"logps/ref_chosen": -76.35124969482422,
"logps/ref_rejected": -89.96072387695312,
"logps/rejected": -296.20147705078125,
"loss": 1.0911,
"margin_dpo/margin_mean": 66.83275604248047,
"margin_dpo/margin_std": 99.37472534179688,
"step": 524
},
{
"KL/chosen_KL_mean": -158.2174530029297,
"KL/mean": -185.13754272460938,
"KL/rejected_KL_mean": -212.05763244628906,
"KL/std": 96.78825378417969,
"epoch": 0.7936507936507936,
"fcm_dpo/beta": 0.006735939532518387,
"fcm_dpo/delta": 0.09070742875337601,
"fcm_dpo/margin": 53.840171813964844,
"fcm_dpo/q_t": 0.4220663607120514,
"grad_norm": 21.1476993560791,
"learning_rate": 6.280488279429185e-08,
"logits/chosen": 0.5505712032318115,
"logits/rejected": 0.5474892854690552,
"logps/chosen": -233.71324157714844,
"logps/ref_chosen": -75.49578857421875,
"logps/ref_rejected": -84.04852294921875,
"logps/rejected": -296.10614013671875,
"loss": 1.1904,
"margin_dpo/margin_mean": 53.840171813964844,
"margin_dpo/margin_std": 114.42605590820312,
"step": 525
},
{
"KL/chosen_KL_mean": -165.60940551757812,
"KL/mean": -190.29290771484375,
"KL/rejected_KL_mean": -214.9763946533203,
"KL/std": 99.83271789550781,
"epoch": 0.7951625094482238,
"fcm_dpo/beta": 0.00672984030097723,
"fcm_dpo/delta": -0.03424973785877228,
"fcm_dpo/margin": 49.36698913574219,
"fcm_dpo/q_t": 0.42518430948257446,
"grad_norm": 15.3506498336792,
"learning_rate": 6.193111425735515e-08,
"logits/chosen": 0.766860842704773,
"logits/rejected": 0.6968935132026672,
"logps/chosen": -226.90182495117188,
"logps/ref_chosen": -61.29241943359375,
"logps/ref_rejected": -82.47763061523438,
"logps/rejected": -297.45404052734375,
"loss": 1.1969,
"margin_dpo/margin_mean": 49.36698913574219,
"margin_dpo/margin_std": 101.69459533691406,
"step": 526
},
{
"KL/chosen_KL_mean": -173.23233032226562,
"KL/mean": -194.60577392578125,
"KL/rejected_KL_mean": -215.97921752929688,
"KL/std": 98.76846313476562,
"epoch": 0.7966742252456538,
"fcm_dpo/beta": 0.006750140339136124,
"fcm_dpo/delta": 0.030073152855038643,
"fcm_dpo/margin": 42.74687957763672,
"fcm_dpo/q_t": 0.4377078413963318,
"grad_norm": 18.568347930908203,
"learning_rate": 6.106260641143546e-08,
"logits/chosen": 0.8117989301681519,
"logits/rejected": 0.728463888168335,
"logps/chosen": -234.7049560546875,
"logps/ref_chosen": -61.472625732421875,
"logps/ref_rejected": -90.52831268310547,
"logps/rejected": -306.50750732421875,
"loss": 1.2515,
"margin_dpo/margin_mean": 42.74687957763672,
"margin_dpo/margin_std": 111.45413208007812,
"step": 527
},
{
"KL/chosen_KL_mean": -153.25848388671875,
"KL/mean": -178.26425170898438,
"KL/rejected_KL_mean": -203.27000427246094,
"KL/std": 95.63511657714844,
"epoch": 0.7981859410430839,
"fcm_dpo/beta": 0.006859553977847099,
"fcm_dpo/delta": 0.11088088154792786,
"fcm_dpo/margin": 50.01153564453125,
"fcm_dpo/q_t": 0.42585062980651855,
"grad_norm": 18.767499923706055,
"learning_rate": 6.019938355056422e-08,
"logits/chosen": 0.679128885269165,
"logits/rejected": 0.6063876748085022,
"logps/chosen": -212.0504913330078,
"logps/ref_chosen": -58.792015075683594,
"logps/ref_rejected": -71.82516479492188,
"logps/rejected": -275.09515380859375,
"loss": 1.2378,
"margin_dpo/margin_mean": 50.011531829833984,
"margin_dpo/margin_std": 122.185546875,
"step": 528
},
{
"KL/chosen_KL_mean": -146.73153686523438,
"KL/mean": -195.88995361328125,
"KL/rejected_KL_mean": -245.04833984375,
"KL/std": 95.25759887695312,
"epoch": 0.799697656840514,
"fcm_dpo/beta": 0.00668944651260972,
"fcm_dpo/delta": -0.22261568903923035,
"fcm_dpo/margin": 98.31681823730469,
"fcm_dpo/q_t": 0.3527362644672394,
"grad_norm": 15.684530258178711,
"learning_rate": 5.934146982094049e-08,
"logits/chosen": 0.6630944013595581,
"logits/rejected": 0.6121984720230103,
"logps/chosen": -201.802490234375,
"logps/ref_chosen": -55.070960998535156,
"logps/ref_rejected": -75.44007873535156,
"logps/rejected": -320.4884033203125,
"loss": 0.9411,
"margin_dpo/margin_mean": 98.31681823730469,
"margin_dpo/margin_std": 103.48916625976562,
"step": 529
},
{
"KL/chosen_KL_mean": -148.173828125,
"KL/mean": -179.78138732910156,
"KL/rejected_KL_mean": -211.3889617919922,
"KL/std": 100.672607421875,
"epoch": 0.8012093726379441,
"fcm_dpo/beta": 0.00664612278342247,
"fcm_dpo/delta": 0.031186042353510857,
"fcm_dpo/margin": 63.215126037597656,
"fcm_dpo/q_t": 0.4094647169113159,
"grad_norm": 19.30486297607422,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": 0.7495627403259277,
"logits/rejected": 0.7060422897338867,
"logps/chosen": -204.91763305664062,
"logps/ref_chosen": -56.743812561035156,
"logps/ref_rejected": -76.6692123413086,
"logps/rejected": -288.05816650390625,
"loss": 1.1221,
"margin_dpo/margin_mean": 63.215126037597656,
"margin_dpo/margin_std": 104.92845153808594,
"step": 530
},
{
"KL/chosen_KL_mean": -147.54202270507812,
"KL/mean": -180.4340057373047,
"KL/rejected_KL_mean": -213.32598876953125,
"KL/std": 99.5765380859375,
"epoch": 0.8027210884353742,
"fcm_dpo/beta": 0.006687507964670658,
"fcm_dpo/delta": 0.01040719449520111,
"fcm_dpo/margin": 65.78396606445312,
"fcm_dpo/q_t": 0.40361833572387695,
"grad_norm": 15.21304988861084,
"learning_rate": 5.7641665597021435e-08,
"logits/chosen": 0.7009578347206116,
"logits/rejected": 0.6236515045166016,
"logps/chosen": -198.65847778320312,
"logps/ref_chosen": -51.116455078125,
"logps/ref_rejected": -79.52884674072266,
"logps/rejected": -292.8548278808594,
"loss": 1.113,
"margin_dpo/margin_mean": 65.78396606445312,
"margin_dpo/margin_std": 107.75035095214844,
"step": 531
},
{
"KL/chosen_KL_mean": -171.36239624023438,
"KL/mean": -206.0482635498047,
"KL/rejected_KL_mean": -240.734130859375,
"KL/std": 98.20024871826172,
"epoch": 0.8042328042328042,
"fcm_dpo/beta": 0.006664278917014599,
"fcm_dpo/delta": -0.012907939031720161,
"fcm_dpo/margin": 69.37173461914062,
"fcm_dpo/q_t": 0.3980584144592285,
"grad_norm": 15.120401382446289,
"learning_rate": 5.679982264990424e-08,
"logits/chosen": 0.6691586971282959,
"logits/rejected": 0.6198612451553345,
"logps/chosen": -229.64234924316406,
"logps/ref_chosen": -58.279945373535156,
"logps/ref_rejected": -78.05426788330078,
"logps/rejected": -318.78839111328125,
"loss": 1.1031,
"margin_dpo/margin_mean": 69.37173461914062,
"margin_dpo/margin_std": 111.1090087890625,
"step": 532
},
{
"KL/chosen_KL_mean": -123.17243957519531,
"KL/mean": -161.1618194580078,
"KL/rejected_KL_mean": -199.15118408203125,
"KL/std": 104.36534118652344,
"epoch": 0.8057445200302343,
"fcm_dpo/beta": 0.006617809645831585,
"fcm_dpo/delta": -0.05558159202337265,
"fcm_dpo/margin": 75.97874450683594,
"fcm_dpo/q_t": 0.3897656202316284,
"grad_norm": 14.80583381652832,
"learning_rate": 5.596338392706076e-08,
"logits/chosen": 0.8223803639411926,
"logits/rejected": 0.7595813274383545,
"logps/chosen": -179.5904541015625,
"logps/ref_chosen": -56.41801071166992,
"logps/ref_rejected": -73.89324951171875,
"logps/rejected": -273.04443359375,
"loss": 1.0669,
"margin_dpo/margin_mean": 75.97874450683594,
"margin_dpo/margin_std": 110.16397094726562,
"step": 533
},
{
"KL/chosen_KL_mean": -152.75064086914062,
"KL/mean": -184.16172790527344,
"KL/rejected_KL_mean": -215.57281494140625,
"KL/std": 96.92642974853516,
"epoch": 0.8072562358276644,
"fcm_dpo/beta": 0.006597267463803291,
"fcm_dpo/delta": 0.036938875913619995,
"fcm_dpo/margin": 62.82218551635742,
"fcm_dpo/q_t": 0.410610169172287,
"grad_norm": 14.558494567871094,
"learning_rate": 5.513237282548033e-08,
"logits/chosen": 0.7041269540786743,
"logits/rejected": 0.6697344183921814,
"logps/chosen": -213.49932861328125,
"logps/ref_chosen": -60.748687744140625,
"logps/ref_rejected": -73.8623046875,
"logps/rejected": -289.43511962890625,
"loss": 1.1546,
"margin_dpo/margin_mean": 62.82218551635742,
"margin_dpo/margin_std": 118.70053100585938,
"step": 534
},
{
"KL/chosen_KL_mean": -163.18838500976562,
"KL/mean": -192.66116333007812,
"KL/rejected_KL_mean": -222.1339111328125,
"KL/std": 103.63077545166016,
"epoch": 0.8087679516250945,
"fcm_dpo/beta": 0.006703161634504795,
"fcm_dpo/delta": 0.0569588765501976,
"fcm_dpo/margin": 58.94554138183594,
"fcm_dpo/q_t": 0.41441118717193604,
"grad_norm": 17.145980834960938,
"learning_rate": 5.430681259032957e-08,
"logits/chosen": 0.6563804149627686,
"logits/rejected": 0.5968525409698486,
"logps/chosen": -224.8258056640625,
"logps/ref_chosen": -61.637413024902344,
"logps/ref_rejected": -80.93138885498047,
"logps/rejected": -303.0653076171875,
"loss": 1.1646,
"margin_dpo/margin_mean": 58.94554138183594,
"margin_dpo/margin_std": 113.44522857666016,
"step": 535
},
{
"KL/chosen_KL_mean": -142.1412353515625,
"KL/mean": -186.98748779296875,
"KL/rejected_KL_mean": -231.83370971679688,
"KL/std": 103.05687713623047,
"epoch": 0.8102796674225246,
"fcm_dpo/beta": 0.0065511735156178474,
"fcm_dpo/delta": -0.14660747349262238,
"fcm_dpo/margin": 89.69249725341797,
"fcm_dpo/q_t": 0.3693081736564636,
"grad_norm": 11.731171607971191,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": 0.7670760154724121,
"logits/rejected": 0.6842254996299744,
"logps/chosen": -194.03021240234375,
"logps/ref_chosen": -51.88897705078125,
"logps/ref_rejected": -73.34864044189453,
"logps/rejected": -305.182373046875,
"loss": 0.9762,
"margin_dpo/margin_mean": 89.69249725341797,
"margin_dpo/margin_std": 99.07676696777344,
"step": 536
},
{
"KL/chosen_KL_mean": -161.25852966308594,
"KL/mean": -199.32022094726562,
"KL/rejected_KL_mean": -237.38189697265625,
"KL/std": 110.19611358642578,
"epoch": 0.8117913832199547,
"fcm_dpo/beta": 0.006451202556490898,
"fcm_dpo/delta": -0.043621353805065155,
"fcm_dpo/margin": 76.12335205078125,
"fcm_dpo/q_t": 0.39416471123695374,
"grad_norm": 14.4218168258667,
"learning_rate": 5.267213693697695e-08,
"logits/chosen": 0.8012008666992188,
"logits/rejected": 0.7109647393226624,
"logps/chosen": -215.50714111328125,
"logps/ref_chosen": -54.248619079589844,
"logps/ref_rejected": -94.94343566894531,
"logps/rejected": -332.3253173828125,
"loss": 1.0885,
"margin_dpo/margin_mean": 76.12335205078125,
"margin_dpo/margin_std": 119.11538696289062,
"step": 537
},
{
"KL/chosen_KL_mean": -160.41598510742188,
"KL/mean": -198.71536254882812,
"KL/rejected_KL_mean": -237.01473999023438,
"KL/std": 106.37571716308594,
"epoch": 0.8133030990173847,
"fcm_dpo/beta": 0.006427483633160591,
"fcm_dpo/delta": -0.044511329382658005,
"fcm_dpo/margin": 76.59874725341797,
"fcm_dpo/q_t": 0.3906959891319275,
"grad_norm": 15.044438362121582,
"learning_rate": 5.1863067244167144e-08,
"logits/chosen": 0.7274729013442993,
"logits/rejected": 0.7041901350021362,
"logps/chosen": -230.509521484375,
"logps/ref_chosen": -70.09353637695312,
"logps/ref_rejected": -79.49833679199219,
"logps/rejected": -316.5130615234375,
"loss": 1.0591,
"margin_dpo/margin_mean": 76.5987548828125,
"margin_dpo/margin_std": 106.91598510742188,
"step": 538
},
{
"KL/chosen_KL_mean": -166.6041259765625,
"KL/mean": -195.6445770263672,
"KL/rejected_KL_mean": -224.68502807617188,
"KL/std": 99.87994384765625,
"epoch": 0.8148148148148148,
"fcm_dpo/beta": 0.0064649423584342,
"fcm_dpo/delta": 0.07749071717262268,
"fcm_dpo/margin": 58.080909729003906,
"fcm_dpo/q_t": 0.41956430673599243,
"grad_norm": 16.343154907226562,
"learning_rate": 5.105953986729195e-08,
"logits/chosen": 0.6749646663665771,
"logits/rejected": 0.594658374786377,
"logps/chosen": -228.5358123779297,
"logps/ref_chosen": -61.93169403076172,
"logps/ref_rejected": -84.08946228027344,
"logps/rejected": -308.77447509765625,
"loss": 1.1496,
"margin_dpo/margin_mean": 58.080909729003906,
"margin_dpo/margin_std": 105.43345642089844,
"step": 539
},
{
"KL/chosen_KL_mean": -153.3197021484375,
"KL/mean": -197.894287109375,
"KL/rejected_KL_mean": -242.4688720703125,
"KL/std": 114.05096435546875,
"epoch": 0.8163265306122449,
"fcm_dpo/beta": 0.006410318426787853,
"fcm_dpo/delta": -0.1286935806274414,
"fcm_dpo/margin": 89.14921569824219,
"fcm_dpo/q_t": 0.3729906976222992,
"grad_norm": 13.589789390563965,
"learning_rate": 5.026157728273966e-08,
"logits/chosen": 0.7904075980186462,
"logits/rejected": 0.6958364248275757,
"logps/chosen": -216.02394104003906,
"logps/ref_chosen": -62.704254150390625,
"logps/ref_rejected": -95.63597106933594,
"logps/rejected": -338.1048583984375,
"loss": 1.0036,
"margin_dpo/margin_mean": 89.14920806884766,
"margin_dpo/margin_std": 107.2660903930664,
"step": 540
},
{
"KL/chosen_KL_mean": -148.68814086914062,
"KL/mean": -186.67027282714844,
"KL/rejected_KL_mean": -224.65240478515625,
"KL/std": 100.93985748291016,
"epoch": 0.817838246409675,
"fcm_dpo/beta": 0.00628033559769392,
"fcm_dpo/delta": -0.029025848954916,
"fcm_dpo/margin": 75.96427917480469,
"fcm_dpo/q_t": 0.3912792205810547,
"grad_norm": 13.795503616333008,
"learning_rate": 4.9469201811239035e-08,
"logits/chosen": 0.7655197381973267,
"logits/rejected": 0.7934525012969971,
"logps/chosen": -211.16897583007812,
"logps/ref_chosen": -62.48084259033203,
"logps/ref_rejected": -57.55541229248047,
"logps/rejected": -282.20782470703125,
"loss": 1.0564,
"margin_dpo/margin_mean": 75.96427154541016,
"margin_dpo/margin_std": 98.35800170898438,
"step": 541
},
{
"KL/chosen_KL_mean": -129.79156494140625,
"KL/mean": -171.60101318359375,
"KL/rejected_KL_mean": -213.4104461669922,
"KL/std": 96.41845703125,
"epoch": 0.8193499622071051,
"fcm_dpo/beta": 0.0062217069789767265,
"fcm_dpo/delta": -0.07427109777927399,
"fcm_dpo/margin": 83.61888122558594,
"fcm_dpo/q_t": 0.38352102041244507,
"grad_norm": 14.153130531311035,
"learning_rate": 4.868243561723534e-08,
"logits/chosen": 0.7965934872627258,
"logits/rejected": 0.7499839067459106,
"logps/chosen": -179.2464599609375,
"logps/ref_chosen": -49.454891204833984,
"logps/ref_rejected": -65.33275604248047,
"logps/rejected": -278.74322509765625,
"loss": 1.0525,
"margin_dpo/margin_mean": 83.61888122558594,
"margin_dpo/margin_std": 116.61778259277344,
"step": 542
},
{
"KL/chosen_KL_mean": -141.03268432617188,
"KL/mean": -181.77450561523438,
"KL/rejected_KL_mean": -222.51632690429688,
"KL/std": 98.75093841552734,
"epoch": 0.8208616780045351,
"fcm_dpo/beta": 0.006175430491566658,
"fcm_dpo/delta": -0.05604030564427376,
"fcm_dpo/margin": 81.48362731933594,
"fcm_dpo/q_t": 0.38659557700157166,
"grad_norm": 11.417715072631836,
"learning_rate": 4.790130070827028e-08,
"logits/chosen": 0.7406554222106934,
"logits/rejected": 0.6567230224609375,
"logps/chosen": -192.133544921875,
"logps/ref_chosen": -51.100860595703125,
"logps/ref_rejected": -76.06130981445312,
"logps/rejected": -298.57763671875,
"loss": 1.0504,
"margin_dpo/margin_mean": 81.48362731933594,
"margin_dpo/margin_std": 107.75604248046875,
"step": 543
},
{
"KL/chosen_KL_mean": -151.23812866210938,
"KL/mean": -197.67628479003906,
"KL/rejected_KL_mean": -244.11444091796875,
"KL/std": 108.5754165649414,
"epoch": 0.8223733938019653,
"fcm_dpo/beta": 0.006031567230820656,
"fcm_dpo/delta": -0.11675632745027542,
"fcm_dpo/margin": 92.87631225585938,
"fcm_dpo/q_t": 0.3768364191055298,
"grad_norm": 15.823902130126953,
"learning_rate": 4.7125818934366454e-08,
"logits/chosen": 0.7370050549507141,
"logits/rejected": 0.6611154675483704,
"logps/chosen": -211.51535034179688,
"logps/ref_chosen": -60.2772331237793,
"logps/ref_rejected": -88.40553283691406,
"logps/rejected": -332.51995849609375,
"loss": 1.0305,
"margin_dpo/margin_mean": 92.87631225585938,
"margin_dpo/margin_std": 125.10038757324219,
"step": 544
},
{
"KL/chosen_KL_mean": -164.6317901611328,
"KL/mean": -191.05908203125,
"KL/rejected_KL_mean": -217.48635864257812,
"KL/std": 101.46080017089844,
"epoch": 0.8238851095993953,
"fcm_dpo/beta": 0.006120487116277218,
"fcm_dpo/delta": 0.1305941343307495,
"fcm_dpo/margin": 52.85456848144531,
"fcm_dpo/q_t": 0.4277143180370331,
"grad_norm": 15.269746780395508,
"learning_rate": 4.635601198741607e-08,
"logits/chosen": 0.7444198131561279,
"logits/rejected": 0.6830329895019531,
"logps/chosen": -226.24703979492188,
"logps/ref_chosen": -61.61524963378906,
"logps/ref_rejected": -78.71266174316406,
"logps/rejected": -296.19903564453125,
"loss": 1.1929,
"margin_dpo/margin_mean": 52.85456848144531,
"margin_dpo/margin_std": 108.56221771240234,
"step": 545
},
{
"KL/chosen_KL_mean": -155.8973388671875,
"KL/mean": -185.46197509765625,
"KL/rejected_KL_mean": -215.02658081054688,
"KL/std": 95.04618835449219,
"epoch": 0.8253968253968254,
"fcm_dpo/beta": 0.0062283361330628395,
"fcm_dpo/delta": 0.08490745723247528,
"fcm_dpo/margin": 59.12923812866211,
"fcm_dpo/q_t": 0.4165213108062744,
"grad_norm": 18.64433479309082,
"learning_rate": 4.559190140057428e-08,
"logits/chosen": 0.8147549629211426,
"logits/rejected": 0.8096719980239868,
"logps/chosen": -215.21060180664062,
"logps/ref_chosen": -59.313262939453125,
"logps/ref_rejected": -64.73631286621094,
"logps/rejected": -279.76287841796875,
"loss": 1.1728,
"margin_dpo/margin_mean": 59.129241943359375,
"margin_dpo/margin_std": 115.27635192871094,
"step": 546
},
{
"KL/chosen_KL_mean": -137.16912841796875,
"KL/mean": -179.7300567626953,
"KL/rejected_KL_mean": -222.29100036621094,
"KL/std": 101.09002685546875,
"epoch": 0.8269085411942555,
"fcm_dpo/beta": 0.006165103986859322,
"fcm_dpo/delta": -0.07947796583175659,
"fcm_dpo/margin": 85.12187194824219,
"fcm_dpo/q_t": 0.38214462995529175,
"grad_norm": 14.160149574279785,
"learning_rate": 4.483350854765672e-08,
"logits/chosen": 0.6544277667999268,
"logits/rejected": 0.5950570106506348,
"logps/chosen": -192.1458740234375,
"logps/ref_chosen": -54.97674560546875,
"logps/ref_rejected": -75.35922241210938,
"logps/rejected": -297.65020751953125,
"loss": 1.0434,
"margin_dpo/margin_mean": 85.12186431884766,
"margin_dpo/margin_std": 114.97239685058594,
"step": 547
},
{
"KL/chosen_KL_mean": -162.11642456054688,
"KL/mean": -187.776611328125,
"KL/rejected_KL_mean": -213.43678283691406,
"KL/std": 100.71829223632812,
"epoch": 0.8284202569916855,
"fcm_dpo/beta": 0.006303782574832439,
"fcm_dpo/delta": 0.13005885481834412,
"fcm_dpo/margin": 51.32037353515625,
"fcm_dpo/q_t": 0.4288024604320526,
"grad_norm": 17.33949851989746,
"learning_rate": 4.4080854642541826e-08,
"logits/chosen": 0.6553746461868286,
"logits/rejected": 0.5998683571815491,
"logps/chosen": -225.32708740234375,
"logps/ref_chosen": -63.21067428588867,
"logps/ref_rejected": -81.23347473144531,
"logps/rejected": -294.6702575683594,
"loss": 1.2002,
"margin_dpo/margin_mean": 51.32037353515625,
"margin_dpo/margin_std": 109.20315551757812,
"step": 548
},
{
"KL/chosen_KL_mean": -158.01779174804688,
"KL/mean": -190.70941162109375,
"KL/rejected_KL_mean": -223.4010467529297,
"KL/std": 108.44243621826172,
"epoch": 0.8299319727891157,
"fcm_dpo/beta": 0.006333451718091965,
"fcm_dpo/delta": 0.03725190833210945,
"fcm_dpo/margin": 65.38323211669922,
"fcm_dpo/q_t": 0.411038339138031,
"grad_norm": 18.26466941833496,
"learning_rate": 4.333396073857723e-08,
"logits/chosen": 0.7768402099609375,
"logits/rejected": 0.7113950252532959,
"logps/chosen": -222.2913055419922,
"logps/ref_chosen": -64.27351379394531,
"logps/ref_rejected": -92.31663513183594,
"logps/rejected": -315.7176818847656,
"loss": 1.1632,
"margin_dpo/margin_mean": 65.38322448730469,
"margin_dpo/margin_std": 127.63583374023438,
"step": 549
},
{
"KL/chosen_KL_mean": -163.54788208007812,
"KL/mean": -186.1416015625,
"KL/rejected_KL_mean": -208.73532104492188,
"KL/std": 97.8836669921875,
"epoch": 0.8314436885865457,
"fcm_dpo/beta": 0.006398816127330065,
"fcm_dpo/delta": 0.02116929367184639,
"fcm_dpo/margin": 45.18742370605469,
"fcm_dpo/q_t": 0.435605525970459,
"grad_norm": 16.312294006347656,
"learning_rate": 4.259284772799099e-08,
"logits/chosen": 0.7508084774017334,
"logits/rejected": 0.7250527143478394,
"logps/chosen": -219.7783203125,
"logps/ref_chosen": -56.230438232421875,
"logps/ref_rejected": -62.59788513183594,
"logps/rejected": -271.33319091796875,
"loss": 1.228,
"margin_dpo/margin_mean": 45.18741989135742,
"margin_dpo/margin_std": 105.23531341552734,
"step": 550
},
{
"KL/chosen_KL_mean": -165.9162139892578,
"KL/mean": -195.33651733398438,
"KL/rejected_KL_mean": -224.75680541992188,
"KL/std": 105.70631408691406,
"epoch": 0.8329554043839759,
"fcm_dpo/beta": 0.006481049582362175,
"fcm_dpo/delta": 0.07112284004688263,
"fcm_dpo/margin": 58.84058380126953,
"fcm_dpo/q_t": 0.4186520576477051,
"grad_norm": 15.41645622253418,
"learning_rate": 4.1857536341307176e-08,
"logits/chosen": 0.7407591342926025,
"logits/rejected": 0.7093254923820496,
"logps/chosen": -233.66342163085938,
"logps/ref_chosen": -67.74720764160156,
"logps/ref_rejected": -87.04285430908203,
"logps/rejected": -311.7996826171875,
"loss": 1.1393,
"margin_dpo/margin_mean": 58.840579986572266,
"margin_dpo/margin_std": 104.4417724609375,
"step": 551
},
{
"KL/chosen_KL_mean": -154.389892578125,
"KL/mean": -186.97584533691406,
"KL/rejected_KL_mean": -219.56179809570312,
"KL/std": 104.70506286621094,
"epoch": 0.8344671201814059,
"fcm_dpo/beta": 0.006548226810991764,
"fcm_dpo/delta": 0.023566871881484985,
"fcm_dpo/margin": 65.17192077636719,
"fcm_dpo/q_t": 0.40180903673171997,
"grad_norm": 18.85803985595703,
"learning_rate": 4.112804714676593e-08,
"logits/chosen": 0.7013387680053711,
"logits/rejected": 0.6492581367492676,
"logps/chosen": -217.31614685058594,
"logps/ref_chosen": -62.92625427246094,
"logps/ref_rejected": -82.98365783691406,
"logps/rejected": -302.5454406738281,
"loss": 1.1028,
"margin_dpo/margin_mean": 65.17192077636719,
"margin_dpo/margin_std": 98.27408599853516,
"step": 552
},
{
"KL/chosen_KL_mean": -167.21725463867188,
"KL/mean": -199.59527587890625,
"KL/rejected_KL_mean": -231.97332763671875,
"KL/std": 102.98626708984375,
"epoch": 0.8359788359788359,
"fcm_dpo/beta": 0.0065464479848742485,
"fcm_dpo/delta": 0.027236608788371086,
"fcm_dpo/margin": 64.75605773925781,
"fcm_dpo/q_t": 0.41023218631744385,
"grad_norm": 15.29988956451416,
"learning_rate": 4.0404400549748144e-08,
"logits/chosen": 0.7155384421348572,
"logits/rejected": 0.61528480052948,
"logps/chosen": -223.2557373046875,
"logps/ref_chosen": -56.038490295410156,
"logps/ref_rejected": -84.48454284667969,
"logps/rejected": -316.4578552246094,
"loss": 1.1541,
"margin_dpo/margin_mean": 64.75605773925781,
"margin_dpo/margin_std": 124.07582092285156,
"step": 553
},
{
"KL/chosen_KL_mean": -151.11111450195312,
"KL/mean": -190.245361328125,
"KL/rejected_KL_mean": -229.37960815429688,
"KL/std": 99.22693634033203,
"epoch": 0.8374905517762661,
"fcm_dpo/beta": 0.006496269255876541,
"fcm_dpo/delta": -0.061591919511556625,
"fcm_dpo/margin": 78.26849365234375,
"fcm_dpo/q_t": 0.384976327419281,
"grad_norm": 14.839645385742188,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": 0.6921082735061646,
"logits/rejected": 0.6812867522239685,
"logps/chosen": -215.64170837402344,
"logps/ref_chosen": -64.53059387207031,
"logps/ref_rejected": -71.2155990600586,
"logps/rejected": -300.59521484375,
"loss": 1.0557,
"margin_dpo/margin_mean": 78.26849365234375,
"margin_dpo/margin_std": 108.02368927001953,
"step": 554
},
{
"KL/chosen_KL_mean": -164.1957244873047,
"KL/mean": -194.96737670898438,
"KL/rejected_KL_mean": -225.73904418945312,
"KL/std": 99.13853454589844,
"epoch": 0.8390022675736961,
"fcm_dpo/beta": 0.006575894542038441,
"fcm_dpo/delta": 0.045365020632743835,
"fcm_dpo/margin": 61.54331970214844,
"fcm_dpo/q_t": 0.4076317250728607,
"grad_norm": 15.397841453552246,
"learning_rate": 3.89747159520904e-08,
"logits/chosen": 0.7093448638916016,
"logits/rejected": 0.689207911491394,
"logps/chosen": -230.84764099121094,
"logps/ref_chosen": -66.65191650390625,
"logps/ref_rejected": -68.6667251586914,
"logps/rejected": -294.40576171875,
"loss": 1.1715,
"margin_dpo/margin_mean": 61.54331970214844,
"margin_dpo/margin_std": 118.01376342773438,
"step": 555
},
{
"KL/chosen_KL_mean": -163.47509765625,
"KL/mean": -192.88812255859375,
"KL/rejected_KL_mean": -222.30117797851562,
"KL/std": 104.06493377685547,
"epoch": 0.8405139833711263,
"fcm_dpo/beta": 0.006583062466233969,
"fcm_dpo/delta": 0.06530771404504776,
"fcm_dpo/margin": 58.82606506347656,
"fcm_dpo/q_t": 0.416615754365921,
"grad_norm": 15.477357864379883,
"learning_rate": 3.826871794280192e-08,
"logits/chosen": 0.771374523639679,
"logits/rejected": 0.7252951860427856,
"logps/chosen": -216.30746459960938,
"logps/ref_chosen": -52.832366943359375,
"logps/ref_rejected": -64.49044036865234,
"logps/rejected": -286.7916259765625,
"loss": 1.1748,
"margin_dpo/margin_mean": 58.82606887817383,
"margin_dpo/margin_std": 116.84832763671875,
"step": 556
},
{
"KL/chosen_KL_mean": -161.71389770507812,
"KL/mean": -203.29481506347656,
"KL/rejected_KL_mean": -244.87571716308594,
"KL/std": 100.98245239257812,
"epoch": 0.8420256991685563,
"fcm_dpo/beta": 0.006499768700450659,
"fcm_dpo/delta": -0.09647659212350845,
"fcm_dpo/margin": 83.16181945800781,
"fcm_dpo/q_t": 0.3794446587562561,
"grad_norm": 12.793874740600586,
"learning_rate": 3.756864251262143e-08,
"logits/chosen": 0.8473179340362549,
"logits/rejected": 0.7723125219345093,
"logps/chosen": -216.74989318847656,
"logps/ref_chosen": -55.03598403930664,
"logps/ref_rejected": -75.80644989013672,
"logps/rejected": -320.68218994140625,
"loss": 1.0115,
"margin_dpo/margin_mean": 83.16181945800781,
"margin_dpo/margin_std": 97.54216003417969,
"step": 557
},
{
"KL/chosen_KL_mean": -148.48394775390625,
"KL/mean": -194.65689086914062,
"KL/rejected_KL_mean": -240.8298797607422,
"KL/std": 108.04835510253906,
"epoch": 0.8435374149659864,
"fcm_dpo/beta": 0.006330681033432484,
"fcm_dpo/delta": -0.1439516693353653,
"fcm_dpo/margin": 92.34593200683594,
"fcm_dpo/q_t": 0.3717380166053772,
"grad_norm": 12.148514747619629,
"learning_rate": 3.687450924416341e-08,
"logits/chosen": 0.7686564922332764,
"logits/rejected": 0.7162346243858337,
"logps/chosen": -211.71029663085938,
"logps/ref_chosen": -63.226348876953125,
"logps/ref_rejected": -91.46881866455078,
"logps/rejected": -332.2987060546875,
"loss": 0.9954,
"margin_dpo/margin_mean": 92.3459243774414,
"margin_dpo/margin_std": 111.07440185546875,
"step": 558
},
{
"KL/chosen_KL_mean": -153.1084442138672,
"KL/mean": -193.3884735107422,
"KL/rejected_KL_mean": -233.66848754882812,
"KL/std": 106.25447845458984,
"epoch": 0.8450491307634165,
"fcm_dpo/beta": 0.006216675043106079,
"fcm_dpo/delta": -0.05483238771557808,
"fcm_dpo/margin": 80.56004333496094,
"fcm_dpo/q_t": 0.39433592557907104,
"grad_norm": 11.37843132019043,
"learning_rate": 3.6186337553827743e-08,
"logits/chosen": 0.7013551592826843,
"logits/rejected": 0.6412140727043152,
"logps/chosen": -214.63009643554688,
"logps/ref_chosen": -61.521644592285156,
"logps/ref_rejected": -82.83859252929688,
"logps/rejected": -316.507080078125,
"loss": 1.0877,
"margin_dpo/margin_mean": 80.56004333496094,
"margin_dpo/margin_std": 127.60150146484375,
"step": 559
},
{
"KL/chosen_KL_mean": -166.9397430419922,
"KL/mean": -204.06809997558594,
"KL/rejected_KL_mean": -241.1964569091797,
"KL/std": 102.28352355957031,
"epoch": 0.8465608465608465,
"fcm_dpo/beta": 0.006258917972445488,
"fcm_dpo/delta": -0.01593797653913498,
"fcm_dpo/margin": 74.25672912597656,
"fcm_dpo/q_t": 0.39539068937301636,
"grad_norm": 17.991178512573242,
"learning_rate": 3.550414669125573e-08,
"logits/chosen": 0.7269060611724854,
"logits/rejected": 0.6896910667419434,
"logps/chosen": -227.58096313476562,
"logps/ref_chosen": -60.64122009277344,
"logps/ref_rejected": -78.75474548339844,
"logps/rejected": -319.9512023925781,
"loss": 1.064,
"margin_dpo/margin_mean": 74.25672912597656,
"margin_dpo/margin_std": 97.35572052001953,
"step": 560
},
{
"KL/chosen_KL_mean": -152.1344757080078,
"KL/mean": -186.82179260253906,
"KL/rejected_KL_mean": -221.50912475585938,
"KL/std": 100.64840698242188,
"epoch": 0.8480725623582767,
"fcm_dpo/beta": 0.006227419711649418,
"fcm_dpo/delta": 0.01874781772494316,
"fcm_dpo/margin": 69.3746566772461,
"fcm_dpo/q_t": 0.4056755304336548,
"grad_norm": 14.884553909301758,
"learning_rate": 3.482795573879241e-08,
"logits/chosen": 0.6993428468704224,
"logits/rejected": 0.6668655872344971,
"logps/chosen": -214.63307189941406,
"logps/ref_chosen": -62.49859619140625,
"logps/ref_rejected": -78.72064208984375,
"logps/rejected": -300.2297668457031,
"loss": 1.1103,
"margin_dpo/margin_mean": 69.37466430664062,
"margin_dpo/margin_std": 110.47499084472656,
"step": 561
},
{
"KL/chosen_KL_mean": -162.52401733398438,
"KL/mean": -207.02621459960938,
"KL/rejected_KL_mean": -251.52841186523438,
"KL/std": 116.5802001953125,
"epoch": 0.8495842781557067,
"fcm_dpo/beta": 0.006109355017542839,
"fcm_dpo/delta": -0.10087047517299652,
"fcm_dpo/margin": 89.00437927246094,
"fcm_dpo/q_t": 0.3810436725616455,
"grad_norm": 16.537874221801758,
"learning_rate": 3.415778361095226e-08,
"logits/chosen": 0.7176022529602051,
"logits/rejected": 0.6851276159286499,
"logps/chosen": -237.30575561523438,
"logps/ref_chosen": -74.78173828125,
"logps/ref_rejected": -92.63499450683594,
"logps/rejected": -344.16339111328125,
"loss": 1.0226,
"margin_dpo/margin_mean": 89.00437927246094,
"margin_dpo/margin_std": 112.19898986816406,
"step": 562
},
{
"KL/chosen_KL_mean": -135.99513244628906,
"KL/mean": -173.78982543945312,
"KL/rejected_KL_mean": -211.58447265625,
"KL/std": 90.8531494140625,
"epoch": 0.8510959939531368,
"fcm_dpo/beta": 0.00610891031101346,
"fcm_dpo/delta": -0.012339213863015175,
"fcm_dpo/margin": 75.58935546875,
"fcm_dpo/q_t": 0.395784854888916,
"grad_norm": 20.83038330078125,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": 0.8373708128929138,
"logits/rejected": 0.7844498157501221,
"logps/chosen": -186.19363403320312,
"logps/ref_chosen": -50.19850158691406,
"logps/ref_rejected": -66.76687622070312,
"logps/rejected": -278.35137939453125,
"loss": 1.0922,
"margin_dpo/margin_mean": 75.58935546875,
"margin_dpo/margin_std": 115.21420288085938,
"step": 563
},
{
"KL/chosen_KL_mean": -137.42124938964844,
"KL/mean": -185.23602294921875,
"KL/rejected_KL_mean": -233.05078125,
"KL/std": 98.76502990722656,
"epoch": 0.8526077097505669,
"fcm_dpo/beta": 0.005984083749353886,
"fcm_dpo/delta": -0.12979558110237122,
"fcm_dpo/margin": 95.62950897216797,
"fcm_dpo/q_t": 0.37068864703178406,
"grad_norm": 12.641019821166992,
"learning_rate": 3.283557064487785e-08,
"logits/chosen": 0.7246955633163452,
"logits/rejected": 0.6956311464309692,
"logps/chosen": -193.162109375,
"logps/ref_chosen": -55.7408447265625,
"logps/ref_rejected": -74.82323455810547,
"logps/rejected": -307.8740234375,
"loss": 1.0071,
"margin_dpo/margin_mean": 95.6295166015625,
"margin_dpo/margin_std": 117.49134826660156,
"step": 564
},
{
"KL/chosen_KL_mean": -166.86520385742188,
"KL/mean": -201.83352661132812,
"KL/rejected_KL_mean": -236.80184936523438,
"KL/std": 102.82844543457031,
"epoch": 0.854119425547997,
"fcm_dpo/beta": 0.00600344967097044,
"fcm_dpo/delta": 0.030905822291970253,
"fcm_dpo/margin": 69.93665313720703,
"fcm_dpo/q_t": 0.40452465415000916,
"grad_norm": 14.577558517456055,
"learning_rate": 3.218356679178252e-08,
"logits/chosen": 0.7704143524169922,
"logits/rejected": 0.7189521193504333,
"logps/chosen": -225.20257568359375,
"logps/ref_chosen": -58.33738327026367,
"logps/ref_rejected": -78.31776428222656,
"logps/rejected": -315.11962890625,
"loss": 1.0977,
"margin_dpo/margin_mean": 69.93665313720703,
"margin_dpo/margin_std": 101.45330810546875,
"step": 565
},
{
"KL/chosen_KL_mean": -157.60694885253906,
"KL/mean": -189.88328552246094,
"KL/rejected_KL_mean": -222.15963745117188,
"KL/std": 105.84686279296875,
"epoch": 0.8556311413454271,
"fcm_dpo/beta": 0.006068835034966469,
"fcm_dpo/delta": 0.05984632670879364,
"fcm_dpo/margin": 64.55268859863281,
"fcm_dpo/q_t": 0.41507333517074585,
"grad_norm": 14.73648452758789,
"learning_rate": 3.1537655732553764e-08,
"logits/chosen": 0.7199764847755432,
"logits/rejected": 0.7104334831237793,
"logps/chosen": -228.8306884765625,
"logps/ref_chosen": -71.22373962402344,
"logps/ref_rejected": -71.11601257324219,
"logps/rejected": -293.275634765625,
"loss": 1.1818,
"margin_dpo/margin_mean": 64.55268096923828,
"margin_dpo/margin_std": 132.97817993164062,
"step": 566
},
{
"KL/chosen_KL_mean": -150.5260009765625,
"KL/mean": -190.51409912109375,
"KL/rejected_KL_mean": -230.50216674804688,
"KL/std": 99.58441925048828,
"epoch": 0.8571428571428571,
"fcm_dpo/beta": 0.005988474003970623,
"fcm_dpo/delta": -0.03157001733779907,
"fcm_dpo/margin": 79.97616577148438,
"fcm_dpo/q_t": 0.39104723930358887,
"grad_norm": 12.800858497619629,
"learning_rate": 3.089785553471233e-08,
"logits/chosen": 0.7463260889053345,
"logits/rejected": 0.6602100133895874,
"logps/chosen": -203.19528198242188,
"logps/ref_chosen": -52.669273376464844,
"logps/ref_rejected": -74.34785461425781,
"logps/rejected": -304.85003662109375,
"loss": 1.0607,
"margin_dpo/margin_mean": 79.97615814208984,
"margin_dpo/margin_std": 105.27865600585938,
"step": 567
},
{
"KL/chosen_KL_mean": -138.55270385742188,
"KL/mean": -187.60003662109375,
"KL/rejected_KL_mean": -236.64739990234375,
"KL/std": 106.38749694824219,
"epoch": 0.8586545729402872,
"fcm_dpo/beta": 0.005939141847193241,
"fcm_dpo/delta": -0.14087416231632233,
"fcm_dpo/margin": 98.09469604492188,
"fcm_dpo/q_t": 0.37110963463783264,
"grad_norm": 15.389481544494629,
"learning_rate": 3.026418409484513e-08,
"logits/chosen": 0.7434817552566528,
"logits/rejected": 0.66554856300354,
"logps/chosen": -190.73069763183594,
"logps/ref_chosen": -52.178001403808594,
"logps/ref_rejected": -85.8277587890625,
"logps/rejected": -322.47515869140625,
"loss": 0.9842,
"margin_dpo/margin_mean": 98.09469604492188,
"margin_dpo/margin_std": 107.54985046386719,
"step": 568
},
{
"KL/chosen_KL_mean": -163.41534423828125,
"KL/mean": -187.918212890625,
"KL/rejected_KL_mean": -212.42108154296875,
"KL/std": 106.85481262207031,
"epoch": 0.8601662887377173,
"fcm_dpo/beta": 0.005847351625561714,
"fcm_dpo/delta": -0.004478311166167259,
"fcm_dpo/margin": 49.0057258605957,
"fcm_dpo/q_t": 0.43568170070648193,
"grad_norm": 16.01313591003418,
"learning_rate": 2.963665913810451e-08,
"logits/chosen": 0.6617103219032288,
"logits/rejected": 0.6303982734680176,
"logps/chosen": -226.06460571289062,
"logps/ref_chosen": -62.649261474609375,
"logps/ref_rejected": -75.4298324584961,
"logps/rejected": -287.85089111328125,
"loss": 1.2172,
"margin_dpo/margin_mean": 49.0057258605957,
"margin_dpo/margin_std": 106.43392181396484,
"step": 569
},
{
"KL/chosen_KL_mean": -142.8104248046875,
"KL/mean": -189.96177673339844,
"KL/rejected_KL_mean": -237.1131134033203,
"KL/std": 98.74018096923828,
"epoch": 0.8616780045351474,
"fcm_dpo/beta": 0.005793171934783459,
"fcm_dpo/delta": -0.1019757091999054,
"fcm_dpo/margin": 94.30270385742188,
"fcm_dpo/q_t": 0.3777024745941162,
"grad_norm": 13.21369457244873,
"learning_rate": 2.9015298217712453e-08,
"logits/chosen": 0.7050719261169434,
"logits/rejected": 0.6235053539276123,
"logps/chosen": -192.85223388671875,
"logps/ref_chosen": -50.04179382324219,
"logps/ref_rejected": -78.27146911621094,
"logps/rejected": -315.38458251953125,
"loss": 1.0105,
"margin_dpo/margin_mean": 94.30270385742188,
"margin_dpo/margin_std": 112.3070068359375,
"step": 570
},
{
"KL/chosen_KL_mean": -155.06976318359375,
"KL/mean": -181.69549560546875,
"KL/rejected_KL_mean": -208.3212432861328,
"KL/std": 95.75420379638672,
"epoch": 0.8631897203325775,
"fcm_dpo/beta": 0.005875091999769211,
"fcm_dpo/delta": 0.14064154028892517,
"fcm_dpo/margin": 53.25146484375,
"fcm_dpo/q_t": 0.4286976754665375,
"grad_norm": 14.657941818237305,
"learning_rate": 2.840011871446962e-08,
"logits/chosen": 0.7359094023704529,
"logits/rejected": 0.7131412625312805,
"logps/chosen": -208.72657775878906,
"logps/ref_chosen": -53.65681457519531,
"logps/ref_rejected": -66.13298034667969,
"logps/rejected": -274.4542236328125,
"loss": 1.1985,
"margin_dpo/margin_mean": 53.25146484375,
"margin_dpo/margin_std": 108.87671661376953,
"step": 571
},
{
"KL/chosen_KL_mean": -156.80117797851562,
"KL/mean": -187.74636840820312,
"KL/rejected_KL_mean": -218.69155883789062,
"KL/std": 101.79773712158203,
"epoch": 0.8647014361300076,
"fcm_dpo/beta": 0.005991543643176556,
"fcm_dpo/delta": 0.08145475387573242,
"fcm_dpo/margin": 61.89037322998047,
"fcm_dpo/q_t": 0.4158562421798706,
"grad_norm": 14.93289852142334,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": 0.7083245515823364,
"logits/rejected": 0.7496261596679688,
"logps/chosen": -231.61911010742188,
"logps/ref_chosen": -74.81792449951172,
"logps/ref_rejected": -65.88681030273438,
"logps/rejected": -284.578369140625,
"loss": 1.1395,
"margin_dpo/margin_mean": 61.89037322998047,
"margin_dpo/margin_std": 102.08415222167969,
"step": 572
},
{
"KL/chosen_KL_mean": -169.649169921875,
"KL/mean": -205.01141357421875,
"KL/rejected_KL_mean": -240.37362670898438,
"KL/std": 111.67261505126953,
"epoch": 0.8662131519274376,
"fcm_dpo/beta": 0.006012643687427044,
"fcm_dpo/delta": 0.025858554989099503,
"fcm_dpo/margin": 70.72447204589844,
"fcm_dpo/q_t": 0.4091688096523285,
"grad_norm": 14.856910705566406,
"learning_rate": 2.718837261761528e-08,
"logits/chosen": 0.7295148968696594,
"logits/rejected": 0.6869486570358276,
"logps/chosen": -238.37481689453125,
"logps/ref_chosen": -68.72564697265625,
"logps/ref_rejected": -88.16201782226562,
"logps/rejected": -328.53564453125,
"loss": 1.1504,
"margin_dpo/margin_mean": 70.72447204589844,
"margin_dpo/margin_std": 133.98379516601562,
"step": 573
},
{
"KL/chosen_KL_mean": -151.06890869140625,
"KL/mean": -195.67068481445312,
"KL/rejected_KL_mean": -240.2724609375,
"KL/std": 102.99846649169922,
"epoch": 0.8677248677248677,
"fcm_dpo/beta": 0.005980664864182472,
"fcm_dpo/delta": -0.08848445862531662,
"fcm_dpo/margin": 89.20353698730469,
"fcm_dpo/q_t": 0.3796440064907074,
"grad_norm": 11.192665100097656,
"learning_rate": 2.659183991914696e-08,
"logits/chosen": 0.8058528900146484,
"logits/rejected": 0.7434405088424683,
"logps/chosen": -207.38229370117188,
"logps/ref_chosen": -56.31340026855469,
"logps/ref_rejected": -83.91553497314453,
"logps/rejected": -324.18798828125,
"loss": 1.0011,
"margin_dpo/margin_mean": 89.20353698730469,
"margin_dpo/margin_std": 96.93377685546875,
"step": 574
},
{
"KL/chosen_KL_mean": -156.55935668945312,
"KL/mean": -188.05662536621094,
"KL/rejected_KL_mean": -219.55389404296875,
"KL/std": 103.39370727539062,
"epoch": 0.8692365835222978,
"fcm_dpo/beta": 0.005861110985279083,
"fcm_dpo/delta": -0.04574337974190712,
"fcm_dpo/margin": 62.99452590942383,
"fcm_dpo/q_t": 0.42001163959503174,
"grad_norm": 13.875605583190918,
"learning_rate": 2.600155642716606e-08,
"logits/chosen": 0.7368036508560181,
"logits/rejected": 0.661503791809082,
"logps/chosen": -221.14349365234375,
"logps/ref_chosen": -64.5841293334961,
"logps/ref_rejected": -93.47034454345703,
"logps/rejected": -313.02423095703125,
"loss": 1.1786,
"margin_dpo/margin_mean": 62.99452590942383,
"margin_dpo/margin_std": 123.26811981201172,
"step": 575
},
{
"KL/chosen_KL_mean": -138.5884246826172,
"KL/mean": -185.07730102539062,
"KL/rejected_KL_mean": -231.56613159179688,
"KL/std": 105.1303939819336,
"epoch": 0.8707482993197279,
"fcm_dpo/beta": 0.005738668143749237,
"fcm_dpo/delta": -0.0901963859796524,
"fcm_dpo/margin": 92.97773742675781,
"fcm_dpo/q_t": 0.3803936243057251,
"grad_norm": 13.09487533569336,
"learning_rate": 2.5417538653170754e-08,
"logits/chosen": 0.7440035343170166,
"logits/rejected": 0.6456351280212402,
"logps/chosen": -191.86895751953125,
"logps/ref_chosen": -53.28052520751953,
"logps/ref_rejected": -84.2000503540039,
"logps/rejected": -315.76617431640625,
"loss": 1.0266,
"margin_dpo/margin_mean": 92.97773742675781,
"margin_dpo/margin_std": 115.10040283203125,
"step": 576
},
{
"KL/chosen_KL_mean": -154.52947998046875,
"KL/mean": -185.0371856689453,
"KL/rejected_KL_mean": -215.54489135742188,
"KL/std": 100.77213287353516,
"epoch": 0.872260015117158,
"fcm_dpo/beta": 0.005833870265632868,
"fcm_dpo/delta": 0.09760239720344543,
"fcm_dpo/margin": 61.01540756225586,
"fcm_dpo/q_t": 0.4187750220298767,
"grad_norm": 13.304019927978516,
"learning_rate": 2.4839802933393607e-08,
"logits/chosen": 0.7207918763160706,
"logits/rejected": 0.7140774130821228,
"logps/chosen": -216.85415649414062,
"logps/ref_chosen": -62.32468795776367,
"logps/ref_rejected": -67.300537109375,
"logps/rejected": -282.8454284667969,
"loss": 1.1637,
"margin_dpo/margin_mean": 61.01540756225586,
"margin_dpo/margin_std": 109.60237884521484,
"step": 577
},
{
"KL/chosen_KL_mean": -146.52560424804688,
"KL/mean": -176.1517333984375,
"KL/rejected_KL_mean": -205.7778778076172,
"KL/std": 105.66199493408203,
"epoch": 0.873771730914588,
"fcm_dpo/beta": 0.005955866537988186,
"fcm_dpo/delta": 0.10069636255502701,
"fcm_dpo/margin": 59.252296447753906,
"fcm_dpo/q_t": 0.4234742522239685,
"grad_norm": 13.605378150939941,
"learning_rate": 2.4268365428344733e-08,
"logits/chosen": 0.7855771780014038,
"logits/rejected": 0.7710686922073364,
"logps/chosen": -203.18118286132812,
"logps/ref_chosen": -56.65557861328125,
"logps/ref_rejected": -68.21835327148438,
"logps/rejected": -273.9962158203125,
"loss": 1.1829,
"margin_dpo/margin_mean": 59.25229263305664,
"margin_dpo/margin_std": 120.24205017089844,
"step": 578
},
{
"KL/chosen_KL_mean": -152.49461364746094,
"KL/mean": -196.47821044921875,
"KL/rejected_KL_mean": -240.4617919921875,
"KL/std": 98.71546936035156,
"epoch": 0.8752834467120182,
"fcm_dpo/beta": 0.005913109518587589,
"fcm_dpo/delta": -0.0742032453417778,
"fcm_dpo/margin": 87.96717834472656,
"fcm_dpo/q_t": 0.3802490234375,
"grad_norm": 12.734554290771484,
"learning_rate": 2.3703242122359357e-08,
"logits/chosen": 0.6896715760231018,
"logits/rejected": 0.6705402731895447,
"logps/chosen": -209.3042755126953,
"logps/ref_chosen": -56.809661865234375,
"logps/ref_rejected": -68.09613037109375,
"logps/rejected": -308.55792236328125,
"loss": 1.0138,
"margin_dpo/margin_mean": 87.96717834472656,
"margin_dpo/margin_std": 100.14832305908203,
"step": 579
},
{
"KL/chosen_KL_mean": -154.41006469726562,
"KL/mean": -191.92547607421875,
"KL/rejected_KL_mean": -229.44088745117188,
"KL/std": 115.38453674316406,
"epoch": 0.8767951625094482,
"fcm_dpo/beta": 0.005934194196015596,
"fcm_dpo/delta": 0.004452776163816452,
"fcm_dpo/margin": 75.03083801269531,
"fcm_dpo/q_t": 0.4045197367668152,
"grad_norm": 14.501690864562988,
"learning_rate": 2.3144448823151392e-08,
"logits/chosen": 0.7188555002212524,
"logits/rejected": 0.6699697971343994,
"logps/chosen": -212.11016845703125,
"logps/ref_chosen": -57.70011520385742,
"logps/ref_rejected": -77.90664672851562,
"logps/rejected": -307.3475341796875,
"loss": 1.121,
"margin_dpo/margin_mean": 75.03083801269531,
"margin_dpo/margin_std": 128.20059204101562,
"step": 580
},
{
"KL/chosen_KL_mean": -169.00424194335938,
"KL/mean": -203.2427978515625,
"KL/rejected_KL_mean": -237.48133850097656,
"KL/std": 101.91188049316406,
"epoch": 0.8783068783068783,
"fcm_dpo/beta": 0.00592381926253438,
"fcm_dpo/delta": 0.046096865087747574,
"fcm_dpo/margin": 68.47708129882812,
"fcm_dpo/q_t": 0.4113912880420685,
"grad_norm": 14.175301551818848,
"learning_rate": 2.259200116137039e-08,
"logits/chosen": 0.7493467330932617,
"logits/rejected": 0.6887227296829224,
"logps/chosen": -228.33660888671875,
"logps/ref_chosen": -59.332359313964844,
"logps/ref_rejected": -83.64482116699219,
"logps/rejected": -321.12615966796875,
"loss": 1.1411,
"margin_dpo/margin_mean": 68.47708129882812,
"margin_dpo/margin_std": 121.07502746582031,
"step": 581
},
{
"KL/chosen_KL_mean": -154.54629516601562,
"KL/mean": -187.54974365234375,
"KL/rejected_KL_mean": -220.55316162109375,
"KL/std": 97.96818542480469,
"epoch": 0.8798185941043084,
"fcm_dpo/beta": 0.006023784168064594,
"fcm_dpo/delta": 0.05431080237030983,
"fcm_dpo/margin": 66.00688934326172,
"fcm_dpo/q_t": 0.41208571195602417,
"grad_norm": 11.859220504760742,
"learning_rate": 2.204591459016525e-08,
"logits/chosen": 0.7633649706840515,
"logits/rejected": 0.7924540042877197,
"logps/chosen": -218.7091522216797,
"logps/ref_chosen": -64.16285705566406,
"logps/ref_rejected": -58.632896423339844,
"logps/rejected": -279.1860656738281,
"loss": 1.1323,
"margin_dpo/margin_mean": 66.00688934326172,
"margin_dpo/margin_std": 110.81221771240234,
"step": 582
},
{
"KL/chosen_KL_mean": -153.4376220703125,
"KL/mean": -195.10647583007812,
"KL/rejected_KL_mean": -236.77532958984375,
"KL/std": 110.63685607910156,
"epoch": 0.8813303099017384,
"fcm_dpo/beta": 0.005971607752144337,
"fcm_dpo/delta": -0.05028872191905975,
"fcm_dpo/margin": 83.33770751953125,
"fcm_dpo/q_t": 0.3901503384113312,
"grad_norm": 15.806873321533203,
"learning_rate": 2.1506204384751064e-08,
"logits/chosen": 0.8192166090011597,
"logits/rejected": 0.7225203514099121,
"logps/chosen": -205.30999755859375,
"logps/ref_chosen": -51.87239456176758,
"logps/ref_rejected": -83.86331176757812,
"logps/rejected": -320.6386413574219,
"loss": 1.08,
"margin_dpo/margin_mean": 83.33770751953125,
"margin_dpo/margin_std": 126.09477233886719,
"step": 583
},
{
"KL/chosen_KL_mean": -140.74923706054688,
"KL/mean": -176.01983642578125,
"KL/rejected_KL_mean": -211.29043579101562,
"KL/std": 99.2724609375,
"epoch": 0.8828420256991686,
"fcm_dpo/beta": 0.005971814971417189,
"fcm_dpo/delta": 0.02986559458076954,
"fcm_dpo/margin": 70.54119873046875,
"fcm_dpo/q_t": 0.4095681309700012,
"grad_norm": 15.029725074768066,
"learning_rate": 2.09728856419826e-08,
"logits/chosen": 0.889620304107666,
"logits/rejected": 0.784419059753418,
"logps/chosen": -187.32061767578125,
"logps/ref_chosen": -46.571388244628906,
"logps/ref_rejected": -80.67969512939453,
"logps/rejected": -291.9701232910156,
"loss": 1.1435,
"margin_dpo/margin_mean": 70.54119873046875,
"margin_dpo/margin_std": 128.3501434326172,
"step": 584
},
{
"KL/chosen_KL_mean": -163.51336669921875,
"KL/mean": -190.11109924316406,
"KL/rejected_KL_mean": -216.7088623046875,
"KL/std": 110.62606811523438,
"epoch": 0.8843537414965986,
"fcm_dpo/beta": 0.006146551109850407,
"fcm_dpo/delta": 0.12602292001247406,
"fcm_dpo/margin": 53.19551086425781,
"fcm_dpo/q_t": 0.42664623260498047,
"grad_norm": 13.625960350036621,
"learning_rate": 2.044597327993153e-08,
"logits/chosen": 0.7172322869300842,
"logits/rejected": 0.6654783487319946,
"logps/chosen": -221.6378936767578,
"logps/ref_chosen": -58.124534606933594,
"logps/ref_rejected": -79.00538635253906,
"logps/rejected": -295.7142333984375,
"loss": 1.1917,
"margin_dpo/margin_mean": 53.19550323486328,
"margin_dpo/margin_std": 108.43853759765625,
"step": 585
},
{
"KL/chosen_KL_mean": -150.9927978515625,
"KL/mean": -185.2848663330078,
"KL/rejected_KL_mean": -219.57691955566406,
"KL/std": 92.50978088378906,
"epoch": 0.8858654572940288,
"fcm_dpo/beta": 0.006165110971778631,
"fcm_dpo/delta": 0.02831181511282921,
"fcm_dpo/margin": 68.5841064453125,
"fcm_dpo/q_t": 0.40376946330070496,
"grad_norm": 13.801816940307617,
"learning_rate": 1.9925482037469187e-08,
"logits/chosen": 0.7588971853256226,
"logits/rejected": 0.7175912857055664,
"logps/chosen": -205.09445190429688,
"logps/ref_chosen": -54.10163879394531,
"logps/ref_rejected": -63.72113037109375,
"logps/rejected": -283.29803466796875,
"loss": 1.0728,
"margin_dpo/margin_mean": 68.58411407470703,
"margin_dpo/margin_std": 85.91049194335938,
"step": 586
},
{
"KL/chosen_KL_mean": -158.9512481689453,
"KL/mean": -192.81903076171875,
"KL/rejected_KL_mean": -226.68679809570312,
"KL/std": 108.18991088867188,
"epoch": 0.8873771730914588,
"fcm_dpo/beta": 0.006208654493093491,
"fcm_dpo/delta": 0.030746201053261757,
"fcm_dpo/margin": 67.73554229736328,
"fcm_dpo/q_t": 0.4009004235267639,
"grad_norm": 16.25068473815918,
"learning_rate": 1.9411426473854687e-08,
"logits/chosen": 0.7532941102981567,
"logits/rejected": 0.7503918409347534,
"logps/chosen": -222.36843872070312,
"logps/ref_chosen": -63.41719436645508,
"logps/ref_rejected": -63.47003936767578,
"logps/rejected": -290.1568603515625,
"loss": 1.1813,
"margin_dpo/margin_mean": 67.73554229736328,
"margin_dpo/margin_std": 144.77902221679688,
"step": 587
},
{
"KL/chosen_KL_mean": -161.81015014648438,
"KL/mean": -199.49249267578125,
"KL/rejected_KL_mean": -237.1748504638672,
"KL/std": 107.10841369628906,
"epoch": 0.8888888888888888,
"fcm_dpo/beta": 0.006185667589306831,
"fcm_dpo/delta": -0.017407085746526718,
"fcm_dpo/margin": 75.36467742919922,
"fcm_dpo/q_t": 0.3991840183734894,
"grad_norm": 17.871604919433594,
"learning_rate": 1.890382096832699e-08,
"logits/chosen": 0.8103761672973633,
"logits/rejected": 0.7680736780166626,
"logps/chosen": -224.01119995117188,
"logps/ref_chosen": -62.20103454589844,
"logps/ref_rejected": -82.10249328613281,
"logps/rejected": -319.27734375,
"loss": 1.1208,
"margin_dpo/margin_mean": 75.36467742919922,
"margin_dpo/margin_std": 130.9927978515625,
"step": 588
},
{
"KL/chosen_KL_mean": -151.50381469726562,
"KL/mean": -190.66334533691406,
"KL/rejected_KL_mean": -229.8228759765625,
"KL/std": 103.41250610351562,
"epoch": 0.890400604686319,
"fcm_dpo/beta": 0.006190561689436436,
"fcm_dpo/delta": -0.03668287768959999,
"fcm_dpo/margin": 78.31907653808594,
"fcm_dpo/q_t": 0.3893091678619385,
"grad_norm": 12.957806587219238,
"learning_rate": 1.840267971970344e-08,
"logits/chosen": 0.7153310775756836,
"logits/rejected": 0.686347246170044,
"logps/chosen": -208.21742248535156,
"logps/ref_chosen": -56.71361541748047,
"logps/ref_rejected": -76.7366943359375,
"logps/rejected": -306.5595703125,
"loss": 1.0425,
"margin_dpo/margin_mean": 78.31907653808594,
"margin_dpo/margin_std": 96.69180297851562,
"step": 589
},
{
"KL/chosen_KL_mean": -165.386474609375,
"KL/mean": -201.8364715576172,
"KL/rejected_KL_mean": -238.28648376464844,
"KL/std": 94.99177551269531,
"epoch": 0.891912320483749,
"fcm_dpo/beta": 0.006181138567626476,
"fcm_dpo/delta": -0.0008702166378498077,
"fcm_dpo/margin": 72.9000015258789,
"fcm_dpo/q_t": 0.3985363841056824,
"grad_norm": 16.859054565429688,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": 0.6825237274169922,
"logits/rejected": 0.6479649543762207,
"logps/chosen": -231.90029907226562,
"logps/ref_chosen": -66.5138168334961,
"logps/ref_rejected": -85.70820617675781,
"logps/rejected": -323.99468994140625,
"loss": 1.0834,
"margin_dpo/margin_mean": 72.9000015258789,
"margin_dpo/margin_std": 103.98199462890625,
"step": 590
},
{
"KL/chosen_KL_mean": -139.999755859375,
"KL/mean": -186.31524658203125,
"KL/rejected_KL_mean": -232.63076782226562,
"KL/std": 111.80885314941406,
"epoch": 0.8934240362811792,
"fcm_dpo/beta": 0.006001078523695469,
"fcm_dpo/delta": -0.11483590304851532,
"fcm_dpo/margin": 92.63101196289062,
"fcm_dpo/q_t": 0.3807540535926819,
"grad_norm": 14.555268287658691,
"learning_rate": 1.7419845883949098e-08,
"logits/chosen": 0.8046550750732422,
"logits/rejected": 0.7481690645217896,
"logps/chosen": -200.69692993164062,
"logps/ref_chosen": -60.697181701660156,
"logps/ref_rejected": -86.12278747558594,
"logps/rejected": -318.7535400390625,
"loss": 1.0569,
"margin_dpo/margin_mean": 92.63101196289062,
"margin_dpo/margin_std": 135.70516967773438,
"step": 591
},
{
"KL/chosen_KL_mean": -153.17800903320312,
"KL/mean": -188.54193115234375,
"KL/rejected_KL_mean": -223.90585327148438,
"KL/std": 96.40587615966797,
"epoch": 0.8949357520786092,
"fcm_dpo/beta": 0.006005411501973867,
"fcm_dpo/delta": 0.02572956308722496,
"fcm_dpo/margin": 70.72784423828125,
"fcm_dpo/q_t": 0.4077094793319702,
"grad_norm": 14.989432334899902,
"learning_rate": 1.6938180788793556e-08,
"logits/chosen": 0.8055961728096008,
"logits/rejected": 0.6956392526626587,
"logps/chosen": -204.4153289794922,
"logps/ref_chosen": -51.237327575683594,
"logps/ref_rejected": -81.60242462158203,
"logps/rejected": -305.50830078125,
"loss": 1.1064,
"margin_dpo/margin_mean": 70.72784423828125,
"margin_dpo/margin_std": 108.61327362060547,
"step": 592
},
{
"KL/chosen_KL_mean": -127.48636627197266,
"KL/mean": -166.19288635253906,
"KL/rejected_KL_mean": -204.8994140625,
"KL/std": 98.92471313476562,
"epoch": 0.8964474678760394,
"fcm_dpo/beta": 0.006023376248776913,
"fcm_dpo/delta": -0.017201118171215057,
"fcm_dpo/margin": 77.41304016113281,
"fcm_dpo/q_t": 0.3943125307559967,
"grad_norm": 16.600597381591797,
"learning_rate": 1.6463034933723336e-08,
"logits/chosen": 0.7461457848548889,
"logits/rejected": 0.6567406058311462,
"logps/chosen": -169.56637573242188,
"logps/ref_chosen": -42.08000183105469,
"logps/ref_rejected": -68.47499084472656,
"logps/rejected": -273.3743896484375,
"loss": 1.0807,
"margin_dpo/margin_mean": 77.41304016113281,
"margin_dpo/margin_std": 112.75965881347656,
"step": 593
},
{
"KL/chosen_KL_mean": -161.6719970703125,
"KL/mean": -194.70840454101562,
"KL/rejected_KL_mean": -227.74481201171875,
"KL/std": 98.35874938964844,
"epoch": 0.8979591836734694,
"fcm_dpo/beta": 0.006076568737626076,
"fcm_dpo/delta": 0.05052534490823746,
"fcm_dpo/margin": 66.07281494140625,
"fcm_dpo/q_t": 0.40820130705833435,
"grad_norm": 14.272443771362305,
"learning_rate": 1.5994421609589385e-08,
"logits/chosen": 0.6612966060638428,
"logits/rejected": 0.6492611169815063,
"logps/chosen": -225.33065795898438,
"logps/ref_chosen": -63.658668518066406,
"logps/ref_rejected": -70.35597229003906,
"logps/rejected": -298.10076904296875,
"loss": 1.1112,
"margin_dpo/margin_mean": 66.07281494140625,
"margin_dpo/margin_std": 99.67398071289062,
"step": 594
},
{
"KL/chosen_KL_mean": -149.42576599121094,
"KL/mean": -192.97630310058594,
"KL/rejected_KL_mean": -236.52682495117188,
"KL/std": 105.02578735351562,
"epoch": 0.8994708994708994,
"fcm_dpo/beta": 0.006041559390723705,
"fcm_dpo/delta": -0.08044502139091492,
"fcm_dpo/margin": 87.10105895996094,
"fcm_dpo/q_t": 0.38504302501678467,
"grad_norm": 11.805750846862793,
"learning_rate": 1.553235392451377e-08,
"logits/chosen": 0.8043855428695679,
"logits/rejected": 0.717462956905365,
"logps/chosen": -205.64453125,
"logps/ref_chosen": -56.21875762939453,
"logps/ref_rejected": -83.95773315429688,
"logps/rejected": -320.48455810546875,
"loss": 1.0636,
"margin_dpo/margin_mean": 87.10105895996094,
"margin_dpo/margin_std": 128.87863159179688,
"step": 595
},
{
"KL/chosen_KL_mean": -173.06805419921875,
"KL/mean": -190.773193359375,
"KL/rejected_KL_mean": -208.47833251953125,
"KL/std": 95.94255065917969,
"epoch": 0.9009826152683296,
"fcm_dpo/beta": 0.0060940892435610294,
"fcm_dpo/delta": 0.08267121016979218,
"fcm_dpo/margin": 35.410255432128906,
"fcm_dpo/q_t": 0.45276933908462524,
"grad_norm": 16.265701293945312,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": 0.6956692934036255,
"logits/rejected": 0.7181705236434937,
"logps/chosen": -241.54891967773438,
"logps/ref_chosen": -68.48088073730469,
"logps/ref_rejected": -61.732967376708984,
"logps/rejected": -270.2113037109375,
"loss": 1.274,
"margin_dpo/margin_mean": 35.410255432128906,
"margin_dpo/margin_std": 101.649169921875,
"step": 596
},
{
"KL/chosen_KL_mean": -132.4046173095703,
"KL/mean": -166.00335693359375,
"KL/rejected_KL_mean": -199.6020965576172,
"KL/std": 85.7354736328125,
"epoch": 0.9024943310657596,
"fcm_dpo/beta": 0.006142480298876762,
"fcm_dpo/delta": 0.0386665053665638,
"fcm_dpo/margin": 67.1974868774414,
"fcm_dpo/q_t": 0.40726274251937866,
"grad_norm": 11.388983726501465,
"learning_rate": 1.4627906988186111e-08,
"logits/chosen": 0.7288523316383362,
"logits/rejected": 0.7139878273010254,
"logps/chosen": -181.26211547851562,
"logps/ref_chosen": -48.85750961303711,
"logps/ref_rejected": -55.068084716796875,
"logps/rejected": -254.67018127441406,
"loss": 1.1035,
"margin_dpo/margin_mean": 67.19747924804688,
"margin_dpo/margin_std": 99.90011596679688,
"step": 597
},
{
"KL/chosen_KL_mean": -173.84255981445312,
"KL/mean": -195.55624389648438,
"KL/rejected_KL_mean": -217.26992797851562,
"KL/std": 101.8652572631836,
"epoch": 0.9040060468631897,
"fcm_dpo/beta": 0.00618097186088562,
"fcm_dpo/delta": 0.06402421742677689,
"fcm_dpo/margin": 43.42737579345703,
"fcm_dpo/q_t": 0.4409489929676056,
"grad_norm": 14.127278327941895,
"learning_rate": 1.4185553036259095e-08,
"logits/chosen": 0.740046501159668,
"logits/rejected": 0.6648821234703064,
"logps/chosen": -232.72970581054688,
"logps/ref_chosen": -58.88715362548828,
"logps/ref_rejected": -81.43145751953125,
"logps/rejected": -298.7013854980469,
"loss": 1.2401,
"margin_dpo/margin_mean": 43.42737579345703,
"margin_dpo/margin_std": 107.77723693847656,
"step": 598
},
{
"KL/chosen_KL_mean": -173.7288055419922,
"KL/mean": -199.68463134765625,
"KL/rejected_KL_mean": -225.64044189453125,
"KL/std": 97.53303527832031,
"epoch": 0.9055177626606198,
"fcm_dpo/beta": 0.006358982063829899,
"fcm_dpo/delta": 0.1232747808098793,
"fcm_dpo/margin": 51.91165542602539,
"fcm_dpo/q_t": 0.42809587717056274,
"grad_norm": 16.50663185119629,
"learning_rate": 1.3749795321332885e-08,
"logits/chosen": 0.79831862449646,
"logits/rejected": 0.7588490843772888,
"logps/chosen": -231.33599853515625,
"logps/ref_chosen": -57.60719299316406,
"logps/ref_rejected": -71.80469512939453,
"logps/rejected": -297.44512939453125,
"loss": 1.2006,
"margin_dpo/margin_mean": 51.911651611328125,
"margin_dpo/margin_std": 111.52732849121094,
"step": 599
},
{
"KL/chosen_KL_mean": -158.45321655273438,
"KL/mean": -189.308837890625,
"KL/rejected_KL_mean": -220.16444396972656,
"KL/std": 106.662109375,
"epoch": 0.9070294784580499,
"fcm_dpo/beta": 0.00634386483579874,
"fcm_dpo/delta": -0.05036870762705803,
"fcm_dpo/margin": 61.711246490478516,
"fcm_dpo/q_t": 0.41562899947166443,
"grad_norm": 18.35607147216797,
"learning_rate": 1.3320646032487393e-08,
"logits/chosen": 0.7958853244781494,
"logits/rejected": 0.7406545877456665,
"logps/chosen": -216.89552307128906,
"logps/ref_chosen": -58.44231414794922,
"logps/ref_rejected": -83.64639282226562,
"logps/rejected": -303.81085205078125,
"loss": 1.1548,
"margin_dpo/margin_mean": 61.71124267578125,
"margin_dpo/margin_std": 113.1817398071289,
"step": 600
},
{
"KL/chosen_KL_mean": -142.54322814941406,
"KL/mean": -184.44918823242188,
"KL/rejected_KL_mean": -226.35516357421875,
"KL/std": 111.73482513427734,
"epoch": 0.90854119425548,
"fcm_dpo/beta": 0.006213693879544735,
"fcm_dpo/delta": -0.07541825622320175,
"fcm_dpo/margin": 83.81192016601562,
"fcm_dpo/q_t": 0.3867299258708954,
"grad_norm": 12.669417381286621,
"learning_rate": 1.2898117173950868e-08,
"logits/chosen": 0.7443298101425171,
"logits/rejected": 0.669115424156189,
"logps/chosen": -198.13755798339844,
"logps/ref_chosen": -55.59432601928711,
"logps/ref_rejected": -83.68630981445312,
"logps/rejected": -310.04150390625,
"loss": 1.072,
"margin_dpo/margin_mean": 83.81192779541016,
"margin_dpo/margin_std": 126.72483825683594,
"step": 601
},
{
"KL/chosen_KL_mean": -132.24453735351562,
"KL/mean": -170.80950927734375,
"KL/rejected_KL_mean": -209.37448120117188,
"KL/std": 96.84636688232422,
"epoch": 0.91005291005291,
"fcm_dpo/beta": 0.0062125981785357,
"fcm_dpo/delta": -0.030824126675724983,
"fcm_dpo/margin": 77.12992858886719,
"fcm_dpo/q_t": 0.39198559522628784,
"grad_norm": 14.065460205078125,
"learning_rate": 1.2482220564763667e-08,
"logits/chosen": 0.780281126499176,
"logits/rejected": 0.7512023448944092,
"logps/chosen": -188.59371948242188,
"logps/ref_chosen": -56.349185943603516,
"logps/ref_rejected": -71.9959716796875,
"logps/rejected": -281.37042236328125,
"loss": 1.0584,
"margin_dpo/margin_mean": 77.12992858886719,
"margin_dpo/margin_std": 103.31945037841797,
"step": 602
},
{
"KL/chosen_KL_mean": -146.39669799804688,
"KL/mean": -184.16403198242188,
"KL/rejected_KL_mean": -221.93136596679688,
"KL/std": 96.0953140258789,
"epoch": 0.9115646258503401,
"fcm_dpo/beta": 0.006151379551738501,
"fcm_dpo/delta": -0.015490264631807804,
"fcm_dpo/margin": 75.53468322753906,
"fcm_dpo/q_t": 0.3961607813835144,
"grad_norm": 13.6494779586792,
"learning_rate": 1.2072967838448051e-08,
"logits/chosen": 0.7146183252334595,
"logits/rejected": 0.6660758256912231,
"logps/chosen": -199.56509399414062,
"logps/ref_chosen": -53.16838836669922,
"logps/ref_rejected": -73.8604736328125,
"logps/rejected": -295.7918701171875,
"loss": 1.0812,
"margin_dpo/margin_mean": 75.53468322753906,
"margin_dpo/margin_std": 110.68234252929688,
"step": 603
},
{
"KL/chosen_KL_mean": -142.59632873535156,
"KL/mean": -174.91241455078125,
"KL/rejected_KL_mean": -207.22850036621094,
"KL/std": 92.63813781738281,
"epoch": 0.9130763416477702,
"fcm_dpo/beta": 0.00621896144002676,
"fcm_dpo/delta": 0.04987533390522003,
"fcm_dpo/margin": 64.63217163085938,
"fcm_dpo/q_t": 0.41217827796936035,
"grad_norm": 16.16542625427246,
"learning_rate": 1.1670370442682459e-08,
"logits/chosen": 0.7201324701309204,
"logits/rejected": 0.7276899814605713,
"logps/chosen": -215.24575805664062,
"logps/ref_chosen": -72.64942169189453,
"logps/ref_rejected": -69.8792724609375,
"logps/rejected": -277.1077880859375,
"loss": 1.153,
"margin_dpo/margin_mean": 64.63217163085938,
"margin_dpo/margin_std": 120.90769958496094,
"step": 604
},
{
"KL/chosen_KL_mean": -160.7579345703125,
"KL/mean": -195.69827270507812,
"KL/rejected_KL_mean": -230.63864135742188,
"KL/std": 96.85035705566406,
"epoch": 0.9145880574452003,
"fcm_dpo/beta": 0.006217213813215494,
"fcm_dpo/delta": 0.016090279445052147,
"fcm_dpo/margin": 69.88069915771484,
"fcm_dpo/q_t": 0.40307921171188354,
"grad_norm": 15.524698257446289,
"learning_rate": 1.1274439638981532e-08,
"logits/chosen": 0.7722749710083008,
"logits/rejected": 0.7222751975059509,
"logps/chosen": -222.3707733154297,
"logps/ref_chosen": -61.61284637451172,
"logps/ref_rejected": -79.34398651123047,
"logps/rejected": -309.98260498046875,
"loss": 1.1192,
"margin_dpo/margin_mean": 69.88069915771484,
"margin_dpo/margin_std": 115.388671875,
"step": 605
},
{
"KL/chosen_KL_mean": -142.37429809570312,
"KL/mean": -184.49478149414062,
"KL/rejected_KL_mean": -226.61526489257812,
"KL/std": 100.42843627929688,
"epoch": 0.9160997732426304,
"fcm_dpo/beta": 0.006196511909365654,
"fcm_dpo/delta": -0.0759873166680336,
"fcm_dpo/margin": 84.24095916748047,
"fcm_dpo/q_t": 0.38542690873146057,
"grad_norm": 15.91579818725586,
"learning_rate": 1.0885186502381016e-08,
"logits/chosen": 0.710547685623169,
"logits/rejected": 0.648972749710083,
"logps/chosen": -196.83853149414062,
"logps/ref_chosen": -54.46424102783203,
"logps/ref_rejected": -79.62708282470703,
"logps/rejected": -306.2423400878906,
"loss": 1.0475,
"margin_dpo/margin_mean": 84.240966796875,
"margin_dpo/margin_std": 115.52052307128906,
"step": 606
},
{
"KL/chosen_KL_mean": -156.99851989746094,
"KL/mean": -193.15847778320312,
"KL/rejected_KL_mean": -229.3184051513672,
"KL/std": 97.73643493652344,
"epoch": 0.9176114890400605,
"fcm_dpo/beta": 0.006090829148888588,
"fcm_dpo/delta": 0.007767722010612488,
"fcm_dpo/margin": 72.31989288330078,
"fcm_dpo/q_t": 0.40162938833236694,
"grad_norm": 13.030501365661621,
"learning_rate": 1.0502621921127774e-08,
"logits/chosen": 0.7018817663192749,
"logits/rejected": 0.6800510287284851,
"logps/chosen": -219.85939025878906,
"logps/ref_chosen": -62.86086654663086,
"logps/ref_rejected": -72.5501937866211,
"logps/rejected": -301.86859130859375,
"loss": 1.1058,
"margin_dpo/margin_mean": 72.31988525390625,
"margin_dpo/margin_std": 109.04088592529297,
"step": 607
},
{
"KL/chosen_KL_mean": -160.4464111328125,
"KL/mean": -199.41481018066406,
"KL/rejected_KL_mean": -238.38323974609375,
"KL/std": 107.04619598388672,
"epoch": 0.9191232048374905,
"fcm_dpo/beta": 0.006139816716313362,
"fcm_dpo/delta": -0.029955772683024406,
"fcm_dpo/margin": 77.93682098388672,
"fcm_dpo/q_t": 0.39407122135162354,
"grad_norm": 13.91819953918457,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": 0.7119251489639282,
"logits/rejected": 0.6361026167869568,
"logps/chosen": -223.62713623046875,
"logps/ref_chosen": -63.18071746826172,
"logps/ref_rejected": -99.15888214111328,
"logps/rejected": -337.5421142578125,
"loss": 1.0655,
"margin_dpo/margin_mean": 77.93682098388672,
"margin_dpo/margin_std": 108.65492248535156,
"step": 608
},
{
"KL/chosen_KL_mean": -142.321533203125,
"KL/mean": -183.87535095214844,
"KL/rejected_KL_mean": -225.42916870117188,
"KL/std": 96.29520416259766,
"epoch": 0.9206349206349206,
"fcm_dpo/beta": 0.0060390448197722435,
"fcm_dpo/delta": -0.05546504259109497,
"fcm_dpo/margin": 83.10763549804688,
"fcm_dpo/q_t": 0.3844478726387024,
"grad_norm": 12.680366516113281,
"learning_rate": 9.757601041885694e-09,
"logits/chosen": 0.8203383684158325,
"logits/rejected": 0.7830733060836792,
"logps/chosen": -190.9447479248047,
"logps/ref_chosen": -48.62322235107422,
"logps/ref_rejected": -68.28271484375,
"logps/rejected": -293.7118835449219,
"loss": 1.0301,
"margin_dpo/margin_mean": 83.10763549804688,
"margin_dpo/margin_std": 95.72496795654297,
"step": 609
},
{
"KL/chosen_KL_mean": -155.60647583007812,
"KL/mean": -194.73910522460938,
"KL/rejected_KL_mean": -233.8717498779297,
"KL/std": 105.99610900878906,
"epoch": 0.9221466364323507,
"fcm_dpo/beta": 0.006009879522025585,
"fcm_dpo/delta": -0.021769745275378227,
"fcm_dpo/margin": 78.26527404785156,
"fcm_dpo/q_t": 0.39677220582962036,
"grad_norm": 14.142727851867676,
"learning_rate": 9.395165583732379e-09,
"logits/chosen": 0.7283831238746643,
"logits/rejected": 0.7220611572265625,
"logps/chosen": -228.27159118652344,
"logps/ref_chosen": -72.66513061523438,
"logps/ref_rejected": -87.15310668945312,
"logps/rejected": -321.02484130859375,
"loss": 1.0927,
"margin_dpo/margin_mean": 78.26527404785156,
"margin_dpo/margin_std": 121.27104187011719,
"step": 610
},
{
"KL/chosen_KL_mean": -143.68881225585938,
"KL/mean": -174.88534545898438,
"KL/rejected_KL_mean": -206.0819091796875,
"KL/std": 93.48977661132812,
"epoch": 0.9236583522297808,
"fcm_dpo/beta": 0.0060781436040997505,
"fcm_dpo/delta": 0.07361207157373428,
"fcm_dpo/margin": 62.39308166503906,
"fcm_dpo/q_t": 0.4132322669029236,
"grad_norm": 14.4437894821167,
"learning_rate": 9.03946036001449e-09,
"logits/chosen": 0.7655025124549866,
"logits/rejected": 0.7187894582748413,
"logps/chosen": -191.99737548828125,
"logps/ref_chosen": -48.30857849121094,
"logps/ref_rejected": -70.6141128540039,
"logps/rejected": -276.6960144042969,
"loss": 1.1224,
"margin_dpo/margin_mean": 62.3930778503418,
"margin_dpo/margin_std": 95.25655364990234,
"step": 611
},
{
"KL/chosen_KL_mean": -158.34677124023438,
"KL/mean": -204.57286071777344,
"KL/rejected_KL_mean": -250.7989501953125,
"KL/std": 100.17593383789062,
"epoch": 0.9251700680272109,
"fcm_dpo/beta": 0.006016192026436329,
"fcm_dpo/delta": -0.11244214326143265,
"fcm_dpo/margin": 92.45218658447266,
"fcm_dpo/q_t": 0.37426358461380005,
"grad_norm": 12.516098022460938,
"learning_rate": 8.690495320571839e-09,
"logits/chosen": 0.6408207416534424,
"logits/rejected": 0.5755380988121033,
"logps/chosen": -219.5783233642578,
"logps/ref_chosen": -61.23155975341797,
"logps/ref_rejected": -94.37979888916016,
"logps/rejected": -345.1787414550781,
"loss": 1.0041,
"margin_dpo/margin_mean": 92.45218658447266,
"margin_dpo/margin_std": 109.68734741210938,
"step": 612
},
{
"KL/chosen_KL_mean": -136.57814025878906,
"KL/mean": -178.27291870117188,
"KL/rejected_KL_mean": -219.96771240234375,
"KL/std": 107.65634155273438,
"epoch": 0.926681783824641,
"fcm_dpo/beta": 0.0059585003182291985,
"fcm_dpo/delta": -0.04942707717418671,
"fcm_dpo/margin": 83.38955688476562,
"fcm_dpo/q_t": 0.38701191544532776,
"grad_norm": 12.486554145812988,
"learning_rate": 8.348280226706722e-09,
"logits/chosen": 0.6936590075492859,
"logits/rejected": 0.6893400549888611,
"logps/chosen": -190.56124877929688,
"logps/ref_chosen": -53.98310852050781,
"logps/ref_rejected": -58.32208251953125,
"logps/rejected": -278.289794921875,
"loss": 1.0483,
"margin_dpo/margin_mean": 83.38955688476562,
"margin_dpo/margin_std": 108.76756286621094,
"step": 613
},
{
"KL/chosen_KL_mean": -156.5811004638672,
"KL/mean": -194.799560546875,
"KL/rejected_KL_mean": -233.0180206298828,
"KL/std": 92.09423828125,
"epoch": 0.9281934996220711,
"fcm_dpo/beta": 0.0059523265808820724,
"fcm_dpo/delta": -0.005938872694969177,
"fcm_dpo/margin": 76.43690490722656,
"fcm_dpo/q_t": 0.39591896533966064,
"grad_norm": 15.995917320251465,
"learning_rate": 8.012824650910937e-09,
"logits/chosen": 0.7563154101371765,
"logits/rejected": 0.7482579946517944,
"logps/chosen": -216.8241424560547,
"logps/ref_chosen": -60.24303436279297,
"logps/ref_rejected": -72.26258850097656,
"logps/rejected": -305.2806091308594,
"loss": 1.0741,
"margin_dpo/margin_mean": 76.43690490722656,
"margin_dpo/margin_std": 103.75457763671875,
"step": 614
},
{
"KL/chosen_KL_mean": -163.4049072265625,
"KL/mean": -206.7666473388672,
"KL/rejected_KL_mean": -250.12840270996094,
"KL/std": 120.01435852050781,
"epoch": 0.9297052154195011,
"fcm_dpo/beta": 0.005815756507217884,
"fcm_dpo/delta": -0.05904611572623253,
"fcm_dpo/margin": 86.7235107421875,
"fcm_dpo/q_t": 0.38891440629959106,
"grad_norm": 14.163253784179688,
"learning_rate": 7.684137976598088e-09,
"logits/chosen": 0.6970840692520142,
"logits/rejected": 0.6417770385742188,
"logps/chosen": -235.49957275390625,
"logps/ref_chosen": -72.09467315673828,
"logps/ref_rejected": -104.02980041503906,
"logps/rejected": -354.158203125,
"loss": 1.0777,
"margin_dpo/margin_mean": 86.7235107421875,
"margin_dpo/margin_std": 130.05892944335938,
"step": 615
},
{
"KL/chosen_KL_mean": -156.443603515625,
"KL/mean": -189.8626251220703,
"KL/rejected_KL_mean": -223.28164672851562,
"KL/std": 110.390869140625,
"epoch": 0.9312169312169312,
"fcm_dpo/beta": 0.005887066014111042,
"fcm_dpo/delta": 0.058877281844615936,
"fcm_dpo/margin": 66.8380126953125,
"fcm_dpo/q_t": 0.41178950667381287,
"grad_norm": 13.02852725982666,
"learning_rate": 7.36222939784098e-09,
"logits/chosen": 0.760650634765625,
"logits/rejected": 0.6907193660736084,
"logps/chosen": -214.97433471679688,
"logps/ref_chosen": -58.530723571777344,
"logps/ref_rejected": -75.48025512695312,
"logps/rejected": -298.76190185546875,
"loss": 1.1227,
"margin_dpo/margin_mean": 66.83802032470703,
"margin_dpo/margin_std": 106.22430419921875,
"step": 616
},
{
"KL/chosen_KL_mean": -164.33099365234375,
"KL/mean": -196.11001586914062,
"KL/rejected_KL_mean": -227.88905334472656,
"KL/std": 100.60113525390625,
"epoch": 0.9327286470143613,
"fcm_dpo/beta": 0.005934374872595072,
"fcm_dpo/delta": 0.07501719892024994,
"fcm_dpo/margin": 63.55805969238281,
"fcm_dpo/q_t": 0.4166525602340698,
"grad_norm": 16.678241729736328,
"learning_rate": 7.047107919114586e-09,
"logits/chosen": 0.7422701120376587,
"logits/rejected": 0.6935194730758667,
"logps/chosen": -221.93966674804688,
"logps/ref_chosen": -57.608673095703125,
"logps/ref_rejected": -81.22109985351562,
"logps/rejected": -309.11016845703125,
"loss": 1.1452,
"margin_dpo/margin_mean": 63.55805969238281,
"margin_dpo/margin_std": 107.30416870117188,
"step": 617
},
{
"KL/chosen_KL_mean": -151.17364501953125,
"KL/mean": -188.66285705566406,
"KL/rejected_KL_mean": -226.15208435058594,
"KL/std": 111.13240051269531,
"epoch": 0.9342403628117913,
"fcm_dpo/beta": 0.005985327996313572,
"fcm_dpo/delta": 0.0010824277997016907,
"fcm_dpo/margin": 74.97844696044922,
"fcm_dpo/q_t": 0.4020419120788574,
"grad_norm": 15.610330581665039,
"learning_rate": 6.738782355044048e-09,
"logits/chosen": 0.7219952940940857,
"logits/rejected": 0.6295669674873352,
"logps/chosen": -207.8695831298828,
"logps/ref_chosen": -56.69594192504883,
"logps/ref_rejected": -85.92362976074219,
"logps/rejected": -312.0757141113281,
"loss": 1.085,
"margin_dpo/margin_mean": 74.97845458984375,
"margin_dpo/margin_std": 110.57818603515625,
"step": 618
},
{
"KL/chosen_KL_mean": -145.59751892089844,
"KL/mean": -185.35507202148438,
"KL/rejected_KL_mean": -225.11264038085938,
"KL/std": 103.94949340820312,
"epoch": 0.9357520786092215,
"fcm_dpo/beta": 0.005967982113361359,
"fcm_dpo/delta": -0.025933532044291496,
"fcm_dpo/margin": 79.51509094238281,
"fcm_dpo/q_t": 0.39350175857543945,
"grad_norm": 13.841119766235352,
"learning_rate": 6.437261330158206e-09,
"logits/chosen": 0.8068792223930359,
"logits/rejected": 0.7342312335968018,
"logps/chosen": -199.65594482421875,
"logps/ref_chosen": -54.05841827392578,
"logps/ref_rejected": -83.55493927001953,
"logps/rejected": -308.6675720214844,
"loss": 1.0728,
"margin_dpo/margin_mean": 79.51509857177734,
"margin_dpo/margin_std": 114.1060562133789,
"step": 619
},
{
"KL/chosen_KL_mean": -148.67767333984375,
"KL/mean": -179.8364715576172,
"KL/rejected_KL_mean": -210.99526977539062,
"KL/std": 95.24564361572266,
"epoch": 0.9372637944066515,
"fcm_dpo/beta": 0.005929121747612953,
"fcm_dpo/delta": -0.07515688985586166,
"fcm_dpo/margin": 62.317569732666016,
"fcm_dpo/q_t": 0.41601672768592834,
"grad_norm": 14.274309158325195,
"learning_rate": 6.142553278648238e-09,
"logits/chosen": 0.7464447021484375,
"logits/rejected": 0.7477549314498901,
"logps/chosen": -212.04739379882812,
"logps/ref_chosen": -63.36971664428711,
"logps/ref_rejected": -65.68269348144531,
"logps/rejected": -276.6779479980469,
"loss": 1.1448,
"margin_dpo/margin_mean": 62.31756591796875,
"margin_dpo/margin_std": 99.23009490966797,
"step": 620
},
{
"KL/chosen_KL_mean": -147.39321899414062,
"KL/mean": -183.0356903076172,
"KL/rejected_KL_mean": -218.67816162109375,
"KL/std": 103.13090515136719,
"epoch": 0.9387755102040817,
"fcm_dpo/beta": 0.005933044478297234,
"fcm_dpo/delta": 0.027808479964733124,
"fcm_dpo/margin": 71.2849349975586,
"fcm_dpo/q_t": 0.4077685475349426,
"grad_norm": 13.713377952575684,
"learning_rate": 5.854666444131934e-09,
"logits/chosen": 0.764801025390625,
"logits/rejected": 0.6597084999084473,
"logps/chosen": -199.71444702148438,
"logps/ref_chosen": -52.321224212646484,
"logps/ref_rejected": -88.09001159667969,
"logps/rejected": -306.7681884765625,
"loss": 1.1263,
"margin_dpo/margin_mean": 71.2849349975586,
"margin_dpo/margin_std": 120.7091293334961,
"step": 621
},
{
"KL/chosen_KL_mean": -162.29052734375,
"KL/mean": -197.90365600585938,
"KL/rejected_KL_mean": -233.51678466796875,
"KL/std": 108.23118591308594,
"epoch": 0.9402872260015117,
"fcm_dpo/beta": 0.005962342023849487,
"fcm_dpo/delta": 0.026074469089508057,
"fcm_dpo/margin": 71.22624206542969,
"fcm_dpo/q_t": 0.4056706428527832,
"grad_norm": 16.656646728515625,
"learning_rate": 5.573608879422875e-09,
"logits/chosen": 0.6871777772903442,
"logits/rejected": 0.6430974006652832,
"logps/chosen": -222.15599060058594,
"logps/ref_chosen": -59.86545944213867,
"logps/ref_rejected": -81.86668395996094,
"logps/rejected": -315.38348388671875,
"loss": 1.1095,
"margin_dpo/margin_mean": 71.22624969482422,
"margin_dpo/margin_std": 111.18820190429688,
"step": 622
},
{
"KL/chosen_KL_mean": -171.44595336914062,
"KL/mean": -209.40542602539062,
"KL/rejected_KL_mean": -247.36492919921875,
"KL/std": 113.62255096435547,
"epoch": 0.9417989417989417,
"fcm_dpo/beta": 0.005917124915868044,
"fcm_dpo/delta": 0.00023915618658065796,
"fcm_dpo/margin": 75.9189682006836,
"fcm_dpo/q_t": 0.39986979961395264,
"grad_norm": 14.932882308959961,
"learning_rate": 5.299388446305342e-09,
"logits/chosen": 0.7615326046943665,
"logits/rejected": 0.7113196849822998,
"logps/chosen": -238.8144073486328,
"logps/ref_chosen": -67.36846160888672,
"logps/ref_rejected": -82.02733612060547,
"logps/rejected": -329.39227294921875,
"loss": 1.0966,
"margin_dpo/margin_mean": 75.9189682006836,
"margin_dpo/margin_std": 115.97571563720703,
"step": 623
},
{
"KL/chosen_KL_mean": -144.95953369140625,
"KL/mean": -184.875,
"KL/rejected_KL_mean": -224.79046630859375,
"KL/std": 106.45075988769531,
"epoch": 0.9433106575963719,
"fcm_dpo/beta": 0.00591125525534153,
"fcm_dpo/delta": -0.023217620328068733,
"fcm_dpo/margin": 79.8309326171875,
"fcm_dpo/q_t": 0.3964461088180542,
"grad_norm": 14.505139350891113,
"learning_rate": 5.03201281531429e-09,
"logits/chosen": 0.7227747440338135,
"logits/rejected": 0.6352113485336304,
"logps/chosen": -195.986083984375,
"logps/ref_chosen": -51.02655029296875,
"logps/ref_rejected": -76.49203491210938,
"logps/rejected": -301.282470703125,
"loss": 1.0784,
"margin_dpo/margin_mean": 79.8309326171875,
"margin_dpo/margin_std": 116.97268676757812,
"step": 624
},
{
"KL/chosen_KL_mean": -147.682861328125,
"KL/mean": -178.62127685546875,
"KL/rejected_KL_mean": -209.5596923828125,
"KL/std": 101.56578063964844,
"epoch": 0.9448223733938019,
"fcm_dpo/beta": 0.005996185354888439,
"fcm_dpo/delta": 0.08198390901088715,
"fcm_dpo/margin": 61.876800537109375,
"fcm_dpo/q_t": 0.42064088582992554,
"grad_norm": 13.062463760375977,
"learning_rate": 4.7714894655209174e-09,
"logits/chosen": 0.8105379343032837,
"logits/rejected": 0.7237575054168701,
"logps/chosen": -201.89048767089844,
"logps/ref_chosen": -54.20761489868164,
"logps/ref_rejected": -84.93669128417969,
"logps/rejected": -294.49639892578125,
"loss": 1.1721,
"margin_dpo/margin_mean": 61.87679672241211,
"margin_dpo/margin_std": 122.6982421875,
"step": 625
},
{
"KL/chosen_KL_mean": -138.61166381835938,
"KL/mean": -181.34934997558594,
"KL/rejected_KL_mean": -224.0870361328125,
"KL/std": 108.27032470703125,
"epoch": 0.9463340891912321,
"fcm_dpo/beta": 0.005954947788268328,
"fcm_dpo/delta": -0.06229614093899727,
"fcm_dpo/margin": 85.47537231445312,
"fcm_dpo/q_t": 0.391404390335083,
"grad_norm": 14.3599271774292,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": 0.8098399639129639,
"logits/rejected": 0.6851488351821899,
"logps/chosen": -183.67367553710938,
"logps/ref_chosen": -45.06201934814453,
"logps/ref_rejected": -89.66368103027344,
"logps/rejected": -313.750732421875,
"loss": 1.0736,
"margin_dpo/margin_mean": 85.47537231445312,
"margin_dpo/margin_std": 131.77513122558594,
"step": 626
},
{
"KL/chosen_KL_mean": -158.8922119140625,
"KL/mean": -203.9259033203125,
"KL/rejected_KL_mean": -248.9595947265625,
"KL/std": 111.73031616210938,
"epoch": 0.9478458049886621,
"fcm_dpo/beta": 0.0058575589209795,
"fcm_dpo/delta": -0.08222895115613937,
"fcm_dpo/margin": 90.06739044189453,
"fcm_dpo/q_t": 0.3802756369113922,
"grad_norm": 14.234342575073242,
"learning_rate": 4.271028567242818e-09,
"logits/chosen": 0.6816971898078918,
"logits/rejected": 0.5688222646713257,
"logps/chosen": -217.68325805664062,
"logps/ref_chosen": -58.791053771972656,
"logps/ref_rejected": -94.90802001953125,
"logps/rejected": -343.86761474609375,
"loss": 1.0285,
"margin_dpo/margin_mean": 90.06739044189453,
"margin_dpo/margin_std": 113.25086975097656,
"step": 627
},
{
"KL/chosen_KL_mean": -146.272216796875,
"KL/mean": -189.48312377929688,
"KL/rejected_KL_mean": -232.6940460205078,
"KL/std": 106.63853454589844,
"epoch": 0.9493575207860923,
"fcm_dpo/beta": 0.005843169055879116,
"fcm_dpo/delta": -0.0587296336889267,
"fcm_dpo/margin": 86.42182159423828,
"fcm_dpo/q_t": 0.38549649715423584,
"grad_norm": 14.816391944885254,
"learning_rate": 4.0311050177251895e-09,
"logits/chosen": 0.7511180639266968,
"logits/rejected": 0.7128807306289673,
"logps/chosen": -199.0758056640625,
"logps/ref_chosen": -52.80357360839844,
"logps/ref_rejected": -76.49468994140625,
"logps/rejected": -309.188720703125,
"loss": 1.0765,
"margin_dpo/margin_mean": 86.42181396484375,
"margin_dpo/margin_std": 121.38540649414062,
"step": 628
},
{
"KL/chosen_KL_mean": -155.6796112060547,
"KL/mean": -187.944091796875,
"KL/rejected_KL_mean": -220.2086181640625,
"KL/std": 97.90316772460938,
"epoch": 0.9508692365835223,
"fcm_dpo/beta": 0.00585499033331871,
"fcm_dpo/delta": 0.07480396330356598,
"fcm_dpo/margin": 64.52902221679688,
"fcm_dpo/q_t": 0.4162479639053345,
"grad_norm": 13.809697151184082,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": 0.7253426313400269,
"logits/rejected": 0.7244319915771484,
"logps/chosen": -226.3970947265625,
"logps/ref_chosen": -70.71749877929688,
"logps/ref_rejected": -78.96273803710938,
"logps/rejected": -299.1713562011719,
"loss": 1.1289,
"margin_dpo/margin_mean": 64.52902221679688,
"margin_dpo/margin_std": 103.2763671875,
"step": 629
},
{
"KL/chosen_KL_mean": -145.9853515625,
"KL/mean": -188.92074584960938,
"KL/rejected_KL_mean": -231.8561553955078,
"KL/std": 106.13909149169922,
"epoch": 0.9523809523809523,
"fcm_dpo/beta": 0.005835742689669132,
"fcm_dpo/delta": -0.053825244307518005,
"fcm_dpo/margin": 85.87080383300781,
"fcm_dpo/q_t": 0.38744089007377625,
"grad_norm": 11.524528503417969,
"learning_rate": 3.5719052736323806e-09,
"logits/chosen": 0.6896286010742188,
"logits/rejected": 0.6513473391532898,
"logps/chosen": -202.186767578125,
"logps/ref_chosen": -56.201412200927734,
"logps/ref_rejected": -74.69807434082031,
"logps/rejected": -306.5542297363281,
"loss": 1.048,
"margin_dpo/margin_mean": 85.87081146240234,
"margin_dpo/margin_std": 113.66392517089844,
"step": 630
},
{
"KL/chosen_KL_mean": -139.0296630859375,
"KL/mean": -189.2365264892578,
"KL/rejected_KL_mean": -239.44338989257812,
"KL/std": 112.87970733642578,
"epoch": 0.9538926681783825,
"fcm_dpo/beta": 0.005641968920826912,
"fcm_dpo/delta": -0.12655234336853027,
"fcm_dpo/margin": 100.41372680664062,
"fcm_dpo/q_t": 0.37543022632598877,
"grad_norm": 13.914621353149414,
"learning_rate": 3.352641923861144e-09,
"logits/chosen": 0.8196430206298828,
"logits/rejected": 0.7138614654541016,
"logps/chosen": -197.85025024414062,
"logps/ref_chosen": -58.82059860229492,
"logps/ref_rejected": -96.51437377929688,
"logps/rejected": -335.957763671875,
"loss": 1.0156,
"margin_dpo/margin_mean": 100.41372680664062,
"margin_dpo/margin_std": 124.60462951660156,
"step": 631
},
{
"KL/chosen_KL_mean": -142.07379150390625,
"KL/mean": -187.84475708007812,
"KL/rejected_KL_mean": -233.61572265625,
"KL/std": 99.48883056640625,
"epoch": 0.9554043839758125,
"fcm_dpo/beta": 0.005615268833935261,
"fcm_dpo/delta": -0.06745753437280655,
"fcm_dpo/margin": 91.54193878173828,
"fcm_dpo/q_t": 0.3823985457420349,
"grad_norm": 13.333562850952148,
"learning_rate": 3.140277830901428e-09,
"logits/chosen": 0.7539972066879272,
"logits/rejected": 0.7377300262451172,
"logps/chosen": -200.85983276367188,
"logps/ref_chosen": -58.786048889160156,
"logps/ref_rejected": -67.21923828125,
"logps/rejected": -300.8349609375,
"loss": 1.0209,
"margin_dpo/margin_mean": 91.54194641113281,
"margin_dpo/margin_std": 106.66963195800781,
"step": 632
},
{
"KL/chosen_KL_mean": -137.31719970703125,
"KL/mean": -173.36279296875,
"KL/rejected_KL_mean": -209.40838623046875,
"KL/std": 96.00003051757812,
"epoch": 0.9569160997732427,
"fcm_dpo/beta": 0.005642236210405827,
"fcm_dpo/delta": 0.04457578808069229,
"fcm_dpo/margin": 72.09117889404297,
"fcm_dpo/q_t": 0.4096265137195587,
"grad_norm": 13.390053749084473,
"learning_rate": 2.9348189350335007e-09,
"logits/chosen": 0.7287853956222534,
"logits/rejected": 0.6764748096466064,
"logps/chosen": -189.4473876953125,
"logps/ref_chosen": -52.13019561767578,
"logps/ref_rejected": -67.23016357421875,
"logps/rejected": -276.6385498046875,
"loss": 1.1101,
"margin_dpo/margin_mean": 72.09117889404297,
"margin_dpo/margin_std": 109.14178466796875,
"step": 633
},
{
"KL/chosen_KL_mean": -165.64334106445312,
"KL/mean": -179.42112731933594,
"KL/rejected_KL_mean": -193.19894409179688,
"KL/std": 99.49234008789062,
"epoch": 0.9584278155706727,
"fcm_dpo/beta": 0.005632580257952213,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 27.555599212646484,
"fcm_dpo/q_t": 0.4662253260612488,
"grad_norm": 16.822216033935547,
"learning_rate": 2.736270983384276e-09,
"logits/chosen": 0.8007980585098267,
"logits/rejected": 0.8156324625015259,
"logps/chosen": -226.62313842773438,
"logps/ref_chosen": -60.97979736328125,
"logps/ref_rejected": -58.50825119018555,
"logps/rejected": -251.70718383789062,
"loss": 1.3355,
"margin_dpo/margin_mean": 27.555599212646484,
"margin_dpo/margin_std": 114.27915954589844,
"step": 634
},
{
"KL/chosen_KL_mean": -180.67120361328125,
"KL/mean": -209.85873413085938,
"KL/rejected_KL_mean": -239.0462646484375,
"KL/std": 98.8995361328125,
"epoch": 0.9599395313681028,
"fcm_dpo/beta": 0.0057508512400090694,
"fcm_dpo/delta": 0.11767241358757019,
"fcm_dpo/margin": 58.37506866455078,
"fcm_dpo/q_t": 0.4253365993499756,
"grad_norm": 12.797456741333008,
"learning_rate": 2.5446395297668287e-09,
"logits/chosen": 0.6555688381195068,
"logits/rejected": 0.6032085418701172,
"logps/chosen": -246.64422607421875,
"logps/ref_chosen": -65.9730224609375,
"logps/ref_rejected": -85.61317443847656,
"logps/rejected": -324.659423828125,
"loss": 1.1911,
"margin_dpo/margin_mean": 58.37506866455078,
"margin_dpo/margin_std": 119.31085205078125,
"step": 635
},
{
"KL/chosen_KL_mean": -141.2942657470703,
"KL/mean": -181.30242919921875,
"KL/rejected_KL_mean": -221.31060791015625,
"KL/std": 100.77838134765625,
"epoch": 0.9614512471655329,
"fcm_dpo/beta": 0.005744201131165028,
"fcm_dpo/delta": -0.010235387831926346,
"fcm_dpo/margin": 80.01634979248047,
"fcm_dpo/q_t": 0.39512956142425537,
"grad_norm": 10.96211051940918,
"learning_rate": 2.359929934524829e-09,
"logits/chosen": 0.7572035789489746,
"logits/rejected": 0.661196231842041,
"logps/chosen": -190.43443298339844,
"logps/ref_chosen": -49.140167236328125,
"logps/ref_rejected": -81.26971435546875,
"logps/rejected": -302.580322265625,
"loss": 1.0661,
"margin_dpo/margin_mean": 80.01634979248047,
"margin_dpo/margin_std": 106.34614562988281,
"step": 636
},
{
"KL/chosen_KL_mean": -188.81138610839844,
"KL/mean": -218.619873046875,
"KL/rejected_KL_mean": -248.4283447265625,
"KL/std": 99.71638488769531,
"epoch": 0.9629629629629629,
"fcm_dpo/beta": 0.005864589475095272,
"fcm_dpo/delta": 0.10340671241283417,
"fcm_dpo/margin": 59.616939544677734,
"fcm_dpo/q_t": 0.4231225848197937,
"grad_norm": 14.910876274108887,
"learning_rate": 2.1821473643827137e-09,
"logits/chosen": 0.7136276960372925,
"logits/rejected": 0.6566417217254639,
"logps/chosen": -262.5079650878906,
"logps/ref_chosen": -73.69658660888672,
"logps/ref_rejected": -83.01487731933594,
"logps/rejected": -331.4432373046875,
"loss": 1.1901,
"margin_dpo/margin_mean": 59.616943359375,
"margin_dpo/margin_std": 124.95484924316406,
"step": 637
},
{
"KL/chosen_KL_mean": -163.7374267578125,
"KL/mean": -199.96839904785156,
"KL/rejected_KL_mean": -236.19937133789062,
"KL/std": 100.8640365600586,
"epoch": 0.9644746787603931,
"fcm_dpo/beta": 0.005893657449632883,
"fcm_dpo/delta": 0.023963581770658493,
"fcm_dpo/margin": 72.46197509765625,
"fcm_dpo/q_t": 0.40526115894317627,
"grad_norm": 13.63633918762207,
"learning_rate": 2.0112967923011646e-09,
"logits/chosen": 0.7319703698158264,
"logits/rejected": 0.6906919479370117,
"logps/chosen": -226.5189971923828,
"logps/ref_chosen": -62.78158187866211,
"logps/ref_rejected": -85.40478515625,
"logps/rejected": -321.6041564941406,
"loss": 1.1007,
"margin_dpo/margin_mean": 72.46197509765625,
"margin_dpo/margin_std": 110.03543090820312,
"step": 638
},
{
"KL/chosen_KL_mean": -146.77230834960938,
"KL/mean": -188.85699462890625,
"KL/rejected_KL_mean": -230.94168090820312,
"KL/std": 107.85531616210938,
"epoch": 0.9659863945578231,
"fcm_dpo/beta": 0.005871212109923363,
"fcm_dpo/delta": -0.04644831269979477,
"fcm_dpo/margin": 84.16936492919922,
"fcm_dpo/q_t": 0.39135512709617615,
"grad_norm": 17.854429244995117,
"learning_rate": 1.847382997337943e-09,
"logits/chosen": 0.7332069873809814,
"logits/rejected": 0.6473530530929565,
"logps/chosen": -200.53887939453125,
"logps/ref_chosen": -53.76658630371094,
"logps/ref_rejected": -72.30009460449219,
"logps/rejected": -303.24176025390625,
"loss": 1.0569,
"margin_dpo/margin_mean": 84.16937255859375,
"margin_dpo/margin_std": 116.0023422241211,
"step": 639
},
{
"KL/chosen_KL_mean": -153.05740356445312,
"KL/mean": -189.9259796142578,
"KL/rejected_KL_mean": -226.7945556640625,
"KL/std": 101.85699462890625,
"epoch": 0.9674981103552532,
"fcm_dpo/beta": 0.005848909728229046,
"fcm_dpo/delta": 0.01944734714925289,
"fcm_dpo/margin": 73.73715209960938,
"fcm_dpo/q_t": 0.40298590064048767,
"grad_norm": 13.762663841247559,
"learning_rate": 1.690410564514244e-09,
"logits/chosen": 0.7831774950027466,
"logits/rejected": 0.7236679792404175,
"logps/chosen": -204.47518920898438,
"logps/ref_chosen": -51.41777801513672,
"logps/ref_rejected": -77.27879333496094,
"logps/rejected": -304.0733642578125,
"loss": 1.1012,
"margin_dpo/margin_mean": 73.73715209960938,
"margin_dpo/margin_std": 112.12767028808594,
"step": 640
},
{
"KL/chosen_KL_mean": -159.89456176757812,
"KL/mean": -193.94393920898438,
"KL/rejected_KL_mean": -227.99334716796875,
"KL/std": 94.81710815429688,
"epoch": 0.9690098261526833,
"fcm_dpo/beta": 0.005902102682739496,
"fcm_dpo/delta": 0.05009445920586586,
"fcm_dpo/margin": 68.09879302978516,
"fcm_dpo/q_t": 0.40735727548599243,
"grad_norm": 14.137347221374512,
"learning_rate": 1.5403838846864692e-09,
"logits/chosen": 0.6990131139755249,
"logits/rejected": 0.6868042945861816,
"logps/chosen": -230.94921875,
"logps/ref_chosen": -71.0546646118164,
"logps/ref_rejected": -82.2440185546875,
"logps/rejected": -310.23736572265625,
"loss": 1.0898,
"margin_dpo/margin_mean": 68.09879302978516,
"margin_dpo/margin_std": 89.69477081298828,
"step": 641
},
{
"KL/chosen_KL_mean": -167.1101837158203,
"KL/mean": -193.14244079589844,
"KL/rejected_KL_mean": -219.17469787597656,
"KL/std": 93.20387268066406,
"epoch": 0.9705215419501134,
"fcm_dpo/beta": 0.00595252588391304,
"fcm_dpo/delta": 0.018339350819587708,
"fcm_dpo/margin": 52.064510345458984,
"fcm_dpo/q_t": 0.43100064992904663,
"grad_norm": 14.998382568359375,
"learning_rate": 1.3973071544233218e-09,
"logits/chosen": 0.6767123937606812,
"logits/rejected": 0.6978293657302856,
"logps/chosen": -236.03945922851562,
"logps/ref_chosen": -68.92927551269531,
"logps/ref_rejected": -70.85682678222656,
"logps/rejected": -290.0315246582031,
"loss": 1.2075,
"margin_dpo/margin_mean": 52.064510345458984,
"margin_dpo/margin_std": 112.48196411132812,
"step": 642
},
{
"KL/chosen_KL_mean": -152.78616333007812,
"KL/mean": -192.1639404296875,
"KL/rejected_KL_mean": -231.5417022705078,
"KL/std": 100.897216796875,
"epoch": 0.9720332577475435,
"fcm_dpo/beta": 0.005934578832238913,
"fcm_dpo/delta": -0.018240533769130707,
"fcm_dpo/margin": 78.75552368164062,
"fcm_dpo/q_t": 0.3963284492492676,
"grad_norm": 19.15582847595215,
"learning_rate": 1.261184375888541e-09,
"logits/chosen": 0.7067471742630005,
"logits/rejected": 0.627659261226654,
"logps/chosen": -218.09521484375,
"logps/ref_chosen": -65.30903625488281,
"logps/ref_rejected": -83.61613464355469,
"logps/rejected": -315.1578369140625,
"loss": 1.0852,
"margin_dpo/margin_mean": 78.75552368164062,
"margin_dpo/margin_std": 117.13465881347656,
"step": 643
},
{
"KL/chosen_KL_mean": -144.2310333251953,
"KL/mean": -170.76400756835938,
"KL/rejected_KL_mean": -197.2969970703125,
"KL/std": 90.6378402709961,
"epoch": 0.9735449735449735,
"fcm_dpo/beta": 0.005933812819421291,
"fcm_dpo/delta": 0.004993738606572151,
"fcm_dpo/margin": 53.065940856933594,
"fcm_dpo/q_t": 0.42780280113220215,
"grad_norm": 11.877394676208496,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": 0.8293318748474121,
"logits/rejected": 0.7982121706008911,
"logps/chosen": -195.233642578125,
"logps/ref_chosen": -51.002601623535156,
"logps/ref_rejected": -64.46372985839844,
"logps/rejected": -261.7607421875,
"loss": 1.2014,
"margin_dpo/margin_mean": 53.06593704223633,
"margin_dpo/margin_std": 110.60516357421875,
"step": 644
},
{
"KL/chosen_KL_mean": -157.09957885742188,
"KL/mean": -190.3580780029297,
"KL/rejected_KL_mean": -223.61656188964844,
"KL/std": 98.42587280273438,
"epoch": 0.9750566893424036,
"fcm_dpo/beta": 0.005985003896057606,
"fcm_dpo/delta": 0.05404530465602875,
"fcm_dpo/margin": 66.51698303222656,
"fcm_dpo/q_t": 0.409078985452652,
"grad_norm": 15.517080307006836,
"learning_rate": 1.0098157099674987e-09,
"logits/chosen": 0.6929702758789062,
"logits/rejected": 0.6790943145751953,
"logps/chosen": -218.06298828125,
"logps/ref_chosen": -60.963409423828125,
"logps/ref_rejected": -69.73353576660156,
"logps/rejected": -293.35009765625,
"loss": 1.111,
"margin_dpo/margin_mean": 66.51698303222656,
"margin_dpo/margin_std": 100.13095092773438,
"step": 645
},
{
"KL/chosen_KL_mean": -173.18130493164062,
"KL/mean": -204.287841796875,
"KL/rejected_KL_mean": -235.39434814453125,
"KL/std": 114.00927734375,
"epoch": 0.9765684051398337,
"fcm_dpo/beta": 0.006071906536817551,
"fcm_dpo/delta": 0.07504133880138397,
"fcm_dpo/margin": 62.21302032470703,
"fcm_dpo/q_t": 0.41674911975860596,
"grad_norm": 13.308674812316895,
"learning_rate": 8.945768539031783e-10,
"logits/chosen": 0.7898342609405518,
"logits/rejected": 0.732674241065979,
"logps/chosen": -235.47137451171875,
"logps/ref_chosen": -62.290069580078125,
"logps/ref_rejected": -85.54812622070312,
"logps/rejected": -320.9424743652344,
"loss": 1.1574,
"margin_dpo/margin_mean": 62.21302795410156,
"margin_dpo/margin_std": 115.69244384765625,
"step": 646
},
{
"KL/chosen_KL_mean": -169.27145385742188,
"KL/mean": -218.55142211914062,
"KL/rejected_KL_mean": -267.83135986328125,
"KL/std": 112.50706481933594,
"epoch": 0.9780801209372638,
"fcm_dpo/beta": 0.005969959311187267,
"fcm_dpo/delta": -0.14687490463256836,
"fcm_dpo/margin": 98.55989837646484,
"fcm_dpo/q_t": 0.36799299716949463,
"grad_norm": 15.381766319274902,
"learning_rate": 7.863060120144316e-10,
"logits/chosen": 0.7591278553009033,
"logits/rejected": 0.6683089733123779,
"logps/chosen": -236.78732299804688,
"logps/ref_chosen": -67.515869140625,
"logps/ref_rejected": -101.50871276855469,
"logps/rejected": -369.340087890625,
"loss": 0.982,
"margin_dpo/margin_mean": 98.55990600585938,
"margin_dpo/margin_std": 111.36669921875,
"step": 647
},
{
"KL/chosen_KL_mean": -169.76194763183594,
"KL/mean": -204.1524658203125,
"KL/rejected_KL_mean": -238.54298400878906,
"KL/std": 98.05025482177734,
"epoch": 0.9795918367346939,
"fcm_dpo/beta": 0.005927952937781811,
"fcm_dpo/delta": 0.04385180026292801,
"fcm_dpo/margin": 68.78103637695312,
"fcm_dpo/q_t": 0.41050416231155396,
"grad_norm": 13.526182174682617,
"learning_rate": 6.850062128694045e-10,
"logits/chosen": 0.6800918579101562,
"logits/rejected": 0.6235531568527222,
"logps/chosen": -234.35787963867188,
"logps/ref_chosen": -64.59593963623047,
"logps/ref_rejected": -83.384033203125,
"logps/rejected": -321.9270324707031,
"loss": 1.1538,
"margin_dpo/margin_mean": 68.78103637695312,
"margin_dpo/margin_std": 125.53382873535156,
"step": 648
},
{
"KL/chosen_KL_mean": -153.29359436035156,
"KL/mean": -189.05458068847656,
"KL/rejected_KL_mean": -224.81556701660156,
"KL/std": 97.22737121582031,
"epoch": 0.981103552532124,
"fcm_dpo/beta": 0.005967048928141594,
"fcm_dpo/delta": 0.024066461250185966,
"fcm_dpo/margin": 71.52197265625,
"fcm_dpo/q_t": 0.4042567014694214,
"grad_norm": 17.365737915039062,
"learning_rate": 5.906802900412788e-10,
"logits/chosen": 0.762288510799408,
"logits/rejected": 0.7033209800720215,
"logps/chosen": -202.60324096679688,
"logps/ref_chosen": -49.30964660644531,
"logps/ref_rejected": -73.73710632324219,
"logps/rejected": -298.55267333984375,
"loss": 1.118,
"margin_dpo/margin_mean": 71.52197265625,
"margin_dpo/margin_std": 116.39321899414062,
"step": 649
},
{
"KL/chosen_KL_mean": -149.021728515625,
"KL/mean": -186.00106811523438,
"KL/rejected_KL_mean": -222.98040771484375,
"KL/std": 99.01878356933594,
"epoch": 0.982615268329554,
"fcm_dpo/beta": 0.005991187412291765,
"fcm_dpo/delta": 0.007154183462262154,
"fcm_dpo/margin": 73.95868682861328,
"fcm_dpo/q_t": 0.4018860459327698,
"grad_norm": 13.914726257324219,
"learning_rate": 5.033308820289184e-10,
"logits/chosen": 0.7910748720169067,
"logits/rejected": 0.732347309589386,
"logps/chosen": -204.08499145507812,
"logps/ref_chosen": -55.06325912475586,
"logps/ref_rejected": -77.39610290527344,
"logps/rejected": -300.37652587890625,
"loss": 1.1102,
"margin_dpo/margin_mean": 73.95867919921875,
"margin_dpo/margin_std": 119.37321472167969,
"step": 650
},
{
"KL/chosen_KL_mean": -147.79086303710938,
"KL/mean": -179.89984130859375,
"KL/rejected_KL_mean": -212.00881958007812,
"KL/std": 98.75411987304688,
"epoch": 0.9841269841269841,
"fcm_dpo/beta": 0.006077755242586136,
"fcm_dpo/delta": 0.061789728701114655,
"fcm_dpo/margin": 64.21796417236328,
"fcm_dpo/q_t": 0.4144824147224426,
"grad_norm": 13.73145580291748,
"learning_rate": 4.2296043218295606e-10,
"logits/chosen": 0.783795952796936,
"logits/rejected": 0.7152203321456909,
"logps/chosen": -201.85601806640625,
"logps/ref_chosen": -54.065162658691406,
"logps/ref_rejected": -77.79080200195312,
"logps/rejected": -289.79962158203125,
"loss": 1.1266,
"margin_dpo/margin_mean": 64.21795654296875,
"margin_dpo/margin_std": 103.98927307128906,
"step": 651
},
{
"KL/chosen_KL_mean": -174.13955688476562,
"KL/mean": -204.1993408203125,
"KL/rejected_KL_mean": -234.25912475585938,
"KL/std": 107.51026153564453,
"epoch": 0.9856386999244142,
"fcm_dpo/beta": 0.006125830113887787,
"fcm_dpo/delta": 0.0847388356924057,
"fcm_dpo/margin": 60.11957931518555,
"fcm_dpo/q_t": 0.42180266976356506,
"grad_norm": 15.607612609863281,
"learning_rate": 3.4957118863768176e-10,
"logits/chosen": 0.7536121606826782,
"logits/rejected": 0.7091095447540283,
"logps/chosen": -237.77984619140625,
"logps/ref_chosen": -63.64030456542969,
"logps/ref_rejected": -78.86882019042969,
"logps/rejected": -313.1279602050781,
"loss": 1.192,
"margin_dpo/margin_mean": 60.11957550048828,
"margin_dpo/margin_std": 127.72785949707031,
"step": 652
},
{
"KL/chosen_KL_mean": -154.40162658691406,
"KL/mean": -190.7852783203125,
"KL/rejected_KL_mean": -227.1689453125,
"KL/std": 96.48133850097656,
"epoch": 0.9871504157218443,
"fcm_dpo/beta": 0.0062039922922849655,
"fcm_dpo/delta": -0.001763913780450821,
"fcm_dpo/margin": 72.76734161376953,
"fcm_dpo/q_t": 0.3998035192489624,
"grad_norm": 15.49831771850586,
"learning_rate": 2.831652042480093e-10,
"logits/chosen": 0.7264477014541626,
"logits/rejected": 0.6846983432769775,
"logps/chosen": -216.07000732421875,
"logps/ref_chosen": -61.668373107910156,
"logps/ref_rejected": -73.83012390136719,
"logps/rejected": -300.99908447265625,
"loss": 1.0966,
"margin_dpo/margin_mean": 72.76734161376953,
"margin_dpo/margin_std": 111.39743041992188,
"step": 653
},
{
"KL/chosen_KL_mean": -161.70181274414062,
"KL/mean": -195.36509704589844,
"KL/rejected_KL_mean": -229.02838134765625,
"KL/std": 114.16594696044922,
"epoch": 0.9886621315192744,
"fcm_dpo/beta": 0.006077038589864969,
"fcm_dpo/delta": -0.08529359847307205,
"fcm_dpo/margin": 67.32656860351562,
"fcm_dpo/q_t": 0.40859442949295044,
"grad_norm": 14.157605171203613,
"learning_rate": 2.2374433653205016e-10,
"logits/chosen": 0.7062954306602478,
"logits/rejected": 0.6156010627746582,
"logps/chosen": -219.27008056640625,
"logps/ref_chosen": -57.568267822265625,
"logps/ref_rejected": -87.74789428710938,
"logps/rejected": -316.7762756347656,
"loss": 1.1233,
"margin_dpo/margin_mean": 67.32656860351562,
"margin_dpo/margin_std": 105.35501098632812,
"step": 654
},
{
"KL/chosen_KL_mean": -127.7425308227539,
"KL/mean": -176.96739196777344,
"KL/rejected_KL_mean": -226.1922607421875,
"KL/std": 99.56523895263672,
"epoch": 0.9901738473167044,
"fcm_dpo/beta": 0.0058916169218719006,
"fcm_dpo/delta": -0.14149729907512665,
"fcm_dpo/margin": 98.44974517822266,
"fcm_dpo/q_t": 0.3675011992454529,
"grad_norm": 12.291767120361328,
"learning_rate": 1.7131024761923852e-10,
"logits/chosen": 0.6882774829864502,
"logits/rejected": 0.6106013059616089,
"logps/chosen": -179.88967895507812,
"logps/ref_chosen": -52.14714813232422,
"logps/ref_rejected": -80.85014343261719,
"logps/rejected": -307.04241943359375,
"loss": 0.9693,
"margin_dpo/margin_mean": 98.44974517822266,
"margin_dpo/margin_std": 96.58932495117188,
"step": 655
},
{
"KL/chosen_KL_mean": -167.70416259765625,
"KL/mean": -204.19580078125,
"KL/rejected_KL_mean": -240.68743896484375,
"KL/std": 101.8294677734375,
"epoch": 0.9916855631141346,
"fcm_dpo/beta": 0.005923710763454437,
"fcm_dpo/delta": 0.018466537818312645,
"fcm_dpo/margin": 72.98329162597656,
"fcm_dpo/q_t": 0.4031839966773987,
"grad_norm": 11.903160095214844,
"learning_rate": 1.2586440420372934e-10,
"logits/chosen": 0.656846821308136,
"logits/rejected": 0.617106556892395,
"logps/chosen": -240.96087646484375,
"logps/ref_chosen": -73.25672912597656,
"logps/ref_rejected": -85.35127258300781,
"logps/rejected": -326.0386962890625,
"loss": 1.1019,
"margin_dpo/margin_mean": 72.98329162597656,
"margin_dpo/margin_std": 112.23123168945312,
"step": 656
},
{
"KL/chosen_KL_mean": -148.2847900390625,
"KL/mean": -192.8415069580078,
"KL/rejected_KL_mean": -237.39822387695312,
"KL/std": 112.21992492675781,
"epoch": 0.9931972789115646,
"fcm_dpo/beta": 0.005868775770068169,
"fcm_dpo/delta": -0.07695280015468597,
"fcm_dpo/margin": 89.11343383789062,
"fcm_dpo/q_t": 0.3840072751045227,
"grad_norm": 12.307347297668457,
"learning_rate": 8.740807750345913e-11,
"logits/chosen": 0.8304808735847473,
"logits/rejected": 0.7517132759094238,
"logps/chosen": -198.0081787109375,
"logps/ref_chosen": -49.72339630126953,
"logps/ref_rejected": -75.1568603515625,
"logps/rejected": -312.5550842285156,
"loss": 1.0473,
"margin_dpo/margin_mean": 89.11343383789062,
"margin_dpo/margin_std": 122.4747314453125,
"step": 657
},
{
"KL/chosen_KL_mean": -158.35809326171875,
"KL/mean": -194.7518768310547,
"KL/rejected_KL_mean": -231.14566040039062,
"KL/std": 115.96528625488281,
"epoch": 0.9947089947089947,
"fcm_dpo/beta": 0.005895557813346386,
"fcm_dpo/delta": 0.020794283598661423,
"fcm_dpo/margin": 72.78756713867188,
"fcm_dpo/q_t": 0.40627312660217285,
"grad_norm": 12.473671913146973,
"learning_rate": 5.594234322453539e-11,
"logits/chosen": 0.7611560225486755,
"logits/rejected": 0.7182252407073975,
"logps/chosen": -221.40443420410156,
"logps/ref_chosen": -63.04634094238281,
"logps/ref_rejected": -83.44963073730469,
"logps/rejected": -314.59527587890625,
"loss": 1.1497,
"margin_dpo/margin_mean": 72.78756713867188,
"margin_dpo/margin_std": 135.44158935546875,
"step": 658
},
{
"KL/chosen_KL_mean": -161.2940673828125,
"KL/mean": -189.77117919921875,
"KL/rejected_KL_mean": -218.24826049804688,
"KL/std": 100.78056335449219,
"epoch": 0.9962207105064248,
"fcm_dpo/beta": 0.00593742448836565,
"fcm_dpo/delta": 0.11584506928920746,
"fcm_dpo/margin": 56.954185485839844,
"fcm_dpo/q_t": 0.4244096875190735,
"grad_norm": 17.622331619262695,
"learning_rate": 3.146808153123293e-11,
"logits/chosen": 0.7817635536193848,
"logits/rejected": 0.7224591970443726,
"logps/chosen": -216.374267578125,
"logps/ref_chosen": -55.0802001953125,
"logps/ref_rejected": -71.91049194335938,
"logps/rejected": -290.15875244140625,
"loss": 1.1874,
"margin_dpo/margin_mean": 56.95417785644531,
"margin_dpo/margin_std": 114.34848022460938,
"step": 659
},
{
"KL/chosen_KL_mean": -152.11578369140625,
"KL/mean": -198.0364990234375,
"KL/rejected_KL_mean": -243.95724487304688,
"KL/std": 109.25594329833984,
"epoch": 0.9977324263038548,
"fcm_dpo/beta": 0.005916485097259283,
"fcm_dpo/delta": -0.09869644045829773,
"fcm_dpo/margin": 91.84146118164062,
"fcm_dpo/q_t": 0.37992629408836365,
"grad_norm": 14.751081466674805,
"learning_rate": 1.3985977021235829e-11,
"logits/chosen": 0.8116575479507446,
"logits/rejected": 0.7513267993927002,
"logps/chosen": -206.64169311523438,
"logps/ref_chosen": -54.525917053222656,
"logps/ref_rejected": -81.23604583740234,
"logps/rejected": -325.19329833984375,
"loss": 1.0144,
"margin_dpo/margin_mean": 91.84146118164062,
"margin_dpo/margin_std": 115.3442611694336,
"step": 660
},
{
"KL/chosen_KL_mean": -175.55447387695312,
"KL/mean": -204.49343872070312,
"KL/rejected_KL_mean": -233.43243408203125,
"KL/std": 109.31103515625,
"epoch": 0.999244142101285,
"fcm_dpo/beta": 0.005994308274239302,
"fcm_dpo/delta": 0.1062980443239212,
"fcm_dpo/margin": 57.877967834472656,
"fcm_dpo/q_t": 0.42280131578445435,
"grad_norm": 12.12573528289795,
"learning_rate": 3.4965187065971735e-12,
"logits/chosen": 0.693326473236084,
"logits/rejected": 0.6238353252410889,
"logps/chosen": -235.92709350585938,
"logps/ref_chosen": -60.37263870239258,
"logps/ref_rejected": -77.42874145507812,
"logps/rejected": -310.8612060546875,
"loss": 1.1979,
"margin_dpo/margin_mean": 57.877967834472656,
"margin_dpo/margin_std": 124.64874267578125,
"step": 661
},
{
"epoch": 0.999244142101285,
"step": 661,
"total_flos": 0.0,
"train_loss": 1.143015539862565,
"train_runtime": 1644.2601,
"train_samples_per_second": 25.748,
"train_steps_per_second": 0.402
}
],
"logging_steps": 1,
"max_steps": 661,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink