W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.8-20260428-045924
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC 9d6a1ff5fb 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.8-20260428-045924
Source: Original Platform
2026-05-22 20:25:40 +08:00

15247 lines
572 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.999244142101285,
"eval_steps": 200,
"global_step": 661,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.02867889404296875,
"KL/mean": 0.029354453086853027,
"KL/rejected_KL_mean": 0.030029296875,
"KL/std": 0.2071000635623932,
"epoch": 0.0015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0013532638549804688,
"fcm_dpo/q_t": 0.5000336766242981,
"grad_norm": 28.21991729736328,
"learning_rate": 0.0,
"logits/chosen": 0.13337239623069763,
"logits/rejected": 0.12492949515581131,
"logps/chosen": -64.5841293334961,
"logps/ref_chosen": -64.61280822753906,
"logps/ref_rejected": -64.17195129394531,
"logps/rejected": -64.14192199707031,
"loss": 1.3866,
"margin_dpo/margin_mean": -0.0013527870178222656,
"margin_dpo/margin_std": 0.2561596930027008,
"step": 1
},
{
"KL/chosen_KL_mean": -0.00289154052734375,
"KL/mean": -0.021616414189338684,
"KL/rejected_KL_mean": -0.04033660888671875,
"KL/std": 0.19624735414981842,
"epoch": 0.0030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.037450045347213745,
"fcm_dpo/q_t": 0.4990639090538025,
"grad_norm": 27.826953887939453,
"learning_rate": 7.462686567164179e-09,
"logits/chosen": 0.09414851665496826,
"logits/rejected": 0.07363267242908478,
"logps/chosen": -56.101890563964844,
"logps/ref_chosen": -56.0989990234375,
"logps/ref_rejected": -66.59971618652344,
"logps/rejected": -66.64006042480469,
"loss": 1.3828,
"margin_dpo/margin_mean": 0.03744968771934509,
"margin_dpo/margin_std": 0.27811938524246216,
"step": 2
},
{
"KL/chosen_KL_mean": 0.04581451416015625,
"KL/mean": 0.04929549992084503,
"KL/rejected_KL_mean": 0.052776336669921875,
"KL/std": 0.24392041563987732,
"epoch": 0.0045351473922902496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.006956905126571655,
"fcm_dpo/q_t": 0.5001732110977173,
"grad_norm": 31.296030044555664,
"learning_rate": 1.4925373134328357e-08,
"logits/chosen": 0.09936262667179108,
"logits/rejected": 0.061408448964357376,
"logps/chosen": -65.41144561767578,
"logps/ref_chosen": -65.45726013183594,
"logps/ref_rejected": -90.82853698730469,
"logps/rejected": -90.7757568359375,
"loss": 1.3872,
"margin_dpo/margin_mean": -0.006957113742828369,
"margin_dpo/margin_std": 0.30191972851753235,
"step": 3
},
{
"KL/chosen_KL_mean": -0.03472900390625,
"KL/mean": -0.02750520408153534,
"KL/rejected_KL_mean": -0.02028656005859375,
"KL/std": 0.22726918756961823,
"epoch": 0.006046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.014443963766098022,
"fcm_dpo/q_t": 0.5003613233566284,
"grad_norm": 34.52410125732422,
"learning_rate": 2.2388059701492534e-08,
"logits/chosen": 0.11272424459457397,
"logits/rejected": 0.09639942646026611,
"logps/chosen": -76.89491271972656,
"logps/ref_chosen": -76.86018371582031,
"logps/ref_rejected": -79.91523742675781,
"logps/rejected": -79.9355239868164,
"loss": 1.388,
"margin_dpo/margin_mean": -0.014444082975387573,
"margin_dpo/margin_std": 0.33552950620651245,
"step": 4
},
{
"KL/chosen_KL_mean": -0.0019626617431640625,
"KL/mean": 0.013008832931518555,
"KL/rejected_KL_mean": 0.027984619140625,
"KL/std": 0.20796799659729004,
"epoch": 0.007558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.029944956302642822,
"fcm_dpo/q_t": 0.500747561454773,
"grad_norm": 29.703157424926758,
"learning_rate": 2.9850746268656714e-08,
"logits/chosen": 0.06775817275047302,
"logits/rejected": 0.029598888009786606,
"logps/chosen": -62.973304748535156,
"logps/ref_chosen": -62.97134017944336,
"logps/ref_rejected": -79.9192123413086,
"logps/rejected": -79.89122772216797,
"loss": 1.3896,
"margin_dpo/margin_mean": -0.02994471788406372,
"margin_dpo/margin_std": 0.3299190402030945,
"step": 5
},
{
"KL/chosen_KL_mean": -0.018129348754882812,
"KL/mean": 0.0032161325216293335,
"KL/rejected_KL_mean": 0.024560928344726562,
"KL/std": 0.22389458119869232,
"epoch": 0.009070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.042690426111221313,
"fcm_dpo/q_t": 0.5010668039321899,
"grad_norm": 29.70970344543457,
"learning_rate": 3.731343283582089e-08,
"logits/chosen": 0.14717215299606323,
"logits/rejected": 0.10750843584537506,
"logps/chosen": -51.32549285888672,
"logps/ref_chosen": -51.30736541748047,
"logps/ref_rejected": -82.77239227294922,
"logps/rejected": -82.74783325195312,
"loss": 1.3908,
"margin_dpo/margin_mean": -0.042690128087997437,
"margin_dpo/margin_std": 0.3145572245121002,
"step": 6
},
{
"KL/chosen_KL_mean": 0.0044879913330078125,
"KL/mean": -0.0019683837890625,
"KL/rejected_KL_mean": -0.008426666259765625,
"KL/std": 0.21890875697135925,
"epoch": 0.010582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.012911856174468994,
"fcm_dpo/q_t": 0.4996773600578308,
"grad_norm": 27.1543025970459,
"learning_rate": 4.477611940298507e-08,
"logits/chosen": 0.018971581012010574,
"logits/rejected": -0.02494441345334053,
"logps/chosen": -51.454925537109375,
"logps/ref_chosen": -51.45941162109375,
"logps/ref_rejected": -66.3828125,
"logps/rejected": -66.3912353515625,
"loss": 1.3852,
"margin_dpo/margin_mean": 0.012911945581436157,
"margin_dpo/margin_std": 0.26080113649368286,
"step": 7
},
{
"KL/chosen_KL_mean": -0.011938095092773438,
"KL/mean": -0.01912282407283783,
"KL/rejected_KL_mean": -0.026304244995117188,
"KL/std": 0.23343831300735474,
"epoch": 0.012093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014365166425704956,
"fcm_dpo/q_t": 0.4996413290500641,
"grad_norm": 28.60932731628418,
"learning_rate": 5.223880597014925e-08,
"logits/chosen": 0.0810195729136467,
"logits/rejected": 0.05867453292012215,
"logps/chosen": -62.2094841003418,
"logps/ref_chosen": -62.197547912597656,
"logps/ref_rejected": -74.66180419921875,
"logps/rejected": -74.6881103515625,
"loss": 1.3851,
"margin_dpo/margin_mean": 0.014364898204803467,
"margin_dpo/margin_std": 0.3226226270198822,
"step": 8
},
{
"KL/chosen_KL_mean": -0.01309967041015625,
"KL/mean": -0.027955979108810425,
"KL/rejected_KL_mean": -0.042816162109375,
"KL/std": 0.23443172872066498,
"epoch": 0.013605442176870748,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02971470355987549,
"fcm_dpo/q_t": 0.4992576837539673,
"grad_norm": 31.4260311126709,
"learning_rate": 5.970149253731343e-08,
"logits/chosen": 0.17611047625541687,
"logits/rejected": 0.11633238196372986,
"logps/chosen": -55.642822265625,
"logps/ref_chosen": -55.629722595214844,
"logps/ref_rejected": -86.21221923828125,
"logps/rejected": -86.25503540039062,
"loss": 1.3836,
"margin_dpo/margin_mean": 0.029714584350585938,
"margin_dpo/margin_std": 0.33428698778152466,
"step": 9
},
{
"KL/chosen_KL_mean": 0.0005950927734375,
"KL/mean": -0.003879353404045105,
"KL/rejected_KL_mean": -0.00835418701171875,
"KL/std": 0.23718583583831787,
"epoch": 0.015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.008948296308517456,
"fcm_dpo/q_t": 0.49977636337280273,
"grad_norm": 29.694154739379883,
"learning_rate": 6.71641791044776e-08,
"logits/chosen": 0.13127511739730835,
"logits/rejected": 0.10026749223470688,
"logps/chosen": -62.690006256103516,
"logps/ref_chosen": -62.69060134887695,
"logps/ref_rejected": -90.610107421875,
"logps/rejected": -90.61846923828125,
"loss": 1.3857,
"margin_dpo/margin_mean": 0.008948534727096558,
"margin_dpo/margin_std": 0.3706110715866089,
"step": 10
},
{
"KL/chosen_KL_mean": 0.015949249267578125,
"KL/mean": 0.014684125781059265,
"KL/rejected_KL_mean": 0.013416290283203125,
"KL/std": 0.1973499059677124,
"epoch": 0.016628873771730914,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0025348961353302,
"fcm_dpo/q_t": 0.49993592500686646,
"grad_norm": 29.386581420898438,
"learning_rate": 7.462686567164178e-08,
"logits/chosen": 0.11886583268642426,
"logits/rejected": 0.11185969412326813,
"logps/chosen": -65.75116729736328,
"logps/ref_chosen": -65.76712036132812,
"logps/ref_rejected": -72.4764633178711,
"logps/rejected": -72.46304321289062,
"loss": 1.3863,
"margin_dpo/margin_mean": 0.002535104751586914,
"margin_dpo/margin_std": 0.299337238073349,
"step": 11
},
{
"KL/chosen_KL_mean": -0.029123306274414062,
"KL/mean": -0.021389245986938477,
"KL/rejected_KL_mean": -0.0136566162109375,
"KL/std": 0.21308918297290802,
"epoch": 0.018140589569160998,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.01546633243560791,
"fcm_dpo/q_t": 0.5003858804702759,
"grad_norm": 28.376405715942383,
"learning_rate": 8.208955223880596e-08,
"logits/chosen": 0.02483578771352768,
"logits/rejected": 0.008677439764142036,
"logps/chosen": -60.73401641845703,
"logps/ref_chosen": -60.704891204833984,
"logps/ref_rejected": -69.41564178466797,
"logps/rejected": -69.4292984008789,
"loss": 1.3881,
"margin_dpo/margin_mean": -0.015465915203094482,
"margin_dpo/margin_std": 0.32422810792922974,
"step": 12
},
{
"KL/chosen_KL_mean": 0.00815582275390625,
"KL/mean": 0.008319318294525146,
"KL/rejected_KL_mean": 0.00849151611328125,
"KL/std": 0.22867777943611145,
"epoch": 0.019652305366591082,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.00033211708068847656,
"fcm_dpo/q_t": 0.5000076293945312,
"grad_norm": 29.0564022064209,
"learning_rate": 8.955223880597014e-08,
"logits/chosen": 0.1282486915588379,
"logits/rejected": 0.06378625333309174,
"logps/chosen": -49.901100158691406,
"logps/ref_chosen": -49.90925598144531,
"logps/ref_rejected": -92.37818145751953,
"logps/rejected": -92.36968994140625,
"loss": 1.3866,
"margin_dpo/margin_mean": -0.00033277273178100586,
"margin_dpo/margin_std": 0.3163772225379944,
"step": 13
},
{
"KL/chosen_KL_mean": 0.020704269409179688,
"KL/mean": -0.00048294663429260254,
"KL/rejected_KL_mean": -0.021671295166015625,
"KL/std": 0.22362124919891357,
"epoch": 0.021164021164021163,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04237312078475952,
"fcm_dpo/q_t": 0.4989417791366577,
"grad_norm": 29.4510498046875,
"learning_rate": 9.701492537313432e-08,
"logits/chosen": 0.08631706982851028,
"logits/rejected": 0.06857284903526306,
"logps/chosen": -60.59809112548828,
"logps/ref_chosen": -60.61879348754883,
"logps/ref_rejected": -71.79306030273438,
"logps/rejected": -71.81472778320312,
"loss": 1.3824,
"margin_dpo/margin_mean": 0.04237350821495056,
"margin_dpo/margin_std": 0.33905673027038574,
"step": 14
},
{
"KL/chosen_KL_mean": -0.028501510620117188,
"KL/mean": -0.025412842631340027,
"KL/rejected_KL_mean": -0.0223236083984375,
"KL/std": 0.21838335692882538,
"epoch": 0.022675736961451247,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0061851441860198975,
"fcm_dpo/q_t": 0.5001543760299683,
"grad_norm": 33.28445816040039,
"learning_rate": 1.044776119402985e-07,
"logits/chosen": 0.08525186032056808,
"logits/rejected": 0.04112376272678375,
"logps/chosen": -63.49803924560547,
"logps/ref_chosen": -63.46953582763672,
"logps/ref_rejected": -88.88951110839844,
"logps/rejected": -88.91183471679688,
"loss": 1.3871,
"margin_dpo/margin_mean": -0.006184697151184082,
"margin_dpo/margin_std": 0.2978639602661133,
"step": 15
},
{
"KL/chosen_KL_mean": -0.00461578369140625,
"KL/mean": -0.003109276294708252,
"KL/rejected_KL_mean": -0.001605987548828125,
"KL/std": 0.21290913224220276,
"epoch": 0.02418745275888133,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.003010869026184082,
"fcm_dpo/q_t": 0.500075101852417,
"grad_norm": 26.85666847229004,
"learning_rate": 1.1194029850746268e-07,
"logits/chosen": 0.11474336683750153,
"logits/rejected": 0.07771545648574829,
"logps/chosen": -46.53691482543945,
"logps/ref_chosen": -46.53229904174805,
"logps/ref_rejected": -74.27533721923828,
"logps/rejected": -74.27694702148438,
"loss": 1.3868,
"margin_dpo/margin_mean": -0.0030111074447631836,
"margin_dpo/margin_std": 0.2670174837112427,
"step": 16
},
{
"KL/chosen_KL_mean": 0.010129928588867188,
"KL/mean": -0.019616693258285522,
"KL/rejected_KL_mean": -0.0493621826171875,
"KL/std": 0.2056957632303238,
"epoch": 0.025699168556311415,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.059494197368621826,
"fcm_dpo/q_t": 0.49851351976394653,
"grad_norm": 32.781803131103516,
"learning_rate": 1.1940298507462686e-07,
"logits/chosen": 0.06784220784902573,
"logits/rejected": 0.04859541356563568,
"logps/chosen": -64.06770324707031,
"logps/ref_chosen": -64.07783508300781,
"logps/ref_rejected": -86.40876770019531,
"logps/rejected": -86.4581298828125,
"loss": 1.3806,
"margin_dpo/margin_mean": 0.0594935417175293,
"margin_dpo/margin_std": 0.29295259714126587,
"step": 17
},
{
"KL/chosen_KL_mean": 0.03248405456542969,
"KL/mean": 0.021163254976272583,
"KL/rejected_KL_mean": 0.009840011596679688,
"KL/std": 0.181782528758049,
"epoch": 0.027210884353741496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02264273166656494,
"fcm_dpo/q_t": 0.4994339942932129,
"grad_norm": 27.759876251220703,
"learning_rate": 1.2686567164179106e-07,
"logits/chosen": 0.06629162281751633,
"logits/rejected": 0.02278873696923256,
"logps/chosen": -44.84185028076172,
"logps/ref_chosen": -44.87433624267578,
"logps/ref_rejected": -70.97604370117188,
"logps/rejected": -70.9662094116211,
"loss": 1.3842,
"margin_dpo/margin_mean": 0.022643178701400757,
"margin_dpo/margin_std": 0.2225417196750641,
"step": 18
},
{
"KL/chosen_KL_mean": 0.041255950927734375,
"KL/mean": 0.006726168096065521,
"KL/rejected_KL_mean": -0.027805328369140625,
"KL/std": 0.25561243295669556,
"epoch": 0.02872260015117158,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.069069042801857,
"fcm_dpo/q_t": 0.4982735514640808,
"grad_norm": 30.727636337280273,
"learning_rate": 1.343283582089552e-07,
"logits/chosen": 0.07066254317760468,
"logits/rejected": 0.057106949388980865,
"logps/chosen": -68.1185531616211,
"logps/ref_chosen": -68.1598129272461,
"logps/ref_rejected": -81.17138671875,
"logps/rejected": -81.1991958618164,
"loss": 1.3796,
"margin_dpo/margin_mean": 0.0690688043832779,
"margin_dpo/margin_std": 0.28775882720947266,
"step": 19
},
{
"KL/chosen_KL_mean": -0.025074005126953125,
"KL/mean": 0.0022208094596862793,
"KL/rejected_KL_mean": 0.02951812744140625,
"KL/std": 0.21140390634536743,
"epoch": 0.030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.054593682289123535,
"fcm_dpo/q_t": 0.5013641119003296,
"grad_norm": 29.338727951049805,
"learning_rate": 1.4179104477611938e-07,
"logits/chosen": 0.13627377152442932,
"logits/rejected": 0.11312912404537201,
"logps/chosen": -53.703636169433594,
"logps/ref_chosen": -53.67856216430664,
"logps/ref_rejected": -74.16911315917969,
"logps/rejected": -74.13960266113281,
"loss": 1.392,
"margin_dpo/margin_mean": -0.05459386110305786,
"margin_dpo/margin_std": 0.3187302052974701,
"step": 20
},
{
"KL/chosen_KL_mean": 0.007659912109375,
"KL/mean": 0.021049678325653076,
"KL/rejected_KL_mean": 0.0344390869140625,
"KL/std": 0.23415425419807434,
"epoch": 0.031746031746031744,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.026774346828460693,
"fcm_dpo/q_t": 0.500669538974762,
"grad_norm": 29.406789779663086,
"learning_rate": 1.4925373134328355e-07,
"logits/chosen": 0.11505385488271713,
"logits/rejected": 0.08910010010004044,
"logps/chosen": -64.69389343261719,
"logps/ref_chosen": -64.70155334472656,
"logps/ref_rejected": -81.02095031738281,
"logps/rejected": -80.98651123046875,
"loss": 1.3893,
"margin_dpo/margin_mean": -0.02677476406097412,
"margin_dpo/margin_std": 0.33221936225891113,
"step": 21
},
{
"KL/chosen_KL_mean": -0.019384384155273438,
"KL/mean": -0.028041943907737732,
"KL/rejected_KL_mean": -0.0366973876953125,
"KL/std": 0.20282882452011108,
"epoch": 0.03325774754346183,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.017315417528152466,
"fcm_dpo/q_t": 0.4995673894882202,
"grad_norm": 28.919084548950195,
"learning_rate": 1.5671641791044775e-07,
"logits/chosen": 0.002440758515149355,
"logits/rejected": -0.01844123937189579,
"logps/chosen": -58.05537414550781,
"logps/ref_chosen": -58.03599166870117,
"logps/ref_rejected": -80.72721862792969,
"logps/rejected": -80.763916015625,
"loss": 1.3848,
"margin_dpo/margin_mean": 0.01731511950492859,
"margin_dpo/margin_std": 0.29331278800964355,
"step": 22
},
{
"KL/chosen_KL_mean": -0.01165771484375,
"KL/mean": -0.003425806760787964,
"KL/rejected_KL_mean": 0.0048065185546875,
"KL/std": 0.2561618685722351,
"epoch": 0.03476946334089191,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.016467154026031494,
"fcm_dpo/q_t": 0.5004115104675293,
"grad_norm": 33.099586486816406,
"learning_rate": 1.6417910447761193e-07,
"logits/chosen": 0.13895675539970398,
"logits/rejected": 0.11350340396165848,
"logps/chosen": -66.36774444580078,
"logps/ref_chosen": -66.35608673095703,
"logps/ref_rejected": -93.02769470214844,
"logps/rejected": -93.02288818359375,
"loss": 1.3882,
"margin_dpo/margin_mean": -0.01646745204925537,
"margin_dpo/margin_std": 0.3485786020755768,
"step": 23
},
{
"KL/chosen_KL_mean": -0.023984909057617188,
"KL/mean": 0.010361678898334503,
"KL/rejected_KL_mean": 0.04471015930175781,
"KL/std": 0.2105405628681183,
"epoch": 0.036281179138321996,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06869538128376007,
"fcm_dpo/q_t": 0.5017166137695312,
"grad_norm": 26.39677619934082,
"learning_rate": 1.716417910447761e-07,
"logits/chosen": 0.12875144183635712,
"logits/rejected": 0.0960879772901535,
"logps/chosen": -54.48522186279297,
"logps/ref_chosen": -54.461238861083984,
"logps/ref_rejected": -68.33817291259766,
"logps/rejected": -68.29346466064453,
"loss": 1.3934,
"margin_dpo/margin_mean": -0.06869524717330933,
"margin_dpo/margin_std": 0.27617546916007996,
"step": 24
},
{
"KL/chosen_KL_mean": -0.023723602294921875,
"KL/mean": -0.036136969923973083,
"KL/rejected_KL_mean": -0.048553466796875,
"KL/std": 0.2357097566127777,
"epoch": 0.03779289493575208,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.024826496839523315,
"fcm_dpo/q_t": 0.49938008189201355,
"grad_norm": 29.66562271118164,
"learning_rate": 1.7910447761194027e-07,
"logits/chosen": 0.12693685293197632,
"logits/rejected": 0.07386107742786407,
"logps/chosen": -60.02792739868164,
"logps/ref_chosen": -60.00420379638672,
"logps/ref_rejected": -90.47376251220703,
"logps/rejected": -90.5223159790039,
"loss": 1.3841,
"margin_dpo/margin_mean": 0.02482694387435913,
"margin_dpo/margin_std": 0.3424012064933777,
"step": 25
},
{
"KL/chosen_KL_mean": -0.027313232421875,
"KL/mean": -0.026278428733348846,
"KL/rejected_KL_mean": -0.02524566650390625,
"KL/std": 0.23390698432922363,
"epoch": 0.039304610733182165,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0020656436681747437,
"fcm_dpo/q_t": 0.500051736831665,
"grad_norm": 29.946218490600586,
"learning_rate": 1.8656716417910447e-07,
"logits/chosen": 0.10996918380260468,
"logits/rejected": 0.09148485213518143,
"logps/chosen": -56.846466064453125,
"logps/ref_chosen": -56.81915283203125,
"logps/ref_rejected": -77.84333038330078,
"logps/rejected": -77.86857604980469,
"loss": 1.3867,
"margin_dpo/margin_mean": -0.0020662248134613037,
"margin_dpo/margin_std": 0.3143938183784485,
"step": 26
},
{
"KL/chosen_KL_mean": -0.01342010498046875,
"KL/mean": -0.03130936622619629,
"KL/rejected_KL_mean": -0.049198150634765625,
"KL/std": 0.2537550628185272,
"epoch": 0.04081632653061224,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03577548265457153,
"fcm_dpo/q_t": 0.49910616874694824,
"grad_norm": 28.97505760192871,
"learning_rate": 1.9402985074626865e-07,
"logits/chosen": 0.10381777584552765,
"logits/rejected": 0.07897449284791946,
"logps/chosen": -62.890445709228516,
"logps/ref_chosen": -62.87702560424805,
"logps/ref_rejected": -71.34437561035156,
"logps/rejected": -71.39356994628906,
"loss": 1.383,
"margin_dpo/margin_mean": 0.035775572061538696,
"margin_dpo/margin_std": 0.3350522220134735,
"step": 27
},
{
"KL/chosen_KL_mean": -0.023956298828125,
"KL/mean": -0.030458927154541016,
"KL/rejected_KL_mean": -0.03696250915527344,
"KL/std": 0.21456307172775269,
"epoch": 0.042328042328042326,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.013008713722229004,
"fcm_dpo/q_t": 0.499674916267395,
"grad_norm": 27.615636825561523,
"learning_rate": 2.0149253731343282e-07,
"logits/chosen": 0.05342138558626175,
"logits/rejected": 0.044701073318719864,
"logps/chosen": -59.857330322265625,
"logps/ref_chosen": -59.8333740234375,
"logps/ref_rejected": -70.39804077148438,
"logps/rejected": -70.43499755859375,
"loss": 1.3852,
"margin_dpo/margin_mean": 0.013009041547775269,
"margin_dpo/margin_std": 0.2985638976097107,
"step": 28
},
{
"KL/chosen_KL_mean": -0.0227813720703125,
"KL/mean": -0.03702476620674133,
"KL/rejected_KL_mean": -0.05126190185546875,
"KL/std": 0.20070354640483856,
"epoch": 0.04383975812547241,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.028487086296081543,
"fcm_dpo/q_t": 0.4992881417274475,
"grad_norm": 32.572021484375,
"learning_rate": 2.08955223880597e-07,
"logits/chosen": 0.1341443955898285,
"logits/rejected": 0.11656548082828522,
"logps/chosen": -74.14298248291016,
"logps/ref_chosen": -74.12020111083984,
"logps/ref_rejected": -83.33099365234375,
"logps/rejected": -83.38224792480469,
"loss": 1.3837,
"margin_dpo/margin_mean": 0.028487682342529297,
"margin_dpo/margin_std": 0.29014480113983154,
"step": 29
},
{
"KL/chosen_KL_mean": -0.020137786865234375,
"KL/mean": -0.029683440923690796,
"KL/rejected_KL_mean": -0.039226531982421875,
"KL/std": 0.2251281887292862,
"epoch": 0.045351473922902494,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.019087255001068115,
"fcm_dpo/q_t": 0.49952298402786255,
"grad_norm": 30.433258056640625,
"learning_rate": 2.1641791044776117e-07,
"logits/chosen": 0.12191575765609741,
"logits/rejected": 0.06771814823150635,
"logps/chosen": -50.771427154541016,
"logps/ref_chosen": -50.75128936767578,
"logps/ref_rejected": -89.29063415527344,
"logps/rejected": -89.32986450195312,
"loss": 1.3846,
"margin_dpo/margin_mean": 0.01908770203590393,
"margin_dpo/margin_std": 0.3208455443382263,
"step": 30
},
{
"KL/chosen_KL_mean": -0.019695281982421875,
"KL/mean": -0.04813128709793091,
"KL/rejected_KL_mean": -0.07656478881835938,
"KL/std": 0.2744419574737549,
"epoch": 0.04686318972033258,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.056871771812438965,
"fcm_dpo/q_t": 0.49857938289642334,
"grad_norm": 34.18708801269531,
"learning_rate": 2.2388059701492537e-07,
"logits/chosen": 0.08505978435277939,
"logits/rejected": 0.040289558470249176,
"logps/chosen": -65.3564453125,
"logps/ref_chosen": -65.33675384521484,
"logps/ref_rejected": -100.76666259765625,
"logps/rejected": -100.84323120117188,
"loss": 1.381,
"margin_dpo/margin_mean": 0.05687209963798523,
"margin_dpo/margin_std": 0.36973148584365845,
"step": 31
},
{
"KL/chosen_KL_mean": -0.0136260986328125,
"KL/mean": 0.004444405436515808,
"KL/rejected_KL_mean": 0.02251434326171875,
"KL/std": 0.2495591938495636,
"epoch": 0.04837490551776266,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.036137908697128296,
"fcm_dpo/q_t": 0.500903308391571,
"grad_norm": 30.191768646240234,
"learning_rate": 2.3134328358208954e-07,
"logits/chosen": 0.0718865916132927,
"logits/rejected": 0.06413871794939041,
"logps/chosen": -67.19696044921875,
"logps/ref_chosen": -67.18333435058594,
"logps/ref_rejected": -82.80763244628906,
"logps/rejected": -82.78512573242188,
"loss": 1.3902,
"margin_dpo/margin_mean": -0.03613784909248352,
"margin_dpo/margin_std": 0.31982362270355225,
"step": 32
},
{
"KL/chosen_KL_mean": -0.05531883239746094,
"KL/mean": -0.05424413084983826,
"KL/rejected_KL_mean": -0.053165435791015625,
"KL/std": 0.22134271264076233,
"epoch": 0.049886621315192746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.002153933048248291,
"fcm_dpo/q_t": 0.5000538229942322,
"grad_norm": 31.012001037597656,
"learning_rate": 2.388059701492537e-07,
"logits/chosen": 0.03484850376844406,
"logits/rejected": 0.008806673809885979,
"logps/chosen": -64.09480285644531,
"logps/ref_chosen": -64.03948211669922,
"logps/ref_rejected": -75.68357849121094,
"logps/rejected": -75.73674774169922,
"loss": 1.3867,
"margin_dpo/margin_mean": -0.0021534860134124756,
"margin_dpo/margin_std": 0.2684057652950287,
"step": 33
},
{
"KL/chosen_KL_mean": -0.04139518737792969,
"KL/mean": -0.06535254418849945,
"KL/rejected_KL_mean": -0.0893096923828125,
"KL/std": 0.17330102622509003,
"epoch": 0.05139833711262283,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04792061448097229,
"fcm_dpo/q_t": 0.4988023340702057,
"grad_norm": 28.53345489501953,
"learning_rate": 2.4626865671641786e-07,
"logits/chosen": 0.11218427121639252,
"logits/rejected": 0.08181029558181763,
"logps/chosen": -53.7056884765625,
"logps/ref_chosen": -53.6642951965332,
"logps/ref_rejected": -65.77989959716797,
"logps/rejected": -65.86920928955078,
"loss": 1.3816,
"margin_dpo/margin_mean": 0.047920435667037964,
"margin_dpo/margin_std": 0.22822073101997375,
"step": 34
},
{
"KL/chosen_KL_mean": -0.10145187377929688,
"KL/mean": -0.10271087288856506,
"KL/rejected_KL_mean": -0.10396957397460938,
"KL/std": 0.251731276512146,
"epoch": 0.05291005291005291,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.002517104148864746,
"fcm_dpo/q_t": 0.4999368190765381,
"grad_norm": 27.890995025634766,
"learning_rate": 2.537313432835821e-07,
"logits/chosen": 0.04327782988548279,
"logits/rejected": 0.02118547447025776,
"logps/chosen": -61.118316650390625,
"logps/ref_chosen": -61.01686096191406,
"logps/ref_rejected": -72.78598022460938,
"logps/rejected": -72.88995361328125,
"loss": 1.3864,
"margin_dpo/margin_mean": 0.0025173723697662354,
"margin_dpo/margin_std": 0.36896008253097534,
"step": 35
},
{
"KL/chosen_KL_mean": -0.06800651550292969,
"KL/mean": -0.07391357421875,
"KL/rejected_KL_mean": -0.07982254028320312,
"KL/std": 0.23057693243026733,
"epoch": 0.05442176870748299,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.011810481548309326,
"fcm_dpo/q_t": 0.49970513582229614,
"grad_norm": 28.75467872619629,
"learning_rate": 2.611940298507462e-07,
"logits/chosen": 0.09926285594701767,
"logits/rejected": 0.046571869403123856,
"logps/chosen": -50.605369567871094,
"logps/ref_chosen": -50.53736114501953,
"logps/ref_rejected": -78.11678314208984,
"logps/rejected": -78.19660949707031,
"loss": 1.3854,
"margin_dpo/margin_mean": 0.011810898780822754,
"margin_dpo/margin_std": 0.35508257150650024,
"step": 36
},
{
"KL/chosen_KL_mean": -0.033443450927734375,
"KL/mean": -0.09048253297805786,
"KL/rejected_KL_mean": -0.14752197265625,
"KL/std": 0.2617317736148834,
"epoch": 0.055933484504913075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11407029628753662,
"fcm_dpo/q_t": 0.49715059995651245,
"grad_norm": 37.495147705078125,
"learning_rate": 2.686567164179104e-07,
"logits/chosen": 0.1185469850897789,
"logits/rejected": 0.03703591972589493,
"logps/chosen": -59.5873908996582,
"logps/ref_chosen": -59.55394744873047,
"logps/ref_rejected": -108.27702331542969,
"logps/rejected": -108.42454528808594,
"loss": 1.3753,
"margin_dpo/margin_mean": 0.11406975984573364,
"margin_dpo/margin_std": 0.3664522171020508,
"step": 37
},
{
"KL/chosen_KL_mean": -0.0837249755859375,
"KL/mean": -0.09539508819580078,
"KL/rejected_KL_mean": -0.10706520080566406,
"KL/std": 0.2474774271249771,
"epoch": 0.05744520030234316,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02333807945251465,
"fcm_dpo/q_t": 0.499416708946228,
"grad_norm": 29.70965576171875,
"learning_rate": 2.761194029850746e-07,
"logits/chosen": 0.08251446485519409,
"logits/rejected": 0.06839097291231155,
"logps/chosen": -65.87208557128906,
"logps/ref_chosen": -65.78836059570312,
"logps/ref_rejected": -76.1619873046875,
"logps/rejected": -76.26905059814453,
"loss": 1.3843,
"margin_dpo/margin_mean": 0.023338019847869873,
"margin_dpo/margin_std": 0.35831284523010254,
"step": 38
},
{
"KL/chosen_KL_mean": -0.0574493408203125,
"KL/mean": -0.08865617215633392,
"KL/rejected_KL_mean": -0.11986541748046875,
"KL/std": 0.25045037269592285,
"epoch": 0.05895691609977324,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06241282820701599,
"fcm_dpo/q_t": 0.49844029545783997,
"grad_norm": 29.232280731201172,
"learning_rate": 2.8358208955223876e-07,
"logits/chosen": 0.16253948211669922,
"logits/rejected": 0.1356792151927948,
"logps/chosen": -57.23426055908203,
"logps/ref_chosen": -57.17681121826172,
"logps/ref_rejected": -79.486328125,
"logps/rejected": -79.606201171875,
"loss": 1.3804,
"margin_dpo/margin_mean": 0.062412917613983154,
"margin_dpo/margin_std": 0.3511146306991577,
"step": 39
},
{
"KL/chosen_KL_mean": -0.12008476257324219,
"KL/mean": -0.1282183676958084,
"KL/rejected_KL_mean": -0.1363525390625,
"KL/std": 0.25607573986053467,
"epoch": 0.06046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.016262918710708618,
"fcm_dpo/q_t": 0.4995928108692169,
"grad_norm": 31.482582092285156,
"learning_rate": 2.9104477611940296e-07,
"logits/chosen": 0.1266281008720398,
"logits/rejected": 0.07608100771903992,
"logps/chosen": -61.454254150390625,
"logps/ref_chosen": -61.33416748046875,
"logps/ref_rejected": -79.10697174072266,
"logps/rejected": -79.24332427978516,
"loss": 1.385,
"margin_dpo/margin_mean": 0.016263514757156372,
"margin_dpo/margin_std": 0.35121607780456543,
"step": 40
},
{
"KL/chosen_KL_mean": -0.07088088989257812,
"KL/mean": -0.14460709691047668,
"KL/rejected_KL_mean": -0.21832656860351562,
"KL/std": 0.3124772906303406,
"epoch": 0.06198034769463341,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.14744830131530762,
"fcm_dpo/q_t": 0.49631595611572266,
"grad_norm": 30.304224014282227,
"learning_rate": 2.985074626865671e-07,
"logits/chosen": 0.028469331562519073,
"logits/rejected": 0.009036978706717491,
"logps/chosen": -67.61761474609375,
"logps/ref_chosen": -67.5467300415039,
"logps/ref_rejected": -83.87788391113281,
"logps/rejected": -84.0962142944336,
"loss": 1.3721,
"margin_dpo/margin_mean": 0.14744865894317627,
"margin_dpo/margin_std": 0.4309389591217041,
"step": 41
},
{
"KL/chosen_KL_mean": -0.10775375366210938,
"KL/mean": -0.13053923845291138,
"KL/rejected_KL_mean": -0.15332794189453125,
"KL/std": 0.2780609726905823,
"epoch": 0.06349206349206349,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04558022320270538,
"fcm_dpo/q_t": 0.4988611340522766,
"grad_norm": 29.170860290527344,
"learning_rate": 3.059701492537313e-07,
"logits/chosen": 0.03401852399110794,
"logits/rejected": 0.0130385123193264,
"logps/chosen": -61.37261199951172,
"logps/ref_chosen": -61.26485824584961,
"logps/ref_rejected": -76.3629150390625,
"logps/rejected": -76.51624298095703,
"loss": 1.3821,
"margin_dpo/margin_mean": 0.04557971656322479,
"margin_dpo/margin_std": 0.39606839418411255,
"step": 42
},
{
"KL/chosen_KL_mean": -0.10474014282226562,
"KL/mean": -0.1389007568359375,
"KL/rejected_KL_mean": -0.17305946350097656,
"KL/std": 0.30590346455574036,
"epoch": 0.06500377928949358,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06832152605056763,
"fcm_dpo/q_t": 0.49829238653182983,
"grad_norm": 34.56472396850586,
"learning_rate": 3.134328358208955e-07,
"logits/chosen": 0.09799328446388245,
"logits/rejected": 0.08677279949188232,
"logps/chosen": -71.91377258300781,
"logps/ref_chosen": -71.80902862548828,
"logps/ref_rejected": -81.12464141845703,
"logps/rejected": -81.29769897460938,
"loss": 1.38,
"margin_dpo/margin_mean": 0.06832125782966614,
"margin_dpo/margin_std": 0.44653332233428955,
"step": 43
},
{
"KL/chosen_KL_mean": -0.14278602600097656,
"KL/mean": -0.14451465010643005,
"KL/rejected_KL_mean": -0.14624786376953125,
"KL/std": 0.32325345277786255,
"epoch": 0.06651549508692366,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0034592151641845703,
"fcm_dpo/q_t": 0.4999151825904846,
"grad_norm": 32.58658218383789,
"learning_rate": 3.2089552238805965e-07,
"logits/chosen": 0.0625898465514183,
"logits/rejected": 0.031519316136837006,
"logps/chosen": -66.6932144165039,
"logps/ref_chosen": -66.55043029785156,
"logps/ref_rejected": -85.06198120117188,
"logps/rejected": -85.2082290649414,
"loss": 1.3865,
"margin_dpo/margin_mean": 0.003458946943283081,
"margin_dpo/margin_std": 0.4451693296432495,
"step": 44
},
{
"KL/chosen_KL_mean": -0.10950851440429688,
"KL/mean": -0.1634192317724228,
"KL/rejected_KL_mean": -0.21733474731445312,
"KL/std": 0.2610400915145874,
"epoch": 0.06802721088435375,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10782626271247864,
"fcm_dpo/q_t": 0.4973050355911255,
"grad_norm": 31.791259765625,
"learning_rate": 3.2835820895522385e-07,
"logits/chosen": 0.11043379455804825,
"logits/rejected": 0.05831673741340637,
"logps/chosen": -62.353363037109375,
"logps/ref_chosen": -62.24385452270508,
"logps/ref_rejected": -92.96665954589844,
"logps/rejected": -93.18399810791016,
"loss": 1.3759,
"margin_dpo/margin_mean": 0.10782578587532043,
"margin_dpo/margin_std": 0.3529035449028015,
"step": 45
},
{
"KL/chosen_KL_mean": -0.10295486450195312,
"KL/mean": -0.18219387531280518,
"KL/rejected_KL_mean": -0.26143646240234375,
"KL/std": 0.3079463541507721,
"epoch": 0.06953892668178382,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15847516059875488,
"fcm_dpo/q_t": 0.49603956937789917,
"grad_norm": 31.26382827758789,
"learning_rate": 3.3582089552238805e-07,
"logits/chosen": 0.12944480776786804,
"logits/rejected": 0.08334116637706757,
"logps/chosen": -61.60186004638672,
"logps/ref_chosen": -61.498905181884766,
"logps/ref_rejected": -78.91172790527344,
"logps/rejected": -79.17316436767578,
"loss": 1.3709,
"margin_dpo/margin_mean": 0.1584753394126892,
"margin_dpo/margin_std": 0.4019874036312103,
"step": 46
},
{
"KL/chosen_KL_mean": -0.13724899291992188,
"KL/mean": -0.19762343168258667,
"KL/rejected_KL_mean": -0.2579994201660156,
"KL/std": 0.29829901456832886,
"epoch": 0.0710506424792139,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12074863910675049,
"fcm_dpo/q_t": 0.49698197841644287,
"grad_norm": 28.516429901123047,
"learning_rate": 3.432835820895522e-07,
"logits/chosen": 0.040763009339571,
"logits/rejected": -0.0024642946664243937,
"logps/chosen": -51.71559524536133,
"logps/ref_chosen": -51.578346252441406,
"logps/ref_rejected": -68.2215576171875,
"logps/rejected": -68.47955322265625,
"loss": 1.3748,
"margin_dpo/margin_mean": 0.1207488477230072,
"margin_dpo/margin_std": 0.4426974654197693,
"step": 47
},
{
"KL/chosen_KL_mean": -0.23521041870117188,
"KL/mean": -0.26210901141166687,
"KL/rejected_KL_mean": -0.28900718688964844,
"KL/std": 0.29804930090904236,
"epoch": 0.07256235827664399,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05379486083984375,
"fcm_dpo/q_t": 0.49865633249282837,
"grad_norm": 26.82840347290039,
"learning_rate": 3.507462686567164e-07,
"logits/chosen": 0.15432414412498474,
"logits/rejected": 0.12441211938858032,
"logps/chosen": -52.02886199951172,
"logps/ref_chosen": -51.79365158081055,
"logps/ref_rejected": -64.22503662109375,
"logps/rejected": -64.51404571533203,
"loss": 1.3813,
"margin_dpo/margin_mean": 0.05379548668861389,
"margin_dpo/margin_std": 0.4050604999065399,
"step": 48
},
{
"KL/chosen_KL_mean": -0.2191181182861328,
"KL/mean": -0.2685723900794983,
"KL/rejected_KL_mean": -0.318023681640625,
"KL/std": 0.3182642459869385,
"epoch": 0.07407407407407407,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09890323877334595,
"fcm_dpo/q_t": 0.49752840399742126,
"grad_norm": 27.29543113708496,
"learning_rate": 3.5820895522388055e-07,
"logits/chosen": 0.02532416582107544,
"logits/rejected": 0.004013156518340111,
"logps/chosen": -58.353721618652344,
"logps/ref_chosen": -58.13460159301758,
"logps/ref_rejected": -64.63206481933594,
"logps/rejected": -64.95008850097656,
"loss": 1.3769,
"margin_dpo/margin_mean": 0.09890377521514893,
"margin_dpo/margin_std": 0.45572179555892944,
"step": 49
},
{
"KL/chosen_KL_mean": -0.2713756561279297,
"KL/mean": -0.33710426092147827,
"KL/rejected_KL_mean": -0.40283203125,
"KL/std": 0.3075346350669861,
"epoch": 0.07558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.13145622611045837,
"fcm_dpo/q_t": 0.49671512842178345,
"grad_norm": 27.953882217407227,
"learning_rate": 3.6567164179104475e-07,
"logits/chosen": 0.11536800861358643,
"logits/rejected": 0.08536919206380844,
"logps/chosen": -53.12781524658203,
"logps/ref_chosen": -52.85643768310547,
"logps/ref_rejected": -72.17460632324219,
"logps/rejected": -72.57743835449219,
"loss": 1.3735,
"margin_dpo/margin_mean": 0.13145634531974792,
"margin_dpo/margin_std": 0.3600861430168152,
"step": 50
},
{
"KL/chosen_KL_mean": -0.31192779541015625,
"KL/mean": -0.411691814661026,
"KL/rejected_KL_mean": -0.5114593505859375,
"KL/std": 0.4130152761936188,
"epoch": 0.07709750566893424,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19952988624572754,
"fcm_dpo/q_t": 0.495017945766449,
"grad_norm": 30.98239517211914,
"learning_rate": 3.7313432835820895e-07,
"logits/chosen": 0.09327873587608337,
"logits/rejected": 0.06517933309078217,
"logps/chosen": -63.96836853027344,
"logps/ref_chosen": -63.65644073486328,
"logps/ref_rejected": -86.13229370117188,
"logps/rejected": -86.64375305175781,
"loss": 1.3671,
"margin_dpo/margin_mean": 0.19953015446662903,
"margin_dpo/margin_std": 0.5144891738891602,
"step": 51
},
{
"KL/chosen_KL_mean": -0.3197002410888672,
"KL/mean": -0.42734378576278687,
"KL/rejected_KL_mean": -0.5349884033203125,
"KL/std": 0.4443569481372833,
"epoch": 0.07860922146636433,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2152886539697647,
"fcm_dpo/q_t": 0.494623064994812,
"grad_norm": 32.54359436035156,
"learning_rate": 3.805970149253731e-07,
"logits/chosen": 0.07348201423883438,
"logits/rejected": 0.025112273171544075,
"logps/chosen": -68.159912109375,
"logps/ref_chosen": -67.8402099609375,
"logps/ref_rejected": -96.97090911865234,
"logps/rejected": -97.50589752197266,
"loss": 1.3657,
"margin_dpo/margin_mean": 0.21528802812099457,
"margin_dpo/margin_std": 0.5663281679153442,
"step": 52
},
{
"KL/chosen_KL_mean": -0.3238105773925781,
"KL/mean": -0.4126301109790802,
"KL/rejected_KL_mean": -0.5014514923095703,
"KL/std": 0.3923019766807556,
"epoch": 0.0801209372637944,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17764323949813843,
"fcm_dpo/q_t": 0.4955608546733856,
"grad_norm": 27.431011199951172,
"learning_rate": 3.880597014925373e-07,
"logits/chosen": 0.09138220548629761,
"logits/rejected": 0.08053071051836014,
"logps/chosen": -57.20195007324219,
"logps/ref_chosen": -56.87813949584961,
"logps/ref_rejected": -60.75569152832031,
"logps/rejected": -61.257144927978516,
"loss": 1.3692,
"margin_dpo/margin_mean": 0.17764326930046082,
"margin_dpo/margin_std": 0.4564116597175598,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3661918640136719,
"KL/mean": -0.45817917585372925,
"KL/rejected_KL_mean": -0.5501670837402344,
"KL/std": 0.39255958795547485,
"epoch": 0.08163265306122448,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.18397372961044312,
"fcm_dpo/q_t": 0.495402991771698,
"grad_norm": 26.49314308166504,
"learning_rate": 3.9552238805970144e-07,
"logits/chosen": 0.061346061527729034,
"logits/rejected": 0.045975200831890106,
"logps/chosen": -47.63311004638672,
"logps/ref_chosen": -47.26692199707031,
"logps/ref_rejected": -62.19426727294922,
"logps/rejected": -62.74443054199219,
"loss": 1.3686,
"margin_dpo/margin_mean": 0.1839737594127655,
"margin_dpo/margin_std": 0.4734251797199249,
"step": 54
},
{
"KL/chosen_KL_mean": -0.3494873046875,
"KL/mean": -0.4886450469493866,
"KL/rejected_KL_mean": -0.6278076171875,
"KL/std": 0.4888648986816406,
"epoch": 0.08314436885865457,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.27831727266311646,
"fcm_dpo/q_t": 0.4930543303489685,
"grad_norm": 31.363264083862305,
"learning_rate": 4.0298507462686564e-07,
"logits/chosen": 0.02471158653497696,
"logits/rejected": -0.050151094794273376,
"logps/chosen": -50.67567825317383,
"logps/ref_chosen": -50.32619094848633,
"logps/ref_rejected": -92.44389343261719,
"logps/rejected": -93.07170104980469,
"loss": 1.3597,
"margin_dpo/margin_mean": 0.2783171534538269,
"margin_dpo/margin_std": 0.6392854452133179,
"step": 55
},
{
"KL/chosen_KL_mean": -0.38335418701171875,
"KL/mean": -0.465893417596817,
"KL/rejected_KL_mean": -0.5484275817871094,
"KL/std": 0.49349987506866455,
"epoch": 0.08465608465608465,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.16507351398468018,
"fcm_dpo/q_t": 0.49587583541870117,
"grad_norm": 27.5733642578125,
"learning_rate": 4.1044776119402984e-07,
"logits/chosen": 0.1194012314081192,
"logits/rejected": 0.09707341343164444,
"logps/chosen": -57.150325775146484,
"logps/ref_chosen": -56.766971588134766,
"logps/ref_rejected": -66.30504608154297,
"logps/rejected": -66.85346984863281,
"loss": 1.371,
"margin_dpo/margin_mean": 0.16507357358932495,
"margin_dpo/margin_std": 0.6623785495758057,
"step": 56
},
{
"KL/chosen_KL_mean": -0.5016689300537109,
"KL/mean": -0.6514174938201904,
"KL/rejected_KL_mean": -0.8011627197265625,
"KL/std": 0.5826833844184875,
"epoch": 0.08616780045351474,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2994988262653351,
"fcm_dpo/q_t": 0.49252432584762573,
"grad_norm": 30.63138771057129,
"learning_rate": 4.17910447761194e-07,
"logits/chosen": 0.08919486403465271,
"logits/rejected": 0.025800924748182297,
"logps/chosen": -58.269412994384766,
"logps/ref_chosen": -57.76774597167969,
"logps/ref_rejected": -82.75698852539062,
"logps/rejected": -83.55815124511719,
"loss": 1.3578,
"margin_dpo/margin_mean": 0.29949846863746643,
"margin_dpo/margin_std": 0.6790944337844849,
"step": 57
},
{
"KL/chosen_KL_mean": -0.5987739562988281,
"KL/mean": -0.6989187598228455,
"KL/rejected_KL_mean": -0.7990646362304688,
"KL/std": 0.6531132459640503,
"epoch": 0.08767951625094482,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20029249787330627,
"fcm_dpo/q_t": 0.4950439929962158,
"grad_norm": 30.253463745117188,
"learning_rate": 4.253731343283582e-07,
"logits/chosen": 0.06011844053864479,
"logits/rejected": 0.04454775154590607,
"logps/chosen": -73.36286163330078,
"logps/ref_chosen": -72.76408386230469,
"logps/ref_rejected": -84.49275207519531,
"logps/rejected": -85.29181671142578,
"loss": 1.3688,
"margin_dpo/margin_mean": 0.20029214024543762,
"margin_dpo/margin_std": 0.9416682720184326,
"step": 58
},
{
"KL/chosen_KL_mean": -0.5233745574951172,
"KL/mean": -0.6699809432029724,
"KL/rejected_KL_mean": -0.8165874481201172,
"KL/std": 0.6026298403739929,
"epoch": 0.08919123204837491,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.29320916533470154,
"fcm_dpo/q_t": 0.49269628524780273,
"grad_norm": 26.622543334960938,
"learning_rate": 4.3283582089552234e-07,
"logits/chosen": 0.10866847634315491,
"logits/rejected": 0.04470418766140938,
"logps/chosen": -50.34415054321289,
"logps/ref_chosen": -49.820777893066406,
"logps/ref_rejected": -77.14368438720703,
"logps/rejected": -77.96027374267578,
"loss": 1.359,
"margin_dpo/margin_mean": 0.2932102382183075,
"margin_dpo/margin_std": 0.7765747308731079,
"step": 59
},
{
"KL/chosen_KL_mean": -0.6319961547851562,
"KL/mean": -0.6532078981399536,
"KL/rejected_KL_mean": -0.6744232177734375,
"KL/std": 0.6227794885635376,
"epoch": 0.09070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04242426156997681,
"fcm_dpo/q_t": 0.498939573764801,
"grad_norm": 29.584983825683594,
"learning_rate": 4.4029850746268654e-07,
"logits/chosen": 0.13120165467262268,
"logits/rejected": 0.12976253032684326,
"logps/chosen": -63.85676956176758,
"logps/ref_chosen": -63.22477340698242,
"logps/ref_rejected": -61.360477447509766,
"logps/rejected": -62.0349006652832,
"loss": 1.3836,
"margin_dpo/margin_mean": 0.04242423176765442,
"margin_dpo/margin_std": 0.7847387790679932,
"step": 60
},
{
"KL/chosen_KL_mean": -0.7138442993164062,
"KL/mean": -0.7507110834121704,
"KL/rejected_KL_mean": -0.7875823974609375,
"KL/std": 0.709343433380127,
"epoch": 0.09221466364323508,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07373163104057312,
"fcm_dpo/q_t": 0.4981583058834076,
"grad_norm": 27.950082778930664,
"learning_rate": 4.4776119402985074e-07,
"logits/chosen": 0.15023085474967957,
"logits/rejected": 0.11694887280464172,
"logps/chosen": -49.73064422607422,
"logps/ref_chosen": -49.01679992675781,
"logps/ref_rejected": -74.90817260742188,
"logps/rejected": -75.69575500488281,
"loss": 1.3814,
"margin_dpo/margin_mean": 0.073731929063797,
"margin_dpo/margin_std": 0.9730736017227173,
"step": 61
},
{
"KL/chosen_KL_mean": -0.7303638458251953,
"KL/mean": -0.8440359830856323,
"KL/rejected_KL_mean": -0.9577064514160156,
"KL/std": 0.6483147144317627,
"epoch": 0.09372637944066516,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.22734490036964417,
"fcm_dpo/q_t": 0.49432116746902466,
"grad_norm": 28.671464920043945,
"learning_rate": 4.552238805970149e-07,
"logits/chosen": 0.09102478623390198,
"logits/rejected": 0.05276797339320183,
"logps/chosen": -63.48223114013672,
"logps/ref_chosen": -62.751869201660156,
"logps/ref_rejected": -78.93360900878906,
"logps/rejected": -79.89131164550781,
"loss": 1.3658,
"margin_dpo/margin_mean": 0.22734498977661133,
"margin_dpo/margin_std": 0.9036822319030762,
"step": 62
},
{
"KL/chosen_KL_mean": -0.5533409118652344,
"KL/mean": -0.8310414552688599,
"KL/rejected_KL_mean": -1.1087455749511719,
"KL/std": 0.6519574522972107,
"epoch": 0.09523809523809523,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5554065704345703,
"fcm_dpo/q_t": 0.48614123463630676,
"grad_norm": 31.55145263671875,
"learning_rate": 4.626865671641791e-07,
"logits/chosen": 0.20062537491321564,
"logits/rejected": 0.17529505491256714,
"logps/chosen": -61.06858825683594,
"logps/ref_chosen": -60.51525115966797,
"logps/ref_rejected": -85.11021423339844,
"logps/rejected": -86.21896362304688,
"loss": 1.3332,
"margin_dpo/margin_mean": 0.5554060935974121,
"margin_dpo/margin_std": 0.8089257478713989,
"step": 63
},
{
"KL/chosen_KL_mean": -0.8106708526611328,
"KL/mean": -0.8460452556610107,
"KL/rejected_KL_mean": -0.8814239501953125,
"KL/std": 0.7306973338127136,
"epoch": 0.09674981103552532,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0707455575466156,
"fcm_dpo/q_t": 0.49822551012039185,
"grad_norm": 26.139772415161133,
"learning_rate": 4.701492537313433e-07,
"logits/chosen": 0.11065845191478729,
"logits/rejected": 0.0851249247789383,
"logps/chosen": -52.01751708984375,
"logps/ref_chosen": -51.20684814453125,
"logps/ref_rejected": -66.93081665039062,
"logps/rejected": -67.81224060058594,
"loss": 1.3812,
"margin_dpo/margin_mean": 0.07074528932571411,
"margin_dpo/margin_std": 0.8756983876228333,
"step": 64
},
{
"KL/chosen_KL_mean": -0.8438053131103516,
"KL/mean": -1.088397741317749,
"KL/rejected_KL_mean": -1.3329925537109375,
"KL/std": 0.892356276512146,
"epoch": 0.0982615268329554,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4891848862171173,
"fcm_dpo/q_t": 0.48782041668891907,
"grad_norm": 30.766826629638672,
"learning_rate": 4.776119402985074e-07,
"logits/chosen": 0.17410069704055786,
"logits/rejected": 0.14507675170898438,
"logps/chosen": -68.13249969482422,
"logps/ref_chosen": -67.2886962890625,
"logps/ref_rejected": -74.44281005859375,
"logps/rejected": -75.77580261230469,
"loss": 1.3414,
"margin_dpo/margin_mean": 0.48918458819389343,
"margin_dpo/margin_std": 1.1623709201812744,
"step": 65
},
{
"KL/chosen_KL_mean": -0.920684814453125,
"KL/mean": -1.086906909942627,
"KL/rejected_KL_mean": -1.2531318664550781,
"KL/std": 0.839600682258606,
"epoch": 0.09977324263038549,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.33244842290878296,
"fcm_dpo/q_t": 0.4917258024215698,
"grad_norm": 29.311410903930664,
"learning_rate": 4.850746268656717e-07,
"logits/chosen": 0.11044877022504807,
"logits/rejected": 0.08624343574047089,
"logps/chosen": -71.66409301757812,
"logps/ref_chosen": -70.743408203125,
"logps/ref_rejected": -77.26499938964844,
"logps/rejected": -78.51813507080078,
"loss": 1.3569,
"margin_dpo/margin_mean": 0.332448273897171,
"margin_dpo/margin_std": 1.181469202041626,
"step": 66
},
{
"KL/chosen_KL_mean": -0.7882022857666016,
"KL/mean": -0.9426577091217041,
"KL/rejected_KL_mean": -1.0971145629882812,
"KL/std": 0.7659956812858582,
"epoch": 0.10128495842781557,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3089101016521454,
"fcm_dpo/q_t": 0.4923003911972046,
"grad_norm": 29.101470947265625,
"learning_rate": 4.925373134328357e-07,
"logits/chosen": 0.07350900769233704,
"logits/rejected": 0.018813468515872955,
"logps/chosen": -61.390804290771484,
"logps/ref_chosen": -60.60260009765625,
"logps/ref_rejected": -75.22235870361328,
"logps/rejected": -76.31947326660156,
"loss": 1.358,
"margin_dpo/margin_mean": 0.3089095652103424,
"margin_dpo/margin_std": 0.9819750189781189,
"step": 67
},
{
"KL/chosen_KL_mean": -1.2221183776855469,
"KL/mean": -1.4071699380874634,
"KL/rejected_KL_mean": -1.5922203063964844,
"KL/std": 0.9413119554519653,
"epoch": 0.10279667422524566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.37010324001312256,
"fcm_dpo/q_t": 0.4907914996147156,
"grad_norm": 31.142738342285156,
"learning_rate": 5e-07,
"logits/chosen": 0.07065162807703018,
"logits/rejected": 0.04052513837814331,
"logps/chosen": -78.75048828125,
"logps/ref_chosen": -77.52836608886719,
"logps/ref_rejected": -93.17778015136719,
"logps/rejected": -94.7699966430664,
"loss": 1.3535,
"margin_dpo/margin_mean": 0.3701035976409912,
"margin_dpo/margin_std": 1.222214698791504,
"step": 68
},
{
"KL/chosen_KL_mean": -1.1215229034423828,
"KL/mean": -1.4538966417312622,
"KL/rejected_KL_mean": -1.7862663269042969,
"KL/std": 1.0045855045318604,
"epoch": 0.10430839002267574,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6647469997406006,
"fcm_dpo/q_t": 0.4834825098514557,
"grad_norm": 30.769756317138672,
"learning_rate": 4.999965034812934e-07,
"logits/chosen": 0.11138215661048889,
"logits/rejected": 0.0678541287779808,
"logps/chosen": -67.0645751953125,
"logps/ref_chosen": -65.94305419921875,
"logps/ref_rejected": -89.7735595703125,
"logps/rejected": -91.55982971191406,
"loss": 1.3248,
"margin_dpo/margin_mean": 0.6647467613220215,
"margin_dpo/margin_std": 1.2122726440429688,
"step": 69
},
{
"KL/chosen_KL_mean": -1.1970710754394531,
"KL/mean": -1.3843632936477661,
"KL/rejected_KL_mean": -1.5716571807861328,
"KL/std": 0.9146664142608643,
"epoch": 0.10582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3745828866958618,
"fcm_dpo/q_t": 0.4906681180000305,
"grad_norm": 28.702402114868164,
"learning_rate": 4.999860140229787e-07,
"logits/chosen": 0.15375293791294098,
"logits/rejected": 0.13041463494300842,
"logps/chosen": -63.15498352050781,
"logps/ref_chosen": -61.95791244506836,
"logps/ref_rejected": -75.80945587158203,
"logps/rejected": -77.38111877441406,
"loss": 1.353,
"margin_dpo/margin_mean": 0.3745833933353424,
"margin_dpo/margin_std": 1.2272173166275024,
"step": 70
},
{
"KL/chosen_KL_mean": -1.423257827758789,
"KL/mean": -1.5462465286254883,
"KL/rejected_KL_mean": -1.669240951538086,
"KL/std": 1.0030606985092163,
"epoch": 0.1073318216175359,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24598197638988495,
"fcm_dpo/q_t": 0.4938420355319977,
"grad_norm": 28.510639190673828,
"learning_rate": 4.999685319184688e-07,
"logits/chosen": 0.10399076342582703,
"logits/rejected": 0.0884547010064125,
"logps/chosen": -64.77083587646484,
"logps/ref_chosen": -63.34757995605469,
"logps/ref_rejected": -67.49658203125,
"logps/rejected": -69.16581726074219,
"loss": 1.367,
"margin_dpo/margin_mean": 0.24598146975040436,
"margin_dpo/margin_std": 1.4251511096954346,
"step": 71
},
{
"KL/chosen_KL_mean": -1.1918869018554688,
"KL/mean": -1.5472865104675293,
"KL/rejected_KL_mean": -1.9026832580566406,
"KL/std": 1.048713207244873,
"epoch": 0.10884353741496598,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7107953429222107,
"fcm_dpo/q_t": 0.4823157489299774,
"grad_norm": 30.407228469848633,
"learning_rate": 4.999440576567755e-07,
"logits/chosen": 0.11062829196453094,
"logits/rejected": 0.04809199273586273,
"logps/chosen": -57.051185607910156,
"logps/ref_chosen": -55.85929870605469,
"logps/ref_rejected": -68.45423889160156,
"logps/rejected": -70.35692596435547,
"loss": 1.3207,
"margin_dpo/margin_mean": 0.7107953429222107,
"margin_dpo/margin_std": 1.270460605621338,
"step": 72
},
{
"KL/chosen_KL_mean": -1.8556289672851562,
"KL/mean": -1.8964017629623413,
"KL/rejected_KL_mean": -1.9371776580810547,
"KL/std": 1.2240742444992065,
"epoch": 0.11035525321239607,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08155509829521179,
"fcm_dpo/q_t": 0.4981040060520172,
"grad_norm": 32.02695846557617,
"learning_rate": 4.999125919224965e-07,
"logits/chosen": 0.10665452480316162,
"logits/rejected": 0.09263806790113449,
"logps/chosen": -70.99443054199219,
"logps/ref_chosen": -69.13880920410156,
"logps/ref_rejected": -79.04586791992188,
"logps/rejected": -80.98304748535156,
"loss": 1.3849,
"margin_dpo/margin_mean": 0.08155572414398193,
"margin_dpo/margin_std": 1.5625742673873901,
"step": 73
},
{
"KL/chosen_KL_mean": -1.274038314819336,
"KL/mean": -1.612091064453125,
"KL/rejected_KL_mean": -1.9501457214355469,
"KL/std": 1.0038673877716064,
"epoch": 0.11186696900982615,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6761044859886169,
"fcm_dpo/q_t": 0.4832031726837158,
"grad_norm": 27.87704086303711,
"learning_rate": 4.998741355957963e-07,
"logits/chosen": 0.16232308745384216,
"logits/rejected": 0.10681352019309998,
"logps/chosen": -51.197776794433594,
"logps/ref_chosen": -49.923736572265625,
"logps/ref_rejected": -81.73213958740234,
"logps/rejected": -83.68228149414062,
"loss": 1.3242,
"margin_dpo/margin_mean": 0.6761043071746826,
"margin_dpo/margin_std": 1.2969073057174683,
"step": 74
},
{
"KL/chosen_KL_mean": -1.4171257019042969,
"KL/mean": -1.7749662399291992,
"KL/rejected_KL_mean": -2.132802963256836,
"KL/std": 1.1096692085266113,
"epoch": 0.11337868480725624,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7156823873519897,
"fcm_dpo/q_t": 0.48222440481185913,
"grad_norm": 26.229402542114258,
"learning_rate": 4.998286897523808e-07,
"logits/chosen": 0.13490338623523712,
"logits/rejected": 0.10130466520786285,
"logps/chosen": -47.485877990722656,
"logps/ref_chosen": -46.06875228881836,
"logps/ref_rejected": -66.1181411743164,
"logps/rejected": -68.25094604492188,
"loss": 1.3206,
"margin_dpo/margin_mean": 0.7156822681427002,
"margin_dpo/margin_std": 1.358628749847412,
"step": 75
},
{
"KL/chosen_KL_mean": -1.6028976440429688,
"KL/mean": -1.7485299110412598,
"KL/rejected_KL_mean": -1.8941612243652344,
"KL/std": 1.1897375583648682,
"epoch": 0.11489040060468632,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.29126521944999695,
"fcm_dpo/q_t": 0.4927474558353424,
"grad_norm": 29.370731353759766,
"learning_rate": 4.997762556634679e-07,
"logits/chosen": 0.13600164651870728,
"logits/rejected": 0.09152773022651672,
"logps/chosen": -55.6656494140625,
"logps/ref_chosen": -54.06275177001953,
"logps/ref_rejected": -74.87464141845703,
"logps/rejected": -76.76880645751953,
"loss": 1.3633,
"margin_dpo/margin_mean": 0.2912653684616089,
"margin_dpo/margin_std": 1.5319194793701172,
"step": 76
},
{
"KL/chosen_KL_mean": -1.63665771484375,
"KL/mean": -2.04000186920166,
"KL/rejected_KL_mean": -2.4433441162109375,
"KL/std": 1.240219235420227,
"epoch": 0.1164021164021164,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8066871166229248,
"fcm_dpo/q_t": 0.4799628257751465,
"grad_norm": 29.07001495361328,
"learning_rate": 4.99716834795752e-07,
"logits/chosen": 0.14318928122520447,
"logits/rejected": 0.10253836214542389,
"logps/chosen": -54.71274948120117,
"logps/ref_chosen": -53.07609176635742,
"logps/ref_rejected": -74.45601654052734,
"logps/rejected": -76.89936065673828,
"loss": 1.3128,
"margin_dpo/margin_mean": 0.8066874742507935,
"margin_dpo/margin_std": 1.4628499746322632,
"step": 77
},
{
"KL/chosen_KL_mean": -1.8630790710449219,
"KL/mean": -2.0672872066497803,
"KL/rejected_KL_mean": -2.271495819091797,
"KL/std": 1.271705150604248,
"epoch": 0.11791383219954649,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.40840888023376465,
"fcm_dpo/q_t": 0.4898289144039154,
"grad_norm": 29.537355422973633,
"learning_rate": 4.996504288113623e-07,
"logits/chosen": 0.13026684522628784,
"logits/rejected": 0.10955053567886353,
"logps/chosen": -69.5885009765625,
"logps/ref_chosen": -67.72541809082031,
"logps/ref_rejected": -79.03926849365234,
"logps/rejected": -81.31076049804688,
"loss": 1.351,
"margin_dpo/margin_mean": 0.4084087610244751,
"margin_dpo/margin_std": 1.4295289516448975,
"step": 78
},
{
"KL/chosen_KL_mean": -1.7345352172851562,
"KL/mean": -2.2997708320617676,
"KL/rejected_KL_mean": -2.8650054931640625,
"KL/std": 1.5332820415496826,
"epoch": 0.11942554799697656,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1304694414138794,
"fcm_dpo/q_t": 0.4722587466239929,
"grad_norm": 30.759552001953125,
"learning_rate": 4.995770395678171e-07,
"logits/chosen": 0.15435688197612762,
"logits/rejected": 0.09562011808156967,
"logps/chosen": -53.895179748535156,
"logps/ref_chosen": -52.16064453125,
"logps/ref_rejected": -83.31062316894531,
"logps/rejected": -86.17562866210938,
"loss": 1.2866,
"margin_dpo/margin_mean": 1.1304690837860107,
"margin_dpo/margin_std": 2.0189428329467773,
"step": 79
},
{
"KL/chosen_KL_mean": -2.062671661376953,
"KL/mean": -2.3099541664123535,
"KL/rejected_KL_mean": -2.5572357177734375,
"KL/std": 1.3525840044021606,
"epoch": 0.12093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4945679008960724,
"fcm_dpo/q_t": 0.4877622127532959,
"grad_norm": 28.568126678466797,
"learning_rate": 4.994966691179711e-07,
"logits/chosen": 0.1576176881790161,
"logits/rejected": 0.09683984518051147,
"logps/chosen": -63.47323226928711,
"logps/ref_chosen": -61.410560607910156,
"logps/ref_rejected": -78.66004943847656,
"logps/rejected": -81.21728515625,
"loss": 1.3467,
"margin_dpo/margin_mean": 0.4945681691169739,
"margin_dpo/margin_std": 1.8858641386032104,
"step": 80
},
{
"KL/chosen_KL_mean": -2.1507835388183594,
"KL/mean": -2.569912910461426,
"KL/rejected_KL_mean": -2.9890403747558594,
"KL/std": 1.4435718059539795,
"epoch": 0.12244897959183673,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8382568955421448,
"fcm_dpo/q_t": 0.47932901978492737,
"grad_norm": 29.092945098876953,
"learning_rate": 4.994093197099587e-07,
"logits/chosen": 0.12218126654624939,
"logits/rejected": 0.08859767019748688,
"logps/chosen": -65.95515441894531,
"logps/ref_chosen": -63.80437088012695,
"logps/ref_rejected": -79.3484115600586,
"logps/rejected": -82.33744812011719,
"loss": 1.3118,
"margin_dpo/margin_mean": 0.8382560610771179,
"margin_dpo/margin_std": 1.6820642948150635,
"step": 81
},
{
"KL/chosen_KL_mean": -1.7358322143554688,
"KL/mean": -2.2893571853637695,
"KL/rejected_KL_mean": -2.842884063720703,
"KL/std": 1.3926236629486084,
"epoch": 0.12396069538926682,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1070516109466553,
"fcm_dpo/q_t": 0.4725184440612793,
"grad_norm": 26.486122131347656,
"learning_rate": 4.993149937871306e-07,
"logits/chosen": 0.12228363752365112,
"logits/rejected": 0.058549731969833374,
"logps/chosen": -50.55372619628906,
"logps/ref_chosen": -48.817893981933594,
"logps/ref_rejected": -70.31497955322266,
"logps/rejected": -73.15786743164062,
"loss": 1.2846,
"margin_dpo/margin_mean": 1.107051134109497,
"margin_dpo/margin_std": 1.5328946113586426,
"step": 82
},
{
"KL/chosen_KL_mean": -2.0348987579345703,
"KL/mean": -2.634671211242676,
"KL/rejected_KL_mean": -3.2344436645507812,
"KL/std": 1.4608440399169922,
"epoch": 0.1254724111866969,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.199547529220581,
"fcm_dpo/q_t": 0.4703766703605652,
"grad_norm": 29.12187385559082,
"learning_rate": 4.992136939879856e-07,
"logits/chosen": 0.18589824438095093,
"logits/rejected": 0.13650354743003845,
"logps/chosen": -59.185672760009766,
"logps/ref_chosen": -57.15077209472656,
"logps/ref_rejected": -75.1710205078125,
"logps/rejected": -78.40547180175781,
"loss": 1.278,
"margin_dpo/margin_mean": 1.1995470523834229,
"margin_dpo/margin_std": 1.78159499168396,
"step": 83
},
{
"KL/chosen_KL_mean": -2.5271358489990234,
"KL/mean": -2.9374001026153564,
"KL/rejected_KL_mean": -3.347667694091797,
"KL/std": 1.8282861709594727,
"epoch": 0.12698412698412698,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8205302953720093,
"fcm_dpo/q_t": 0.47971442341804504,
"grad_norm": 30.673738479614258,
"learning_rate": 4.991054231460969e-07,
"logits/chosen": 0.13898839056491852,
"logits/rejected": 0.09928076714277267,
"logps/chosen": -67.30443572998047,
"logps/ref_chosen": -64.77729797363281,
"logps/ref_rejected": -84.71949768066406,
"logps/rejected": -88.06716918945312,
"loss": 1.3149,
"margin_dpo/margin_mean": 0.820530116558075,
"margin_dpo/margin_std": 1.8985121250152588,
"step": 84
},
{
"KL/chosen_KL_mean": -2.2237377166748047,
"KL/mean": -2.8801236152648926,
"KL/rejected_KL_mean": -3.536510467529297,
"KL/std": 1.7785743474960327,
"epoch": 0.12849584278155707,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3127713203430176,
"fcm_dpo/q_t": 0.46756821870803833,
"grad_norm": 26.659639358520508,
"learning_rate": 4.989901842900325e-07,
"logits/chosen": 0.1752845197916031,
"logits/rejected": 0.13079001009464264,
"logps/chosen": -52.475433349609375,
"logps/ref_chosen": -50.25169372558594,
"logps/ref_rejected": -66.55439758300781,
"logps/rejected": -70.09090423583984,
"loss": 1.2692,
"margin_dpo/margin_mean": 1.3127710819244385,
"margin_dpo/margin_std": 1.9418845176696777,
"step": 85
},
{
"KL/chosen_KL_mean": -2.8079910278320312,
"KL/mean": -3.3052587509155273,
"KL/rejected_KL_mean": -3.8025245666503906,
"KL/std": 1.901952862739563,
"epoch": 0.13000755857898716,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9945327639579773,
"fcm_dpo/q_t": 0.4754181504249573,
"grad_norm": 27.24299430847168,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": 0.20016081631183624,
"logits/rejected": 0.18127304315567017,
"logps/chosen": -63.53717041015625,
"logps/ref_chosen": -60.72917938232422,
"logps/ref_rejected": -72.30961608886719,
"logps/rejected": -76.11213684082031,
"loss": 1.2994,
"margin_dpo/margin_mean": 0.9945329427719116,
"margin_dpo/margin_std": 2.011247396469116,
"step": 86
},
{
"KL/chosen_KL_mean": -3.0212860107421875,
"KL/mean": -3.5553457736968994,
"KL/rejected_KL_mean": -4.0894012451171875,
"KL/std": 2.0620341300964355,
"epoch": 0.13151927437641722,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.068122148513794,
"fcm_dpo/q_t": 0.4736815392971039,
"grad_norm": 30.50346565246582,
"learning_rate": 4.987388156241114e-07,
"logits/chosen": 0.1732024997472763,
"logits/rejected": 0.1137414500117302,
"logps/chosen": -68.77925109863281,
"logps/ref_chosen": -65.75796508789062,
"logps/ref_rejected": -84.81159973144531,
"logps/rejected": -88.9010009765625,
"loss": 1.2969,
"margin_dpo/margin_mean": 1.0681220293045044,
"margin_dpo/margin_std": 2.4039626121520996,
"step": 87
},
{
"KL/chosen_KL_mean": -3.031696319580078,
"KL/mean": -3.444826126098633,
"KL/rejected_KL_mean": -3.857952117919922,
"KL/std": 2.2947521209716797,
"epoch": 0.1330309901738473,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8262563347816467,
"fcm_dpo/q_t": 0.47979599237442017,
"grad_norm": 30.31769561767578,
"learning_rate": 4.986026928455767e-07,
"logits/chosen": 0.20679256319999695,
"logits/rejected": 0.1803770810365677,
"logps/chosen": -65.85572814941406,
"logps/ref_chosen": -62.82402801513672,
"logps/ref_rejected": -74.9607162475586,
"logps/rejected": -78.81866455078125,
"loss": 1.3276,
"margin_dpo/margin_mean": 0.8262564539909363,
"margin_dpo/margin_std": 2.897235870361328,
"step": 88
},
{
"KL/chosen_KL_mean": -2.560823440551758,
"KL/mean": -3.250973701477051,
"KL/rejected_KL_mean": -3.9411239624023438,
"KL/std": 2.1574809551239014,
"epoch": 0.1345427059712774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3802971839904785,
"fcm_dpo/q_t": 0.46636801958084106,
"grad_norm": 29.29635238647461,
"learning_rate": 4.984596161153135e-07,
"logits/chosen": 0.2754148244857788,
"logits/rejected": 0.1902625858783722,
"logps/chosen": -43.75225830078125,
"logps/ref_chosen": -41.191436767578125,
"logps/ref_rejected": -85.44769287109375,
"logps/rejected": -89.38882446289062,
"loss": 1.2696,
"margin_dpo/margin_mean": 1.3802977800369263,
"margin_dpo/margin_std": 2.5125932693481445,
"step": 89
},
{
"KL/chosen_KL_mean": -3.1273326873779297,
"KL/mean": -3.7636611461639404,
"KL/rejected_KL_mean": -4.399990081787109,
"KL/std": 2.468470335006714,
"epoch": 0.1360544217687075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.27265465259552,
"fcm_dpo/q_t": 0.4687526822090149,
"grad_norm": 29.193382263183594,
"learning_rate": 4.983095894354857e-07,
"logits/chosen": 0.14260095357894897,
"logits/rejected": 0.08857794851064682,
"logps/chosen": -59.71124267578125,
"logps/ref_chosen": -56.58390808105469,
"logps/ref_rejected": -86.86978149414062,
"logps/rejected": -91.269775390625,
"loss": 1.2791,
"margin_dpo/margin_mean": 1.272654414176941,
"margin_dpo/margin_std": 2.4919843673706055,
"step": 90
},
{
"KL/chosen_KL_mean": -3.0993270874023438,
"KL/mean": -3.6860151290893555,
"KL/rejected_KL_mean": -4.272705078125,
"KL/std": 2.293879985809326,
"epoch": 0.13756613756613756,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.173375129699707,
"fcm_dpo/q_t": 0.4715588688850403,
"grad_norm": 25.667625427246094,
"learning_rate": 4.98152617002662e-07,
"logits/chosen": 0.15594482421875,
"logits/rejected": 0.11233173310756683,
"logps/chosen": -55.48167037963867,
"logps/ref_chosen": -52.38234329223633,
"logps/ref_rejected": -72.17642211914062,
"logps/rejected": -76.44912719726562,
"loss": 1.2915,
"margin_dpo/margin_mean": 1.173375129699707,
"margin_dpo/margin_std": 2.77371883392334,
"step": 91
},
{
"KL/chosen_KL_mean": -3.2288036346435547,
"KL/mean": -4.0197906494140625,
"KL/rejected_KL_mean": -4.810783386230469,
"KL/std": 2.6627650260925293,
"epoch": 0.13907785336356765,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5819768905639648,
"fcm_dpo/q_t": 0.46180659532546997,
"grad_norm": 26.956077575683594,
"learning_rate": 4.979887032076988e-07,
"logits/chosen": 0.19724005460739136,
"logits/rejected": 0.15849006175994873,
"logps/chosen": -56.23750305175781,
"logps/ref_chosen": -53.00870132446289,
"logps/ref_rejected": -79.77812957763672,
"logps/rejected": -84.58891296386719,
"loss": 1.2584,
"margin_dpo/margin_mean": 1.5819770097732544,
"margin_dpo/margin_std": 3.115145206451416,
"step": 92
},
{
"KL/chosen_KL_mean": -3.5571727752685547,
"KL/mean": -4.073748588562012,
"KL/rejected_KL_mean": -4.590320587158203,
"KL/std": 2.6155309677124023,
"epoch": 0.14058956916099774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0331499576568604,
"fcm_dpo/q_t": 0.4753725230693817,
"grad_norm": 24.509965896606445,
"learning_rate": 4.978178526356172e-07,
"logits/chosen": 0.19289353489875793,
"logits/rejected": 0.16430367529392242,
"logps/chosen": -48.46422576904297,
"logps/ref_chosen": -44.90705108642578,
"logps/ref_rejected": -58.7879524230957,
"logps/rejected": -63.378273010253906,
"loss": 1.3113,
"margin_dpo/margin_mean": 1.03315007686615,
"margin_dpo/margin_std": 3.2502970695495605,
"step": 93
},
{
"KL/chosen_KL_mean": -3.304903030395508,
"KL/mean": -4.201479434967041,
"KL/rejected_KL_mean": -5.098056793212891,
"KL/std": 3.1756069660186768,
"epoch": 0.1421012849584278,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.793152928352356,
"fcm_dpo/q_t": 0.4567800760269165,
"grad_norm": 27.731033325195312,
"learning_rate": 4.976400700654751e-07,
"logits/chosen": 0.26267024874687195,
"logits/rejected": 0.2218886911869049,
"logps/chosen": -63.24267578125,
"logps/ref_chosen": -59.93777084350586,
"logps/ref_rejected": -79.3138427734375,
"logps/rejected": -84.41189575195312,
"loss": 1.2533,
"margin_dpo/margin_mean": 1.7931523323059082,
"margin_dpo/margin_std": 3.9416329860687256,
"step": 94
},
{
"KL/chosen_KL_mean": -4.260015487670898,
"KL/mean": -5.063937187194824,
"KL/rejected_KL_mean": -5.86785888671875,
"KL/std": 2.690230131149292,
"epoch": 0.1436130007558579,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.607844591140747,
"fcm_dpo/q_t": 0.46108755469322205,
"grad_norm": 29.20717430114746,
"learning_rate": 4.974553604702332e-07,
"logits/chosen": 0.14258426427841187,
"logits/rejected": 0.0792468786239624,
"logps/chosen": -64.42850494384766,
"logps/ref_chosen": -60.168487548828125,
"logps/ref_rejected": -90.73665618896484,
"logps/rejected": -96.6045150756836,
"loss": 1.2598,
"margin_dpo/margin_mean": 1.6078438758850098,
"margin_dpo/margin_std": 3.364819049835205,
"step": 95
},
{
"KL/chosen_KL_mean": -4.149753570556641,
"KL/mean": -5.083588123321533,
"KL/rejected_KL_mean": -6.017417907714844,
"KL/std": 3.411038398742676,
"epoch": 0.14512471655328799,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8676685094833374,
"fcm_dpo/q_t": 0.45662426948547363,
"grad_norm": 27.411863327026367,
"learning_rate": 4.972637290166157e-07,
"logits/chosen": 0.18811967968940735,
"logits/rejected": 0.1457143872976303,
"logps/chosen": -64.81852722167969,
"logps/ref_chosen": -60.66877746582031,
"logps/ref_rejected": -88.30673217773438,
"logps/rejected": -94.32415008544922,
"loss": 1.2587,
"margin_dpo/margin_mean": 1.8676682710647583,
"margin_dpo/margin_std": 4.52335786819458,
"step": 96
},
{
"KL/chosen_KL_mean": -5.019113540649414,
"KL/mean": -5.533176422119141,
"KL/rejected_KL_mean": -6.0472412109375,
"KL/std": 3.6687517166137695,
"epoch": 0.14663643235071808,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0281330347061157,
"fcm_dpo/q_t": 0.475546658039093,
"grad_norm": 36.294029235839844,
"learning_rate": 4.970651810649666e-07,
"logits/chosen": 0.15061883628368378,
"logits/rejected": 0.10639595985412598,
"logps/chosen": -70.063232421875,
"logps/ref_chosen": -65.04412078857422,
"logps/ref_rejected": -78.42092895507812,
"logps/rejected": -84.46817016601562,
"loss": 1.347,
"margin_dpo/margin_mean": 1.0281330347061157,
"margin_dpo/margin_std": 5.024996280670166,
"step": 97
},
{
"KL/chosen_KL_mean": -4.630523681640625,
"KL/mean": -4.976384162902832,
"KL/rejected_KL_mean": -5.322246551513672,
"KL/std": 3.221841812133789,
"epoch": 0.14814814814814814,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6917204856872559,
"fcm_dpo/q_t": 0.482790470123291,
"grad_norm": 30.246551513671875,
"learning_rate": 4.968597221690985e-07,
"logits/chosen": 0.2132185399532318,
"logits/rejected": 0.186471626162529,
"logps/chosen": -60.13375473022461,
"logps/ref_chosen": -55.503231048583984,
"logps/ref_rejected": -72.81553649902344,
"logps/rejected": -78.13778686523438,
"loss": 1.3441,
"margin_dpo/margin_mean": 0.6917206048965454,
"margin_dpo/margin_std": 3.1939637660980225,
"step": 98
},
{
"KL/chosen_KL_mean": -4.680410385131836,
"KL/mean": -5.174110412597656,
"KL/rejected_KL_mean": -5.667812347412109,
"KL/std": 3.500779151916504,
"epoch": 0.14965986394557823,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9873988628387451,
"fcm_dpo/q_t": 0.4767622947692871,
"grad_norm": 32.56043243408203,
"learning_rate": 4.966473580761389e-07,
"logits/chosen": 0.24544592201709747,
"logits/rejected": 0.2058737725019455,
"logps/chosen": -63.25605010986328,
"logps/ref_chosen": -58.57563781738281,
"logps/ref_rejected": -78.693603515625,
"logps/rejected": -84.36141967773438,
"loss": 1.3409,
"margin_dpo/margin_mean": 0.9873991012573242,
"margin_dpo/margin_std": 4.55812931060791,
"step": 99
},
{
"KL/chosen_KL_mean": -5.287479400634766,
"KL/mean": -6.03157901763916,
"KL/rejected_KL_mean": -6.775676727294922,
"KL/std": 4.144209384918213,
"epoch": 0.15117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4881947040557861,
"fcm_dpo/q_t": 0.4669296145439148,
"grad_norm": 33.02346420288086,
"learning_rate": 4.964280947263676e-07,
"logits/chosen": 0.2219209372997284,
"logits/rejected": 0.21426787972450256,
"logps/chosen": -84.87091064453125,
"logps/ref_chosen": -79.58343505859375,
"logps/ref_rejected": -92.152587890625,
"logps/rejected": -98.92826080322266,
"loss": 1.3292,
"margin_dpo/margin_mean": 1.4881953001022339,
"margin_dpo/margin_std": 5.861654281616211,
"step": 100
},
{
"KL/chosen_KL_mean": -4.301380157470703,
"KL/mean": -5.327387809753418,
"KL/rejected_KL_mean": -6.353397369384766,
"KL/std": 3.664494514465332,
"epoch": 0.15268329554043839,
"fcm_dpo/beta": 0.10289128869771957,
"fcm_dpo/delta": 0.28107818961143494,
"fcm_dpo/margin": 2.0520172119140625,
"fcm_dpo/q_t": 0.45036324858665466,
"grad_norm": 26.742868423461914,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": 0.25061866641044617,
"logits/rejected": 0.21055373549461365,
"logps/chosen": -56.6341667175293,
"logps/ref_chosen": -52.332786560058594,
"logps/ref_rejected": -69.55589294433594,
"logps/rejected": -75.90929412841797,
"loss": 1.2325,
"margin_dpo/margin_mean": 2.0520172119140625,
"margin_dpo/margin_std": 4.156118869781494,
"step": 101
},
{
"KL/chosen_KL_mean": -5.317193984985352,
"KL/mean": -5.750248908996582,
"KL/rejected_KL_mean": -6.183307647705078,
"KL/std": 3.721498966217041,
"epoch": 0.15419501133786848,
"fcm_dpo/beta": 0.10578257590532303,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8661136627197266,
"fcm_dpo/q_t": 0.4816773533821106,
"grad_norm": 34.89421081542969,
"learning_rate": 4.959688949822748e-07,
"logits/chosen": 0.1457417905330658,
"logits/rejected": 0.1073441207408905,
"logps/chosen": -70.06067657470703,
"logps/ref_chosen": -64.74348449707031,
"logps/ref_rejected": -69.06132507324219,
"logps/rejected": -75.24463653564453,
"loss": 1.375,
"margin_dpo/margin_mean": 0.8661136627197266,
"margin_dpo/margin_std": 5.487614154815674,
"step": 102
},
{
"KL/chosen_KL_mean": -4.824554443359375,
"KL/mean": -5.601681709289551,
"KL/rejected_KL_mean": -6.378810882568359,
"KL/std": 3.8675785064697266,
"epoch": 0.15570672713529857,
"fcm_dpo/beta": 0.10896602272987366,
"fcm_dpo/delta": 0.2922336161136627,
"fcm_dpo/margin": 1.5542576313018799,
"fcm_dpo/q_t": 0.46070000529289246,
"grad_norm": 31.94975471496582,
"learning_rate": 4.957289714327572e-07,
"logits/chosen": 0.2437642365694046,
"logits/rejected": 0.21064837276935577,
"logps/chosen": -68.66119384765625,
"logps/ref_chosen": -63.83664321899414,
"logps/ref_rejected": -79.32362365722656,
"logps/rejected": -85.70243835449219,
"loss": 1.2614,
"margin_dpo/margin_mean": 1.5542573928833008,
"margin_dpo/margin_std": 3.6246559619903564,
"step": 103
},
{
"KL/chosen_KL_mean": -4.671621322631836,
"KL/mean": -5.832412242889404,
"KL/rejected_KL_mean": -6.9932098388671875,
"KL/std": 4.947868347167969,
"epoch": 0.15721844293272866,
"fcm_dpo/beta": 0.11496137082576752,
"fcm_dpo/delta": 0.24464420974254608,
"fcm_dpo/margin": 2.3215885162353516,
"fcm_dpo/q_t": 0.445722758769989,
"grad_norm": 35.56681442260742,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": 0.2890929579734802,
"logits/rejected": 0.19809794425964355,
"logps/chosen": -65.67082977294922,
"logps/ref_chosen": -60.99920654296875,
"logps/ref_rejected": -98.84645080566406,
"logps/rejected": -105.83966064453125,
"loss": 1.2572,
"margin_dpo/margin_mean": 2.3215882778167725,
"margin_dpo/margin_std": 6.310924530029297,
"step": 104
},
{
"KL/chosen_KL_mean": -5.358615875244141,
"KL/mean": -6.24442195892334,
"KL/rejected_KL_mean": -7.130226135253906,
"KL/std": 3.9761853218078613,
"epoch": 0.15873015873015872,
"fcm_dpo/beta": 0.11777327209711075,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.7716116905212402,
"fcm_dpo/q_t": 0.45355188846588135,
"grad_norm": 36.21678161621094,
"learning_rate": 4.952285105344791e-07,
"logits/chosen": 0.20373253524303436,
"logits/rejected": 0.14855940639972687,
"logps/chosen": -76.30888366699219,
"logps/ref_chosen": -70.95027160644531,
"logps/ref_rejected": -87.88340759277344,
"logps/rejected": -95.01364135742188,
"loss": 1.2679,
"margin_dpo/margin_mean": 1.7716121673583984,
"margin_dpo/margin_std": 4.9041643142700195,
"step": 105
},
{
"KL/chosen_KL_mean": -5.010364532470703,
"KL/mean": -5.787876129150391,
"KL/rejected_KL_mean": -6.5653839111328125,
"KL/std": 3.9396629333496094,
"epoch": 0.1602418745275888,
"fcm_dpo/beta": 0.11777327209711075,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5550204515457153,
"fcm_dpo/q_t": 0.4568718373775482,
"grad_norm": 37.559017181396484,
"learning_rate": 4.949679871846857e-07,
"logits/chosen": 0.22271032631397247,
"logits/rejected": 0.20895954966545105,
"logps/chosen": -67.46969604492188,
"logps/ref_chosen": -62.45933151245117,
"logps/ref_rejected": -67.00595092773438,
"logps/rejected": -73.57133483886719,
"loss": 1.3045,
"margin_dpo/margin_mean": 1.5550203323364258,
"margin_dpo/margin_std": 5.320964813232422,
"step": 106
},
{
"KL/chosen_KL_mean": -6.474697113037109,
"KL/mean": -6.880711555480957,
"KL/rejected_KL_mean": -7.2867279052734375,
"KL/std": 4.923356056213379,
"epoch": 0.1617535903250189,
"fcm_dpo/beta": 0.12500976026058197,
"fcm_dpo/delta": 0.2981524169445038,
"fcm_dpo/margin": 0.8120315074920654,
"fcm_dpo/q_t": 0.47507673501968384,
"grad_norm": 55.240623474121094,
"learning_rate": 4.947006115536947e-07,
"logits/chosen": 0.183563694357872,
"logits/rejected": 0.1593797206878662,
"logps/chosen": -82.31266784667969,
"logps/ref_chosen": -75.83796691894531,
"logps/ref_rejected": -87.74038696289062,
"logps/rejected": -95.02711486816406,
"loss": 1.4266,
"margin_dpo/margin_mean": 0.8120315670967102,
"margin_dpo/margin_std": 6.172867774963379,
"step": 107
},
{
"KL/chosen_KL_mean": -5.433837890625,
"KL/mean": -6.352394104003906,
"KL/rejected_KL_mean": -7.2709503173828125,
"KL/std": 4.04887580871582,
"epoch": 0.16326530612244897,
"fcm_dpo/beta": 0.1361444890499115,
"fcm_dpo/delta": 0.5635470151901245,
"fcm_dpo/margin": 1.8371151685714722,
"fcm_dpo/q_t": 0.4484684467315674,
"grad_norm": 40.03304672241211,
"learning_rate": 4.944263911205772e-07,
"logits/chosen": 0.1733136624097824,
"logits/rejected": 0.1424228549003601,
"logps/chosen": -73.82707214355469,
"logps/ref_chosen": -68.39323425292969,
"logps/ref_rejected": -83.24267578125,
"logps/rejected": -90.51362609863281,
"loss": 1.274,
"margin_dpo/margin_mean": 1.8371152877807617,
"margin_dpo/margin_std": 5.285173416137695,
"step": 108
},
{
"KL/chosen_KL_mean": -4.783725738525391,
"KL/mean": -6.253157615661621,
"KL/rejected_KL_mean": -7.722591400146484,
"KL/std": 4.904244899749756,
"epoch": 0.16477702191987906,
"fcm_dpo/beta": 0.14831490814685822,
"fcm_dpo/delta": 0.3801053464412689,
"fcm_dpo/margin": 2.9388718605041504,
"fcm_dpo/q_t": 0.4147114157676697,
"grad_norm": 40.31388854980469,
"learning_rate": 4.941453335558681e-07,
"logits/chosen": 0.17615270614624023,
"logits/rejected": 0.1229727491736412,
"logps/chosen": -60.31120681762695,
"logps/ref_chosen": -55.52748107910156,
"logps/ref_rejected": -83.55218505859375,
"logps/rejected": -91.2747802734375,
"loss": 1.1218,
"margin_dpo/margin_mean": 2.938871383666992,
"margin_dpo/margin_std": 5.313412666320801,
"step": 109
},
{
"KL/chosen_KL_mean": -6.120460510253906,
"KL/mean": -6.251190185546875,
"KL/rejected_KL_mean": -6.381919860839844,
"KL/std": 3.969235420227051,
"epoch": 0.16628873771730915,
"fcm_dpo/beta": 0.15097622573375702,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.26145556569099426,
"fcm_dpo/q_t": 0.48575273156166077,
"grad_norm": 72.10308074951172,
"learning_rate": 4.938574467213517e-07,
"logits/chosen": 0.1714879870414734,
"logits/rejected": 0.18068574368953705,
"logps/chosen": -87.27920532226562,
"logps/ref_chosen": -81.15874481201172,
"logps/ref_rejected": -72.56021118164062,
"logps/rejected": -78.94213104248047,
"loss": 1.4907,
"margin_dpo/margin_mean": 0.26145583391189575,
"margin_dpo/margin_std": 5.336217403411865,
"step": 110
},
{
"KL/chosen_KL_mean": -5.464059829711914,
"KL/mean": -6.4398345947265625,
"KL/rejected_KL_mean": -7.415611267089844,
"KL/std": 4.08194637298584,
"epoch": 0.16780045351473924,
"fcm_dpo/beta": 0.16302645206451416,
"fcm_dpo/delta": 0.4963175654411316,
"fcm_dpo/margin": 1.951550006866455,
"fcm_dpo/q_t": 0.43752434849739075,
"grad_norm": 42.85625457763672,
"learning_rate": 4.935627386698418e-07,
"logits/chosen": 0.3038405776023865,
"logits/rejected": 0.26507824659347534,
"logps/chosen": -57.82304382324219,
"logps/ref_chosen": -52.358985900878906,
"logps/ref_rejected": -77.06150817871094,
"logps/rejected": -84.47711181640625,
"loss": 1.2376,
"margin_dpo/margin_mean": 1.9515496492385864,
"margin_dpo/margin_std": 4.929745674133301,
"step": 111
},
{
"KL/chosen_KL_mean": -5.119453430175781,
"KL/mean": -6.2895002365112305,
"KL/rejected_KL_mean": -7.4595489501953125,
"KL/std": 4.037227630615234,
"epoch": 0.1693121693121693,
"fcm_dpo/beta": 0.17741267383098602,
"fcm_dpo/delta": 0.4004848003387451,
"fcm_dpo/margin": 2.3400979042053223,
"fcm_dpo/q_t": 0.41141277551651,
"grad_norm": 55.50019454956055,
"learning_rate": 4.932612176449559e-07,
"logits/chosen": 0.21610409021377563,
"logits/rejected": 0.1481962949037552,
"logps/chosen": -68.13951873779297,
"logps/ref_chosen": -63.02006530761719,
"logps/ref_rejected": -111.36941528320312,
"logps/rejected": -118.82896423339844,
"loss": 1.1581,
"margin_dpo/margin_mean": 2.340097665786743,
"margin_dpo/margin_std": 4.509957790374756,
"step": 112
},
{
"KL/chosen_KL_mean": -5.140375137329102,
"KL/mean": -5.796399116516113,
"KL/rejected_KL_mean": -6.452423095703125,
"KL/std": 3.3658576011657715,
"epoch": 0.1708238851095994,
"fcm_dpo/beta": 0.18488053977489471,
"fcm_dpo/delta": 0.22963446378707886,
"fcm_dpo/margin": 1.3120431900024414,
"fcm_dpo/q_t": 0.445895791053772,
"grad_norm": 69.64769744873047,
"learning_rate": 4.929528920808854e-07,
"logits/chosen": 0.21564623713493347,
"logits/rejected": 0.1755000799894333,
"logps/chosen": -60.948036193847656,
"logps/ref_chosen": -55.80766296386719,
"logps/ref_rejected": -69.84014129638672,
"logps/rejected": -76.29256439208984,
"loss": 1.3029,
"margin_dpo/margin_mean": 1.3120434284210205,
"margin_dpo/margin_std": 4.295475006103516,
"step": 113
},
{
"KL/chosen_KL_mean": -4.754701614379883,
"KL/mean": -5.911993026733398,
"KL/rejected_KL_mean": -7.069286346435547,
"KL/std": 3.651322841644287,
"epoch": 0.17233560090702948,
"fcm_dpo/beta": 0.19917583465576172,
"fcm_dpo/delta": 0.3558206558227539,
"fcm_dpo/margin": 2.3145804405212402,
"fcm_dpo/q_t": 0.4093940258026123,
"grad_norm": 57.998111724853516,
"learning_rate": 4.92637770602159e-07,
"logits/chosen": 0.2491624653339386,
"logits/rejected": 0.18856176733970642,
"logps/chosen": -71.08747863769531,
"logps/ref_chosen": -66.33277130126953,
"logps/ref_rejected": -71.61489868164062,
"logps/rejected": -78.6841812133789,
"loss": 1.1409,
"margin_dpo/margin_mean": 2.3145809173583984,
"margin_dpo/margin_std": 4.402614593505859,
"step": 114
},
{
"KL/chosen_KL_mean": -5.139909744262695,
"KL/mean": -6.020167350769043,
"KL/rejected_KL_mean": -6.900426864624023,
"KL/std": 3.308725595474243,
"epoch": 0.17384731670445955,
"fcm_dpo/beta": 0.21640095114707947,
"fcm_dpo/delta": 0.43631792068481445,
"fcm_dpo/margin": 1.7605162858963013,
"fcm_dpo/q_t": 0.4244588017463684,
"grad_norm": 63.72927474975586,
"learning_rate": 4.923158620234019e-07,
"logits/chosen": 0.24715614318847656,
"logits/rejected": 0.18282869458198547,
"logps/chosen": -60.888946533203125,
"logps/ref_chosen": -55.74903869628906,
"logps/ref_rejected": -79.59849548339844,
"logps/rejected": -86.4989242553711,
"loss": 1.2101,
"margin_dpo/margin_mean": 1.7605164051055908,
"margin_dpo/margin_std": 4.01201057434082,
"step": 115
},
{
"KL/chosen_KL_mean": -4.754339218139648,
"KL/mean": -6.064699172973633,
"KL/rejected_KL_mean": -7.37506103515625,
"KL/std": 3.3698506355285645,
"epoch": 0.17535903250188964,
"fcm_dpo/beta": 0.2299938201904297,
"fcm_dpo/delta": 0.2077406346797943,
"fcm_dpo/margin": 2.6207261085510254,
"fcm_dpo/q_t": 0.37513041496276855,
"grad_norm": 48.991607666015625,
"learning_rate": 4.91987175349089e-07,
"logits/chosen": 0.21752528846263885,
"logits/rejected": 0.14967337250709534,
"logps/chosen": -54.1195068359375,
"logps/ref_chosen": -49.36516571044922,
"logps/ref_rejected": -72.84671020507812,
"logps/rejected": -80.22177124023438,
"loss": 1.0273,
"margin_dpo/margin_mean": 2.6207261085510254,
"margin_dpo/margin_std": 3.627825975418091,
"step": 116
},
{
"KL/chosen_KL_mean": -4.6624298095703125,
"KL/mean": -5.545193195343018,
"KL/rejected_KL_mean": -6.427959442138672,
"KL/std": 3.1073155403137207,
"epoch": 0.17687074829931973,
"fcm_dpo/beta": 0.23274856805801392,
"fcm_dpo/delta": 0.037866029888391495,
"fcm_dpo/margin": 1.7655225992202759,
"fcm_dpo/q_t": 0.4122212529182434,
"grad_norm": 68.00582122802734,
"learning_rate": 4.916517197732933e-07,
"logits/chosen": 0.23936522006988525,
"logits/rejected": 0.20207154750823975,
"logps/chosen": -62.373329162597656,
"logps/ref_chosen": -57.710899353027344,
"logps/ref_rejected": -69.77253723144531,
"logps/rejected": -76.20050048828125,
"loss": 1.2094,
"margin_dpo/margin_mean": 1.7655227184295654,
"margin_dpo/margin_std": 3.8186731338500977,
"step": 117
},
{
"KL/chosen_KL_mean": -4.468959808349609,
"KL/mean": -5.641507148742676,
"KL/rejected_KL_mean": -6.814048767089844,
"KL/std": 3.3355770111083984,
"epoch": 0.17838246409674982,
"fcm_dpo/beta": 0.2407892644405365,
"fcm_dpo/delta": 0.2500095069408417,
"fcm_dpo/margin": 2.345097541809082,
"fcm_dpo/q_t": 0.3831072747707367,
"grad_norm": 61.70059585571289,
"learning_rate": 4.913095046794281e-07,
"logits/chosen": 0.30035635828971863,
"logits/rejected": 0.2581939697265625,
"logps/chosen": -56.94886016845703,
"logps/ref_chosen": -52.479896545410156,
"logps/ref_rejected": -81.359130859375,
"logps/rejected": -88.17317199707031,
"loss": 1.052,
"margin_dpo/margin_mean": 2.345097064971924,
"margin_dpo/margin_std": 3.4296135902404785,
"step": 118
},
{
"KL/chosen_KL_mean": -5.600677490234375,
"KL/mean": -6.450741291046143,
"KL/rejected_KL_mean": -7.300800323486328,
"KL/std": 3.7528910636901855,
"epoch": 0.17989417989417988,
"fcm_dpo/beta": 0.25984010100364685,
"fcm_dpo/delta": 0.3726928234100342,
"fcm_dpo/margin": 1.7001276016235352,
"fcm_dpo/q_t": 0.4158112704753876,
"grad_norm": 81.90481567382812,
"learning_rate": 4.909605396399855e-07,
"logits/chosen": 0.18883086740970612,
"logits/rejected": 0.15015271306037903,
"logps/chosen": -66.9583511352539,
"logps/ref_chosen": -61.35767364501953,
"logps/ref_rejected": -75.71510314941406,
"logps/rejected": -83.01589965820312,
"loss": 1.283,
"margin_dpo/margin_mean": 1.7001276016235352,
"margin_dpo/margin_std": 4.581892967224121,
"step": 119
},
{
"KL/chosen_KL_mean": -4.532203674316406,
"KL/mean": -5.765423774719238,
"KL/rejected_KL_mean": -6.9986419677734375,
"KL/std": 3.1511075496673584,
"epoch": 0.18140589569160998,
"fcm_dpo/beta": 0.2700057625770569,
"fcm_dpo/delta": 0.1430681198835373,
"fcm_dpo/margin": 2.4664409160614014,
"fcm_dpo/q_t": 0.365251362323761,
"grad_norm": 63.3851203918457,
"learning_rate": 4.906048344162676e-07,
"logits/chosen": 0.18862515687942505,
"logits/rejected": 0.12877434492111206,
"logps/chosen": -64.43977355957031,
"logps/ref_chosen": -59.907569885253906,
"logps/ref_rejected": -79.6910629272461,
"logps/rejected": -86.68970489501953,
"loss": 0.9804,
"margin_dpo/margin_mean": 2.4664411544799805,
"margin_dpo/margin_std": 3.162449836730957,
"step": 120
},
{
"KL/chosen_KL_mean": -4.892913818359375,
"KL/mean": -6.147130966186523,
"KL/rejected_KL_mean": -7.4013519287109375,
"KL/std": 3.5860328674316406,
"epoch": 0.18291761148904007,
"fcm_dpo/beta": 0.27552086114883423,
"fcm_dpo/delta": 0.11710186302661896,
"fcm_dpo/margin": 2.5084357261657715,
"fcm_dpo/q_t": 0.3749258816242218,
"grad_norm": 61.17416000366211,
"learning_rate": 4.902423989581143e-07,
"logits/chosen": 0.30166101455688477,
"logits/rejected": 0.21480508148670197,
"logps/chosen": -60.558956146240234,
"logps/ref_chosen": -55.66604232788086,
"logps/ref_rejected": -101.56233978271484,
"logps/rejected": -108.96369171142578,
"loss": 1.014,
"margin_dpo/margin_mean": 2.5084352493286133,
"margin_dpo/margin_std": 3.9617276191711426,
"step": 121
},
{
"KL/chosen_KL_mean": -5.208187103271484,
"KL/mean": -6.4071879386901855,
"KL/rejected_KL_mean": -7.606189727783203,
"KL/std": 3.7734618186950684,
"epoch": 0.18442932728647016,
"fcm_dpo/beta": 0.284409761428833,
"fcm_dpo/delta": 0.1253754049539566,
"fcm_dpo/margin": 2.397998571395874,
"fcm_dpo/q_t": 0.38381946086883545,
"grad_norm": 89.1806640625,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": 0.19963660836219788,
"logits/rejected": 0.16456535458564758,
"logps/chosen": -68.54255676269531,
"logps/ref_chosen": -63.334373474121094,
"logps/ref_rejected": -73.67523193359375,
"logps/rejected": -81.28142547607422,
"loss": 1.1012,
"margin_dpo/margin_mean": 2.397998332977295,
"margin_dpo/margin_std": 4.460177421569824,
"step": 122
},
{
"KL/chosen_KL_mean": -4.837169647216797,
"KL/mean": -5.956876754760742,
"KL/rejected_KL_mean": -7.0765838623046875,
"KL/std": 3.257021188735962,
"epoch": 0.18594104308390022,
"fcm_dpo/beta": 0.29277366399765015,
"fcm_dpo/delta": 0.15317855775356293,
"fcm_dpo/margin": 2.239412784576416,
"fcm_dpo/q_t": 0.3722848892211914,
"grad_norm": 71.29094696044922,
"learning_rate": 4.894973780788722e-07,
"logits/chosen": 0.22779148817062378,
"logits/rejected": 0.18621155619621277,
"logps/chosen": -61.73591232299805,
"logps/ref_chosen": -56.89874267578125,
"logps/ref_rejected": -78.97028350830078,
"logps/rejected": -86.04686737060547,
"loss": 1.0543,
"margin_dpo/margin_mean": 2.239412307739258,
"margin_dpo/margin_std": 3.5291574001312256,
"step": 123
},
{
"KL/chosen_KL_mean": -5.083835601806641,
"KL/mean": -6.601924896240234,
"KL/rejected_KL_mean": -8.120014190673828,
"KL/std": 3.4692869186401367,
"epoch": 0.1874527588813303,
"fcm_dpo/beta": 0.2885604500770569,
"fcm_dpo/delta": -0.08434109389781952,
"fcm_dpo/margin": 3.036177396774292,
"fcm_dpo/q_t": 0.32999253273010254,
"grad_norm": 60.00226974487305,
"learning_rate": 4.89114813497619e-07,
"logits/chosen": 0.25246724486351013,
"logits/rejected": 0.19060075283050537,
"logps/chosen": -62.199920654296875,
"logps/ref_chosen": -57.116085052490234,
"logps/ref_rejected": -87.93074035644531,
"logps/rejected": -96.0507583618164,
"loss": 0.8918,
"margin_dpo/margin_mean": 3.036177396774292,
"margin_dpo/margin_std": 3.5483925342559814,
"step": 124
},
{
"KL/chosen_KL_mean": -5.379281997680664,
"KL/mean": -6.626376152038574,
"KL/rejected_KL_mean": -7.873466491699219,
"KL/std": 3.7205638885498047,
"epoch": 0.1889644746787604,
"fcm_dpo/beta": 0.2898848354816437,
"fcm_dpo/delta": 0.0822625532746315,
"fcm_dpo/margin": 2.4941811561584473,
"fcm_dpo/q_t": 0.3657096028327942,
"grad_norm": 73.03401947021484,
"learning_rate": 4.887255603610184e-07,
"logits/chosen": 0.2572287619113922,
"logits/rejected": 0.19734513759613037,
"logps/chosen": -71.08546447753906,
"logps/ref_chosen": -65.7061767578125,
"logps/ref_rejected": -91.72711944580078,
"logps/rejected": -99.6005859375,
"loss": 1.0256,
"margin_dpo/margin_mean": 2.4941816329956055,
"margin_dpo/margin_std": 3.7722997665405273,
"step": 125
},
{
"KL/chosen_KL_mean": -4.964244842529297,
"KL/mean": -5.7615766525268555,
"KL/rejected_KL_mean": -6.558906555175781,
"KL/std": 3.618135690689087,
"epoch": 0.19047619047619047,
"fcm_dpo/beta": 0.308197557926178,
"fcm_dpo/delta": 0.3242906332015991,
"fcm_dpo/margin": 1.5946621894836426,
"fcm_dpo/q_t": 0.40892714262008667,
"grad_norm": 82.392333984375,
"learning_rate": 4.883296295573176e-07,
"logits/chosen": 0.09359246492385864,
"logits/rejected": 0.08766859024763107,
"logps/chosen": -73.14033508300781,
"logps/ref_chosen": -68.17608642578125,
"logps/ref_rejected": -65.1175537109375,
"logps/rejected": -71.67646026611328,
"loss": 1.2336,
"margin_dpo/margin_mean": 1.5946623086929321,
"margin_dpo/margin_std": 3.873288631439209,
"step": 126
},
{
"KL/chosen_KL_mean": -5.131870269775391,
"KL/mean": -6.483763217926025,
"KL/rejected_KL_mean": -7.835659027099609,
"KL/std": 3.119049549102783,
"epoch": 0.19198790627362056,
"fcm_dpo/beta": 0.30798083543777466,
"fcm_dpo/delta": -0.038629740476608276,
"fcm_dpo/margin": 2.703786849975586,
"fcm_dpo/q_t": 0.33612674474716187,
"grad_norm": 72.35118103027344,
"learning_rate": 4.87927032161552e-07,
"logits/chosen": 0.19651305675506592,
"logits/rejected": 0.165663942694664,
"logps/chosen": -67.01210021972656,
"logps/ref_chosen": -61.88023376464844,
"logps/ref_rejected": -68.46012878417969,
"logps/rejected": -76.29578399658203,
"loss": 0.9255,
"margin_dpo/margin_mean": 2.703786849975586,
"margin_dpo/margin_std": 3.216360092163086,
"step": 127
},
{
"KL/chosen_KL_mean": -5.400476455688477,
"KL/mean": -6.705332279205322,
"KL/rejected_KL_mean": -8.010189056396484,
"KL/std": 3.893587350845337,
"epoch": 0.19349962207105065,
"fcm_dpo/beta": 0.30849719047546387,
"fcm_dpo/delta": -0.006407309323549271,
"fcm_dpo/margin": 2.6097097396850586,
"fcm_dpo/q_t": 0.36017996072769165,
"grad_norm": 74.5383071899414,
"learning_rate": 4.875177794352363e-07,
"logits/chosen": 0.22811782360076904,
"logits/rejected": 0.169705331325531,
"logps/chosen": -72.10945892333984,
"logps/ref_chosen": -66.708984375,
"logps/ref_rejected": -94.97969055175781,
"logps/rejected": -102.98988342285156,
"loss": 1.1106,
"margin_dpo/margin_mean": 2.6097097396850586,
"margin_dpo/margin_std": 4.57863187789917,
"step": 128
},
{
"KL/chosen_KL_mean": -5.905820846557617,
"KL/mean": -6.7846760749816895,
"KL/rejected_KL_mean": -7.663532257080078,
"KL/std": 3.419311046600342,
"epoch": 0.19501133786848074,
"fcm_dpo/beta": 0.3223820924758911,
"fcm_dpo/delta": 0.2470059096813202,
"fcm_dpo/margin": 1.757707953453064,
"fcm_dpo/q_t": 0.39814797043800354,
"grad_norm": 109.02452850341797,
"learning_rate": 4.871018828260491e-07,
"logits/chosen": 0.1916099488735199,
"logits/rejected": 0.18546706438064575,
"logps/chosen": -71.24464416503906,
"logps/ref_chosen": -65.33882904052734,
"logps/ref_rejected": -68.06109619140625,
"logps/rejected": -75.72462463378906,
"loss": 1.2241,
"margin_dpo/margin_mean": 1.7577086687088013,
"margin_dpo/margin_std": 3.991182565689087,
"step": 129
},
{
"KL/chosen_KL_mean": -5.464776992797852,
"KL/mean": -6.652037620544434,
"KL/rejected_KL_mean": -7.839298248291016,
"KL/std": 3.3182621002197266,
"epoch": 0.1965230536659108,
"fcm_dpo/beta": 0.322995662689209,
"fcm_dpo/delta": 0.0319809764623642,
"fcm_dpo/margin": 2.3745198249816895,
"fcm_dpo/q_t": 0.35143962502479553,
"grad_norm": 92.80889892578125,
"learning_rate": 4.866793539675126e-07,
"logits/chosen": 0.15645794570446014,
"logits/rejected": 0.10603205859661102,
"logps/chosen": -64.12551879882812,
"logps/ref_chosen": -58.660743713378906,
"logps/ref_rejected": -79.24510192871094,
"logps/rejected": -87.08439636230469,
"loss": 0.9934,
"margin_dpo/margin_mean": 2.3745198249816895,
"margin_dpo/margin_std": 3.211923599243164,
"step": 130
},
{
"KL/chosen_KL_mean": -5.23967170715332,
"KL/mean": -6.682486057281494,
"KL/rejected_KL_mean": -8.125301361083984,
"KL/std": 3.897421360015869,
"epoch": 0.1980347694633409,
"fcm_dpo/beta": 0.32098785042762756,
"fcm_dpo/delta": -0.13940098881721497,
"fcm_dpo/margin": 2.8856329917907715,
"fcm_dpo/q_t": 0.3405371308326721,
"grad_norm": 77.46083068847656,
"learning_rate": 4.86250204678667e-07,
"logits/chosen": 0.17860546708106995,
"logits/rejected": 0.1137775182723999,
"logps/chosen": -57.754207611083984,
"logps/ref_chosen": -52.51453399658203,
"logps/ref_rejected": -85.18299865722656,
"logps/rejected": -93.30830383300781,
"loss": 1.0287,
"margin_dpo/margin_mean": 2.8856334686279297,
"margin_dpo/margin_std": 4.351919651031494,
"step": 131
},
{
"KL/chosen_KL_mean": -5.565944671630859,
"KL/mean": -6.807995796203613,
"KL/rejected_KL_mean": -8.050046920776367,
"KL/std": 3.650963544845581,
"epoch": 0.19954648526077098,
"fcm_dpo/beta": 0.31917959451675415,
"fcm_dpo/delta": 0.007726715877652168,
"fcm_dpo/margin": 2.484100818634033,
"fcm_dpo/q_t": 0.3431278467178345,
"grad_norm": 81.25666809082031,
"learning_rate": 4.858144469637408e-07,
"logits/chosen": 0.24190431833267212,
"logits/rejected": 0.20946651697158813,
"logps/chosen": -71.2510757446289,
"logps/ref_chosen": -65.68513488769531,
"logps/ref_rejected": -69.54120635986328,
"logps/rejected": -77.59125518798828,
"loss": 1.0342,
"margin_dpo/margin_mean": 2.484100580215454,
"margin_dpo/margin_std": 3.7712955474853516,
"step": 132
},
{
"KL/chosen_KL_mean": -5.565555572509766,
"KL/mean": -6.669124603271484,
"KL/rejected_KL_mean": -7.7726898193359375,
"KL/std": 3.5532050132751465,
"epoch": 0.20105820105820105,
"fcm_dpo/beta": 0.3241734504699707,
"fcm_dpo/delta": 0.09094859659671783,
"fcm_dpo/margin": 2.207136631011963,
"fcm_dpo/q_t": 0.3639262616634369,
"grad_norm": 92.71206665039062,
"learning_rate": 4.853720930118138e-07,
"logits/chosen": 0.15483415126800537,
"logits/rejected": 0.14521589875221252,
"logps/chosen": -69.16366577148438,
"logps/ref_chosen": -63.598114013671875,
"logps/ref_rejected": -73.72798156738281,
"logps/rejected": -81.50067138671875,
"loss": 1.0788,
"margin_dpo/margin_mean": 2.207136631011963,
"margin_dpo/margin_std": 3.691882610321045,
"step": 133
},
{
"KL/chosen_KL_mean": -5.148525238037109,
"KL/mean": -6.78424072265625,
"KL/rejected_KL_mean": -8.41995620727539,
"KL/std": 4.025635719299316,
"epoch": 0.20256991685563114,
"fcm_dpo/beta": 0.31296706199645996,
"fcm_dpo/delta": -0.2501199245452881,
"fcm_dpo/margin": 3.2714293003082275,
"fcm_dpo/q_t": 0.31163257360458374,
"grad_norm": 70.16565704345703,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": 0.2684386968612671,
"logits/rejected": 0.21187162399291992,
"logps/chosen": -58.94309616088867,
"logps/ref_chosen": -53.79457092285156,
"logps/ref_rejected": -74.16741943359375,
"logps/rejected": -82.58737182617188,
"loss": 0.8797,
"margin_dpo/margin_mean": 3.271430253982544,
"margin_dpo/margin_std": 3.986935615539551,
"step": 134
},
{
"KL/chosen_KL_mean": -5.128990173339844,
"KL/mean": -6.169395446777344,
"KL/rejected_KL_mean": -7.209804534912109,
"KL/std": 3.7049777507781982,
"epoch": 0.20408163265306123,
"fcm_dpo/beta": 0.31707170605659485,
"fcm_dpo/delta": 0.1498485505580902,
"fcm_dpo/margin": 2.080817699432373,
"fcm_dpo/q_t": 0.3767089545726776,
"grad_norm": 72.5808334350586,
"learning_rate": 4.844676460754862e-07,
"logits/chosen": 0.19408661127090454,
"logits/rejected": 0.1618993878364563,
"logps/chosen": -54.570068359375,
"logps/ref_chosen": -49.441078186035156,
"logps/ref_rejected": -65.96878051757812,
"logps/rejected": -73.1785888671875,
"loss": 1.1257,
"margin_dpo/margin_mean": 2.080817222595215,
"margin_dpo/margin_std": 3.858844518661499,
"step": 135
},
{
"KL/chosen_KL_mean": -6.007057189941406,
"KL/mean": -7.334796905517578,
"KL/rejected_KL_mean": -8.66253662109375,
"KL/std": 4.03962516784668,
"epoch": 0.20559334845049132,
"fcm_dpo/beta": 0.3157135248184204,
"fcm_dpo/delta": -0.04242422431707382,
"fcm_dpo/margin": 2.655482530593872,
"fcm_dpo/q_t": 0.36486750841140747,
"grad_norm": 97.05379486083984,
"learning_rate": 4.840055783904106e-07,
"logits/chosen": 0.17138057947158813,
"logits/rejected": 0.10115008801221848,
"logps/chosen": -72.76631927490234,
"logps/ref_chosen": -66.75926208496094,
"logps/ref_rejected": -94.61787414550781,
"logps/rejected": -103.28041076660156,
"loss": 1.1929,
"margin_dpo/margin_mean": 2.655484199523926,
"margin_dpo/margin_std": 5.235541343688965,
"step": 136
},
{
"KL/chosen_KL_mean": -5.561779022216797,
"KL/mean": -6.875943183898926,
"KL/rejected_KL_mean": -8.190105438232422,
"KL/std": 3.721090316772461,
"epoch": 0.20710506424792138,
"fcm_dpo/beta": 0.31635814905166626,
"fcm_dpo/delta": -0.03473253175616264,
"fcm_dpo/margin": 2.628328323364258,
"fcm_dpo/q_t": 0.3503134548664093,
"grad_norm": 74.95806884765625,
"learning_rate": 4.835369650662767e-07,
"logits/chosen": 0.21699392795562744,
"logits/rejected": 0.1880975365638733,
"logps/chosen": -62.345577239990234,
"logps/ref_chosen": -56.78379821777344,
"logps/ref_rejected": -69.89952087402344,
"logps/rejected": -78.08963012695312,
"loss": 1.0336,
"margin_dpo/margin_mean": 2.628328561782837,
"margin_dpo/margin_std": 4.104913711547852,
"step": 137
},
{
"KL/chosen_KL_mean": -6.465946197509766,
"KL/mean": -7.493434906005859,
"KL/rejected_KL_mean": -8.520923614501953,
"KL/std": 3.798523187637329,
"epoch": 0.20861678004535147,
"fcm_dpo/beta": 0.32131582498550415,
"fcm_dpo/delta": 0.14952073991298676,
"fcm_dpo/margin": 2.0549755096435547,
"fcm_dpo/q_t": 0.376254677772522,
"grad_norm": 85.68193054199219,
"learning_rate": 4.830618192112065e-07,
"logits/chosen": 0.2118818461894989,
"logits/rejected": 0.1755654513835907,
"logps/chosen": -65.2319564819336,
"logps/ref_chosen": -58.766014099121094,
"logps/ref_rejected": -68.12371826171875,
"logps/rejected": -76.64463806152344,
"loss": 1.1774,
"margin_dpo/margin_mean": 2.0549752712249756,
"margin_dpo/margin_std": 4.0768890380859375,
"step": 138
},
{
"KL/chosen_KL_mean": -6.410148620605469,
"KL/mean": -7.718695640563965,
"KL/rejected_KL_mean": -9.027240753173828,
"KL/std": 3.6766586303710938,
"epoch": 0.21012849584278157,
"fcm_dpo/beta": 0.3235969543457031,
"fcm_dpo/delta": -0.05197485536336899,
"fcm_dpo/margin": 2.6170876026153564,
"fcm_dpo/q_t": 0.34167400002479553,
"grad_norm": 83.7813720703125,
"learning_rate": 4.825801541160509e-07,
"logits/chosen": 0.15254321694374084,
"logits/rejected": 0.12392173707485199,
"logps/chosen": -77.6357421875,
"logps/ref_chosen": -71.2255859375,
"logps/ref_rejected": -82.1834716796875,
"logps/rejected": -91.21070861816406,
"loss": 0.9695,
"margin_dpo/margin_mean": 2.6170878410339355,
"margin_dpo/margin_std": 3.5879359245300293,
"step": 139
},
{
"KL/chosen_KL_mean": -5.829267501831055,
"KL/mean": -7.498285293579102,
"KL/rejected_KL_mean": -9.167301177978516,
"KL/std": 4.7677483558654785,
"epoch": 0.21164021164021163,
"fcm_dpo/beta": 0.30738502740859985,
"fcm_dpo/delta": -0.25325673818588257,
"fcm_dpo/margin": 3.33803653717041,
"fcm_dpo/q_t": 0.3198111951351166,
"grad_norm": 91.1104965209961,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": 0.18167796730995178,
"logits/rejected": 0.1350601315498352,
"logps/chosen": -69.10693359375,
"logps/ref_chosen": -63.27766418457031,
"logps/ref_rejected": -83.30647277832031,
"logps/rejected": -92.47377014160156,
"loss": 1.0517,
"margin_dpo/margin_mean": 3.3380370140075684,
"margin_dpo/margin_std": 5.130913734436035,
"step": 140
},
{
"KL/chosen_KL_mean": -5.997409820556641,
"KL/mean": -7.6452717781066895,
"KL/rejected_KL_mean": -9.293132781982422,
"KL/std": 3.922736167907715,
"epoch": 0.21315192743764172,
"fcm_dpo/beta": 0.2922168970108032,
"fcm_dpo/delta": -0.1869136393070221,
"fcm_dpo/margin": 3.295717716217041,
"fcm_dpo/q_t": 0.3368871808052063,
"grad_norm": 77.85089111328125,
"learning_rate": 4.815973202802966e-07,
"logits/chosen": 0.20691722631454468,
"logits/rejected": 0.16274985671043396,
"logps/chosen": -67.76417541503906,
"logps/ref_chosen": -61.76676940917969,
"logps/ref_rejected": -88.60601806640625,
"logps/rejected": -97.89915466308594,
"loss": 1.0053,
"margin_dpo/margin_mean": 3.295717716217041,
"margin_dpo/margin_std": 4.843393325805664,
"step": 141
},
{
"KL/chosen_KL_mean": -5.920963287353516,
"KL/mean": -7.147356033325195,
"KL/rejected_KL_mean": -8.37375259399414,
"KL/std": 3.805548667907715,
"epoch": 0.2146636432350718,
"fcm_dpo/beta": 0.2973337769508362,
"fcm_dpo/delta": 0.07598335295915604,
"fcm_dpo/margin": 2.452789783477783,
"fcm_dpo/q_t": 0.36309534311294556,
"grad_norm": 72.36812591552734,
"learning_rate": 4.810961790316729e-07,
"logits/chosen": 0.1847228705883026,
"logits/rejected": 0.16021020710468292,
"logps/chosen": -71.19573974609375,
"logps/ref_chosen": -65.2747802734375,
"logps/ref_rejected": -81.1378173828125,
"logps/rejected": -89.51156616210938,
"loss": 1.0487,
"margin_dpo/margin_mean": 2.452789545059204,
"margin_dpo/margin_std": 3.8934617042541504,
"step": 142
},
{
"KL/chosen_KL_mean": -6.140836715698242,
"KL/mean": -7.2427263259887695,
"KL/rejected_KL_mean": -8.344612121582031,
"KL/std": 4.039183616638184,
"epoch": 0.2161753590325019,
"fcm_dpo/beta": 0.30352532863616943,
"fcm_dpo/delta": 0.14069342613220215,
"fcm_dpo/margin": 2.203770637512207,
"fcm_dpo/q_t": 0.3680863380432129,
"grad_norm": 99.17074584960938,
"learning_rate": 4.805885735261454e-07,
"logits/chosen": 0.19867965579032898,
"logits/rejected": 0.1820322573184967,
"logps/chosen": -68.7586669921875,
"logps/ref_chosen": -62.617828369140625,
"logps/ref_rejected": -70.39239501953125,
"logps/rejected": -78.73699951171875,
"loss": 1.2289,
"margin_dpo/margin_mean": 2.203770637512207,
"margin_dpo/margin_std": 4.742335319519043,
"step": 143
},
{
"KL/chosen_KL_mean": -6.31688117980957,
"KL/mean": -7.679841995239258,
"KL/rejected_KL_mean": -9.042800903320312,
"KL/std": 4.1917219161987305,
"epoch": 0.21768707482993196,
"fcm_dpo/beta": 0.303509384393692,
"fcm_dpo/delta": -0.030424147844314575,
"fcm_dpo/margin": 2.7259182929992676,
"fcm_dpo/q_t": 0.35583314299583435,
"grad_norm": 83.09550476074219,
"learning_rate": 4.800745179625307e-07,
"logits/chosen": 0.17651958763599396,
"logits/rejected": 0.14690782129764557,
"logps/chosen": -67.11956787109375,
"logps/ref_chosen": -60.80268859863281,
"logps/ref_rejected": -79.07284545898438,
"logps/rejected": -88.11564636230469,
"loss": 1.0936,
"margin_dpo/margin_mean": 2.725918769836426,
"margin_dpo/margin_std": 4.664007663726807,
"step": 144
},
{
"KL/chosen_KL_mean": -5.749847412109375,
"KL/mean": -7.1203932762146,
"KL/rejected_KL_mean": -8.490936279296875,
"KL/std": 4.098171234130859,
"epoch": 0.21919879062736206,
"fcm_dpo/beta": 0.30279701948165894,
"fcm_dpo/delta": -0.03274145722389221,
"fcm_dpo/margin": 2.7410945892333984,
"fcm_dpo/q_t": 0.3517426550388336,
"grad_norm": 89.16951751708984,
"learning_rate": 4.795540267200686e-07,
"logits/chosen": 0.1450120210647583,
"logits/rejected": 0.16164004802703857,
"logps/chosen": -80.36131286621094,
"logps/ref_chosen": -74.61146545410156,
"logps/ref_rejected": -83.24461364746094,
"logps/rejected": -91.73554992675781,
"loss": 1.0838,
"margin_dpo/margin_mean": 2.7410945892333984,
"margin_dpo/margin_std": 4.671388626098633,
"step": 145
},
{
"KL/chosen_KL_mean": -5.396156311035156,
"KL/mean": -6.788448810577393,
"KL/rejected_KL_mean": -8.180740356445312,
"KL/std": 3.8954482078552246,
"epoch": 0.22071050642479215,
"fcm_dpo/beta": 0.3020232021808624,
"fcm_dpo/delta": -0.04509184509515762,
"fcm_dpo/margin": 2.7845849990844727,
"fcm_dpo/q_t": 0.3457740843296051,
"grad_norm": 79.14019775390625,
"learning_rate": 4.790271143580173e-07,
"logits/chosen": 0.13101473450660706,
"logits/rejected": 0.11654573678970337,
"logps/chosen": -63.23714065551758,
"logps/ref_chosen": -57.84098434448242,
"logps/ref_rejected": -67.47422790527344,
"logps/rejected": -75.65496826171875,
"loss": 1.0088,
"margin_dpo/margin_mean": 2.7845849990844727,
"margin_dpo/margin_std": 4.094869136810303,
"step": 146
},
{
"KL/chosen_KL_mean": -6.266658782958984,
"KL/mean": -7.423336982727051,
"KL/rejected_KL_mean": -8.580020904541016,
"KL/std": 4.117965221405029,
"epoch": 0.2222222222222222,
"fcm_dpo/beta": 0.30491483211517334,
"fcm_dpo/delta": 0.10152038931846619,
"fcm_dpo/margin": 2.313361167907715,
"fcm_dpo/q_t": 0.37097328901290894,
"grad_norm": 102.72885131835938,
"learning_rate": 4.784937956152489e-07,
"logits/chosen": 0.17838065326213837,
"logits/rejected": 0.1357332020998001,
"logps/chosen": -73.08013153076172,
"logps/ref_chosen": -66.81346893310547,
"logps/ref_rejected": -81.1796875,
"logps/rejected": -89.75971221923828,
"loss": 1.1535,
"margin_dpo/margin_mean": 2.3133602142333984,
"margin_dpo/margin_std": 4.482905387878418,
"step": 147
},
{
"KL/chosen_KL_mean": -5.813823699951172,
"KL/mean": -7.228766441345215,
"KL/rejected_KL_mean": -8.643707275390625,
"KL/std": 3.8219246864318848,
"epoch": 0.2237339380196523,
"fcm_dpo/beta": 0.3052898645401001,
"fcm_dpo/delta": -0.07084967195987701,
"fcm_dpo/margin": 2.829887628555298,
"fcm_dpo/q_t": 0.34750601649284363,
"grad_norm": 67.65303802490234,
"learning_rate": 4.779540854098347e-07,
"logits/chosen": 0.28564712405204773,
"logits/rejected": 0.2142116129398346,
"logps/chosen": -54.50157928466797,
"logps/ref_chosen": -48.6877555847168,
"logps/ref_rejected": -67.50503540039062,
"logps/rejected": -76.14874267578125,
"loss": 1.0565,
"margin_dpo/margin_mean": 2.8298869132995605,
"margin_dpo/margin_std": 4.450958728790283,
"step": 148
},
{
"KL/chosen_KL_mean": -7.005121231079102,
"KL/mean": -8.661083221435547,
"KL/rejected_KL_mean": -10.31704330444336,
"KL/std": 4.533745765686035,
"epoch": 0.2252456538170824,
"fcm_dpo/beta": 0.29332873225212097,
"fcm_dpo/delta": -0.1899138242006302,
"fcm_dpo/margin": 3.311924457550049,
"fcm_dpo/q_t": 0.3203445076942444,
"grad_norm": 66.86536407470703,
"learning_rate": 4.774079988386296e-07,
"logits/chosen": 0.15289664268493652,
"logits/rejected": 0.10625985264778137,
"logps/chosen": -62.14889907836914,
"logps/ref_chosen": -55.143775939941406,
"logps/ref_rejected": -64.79888916015625,
"logps/rejected": -75.11593627929688,
"loss": 0.9195,
"margin_dpo/margin_mean": 3.311924457550049,
"margin_dpo/margin_std": 4.24875545501709,
"step": 149
},
{
"KL/chosen_KL_mean": -5.336969375610352,
"KL/mean": -7.341297149658203,
"KL/rejected_KL_mean": -9.345619201660156,
"KL/std": 4.29542350769043,
"epoch": 0.22675736961451248,
"fcm_dpo/beta": 0.2759903073310852,
"fcm_dpo/delta": -0.34455248713493347,
"fcm_dpo/margin": 4.008656024932861,
"fcm_dpo/q_t": 0.2917425036430359,
"grad_norm": 61.21348190307617,
"learning_rate": 4.768555511768486e-07,
"logits/chosen": 0.1695353239774704,
"logits/rejected": 0.12736281752586365,
"logps/chosen": -72.8077163696289,
"logps/ref_chosen": -67.47074890136719,
"logps/ref_rejected": -89.21170806884766,
"logps/rejected": -98.55732727050781,
"loss": 0.8057,
"margin_dpo/margin_mean": 4.0086565017700195,
"margin_dpo/margin_std": 4.115660667419434,
"step": 150
},
{
"KL/chosen_KL_mean": -5.009391784667969,
"KL/mean": -7.053393840789795,
"KL/rejected_KL_mean": -9.097404479980469,
"KL/std": 4.357527256011963,
"epoch": 0.22826908541194255,
"fcm_dpo/beta": 0.26016658544540405,
"fcm_dpo/delta": -0.2943726181983948,
"fcm_dpo/margin": 4.088005065917969,
"fcm_dpo/q_t": 0.30344998836517334,
"grad_norm": 55.01530456542969,
"learning_rate": 4.762967578776406e-07,
"logits/chosen": 0.16540372371673584,
"logits/rejected": 0.11411911994218826,
"logps/chosen": -57.46893310546875,
"logps/ref_chosen": -52.45954132080078,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -88.16041564941406,
"loss": 0.8328,
"margin_dpo/margin_mean": 4.088005542755127,
"margin_dpo/margin_std": 4.563044548034668,
"step": 151
},
{
"KL/chosen_KL_mean": -6.312803268432617,
"KL/mean": -7.799456596374512,
"KL/rejected_KL_mean": -9.286109924316406,
"KL/std": 4.282715320587158,
"epoch": 0.22978080120937264,
"fcm_dpo/beta": 0.2594345211982727,
"fcm_dpo/delta": 0.028224565088748932,
"fcm_dpo/margin": 2.973308563232422,
"fcm_dpo/q_t": 0.3617689311504364,
"grad_norm": 69.3165054321289,
"learning_rate": 4.757316345716553e-07,
"logits/chosen": 0.24428924918174744,
"logits/rejected": 0.1937820166349411,
"logps/chosen": -62.86663818359375,
"logps/ref_chosen": -56.5538330078125,
"logps/ref_rejected": -76.55074310302734,
"logps/rejected": -85.83685302734375,
"loss": 1.0719,
"margin_dpo/margin_mean": 2.9733083248138428,
"margin_dpo/margin_std": 4.82022762298584,
"step": 152
},
{
"KL/chosen_KL_mean": -5.811321258544922,
"KL/mean": -7.454206943511963,
"KL/rejected_KL_mean": -9.097091674804688,
"KL/std": 4.515659332275391,
"epoch": 0.23129251700680273,
"fcm_dpo/beta": 0.2544195055961609,
"fcm_dpo/delta": -0.03977034240961075,
"fcm_dpo/margin": 3.2857680320739746,
"fcm_dpo/q_t": 0.33868566155433655,
"grad_norm": 61.1727409362793,
"learning_rate": 4.751601970666064e-07,
"logits/chosen": 0.1626579612493515,
"logits/rejected": 0.12607043981552124,
"logps/chosen": -73.8182144165039,
"logps/ref_chosen": -68.00689697265625,
"logps/ref_rejected": -74.83482360839844,
"logps/rejected": -83.93191528320312,
"loss": 0.9371,
"margin_dpo/margin_mean": 3.2857680320739746,
"margin_dpo/margin_std": 4.14332914352417,
"step": 153
},
{
"KL/chosen_KL_mean": -6.67750358581543,
"KL/mean": -7.8356099128723145,
"KL/rejected_KL_mean": -8.993721008300781,
"KL/std": 4.44697904586792,
"epoch": 0.2328042328042328,
"fcm_dpo/beta": 0.26349014043807983,
"fcm_dpo/delta": 0.20121973752975464,
"fcm_dpo/margin": 2.316216468811035,
"fcm_dpo/q_t": 0.3814409077167511,
"grad_norm": 69.11931610107422,
"learning_rate": 4.745824613468292e-07,
"logits/chosen": 0.23633871972560883,
"logits/rejected": 0.2331632822751999,
"logps/chosen": -65.90003967285156,
"logps/ref_chosen": -59.222537994384766,
"logps/ref_rejected": -64.19131469726562,
"logps/rejected": -73.18504333496094,
"loss": 1.1743,
"margin_dpo/margin_mean": 2.3162169456481934,
"margin_dpo/margin_std": 4.630753517150879,
"step": 154
},
{
"KL/chosen_KL_mean": -6.549232482910156,
"KL/mean": -8.094888687133789,
"KL/rejected_KL_mean": -9.64053726196289,
"KL/std": 4.321755886077881,
"epoch": 0.23431594860166288,
"fcm_dpo/beta": 0.2623800039291382,
"fcm_dpo/delta": -0.014081403613090515,
"fcm_dpo/margin": 3.09130597114563,
"fcm_dpo/q_t": 0.3595314919948578,
"grad_norm": 71.75285339355469,
"learning_rate": 4.7399844357283393e-07,
"logits/chosen": 0.22018414735794067,
"logits/rejected": 0.20080286264419556,
"logps/chosen": -75.0039291381836,
"logps/ref_chosen": -68.45469665527344,
"logps/ref_rejected": -77.91763305664062,
"logps/rejected": -87.55817413330078,
"loss": 1.1156,
"margin_dpo/margin_mean": 3.091305732727051,
"margin_dpo/margin_std": 5.317093849182129,
"step": 155
},
{
"KL/chosen_KL_mean": -6.38591194152832,
"KL/mean": -8.252660751342773,
"KL/rejected_KL_mean": -10.119407653808594,
"KL/std": 4.605101585388184,
"epoch": 0.23582766439909297,
"fcm_dpo/beta": 0.25796937942504883,
"fcm_dpo/delta": -0.18036875128746033,
"fcm_dpo/margin": 3.7334985733032227,
"fcm_dpo/q_t": 0.3195089101791382,
"grad_norm": 62.9564323425293,
"learning_rate": 4.7340816008085305e-07,
"logits/chosen": 0.17611095309257507,
"logits/rejected": 0.13236644864082336,
"logps/chosen": -73.65550994873047,
"logps/ref_chosen": -67.26959991455078,
"logps/ref_rejected": -86.95914459228516,
"logps/rejected": -97.07855224609375,
"loss": 0.8812,
"margin_dpo/margin_mean": 3.7334985733032227,
"margin_dpo/margin_std": 4.432024955749512,
"step": 156
},
{
"KL/chosen_KL_mean": -5.90778923034668,
"KL/mean": -7.382824420928955,
"KL/rejected_KL_mean": -8.857852935791016,
"KL/std": 4.559696197509766,
"epoch": 0.23733938019652306,
"fcm_dpo/beta": 0.2535499334335327,
"fcm_dpo/delta": 0.052449166774749756,
"fcm_dpo/margin": 2.9500629901885986,
"fcm_dpo/q_t": 0.35793808102607727,
"grad_norm": 63.09500503540039,
"learning_rate": 4.728116273823847e-07,
"logits/chosen": 0.1454748958349228,
"logits/rejected": 0.12664461135864258,
"logps/chosen": -60.6806640625,
"logps/ref_chosen": -54.77287292480469,
"logps/ref_rejected": -63.87866973876953,
"logps/rejected": -72.73652648925781,
"loss": 1.0299,
"margin_dpo/margin_mean": 2.9500629901885986,
"margin_dpo/margin_std": 4.351293563842773,
"step": 157
},
{
"KL/chosen_KL_mean": -6.305976867675781,
"KL/mean": -7.767054080963135,
"KL/rejected_KL_mean": -9.228130340576172,
"KL/std": 4.4387922286987305,
"epoch": 0.23885109599395313,
"fcm_dpo/beta": 0.25931790471076965,
"fcm_dpo/delta": 0.04571561515331268,
"fcm_dpo/margin": 2.922152280807495,
"fcm_dpo/q_t": 0.35533732175827026,
"grad_norm": 68.03280639648438,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": 0.18727868795394897,
"logits/rejected": 0.15256363153457642,
"logps/chosen": -71.22869873046875,
"logps/ref_chosen": -64.92271423339844,
"logps/ref_rejected": -82.23789978027344,
"logps/rejected": -91.46602630615234,
"loss": 1.0286,
"margin_dpo/margin_mean": 2.922152042388916,
"margin_dpo/margin_std": 4.38568115234375,
"step": 158
},
{
"KL/chosen_KL_mean": -6.437795639038086,
"KL/mean": -8.105405807495117,
"KL/rejected_KL_mean": -9.773014068603516,
"KL/std": 4.696426868438721,
"epoch": 0.24036281179138322,
"fcm_dpo/beta": 0.2608144283294678,
"fcm_dpo/delta": -0.07918387651443481,
"fcm_dpo/margin": 3.3352150917053223,
"fcm_dpo/q_t": 0.3356255888938904,
"grad_norm": 76.94505310058594,
"learning_rate": 4.715998812855304e-07,
"logits/chosen": 0.21540075540542603,
"logits/rejected": 0.17887872457504272,
"logps/chosen": -63.48478698730469,
"logps/ref_chosen": -57.046993255615234,
"logps/ref_rejected": -73.32441711425781,
"logps/rejected": -83.09742736816406,
"loss": 1.0131,
"margin_dpo/margin_mean": 3.3352150917053223,
"margin_dpo/margin_std": 4.913866996765137,
"step": 159
},
{
"KL/chosen_KL_mean": -7.160207748413086,
"KL/mean": -8.652528762817383,
"KL/rejected_KL_mean": -10.144851684570312,
"KL/std": 4.411318778991699,
"epoch": 0.2418745275888133,
"fcm_dpo/beta": 0.25836193561553955,
"fcm_dpo/delta": 0.030919140204787254,
"fcm_dpo/margin": 2.9846417903900146,
"fcm_dpo/q_t": 0.36456459760665894,
"grad_norm": 64.14701843261719,
"learning_rate": 4.7098470178228755e-07,
"logits/chosen": 0.0936279147863388,
"logits/rejected": 0.05139011889696121,
"logps/chosen": -56.96712112426758,
"logps/ref_chosen": -49.806915283203125,
"logps/ref_rejected": -68.3370132446289,
"logps/rejected": -78.48186492919922,
"loss": 1.0898,
"margin_dpo/margin_mean": 2.9846420288085938,
"margin_dpo/margin_std": 5.151812553405762,
"step": 160
},
{
"KL/chosen_KL_mean": -6.983335494995117,
"KL/mean": -8.564802169799805,
"KL/rejected_KL_mean": -10.146270751953125,
"KL/std": 4.450295925140381,
"epoch": 0.24338624338624337,
"fcm_dpo/beta": 0.2577332854270935,
"fcm_dpo/delta": -0.016592275351285934,
"fcm_dpo/margin": 3.1629390716552734,
"fcm_dpo/q_t": 0.35136890411376953,
"grad_norm": 60.320228576660156,
"learning_rate": 4.703633408618955e-07,
"logits/chosen": 0.21055185794830322,
"logits/rejected": 0.17318174242973328,
"logps/chosen": -59.48382568359375,
"logps/ref_chosen": -52.50048828125,
"logps/ref_rejected": -66.04540252685547,
"logps/rejected": -76.1916732788086,
"loss": 1.0397,
"margin_dpo/margin_mean": 3.1629395484924316,
"margin_dpo/margin_std": 4.972609519958496,
"step": 161
},
{
"KL/chosen_KL_mean": -7.363391876220703,
"KL/mean": -9.55433177947998,
"KL/rejected_KL_mean": -11.74527359008789,
"KL/std": 4.978542804718018,
"epoch": 0.24489795918367346,
"fcm_dpo/beta": 0.24483485519886017,
"fcm_dpo/delta": -0.3069154620170593,
"fcm_dpo/margin": 4.381883144378662,
"fcm_dpo/q_t": 0.2978348135948181,
"grad_norm": 55.664127349853516,
"learning_rate": 4.697358159051549e-07,
"logits/chosen": 0.25998812913894653,
"logits/rejected": 0.21173785626888275,
"logps/chosen": -76.83258056640625,
"logps/ref_chosen": -69.46919250488281,
"logps/ref_rejected": -92.00952911376953,
"logps/rejected": -103.75480651855469,
"loss": 0.8312,
"margin_dpo/margin_mean": 4.3818840980529785,
"margin_dpo/margin_std": 4.7379302978515625,
"step": 162
},
{
"KL/chosen_KL_mean": -6.585916519165039,
"KL/mean": -8.661686897277832,
"KL/rejected_KL_mean": -10.73746109008789,
"KL/std": 4.580445289611816,
"epoch": 0.24640967498110355,
"fcm_dpo/beta": 0.23843413591384888,
"fcm_dpo/delta": -0.2134178876876831,
"fcm_dpo/margin": 4.151541709899902,
"fcm_dpo/q_t": 0.3177376091480255,
"grad_norm": 58.10890579223633,
"learning_rate": 4.691021444652876e-07,
"logits/chosen": 0.18294395506381989,
"logits/rejected": 0.13766877353191376,
"logps/chosen": -57.19975280761719,
"logps/ref_chosen": -50.613834381103516,
"logps/ref_rejected": -74.62033081054688,
"logps/rejected": -85.3577880859375,
"loss": 0.9309,
"margin_dpo/margin_mean": 4.1515421867370605,
"margin_dpo/margin_std": 5.052495002746582,
"step": 163
},
{
"KL/chosen_KL_mean": -7.12010383605957,
"KL/mean": -9.122835159301758,
"KL/rejected_KL_mean": -11.125564575195312,
"KL/std": 4.873750686645508,
"epoch": 0.24792139077853365,
"fcm_dpo/beta": 0.22845444083213806,
"fcm_dpo/delta": -0.12654295563697815,
"fcm_dpo/margin": 4.005465030670166,
"fcm_dpo/q_t": 0.33134955167770386,
"grad_norm": 53.08348846435547,
"learning_rate": 4.6846234426744624e-07,
"logits/chosen": 0.17225149273872375,
"logits/rejected": 0.11162863671779633,
"logps/chosen": -61.96821594238281,
"logps/ref_chosen": -54.848114013671875,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -90.1885757446289,
"loss": 0.9499,
"margin_dpo/margin_mean": 4.005465030670166,
"margin_dpo/margin_std": 5.20572566986084,
"step": 164
},
{
"KL/chosen_KL_mean": -7.681882858276367,
"KL/mean": -9.503345489501953,
"KL/rejected_KL_mean": -11.324806213378906,
"KL/std": 4.7919440269470215,
"epoch": 0.2494331065759637,
"fcm_dpo/beta": 0.22671417891979218,
"fcm_dpo/delta": -0.029426731169223785,
"fcm_dpo/margin": 3.6429243087768555,
"fcm_dpo/q_t": 0.33616942167282104,
"grad_norm": 53.045433044433594,
"learning_rate": 4.678164332082175e-07,
"logits/chosen": 0.2291611135005951,
"logits/rejected": 0.17568400502204895,
"logps/chosen": -58.771095275878906,
"logps/ref_chosen": -51.089210510253906,
"logps/ref_rejected": -71.23370361328125,
"logps/rejected": -82.55850219726562,
"loss": 0.9491,
"margin_dpo/margin_mean": 3.6429238319396973,
"margin_dpo/margin_std": 4.521003246307373,
"step": 165
},
{
"KL/chosen_KL_mean": -7.4115447998046875,
"KL/mean": -8.709308624267578,
"KL/rejected_KL_mean": -10.007076263427734,
"KL/std": 4.388735294342041,
"epoch": 0.2509448223733938,
"fcm_dpo/beta": 0.23121148347854614,
"fcm_dpo/delta": 0.2130114734172821,
"fcm_dpo/margin": 2.5955326557159424,
"fcm_dpo/q_t": 0.38784292340278625,
"grad_norm": 65.73187255859375,
"learning_rate": 4.6716442935512214e-07,
"logits/chosen": 0.2012665569782257,
"logits/rejected": 0.11512681841850281,
"logps/chosen": -70.60235595703125,
"logps/ref_chosen": -63.19081115722656,
"logps/ref_rejected": -93.8402099609375,
"logps/rejected": -103.8472900390625,
"loss": 1.0979,
"margin_dpo/margin_mean": 2.5955324172973633,
"margin_dpo/margin_std": 4.49823522567749,
"step": 166
},
{
"KL/chosen_KL_mean": -6.62812614440918,
"KL/mean": -8.661577224731445,
"KL/rejected_KL_mean": -10.695026397705078,
"KL/std": 4.661866188049316,
"epoch": 0.25245653817082386,
"fcm_dpo/beta": 0.22737373411655426,
"fcm_dpo/delta": -0.14043092727661133,
"fcm_dpo/margin": 4.066901206970215,
"fcm_dpo/q_t": 0.3204006850719452,
"grad_norm": 48.181392669677734,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": 0.1581970453262329,
"logits/rejected": 0.12355121970176697,
"logps/chosen": -65.55239868164062,
"logps/ref_chosen": -58.92427062988281,
"logps/ref_rejected": -72.97377014160156,
"logps/rejected": -83.66879272460938,
"loss": 0.8813,
"margin_dpo/margin_mean": 4.066901206970215,
"margin_dpo/margin_std": 4.630178451538086,
"step": 167
},
{
"KL/chosen_KL_mean": -8.006879806518555,
"KL/mean": -9.33842658996582,
"KL/rejected_KL_mean": -10.66998291015625,
"KL/std": 4.7535505294799805,
"epoch": 0.25396825396825395,
"fcm_dpo/beta": 0.23451144993305206,
"fcm_dpo/delta": 0.18685543537139893,
"fcm_dpo/margin": 2.6631035804748535,
"fcm_dpo/q_t": 0.3788146674633026,
"grad_norm": 65.586669921875,
"learning_rate": 4.6584221638904767e-07,
"logits/chosen": 0.17673316597938538,
"logits/rejected": 0.14384135603904724,
"logps/chosen": -73.65826416015625,
"logps/ref_chosen": -65.65138244628906,
"logps/ref_rejected": -79.71418762207031,
"logps/rejected": -90.38417053222656,
"loss": 1.0702,
"margin_dpo/margin_mean": 2.6631031036376953,
"margin_dpo/margin_std": 4.263519287109375,
"step": 168
},
{
"KL/chosen_KL_mean": -6.801374435424805,
"KL/mean": -8.660758972167969,
"KL/rejected_KL_mean": -10.5201416015625,
"KL/std": 4.897111892700195,
"epoch": 0.25547996976568405,
"fcm_dpo/beta": 0.2340339571237564,
"fcm_dpo/delta": -0.07695929706096649,
"fcm_dpo/margin": 3.7187600135803223,
"fcm_dpo/q_t": 0.3465001881122589,
"grad_norm": 60.35015869140625,
"learning_rate": 4.651720442612075e-07,
"logits/chosen": 0.2228689342737198,
"logits/rejected": 0.19062112271785736,
"logps/chosen": -68.22724151611328,
"logps/ref_chosen": -61.425865173339844,
"logps/ref_rejected": -76.09590148925781,
"logps/rejected": -86.61604309082031,
"loss": 0.9808,
"margin_dpo/margin_mean": 3.7187604904174805,
"margin_dpo/margin_std": 5.548947811126709,
"step": 169
},
{
"KL/chosen_KL_mean": -6.976308822631836,
"KL/mean": -8.671058654785156,
"KL/rejected_KL_mean": -10.365806579589844,
"KL/std": 4.641627788543701,
"epoch": 0.25699168556311414,
"fcm_dpo/beta": 0.23378986120224,
"fcm_dpo/delta": 0.008020089939236641,
"fcm_dpo/margin": 3.3894968032836914,
"fcm_dpo/q_t": 0.3479105830192566,
"grad_norm": 51.56340408325195,
"learning_rate": 4.6449585330874425e-07,
"logits/chosen": 0.2002856433391571,
"logits/rejected": 0.19725605845451355,
"logps/chosen": -63.62950134277344,
"logps/ref_chosen": -56.65319061279297,
"logps/ref_rejected": -63.45965576171875,
"logps/rejected": -73.8254623413086,
"loss": 1.0364,
"margin_dpo/margin_mean": 3.3894965648651123,
"margin_dpo/margin_std": 5.16367244720459,
"step": 170
},
{
"KL/chosen_KL_mean": -7.357227325439453,
"KL/mean": -9.279874801635742,
"KL/rejected_KL_mean": -11.202518463134766,
"KL/std": 5.292311668395996,
"epoch": 0.2585034013605442,
"fcm_dpo/beta": 0.22743916511535645,
"fcm_dpo/delta": -0.08598235249519348,
"fcm_dpo/margin": 3.8452939987182617,
"fcm_dpo/q_t": 0.33920013904571533,
"grad_norm": 63.153053283691406,
"learning_rate": 4.6381366244617224e-07,
"logits/chosen": 0.23392340540885925,
"logits/rejected": 0.18576276302337646,
"logps/chosen": -71.09198760986328,
"logps/ref_chosen": -63.73476028442383,
"logps/ref_rejected": -78.50328063964844,
"logps/rejected": -89.70579528808594,
"loss": 1.0377,
"margin_dpo/margin_mean": 3.845294237136841,
"margin_dpo/margin_std": 5.816746711730957,
"step": 171
},
{
"KL/chosen_KL_mean": -7.918674468994141,
"KL/mean": -9.873100280761719,
"KL/rejected_KL_mean": -11.827526092529297,
"KL/std": 5.193835258483887,
"epoch": 0.2600151171579743,
"fcm_dpo/beta": 0.22867974638938904,
"fcm_dpo/delta": -0.10525374859571457,
"fcm_dpo/margin": 3.9088492393493652,
"fcm_dpo/q_t": 0.326849102973938,
"grad_norm": 50.447669982910156,
"learning_rate": 4.631254907558365e-07,
"logits/chosen": 0.2409161925315857,
"logits/rejected": 0.1868460476398468,
"logps/chosen": -60.12043380737305,
"logps/ref_chosen": -52.201759338378906,
"logps/ref_rejected": -82.85285949707031,
"logps/rejected": -94.68038940429688,
"loss": 0.9546,
"margin_dpo/margin_mean": 3.908848762512207,
"margin_dpo/margin_std": 5.0718889236450195,
"step": 172
},
{
"KL/chosen_KL_mean": -7.773256301879883,
"KL/mean": -9.733634948730469,
"KL/rejected_KL_mean": -11.694019317626953,
"KL/std": 5.298603057861328,
"epoch": 0.2615268329554044,
"fcm_dpo/beta": 0.21797175705432892,
"fcm_dpo/delta": -0.06880420446395874,
"fcm_dpo/margin": 3.92075777053833,
"fcm_dpo/q_t": 0.353862464427948,
"grad_norm": 52.90761184692383,
"learning_rate": 4.624313574873786e-07,
"logits/chosen": 0.2440553605556488,
"logits/rejected": 0.15911364555358887,
"logps/chosen": -63.20798110961914,
"logps/ref_chosen": -55.434722900390625,
"logps/ref_rejected": -77.81967163085938,
"logps/rejected": -89.5136947631836,
"loss": 1.1063,
"margin_dpo/margin_mean": 3.920758008956909,
"margin_dpo/margin_std": 6.446454048156738,
"step": 173
},
{
"KL/chosen_KL_mean": -8.353595733642578,
"KL/mean": -10.422002792358398,
"KL/rejected_KL_mean": -12.490409851074219,
"KL/std": 5.196247100830078,
"epoch": 0.26303854875283444,
"fcm_dpo/beta": 0.2176172137260437,
"fcm_dpo/delta": -0.11027532070875168,
"fcm_dpo/margin": 4.136814594268799,
"fcm_dpo/q_t": 0.3365994691848755,
"grad_norm": 58.16306686401367,
"learning_rate": 4.61731282057198e-07,
"logits/chosen": 0.22505061328411102,
"logits/rejected": 0.16084152460098267,
"logps/chosen": -65.52555084228516,
"logps/ref_chosen": -57.17195129394531,
"logps/ref_rejected": -85.47578430175781,
"logps/rejected": -97.9661865234375,
"loss": 1.0065,
"margin_dpo/margin_mean": 4.136814594268799,
"margin_dpo/margin_std": 6.124249458312988,
"step": 174
},
{
"KL/chosen_KL_mean": -8.06877326965332,
"KL/mean": -10.374343872070312,
"KL/rejected_KL_mean": -12.679920196533203,
"KL/std": 5.566990852355957,
"epoch": 0.26455026455026454,
"fcm_dpo/beta": 0.21098440885543823,
"fcm_dpo/delta": -0.19120723009109497,
"fcm_dpo/margin": 4.61114501953125,
"fcm_dpo/q_t": 0.3338923156261444,
"grad_norm": 56.7152099609375,
"learning_rate": 4.6102528404790965e-07,
"logits/chosen": 0.2686522901058197,
"logits/rejected": 0.2384568750858307,
"logps/chosen": -75.73440551757812,
"logps/ref_chosen": -67.6656265258789,
"logps/ref_rejected": -84.36766815185547,
"logps/rejected": -97.04759216308594,
"loss": 0.9772,
"margin_dpo/margin_mean": 4.611144065856934,
"margin_dpo/margin_std": 6.451984405517578,
"step": 175
},
{
"KL/chosen_KL_mean": -9.094661712646484,
"KL/mean": -10.573260307312012,
"KL/rejected_KL_mean": -12.051864624023438,
"KL/std": 5.919432640075684,
"epoch": 0.2660619803476946,
"fcm_dpo/beta": 0.21295952796936035,
"fcm_dpo/delta": 0.18204456567764282,
"fcm_dpo/margin": 2.957207202911377,
"fcm_dpo/q_t": 0.39002811908721924,
"grad_norm": 72.97171783447266,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": 0.1966366022825241,
"logits/rejected": 0.17095749080181122,
"logps/chosen": -86.95342254638672,
"logps/ref_chosen": -77.8587646484375,
"logps/ref_rejected": -81.08732604980469,
"logps/rejected": -93.13919067382812,
"loss": 1.2114,
"margin_dpo/margin_mean": 2.9572067260742188,
"margin_dpo/margin_std": 6.396285533905029,
"step": 176
},
{
"KL/chosen_KL_mean": -7.932289123535156,
"KL/mean": -10.874521255493164,
"KL/rejected_KL_mean": -13.816753387451172,
"KL/std": 6.001509666442871,
"epoch": 0.2675736961451247,
"fcm_dpo/beta": 0.2011113166809082,
"fcm_dpo/delta": -0.43956103920936584,
"fcm_dpo/margin": 5.884462833404541,
"fcm_dpo/q_t": 0.279480516910553,
"grad_norm": 52.27606201171875,
"learning_rate": 4.5959559945025183e-07,
"logits/chosen": 0.3216002583503723,
"logits/rejected": 0.22915410995483398,
"logps/chosen": -63.152687072753906,
"logps/ref_chosen": -55.22039794921875,
"logps/ref_rejected": -92.54973602294922,
"logps/rejected": -106.36648559570312,
"loss": 0.7694,
"margin_dpo/margin_mean": 5.884462356567383,
"margin_dpo/margin_std": 5.901601314544678,
"step": 177
},
{
"KL/chosen_KL_mean": -8.448629379272461,
"KL/mean": -10.219297409057617,
"KL/rejected_KL_mean": -11.98996353149414,
"KL/std": 5.255385875701904,
"epoch": 0.2690854119425548,
"fcm_dpo/beta": 0.1983993649482727,
"fcm_dpo/delta": 0.1025046855211258,
"fcm_dpo/margin": 3.541327476501465,
"fcm_dpo/q_t": 0.36179715394973755,
"grad_norm": 53.7444953918457,
"learning_rate": 4.588719528532341e-07,
"logits/chosen": 0.18969491124153137,
"logits/rejected": 0.14258652925491333,
"logps/chosen": -69.25912475585938,
"logps/ref_chosen": -60.81049346923828,
"logps/ref_rejected": -81.12973022460938,
"logps/rejected": -93.11968994140625,
"loss": 1.027,
"margin_dpo/margin_mean": 3.541327476501465,
"margin_dpo/margin_std": 5.07969331741333,
"step": 178
},
{
"KL/chosen_KL_mean": -9.205842971801758,
"KL/mean": -11.216886520385742,
"KL/rejected_KL_mean": -13.227928161621094,
"KL/std": 5.487806797027588,
"epoch": 0.2705971277399849,
"fcm_dpo/beta": 0.20230549573898315,
"fcm_dpo/delta": -0.015091773122549057,
"fcm_dpo/margin": 4.022085189819336,
"fcm_dpo/q_t": 0.3512629568576813,
"grad_norm": 55.53172302246094,
"learning_rate": 4.581424636586928e-07,
"logits/chosen": 0.22318163514137268,
"logits/rejected": 0.2072812020778656,
"logps/chosen": -74.87755584716797,
"logps/ref_chosen": -65.67171478271484,
"logps/ref_rejected": -75.32586669921875,
"logps/rejected": -88.55379486083984,
"loss": 1.0564,
"margin_dpo/margin_mean": 4.022086143493652,
"margin_dpo/margin_std": 6.479291915893555,
"step": 179
},
{
"KL/chosen_KL_mean": -7.661951065063477,
"KL/mean": -9.359560012817383,
"KL/rejected_KL_mean": -11.057170867919922,
"KL/std": 5.326980113983154,
"epoch": 0.272108843537415,
"fcm_dpo/beta": 0.20592835545539856,
"fcm_dpo/delta": 0.10698837041854858,
"fcm_dpo/margin": 3.3952207565307617,
"fcm_dpo/q_t": 0.3744320273399353,
"grad_norm": 57.96144104003906,
"learning_rate": 4.5740715227200897e-07,
"logits/chosen": 0.09134892374277115,
"logits/rejected": 0.07213237881660461,
"logps/chosen": -64.34475708007812,
"logps/ref_chosen": -56.68280792236328,
"logps/ref_rejected": -64.94414520263672,
"logps/rejected": -76.0013198852539,
"loss": 1.1279,
"margin_dpo/margin_mean": 3.3952207565307617,
"margin_dpo/margin_std": 6.209173679351807,
"step": 180
},
{
"KL/chosen_KL_mean": -7.21190071105957,
"KL/mean": -9.833938598632812,
"KL/rejected_KL_mean": -12.455978393554688,
"KL/std": 5.954924583435059,
"epoch": 0.273620559334845,
"fcm_dpo/beta": 0.19779911637306213,
"fcm_dpo/delta": -0.26495885848999023,
"fcm_dpo/margin": 5.244076728820801,
"fcm_dpo/q_t": 0.29986950755119324,
"grad_norm": 46.34823226928711,
"learning_rate": 4.566660392614228e-07,
"logits/chosen": 0.25384521484375,
"logits/rejected": 0.21548154950141907,
"logps/chosen": -67.98794555664062,
"logps/ref_chosen": -60.77604675292969,
"logps/ref_rejected": -83.98361206054688,
"logps/rejected": -96.43959045410156,
"loss": 0.8039,
"margin_dpo/margin_mean": 5.244076728820801,
"margin_dpo/margin_std": 5.329998016357422,
"step": 181
},
{
"KL/chosen_KL_mean": -8.529552459716797,
"KL/mean": -11.048151016235352,
"KL/rejected_KL_mean": -13.566749572753906,
"KL/std": 6.021827697753906,
"epoch": 0.2751322751322751,
"fcm_dpo/beta": 0.18954861164093018,
"fcm_dpo/delta": -0.17215055227279663,
"fcm_dpo/margin": 5.037204265594482,
"fcm_dpo/q_t": 0.3292804956436157,
"grad_norm": 52.639610290527344,
"learning_rate": 4.5591914535745817e-07,
"logits/chosen": 0.21993078291416168,
"logits/rejected": 0.14366433024406433,
"logps/chosen": -68.78334045410156,
"logps/ref_chosen": -60.2537841796875,
"logps/ref_rejected": -89.7706298828125,
"logps/rejected": -103.3373794555664,
"loss": 0.9768,
"margin_dpo/margin_mean": 5.037204265594482,
"margin_dpo/margin_std": 6.95863151550293,
"step": 182
},
{
"KL/chosen_KL_mean": -10.168844223022461,
"KL/mean": -11.266523361206055,
"KL/rejected_KL_mean": -12.364204406738281,
"KL/std": 6.000180721282959,
"epoch": 0.2766439909297052,
"fcm_dpo/beta": 0.1989203244447708,
"fcm_dpo/delta": 0.3810041546821594,
"fcm_dpo/margin": 2.195363998413086,
"fcm_dpo/q_t": 0.41308414936065674,
"grad_norm": 58.42243194580078,
"learning_rate": 4.551664914523433e-07,
"logits/chosen": 0.21723271906375885,
"logits/rejected": 0.19659434258937836,
"logps/chosen": -71.93026733398438,
"logps/ref_chosen": -61.76142120361328,
"logps/ref_rejected": -72.54627990722656,
"logps/rejected": -84.91049194335938,
"loss": 1.2629,
"margin_dpo/margin_mean": 2.195363759994507,
"margin_dpo/margin_std": 5.551734924316406,
"step": 183
},
{
"KL/chosen_KL_mean": -7.7139739990234375,
"KL/mean": -9.649314880371094,
"KL/rejected_KL_mean": -11.584651947021484,
"KL/std": 5.03806209564209,
"epoch": 0.2781557067271353,
"fcm_dpo/beta": 0.2014952301979065,
"fcm_dpo/delta": 0.018574021756649017,
"fcm_dpo/margin": 3.8706769943237305,
"fcm_dpo/q_t": 0.3508874773979187,
"grad_norm": 43.059913635253906,
"learning_rate": 4.544080985994258e-07,
"logits/chosen": 0.2943282425403595,
"logits/rejected": 0.23392927646636963,
"logps/chosen": -54.55469512939453,
"logps/ref_chosen": -46.840721130371094,
"logps/ref_rejected": -69.3609390258789,
"logps/rejected": -80.94558715820312,
"loss": 0.9506,
"margin_dpo/margin_mean": 3.8706767559051514,
"margin_dpo/margin_std": 4.824084281921387,
"step": 184
},
{
"KL/chosen_KL_mean": -8.752460479736328,
"KL/mean": -10.798396110534668,
"KL/rejected_KL_mean": -12.84432601928711,
"KL/std": 5.784908294677734,
"epoch": 0.2796674225245654,
"fcm_dpo/beta": 0.20158517360687256,
"fcm_dpo/delta": -0.029050692915916443,
"fcm_dpo/margin": 4.091870307922363,
"fcm_dpo/q_t": 0.3572160005569458,
"grad_norm": 51.14906311035156,
"learning_rate": 4.5364398801258394e-07,
"logits/chosen": 0.2673199474811554,
"logits/rejected": 0.22023963928222656,
"logps/chosen": -61.07360076904297,
"logps/ref_chosen": -52.32114028930664,
"logps/ref_rejected": -68.3885726928711,
"logps/rejected": -81.23289489746094,
"loss": 1.1251,
"margin_dpo/margin_mean": 4.091870307922363,
"margin_dpo/margin_std": 7.132951736450195,
"step": 185
},
{
"KL/chosen_KL_mean": -8.121000289916992,
"KL/mean": -10.394832611083984,
"KL/rejected_KL_mean": -12.66866683959961,
"KL/std": 5.741436958312988,
"epoch": 0.2811791383219955,
"fcm_dpo/beta": 0.19917798042297363,
"fcm_dpo/delta": -0.1167006641626358,
"fcm_dpo/margin": 4.547662734985352,
"fcm_dpo/q_t": 0.33835506439208984,
"grad_norm": 63.45903778076172,
"learning_rate": 4.5287418106563354e-07,
"logits/chosen": 0.15486830472946167,
"logits/rejected": 0.11560823023319244,
"logps/chosen": -75.54113006591797,
"logps/ref_chosen": -67.42012786865234,
"logps/ref_rejected": -82.50968933105469,
"logps/rejected": -95.17835998535156,
"loss": 1.0224,
"margin_dpo/margin_mean": 4.547662734985352,
"margin_dpo/margin_std": 6.847630023956299,
"step": 186
},
{
"KL/chosen_KL_mean": -9.027420043945312,
"KL/mean": -11.191137313842773,
"KL/rejected_KL_mean": -13.354854583740234,
"KL/std": 5.968443393707275,
"epoch": 0.28269085411942557,
"fcm_dpo/beta": 0.19584915041923523,
"fcm_dpo/delta": -0.05276474729180336,
"fcm_dpo/margin": 4.327441692352295,
"fcm_dpo/q_t": 0.34239810705184937,
"grad_norm": 62.50242233276367,
"learning_rate": 4.520986992917297e-07,
"logits/chosen": 0.20313122868537903,
"logits/rejected": 0.15035201609134674,
"logps/chosen": -84.55291748046875,
"logps/ref_chosen": -75.52549743652344,
"logps/ref_rejected": -94.76289367675781,
"logps/rejected": -108.11775207519531,
"loss": 1.0374,
"margin_dpo/margin_mean": 4.327441215515137,
"margin_dpo/margin_std": 6.5522308349609375,
"step": 187
},
{
"KL/chosen_KL_mean": -8.111988067626953,
"KL/mean": -10.224084854125977,
"KL/rejected_KL_mean": -12.336181640625,
"KL/std": 5.906611442565918,
"epoch": 0.2842025699168556,
"fcm_dpo/beta": 0.195459246635437,
"fcm_dpo/delta": -0.02797466143965721,
"fcm_dpo/margin": 4.224184036254883,
"fcm_dpo/q_t": 0.34629908204078674,
"grad_norm": 58.858394622802734,
"learning_rate": 4.5131756438276466e-07,
"logits/chosen": 0.21499189734458923,
"logits/rejected": 0.17359653115272522,
"logps/chosen": -79.63531494140625,
"logps/ref_chosen": -71.52333068847656,
"logps/ref_rejected": -78.29949951171875,
"logps/rejected": -90.63568115234375,
"loss": 1.0415,
"margin_dpo/margin_mean": 4.224184036254883,
"margin_dpo/margin_std": 6.645949363708496,
"step": 188
},
{
"KL/chosen_KL_mean": -7.738534927368164,
"KL/mean": -10.014970779418945,
"KL/rejected_KL_mean": -12.291412353515625,
"KL/std": 5.990107536315918,
"epoch": 0.2857142857142857,
"fcm_dpo/beta": 0.1896684467792511,
"fcm_dpo/delta": -0.07736320793628693,
"fcm_dpo/margin": 4.552879333496094,
"fcm_dpo/q_t": 0.33414921164512634,
"grad_norm": 52.09397888183594,
"learning_rate": 4.5053079818876096e-07,
"logits/chosen": 0.22613701224327087,
"logits/rejected": 0.23754778504371643,
"logps/chosen": -79.91480255126953,
"logps/ref_chosen": -72.17626953125,
"logps/ref_rejected": -75.26313781738281,
"logps/rejected": -87.55455017089844,
"loss": 0.975,
"margin_dpo/margin_mean": 4.55288028717041,
"margin_dpo/margin_std": 5.972061634063721,
"step": 189
},
{
"KL/chosen_KL_mean": -7.69354248046875,
"KL/mean": -10.335224151611328,
"KL/rejected_KL_mean": -12.97690200805664,
"KL/std": 5.906277656555176,
"epoch": 0.2872260015117158,
"fcm_dpo/beta": 0.1865214705467224,
"fcm_dpo/delta": -0.20571252703666687,
"fcm_dpo/margin": 5.283355712890625,
"fcm_dpo/q_t": 0.3164942264556885,
"grad_norm": 51.77168273925781,
"learning_rate": 4.4973842271726024e-07,
"logits/chosen": 0.28338149189949036,
"logits/rejected": 0.1495637595653534,
"logps/chosen": -62.317813873291016,
"logps/ref_chosen": -54.624271392822266,
"logps/ref_rejected": -101.47068786621094,
"logps/rejected": -114.44758605957031,
"loss": 0.8768,
"margin_dpo/margin_mean": 5.283356189727783,
"margin_dpo/margin_std": 6.187994480133057,
"step": 190
},
{
"KL/chosen_KL_mean": -9.158432006835938,
"KL/mean": -11.346714973449707,
"KL/rejected_KL_mean": -13.53499984741211,
"KL/std": 6.011017799377441,
"epoch": 0.2887377173091459,
"fcm_dpo/beta": 0.1845850646495819,
"fcm_dpo/delta": -0.008593167178332806,
"fcm_dpo/margin": 4.376575469970703,
"fcm_dpo/q_t": 0.3456087112426758,
"grad_norm": 58.76239776611328,
"learning_rate": 4.48940460132708e-07,
"logits/chosen": 0.2690032124519348,
"logits/rejected": 0.24326932430267334,
"logps/chosen": -82.0909423828125,
"logps/ref_chosen": -72.93251037597656,
"logps/ref_rejected": -89.95103454589844,
"logps/rejected": -103.48603820800781,
"loss": 1.0117,
"margin_dpo/margin_mean": 4.376575469970703,
"margin_dpo/margin_std": 6.37799072265625,
"step": 191
},
{
"KL/chosen_KL_mean": -9.196332931518555,
"KL/mean": -10.66668701171875,
"KL/rejected_KL_mean": -12.137035369873047,
"KL/std": 6.126029014587402,
"epoch": 0.29024943310657597,
"fcm_dpo/beta": 0.19202987849712372,
"fcm_dpo/delta": 0.2476315051317215,
"fcm_dpo/margin": 2.940706968307495,
"fcm_dpo/q_t": 0.3896513879299164,
"grad_norm": 46.325660705566406,
"learning_rate": 4.481369327558329e-07,
"logits/chosen": 0.2842397689819336,
"logits/rejected": 0.2590170204639435,
"logps/chosen": -63.197452545166016,
"logps/ref_chosen": -54.001121520996094,
"logps/ref_rejected": -63.531551361083984,
"logps/rejected": -75.66858673095703,
"loss": 1.145,
"margin_dpo/margin_mean": 2.940706729888916,
"margin_dpo/margin_std": 5.678344249725342,
"step": 192
},
{
"KL/chosen_KL_mean": -8.372961044311523,
"KL/mean": -10.790081024169922,
"KL/rejected_KL_mean": -13.207202911376953,
"KL/std": 6.010296821594238,
"epoch": 0.29176114890400606,
"fcm_dpo/beta": 0.1883402019739151,
"fcm_dpo/delta": -0.12426133453845978,
"fcm_dpo/margin": 4.8342437744140625,
"fcm_dpo/q_t": 0.3302828073501587,
"grad_norm": 45.016563415527344,
"learning_rate": 4.47327863063023e-07,
"logits/chosen": 0.19597403705120087,
"logits/rejected": 0.17333577573299408,
"logps/chosen": -65.12223815917969,
"logps/ref_chosen": -56.74927520751953,
"logps/ref_rejected": -58.80629348754883,
"logps/rejected": -72.01349639892578,
"loss": 0.9163,
"margin_dpo/margin_mean": 4.834243297576904,
"margin_dpo/margin_std": 6.111197471618652,
"step": 193
},
{
"KL/chosen_KL_mean": -7.8690032958984375,
"KL/mean": -9.918621063232422,
"KL/rejected_KL_mean": -11.968246459960938,
"KL/std": 5.435059070587158,
"epoch": 0.29327286470143615,
"fcm_dpo/beta": 0.18736335635185242,
"fcm_dpo/delta": 0.03185667097568512,
"fcm_dpo/margin": 4.099242210388184,
"fcm_dpo/q_t": 0.3544465899467468,
"grad_norm": 43.90078353881836,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": 0.2552631199359894,
"logits/rejected": 0.2272903025150299,
"logps/chosen": -64.51844787597656,
"logps/ref_chosen": -56.64944076538086,
"logps/ref_rejected": -69.98954772949219,
"logps/rejected": -81.95779418945312,
"loss": 1.0746,
"margin_dpo/margin_mean": 4.099242210388184,
"margin_dpo/margin_std": 6.507394313812256,
"step": 194
},
{
"KL/chosen_KL_mean": -9.42181396484375,
"KL/mean": -11.70624828338623,
"KL/rejected_KL_mean": -13.990684509277344,
"KL/std": 6.544086456298828,
"epoch": 0.2947845804988662,
"fcm_dpo/beta": 0.1866236925125122,
"fcm_dpo/delta": -0.059793997555971146,
"fcm_dpo/margin": 4.568871974945068,
"fcm_dpo/q_t": 0.338507741689682,
"grad_norm": 51.90018081665039,
"learning_rate": 4.4569318740967043e-07,
"logits/chosen": 0.14530959725379944,
"logits/rejected": 0.14585906267166138,
"logps/chosen": -79.83158874511719,
"logps/ref_chosen": -70.40977478027344,
"logps/ref_rejected": -74.39448547363281,
"logps/rejected": -88.38516998291016,
"loss": 0.9728,
"margin_dpo/margin_mean": 4.568871974945068,
"margin_dpo/margin_std": 6.183128356933594,
"step": 195
},
{
"KL/chosen_KL_mean": -8.454660415649414,
"KL/mean": -10.585471153259277,
"KL/rejected_KL_mean": -12.71627426147461,
"KL/std": 6.18372917175293,
"epoch": 0.2962962962962963,
"fcm_dpo/beta": 0.18937581777572632,
"fcm_dpo/delta": -0.009729728102684021,
"fcm_dpo/margin": 4.261617183685303,
"fcm_dpo/q_t": 0.33870434761047363,
"grad_norm": 41.523197174072266,
"learning_rate": 4.448676271745197e-07,
"logits/chosen": 0.25778093934059143,
"logits/rejected": 0.21833831071853638,
"logps/chosen": -67.68223571777344,
"logps/ref_chosen": -59.227577209472656,
"logps/ref_rejected": -83.54757690429688,
"logps/rejected": -96.26384735107422,
"loss": 0.9843,
"margin_dpo/margin_mean": 4.261617183685303,
"margin_dpo/margin_std": 5.713452339172363,
"step": 196
},
{
"KL/chosen_KL_mean": -8.017414093017578,
"KL/mean": -10.4995698928833,
"KL/rejected_KL_mean": -12.981727600097656,
"KL/std": 6.454028606414795,
"epoch": 0.29780801209372637,
"fcm_dpo/beta": 0.18606367707252502,
"fcm_dpo/delta": -0.13813000917434692,
"fcm_dpo/margin": 4.9643073081970215,
"fcm_dpo/q_t": 0.3351513743400574,
"grad_norm": 48.245933532714844,
"learning_rate": 4.440366160729392e-07,
"logits/chosen": 0.3265201151371002,
"logits/rejected": 0.2801608443260193,
"logps/chosen": -59.546546936035156,
"logps/ref_chosen": -51.52912902832031,
"logps/ref_rejected": -73.70631408691406,
"logps/rejected": -86.68803405761719,
"loss": 1.0636,
"margin_dpo/margin_mean": 4.964306831359863,
"margin_dpo/margin_std": 7.59263801574707,
"step": 197
},
{
"KL/chosen_KL_mean": -8.19921875,
"KL/mean": -10.83227252960205,
"KL/rejected_KL_mean": -13.465335845947266,
"KL/std": 5.659462928771973,
"epoch": 0.29931972789115646,
"fcm_dpo/beta": 0.17989234626293182,
"fcm_dpo/delta": -0.1634722501039505,
"fcm_dpo/margin": 5.266110420227051,
"fcm_dpo/q_t": 0.31562328338623047,
"grad_norm": 43.244449615478516,
"learning_rate": 4.432001773500957e-07,
"logits/chosen": 0.30426979064941406,
"logits/rejected": 0.26571154594421387,
"logps/chosen": -67.98190307617188,
"logps/ref_chosen": -59.78268051147461,
"logps/ref_rejected": -72.24533081054688,
"logps/rejected": -85.71066284179688,
"loss": 0.8725,
"margin_dpo/margin_mean": 5.266110420227051,
"margin_dpo/margin_std": 5.919608116149902,
"step": 198
},
{
"KL/chosen_KL_mean": -8.999025344848633,
"KL/mean": -11.057474136352539,
"KL/rejected_KL_mean": -13.115921020507812,
"KL/std": 5.817185878753662,
"epoch": 0.30083144368858655,
"fcm_dpo/beta": 0.1805601716041565,
"fcm_dpo/delta": 0.05563047528266907,
"fcm_dpo/margin": 4.116896629333496,
"fcm_dpo/q_t": 0.35699188709259033,
"grad_norm": 49.35909652709961,
"learning_rate": 4.4235833440297856e-07,
"logits/chosen": 0.23536258935928345,
"logits/rejected": 0.15354374051094055,
"logps/chosen": -65.38579559326172,
"logps/ref_chosen": -56.38677215576172,
"logps/ref_rejected": -74.56779479980469,
"logps/rejected": -87.6837158203125,
"loss": 1.1103,
"margin_dpo/margin_mean": 4.116896152496338,
"margin_dpo/margin_std": 6.822243690490723,
"step": 199
},
{
"KL/chosen_KL_mean": -7.210149765014648,
"KL/mean": -10.162313461303711,
"KL/rejected_KL_mean": -13.114479064941406,
"KL/std": 6.341808795928955,
"epoch": 0.30234315948601664,
"fcm_dpo/beta": 0.17075824737548828,
"fcm_dpo/delta": -0.23531164228916168,
"fcm_dpo/margin": 5.9043288230896,
"fcm_dpo/q_t": 0.31821250915527344,
"grad_norm": 43.43456268310547,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": 0.29511919617652893,
"logits/rejected": 0.22751715779304504,
"logps/chosen": -65.03447723388672,
"logps/ref_chosen": -57.82432556152344,
"logps/ref_rejected": -89.28246307373047,
"logps/rejected": -102.39694213867188,
"loss": 0.9253,
"margin_dpo/margin_mean": 5.904328346252441,
"margin_dpo/margin_std": 7.588106155395508,
"step": 200
},
{
"KL/chosen_KL_mean": -9.282886505126953,
"KL/mean": -11.540687561035156,
"KL/rejected_KL_mean": -13.798492431640625,
"KL/std": 6.509415626525879,
"epoch": 0.30385487528344673,
"fcm_dpo/beta": 0.17012134194374084,
"fcm_dpo/delta": 0.034042563289403915,
"fcm_dpo/margin": 4.515607833862305,
"fcm_dpo/q_t": 0.35858190059661865,
"grad_norm": 52.089141845703125,
"learning_rate": 4.4065853017905953e-07,
"logits/chosen": 0.31442439556121826,
"logits/rejected": 0.27165716886520386,
"logps/chosen": -68.28264617919922,
"logps/ref_chosen": -58.999759674072266,
"logps/ref_rejected": -84.67575073242188,
"logps/rejected": -98.4742431640625,
"loss": 1.0222,
"margin_dpo/margin_mean": 4.515607833862305,
"margin_dpo/margin_std": 6.753261566162109,
"step": 201
},
{
"KL/chosen_KL_mean": -7.988605499267578,
"KL/mean": -10.506321907043457,
"KL/rejected_KL_mean": -13.024032592773438,
"KL/std": 6.282382011413574,
"epoch": 0.30536659108087677,
"fcm_dpo/beta": 0.1711854636669159,
"fcm_dpo/delta": -0.06901469826698303,
"fcm_dpo/margin": 5.035429000854492,
"fcm_dpo/q_t": 0.3341776430606842,
"grad_norm": 40.801334381103516,
"learning_rate": 4.3980061644943575e-07,
"logits/chosen": 0.19821295142173767,
"logits/rejected": 0.1341780424118042,
"logps/chosen": -55.649253845214844,
"logps/ref_chosen": -47.660648345947266,
"logps/ref_rejected": -73.63249969482422,
"logps/rejected": -86.65653228759766,
"loss": 0.9692,
"margin_dpo/margin_mean": 5.035429000854492,
"margin_dpo/margin_std": 6.7977294921875,
"step": 202
},
{
"KL/chosen_KL_mean": -8.52560043334961,
"KL/mean": -10.693042755126953,
"KL/rejected_KL_mean": -12.860485076904297,
"KL/std": 6.2385053634643555,
"epoch": 0.30687830687830686,
"fcm_dpo/beta": 0.17108216881752014,
"fcm_dpo/delta": 0.06265536695718765,
"fcm_dpo/margin": 4.334885597229004,
"fcm_dpo/q_t": 0.3605703115463257,
"grad_norm": 49.72651290893555,
"learning_rate": 4.3893739358856455e-07,
"logits/chosen": 0.29747486114501953,
"logits/rejected": 0.22976483404636383,
"logps/chosen": -70.85113525390625,
"logps/ref_chosen": -62.32553482055664,
"logps/ref_rejected": -99.37226104736328,
"logps/rejected": -112.23274230957031,
"loss": 1.0333,
"margin_dpo/margin_mean": 4.334885597229004,
"margin_dpo/margin_std": 6.700814723968506,
"step": 203
},
{
"KL/chosen_KL_mean": -8.656208038330078,
"KL/mean": -11.099645614624023,
"KL/rejected_KL_mean": -13.543079376220703,
"KL/std": 6.6452507972717285,
"epoch": 0.30839002267573695,
"fcm_dpo/beta": 0.16719385981559753,
"fcm_dpo/delta": -0.026872709393501282,
"fcm_dpo/margin": 4.886868953704834,
"fcm_dpo/q_t": 0.3485722541809082,
"grad_norm": 41.345577239990234,
"learning_rate": 4.380688857426449e-07,
"logits/chosen": 0.22623693943023682,
"logits/rejected": 0.16254332661628723,
"logps/chosen": -59.2855224609375,
"logps/ref_chosen": -50.62931823730469,
"logps/ref_rejected": -66.60475158691406,
"logps/rejected": -80.1478271484375,
"loss": 1.0118,
"margin_dpo/margin_mean": 4.886869430541992,
"margin_dpo/margin_std": 6.901353359222412,
"step": 204
},
{
"KL/chosen_KL_mean": -9.14675521850586,
"KL/mean": -11.405117988586426,
"KL/rejected_KL_mean": -13.66347885131836,
"KL/std": 6.326295852661133,
"epoch": 0.30990173847316704,
"fcm_dpo/beta": 0.16965240240097046,
"fcm_dpo/delta": 0.03534369170665741,
"fcm_dpo/margin": 4.516724109649658,
"fcm_dpo/q_t": 0.3603339195251465,
"grad_norm": 61.0053596496582,
"learning_rate": 4.3719511720570814e-07,
"logits/chosen": 0.2996768355369568,
"logits/rejected": 0.24160867929458618,
"logps/chosen": -79.5029296875,
"logps/ref_chosen": -70.3561782836914,
"logps/ref_rejected": -93.39848327636719,
"logps/rejected": -107.06195831298828,
"loss": 1.1121,
"margin_dpo/margin_mean": 4.5167236328125,
"margin_dpo/margin_std": 7.951142311096191,
"step": 205
},
{
"KL/chosen_KL_mean": -9.403274536132812,
"KL/mean": -11.332551956176758,
"KL/rejected_KL_mean": -13.261837005615234,
"KL/std": 6.498051166534424,
"epoch": 0.31141345427059713,
"fcm_dpo/beta": 0.17620447278022766,
"fcm_dpo/delta": 0.12605662643909454,
"fcm_dpo/margin": 3.85856032371521,
"fcm_dpo/q_t": 0.37659770250320435,
"grad_norm": 51.65834426879883,
"learning_rate": 4.363161124189387e-07,
"logits/chosen": 0.30237025022506714,
"logits/rejected": 0.28653794527053833,
"logps/chosen": -77.04875183105469,
"logps/ref_chosen": -67.64547729492188,
"logps/ref_rejected": -79.89584350585938,
"logps/rejected": -93.15768432617188,
"loss": 1.1709,
"margin_dpo/margin_mean": 3.858560562133789,
"margin_dpo/margin_std": 7.424094200134277,
"step": 206
},
{
"KL/chosen_KL_mean": -10.225160598754883,
"KL/mean": -12.855207443237305,
"KL/rejected_KL_mean": -15.48525619506836,
"KL/std": 6.979071140289307,
"epoch": 0.3129251700680272,
"fcm_dpo/beta": 0.17152279615402222,
"fcm_dpo/delta": -0.11443052440881729,
"fcm_dpo/margin": 5.260091781616211,
"fcm_dpo/q_t": 0.3384243845939636,
"grad_norm": 42.885921478271484,
"learning_rate": 4.3543189596998986e-07,
"logits/chosen": 0.22064357995986938,
"logits/rejected": 0.1584734171628952,
"logps/chosen": -77.88935852050781,
"logps/ref_chosen": -67.66419219970703,
"logps/ref_rejected": -85.10249328613281,
"logps/rejected": -100.5877456665039,
"loss": 0.9697,
"margin_dpo/margin_mean": 5.260092735290527,
"margin_dpo/margin_std": 7.317288398742676,
"step": 207
},
{
"KL/chosen_KL_mean": -8.486841201782227,
"KL/mean": -10.045097351074219,
"KL/rejected_KL_mean": -11.603355407714844,
"KL/std": 6.03558349609375,
"epoch": 0.3144368858654573,
"fcm_dpo/beta": 0.17885954678058624,
"fcm_dpo/delta": 0.2562446892261505,
"fcm_dpo/margin": 3.1165127754211426,
"fcm_dpo/q_t": 0.3973674178123474,
"grad_norm": 51.5666389465332,
"learning_rate": 4.3454249259229664e-07,
"logits/chosen": 0.23146170377731323,
"logits/rejected": 0.20724132657051086,
"logps/chosen": -66.21855163574219,
"logps/ref_chosen": -57.731712341308594,
"logps/ref_rejected": -74.19276428222656,
"logps/rejected": -85.7961196899414,
"loss": 1.2168,
"margin_dpo/margin_mean": 3.1165130138397217,
"margin_dpo/margin_std": 6.801269054412842,
"step": 208
},
{
"KL/chosen_KL_mean": -8.83671760559082,
"KL/mean": -11.54526138305664,
"KL/rejected_KL_mean": -14.25381088256836,
"KL/std": 6.972295761108398,
"epoch": 0.31594860166288735,
"fcm_dpo/beta": 0.17789439857006073,
"fcm_dpo/delta": -0.18146634101867676,
"fcm_dpo/margin": 5.417089462280273,
"fcm_dpo/q_t": 0.3251495361328125,
"grad_norm": 53.9567756652832,
"learning_rate": 4.336479271643833e-07,
"logits/chosen": 0.20470848679542542,
"logits/rejected": 0.15327095985412598,
"logps/chosen": -77.38679504394531,
"logps/ref_chosen": -68.55007934570312,
"logps/ref_rejected": -87.90541076660156,
"logps/rejected": -102.15922546386719,
"loss": 1.001,
"margin_dpo/margin_mean": 5.417089939117432,
"margin_dpo/margin_std": 7.826272010803223,
"step": 209
},
{
"KL/chosen_KL_mean": -8.892902374267578,
"KL/mean": -11.775569915771484,
"KL/rejected_KL_mean": -14.65823745727539,
"KL/std": 7.52336311340332,
"epoch": 0.31746031746031744,
"fcm_dpo/beta": 0.16959837079048157,
"fcm_dpo/delta": -0.19686026871204376,
"fcm_dpo/margin": 5.765329360961914,
"fcm_dpo/q_t": 0.33063024282455444,
"grad_norm": 41.87021255493164,
"learning_rate": 4.327482247091679e-07,
"logits/chosen": 0.2841176688671112,
"logits/rejected": 0.1984454244375229,
"logps/chosen": -66.16117858886719,
"logps/ref_chosen": -57.268272399902344,
"logps/ref_rejected": -85.72807312011719,
"logps/rejected": -100.38631439208984,
"loss": 0.9738,
"margin_dpo/margin_mean": 5.765328407287598,
"margin_dpo/margin_std": 8.068181991577148,
"step": 210
},
{
"KL/chosen_KL_mean": -8.13759994506836,
"KL/mean": -10.518904685974121,
"KL/rejected_KL_mean": -12.90020751953125,
"KL/std": 6.33951997756958,
"epoch": 0.31897203325774753,
"fcm_dpo/beta": 0.16893050074577332,
"fcm_dpo/delta": -0.006408661603927612,
"fcm_dpo/margin": 4.7626051902771,
"fcm_dpo/q_t": 0.34970852732658386,
"grad_norm": 51.645408630371094,
"learning_rate": 4.3184341039326217e-07,
"logits/chosen": 0.33264777064323425,
"logits/rejected": 0.24676403403282166,
"logps/chosen": -61.77831268310547,
"logps/ref_chosen": -53.640708923339844,
"logps/ref_rejected": -93.0387954711914,
"logps/rejected": -105.93900299072266,
"loss": 0.978,
"margin_dpo/margin_mean": 4.762604236602783,
"margin_dpo/margin_std": 6.6331682205200195,
"step": 211
},
{
"KL/chosen_KL_mean": -9.244085311889648,
"KL/mean": -11.903303146362305,
"KL/rejected_KL_mean": -14.562515258789062,
"KL/std": 6.864345550537109,
"epoch": 0.3204837490551776,
"fcm_dpo/beta": 0.16524195671081543,
"fcm_dpo/delta": -0.08646350353956223,
"fcm_dpo/margin": 5.318434238433838,
"fcm_dpo/q_t": 0.3367248773574829,
"grad_norm": 43.68009948730469,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": 0.2854962944984436,
"logits/rejected": 0.2196652889251709,
"logps/chosen": -66.61082458496094,
"logps/ref_chosen": -57.36674499511719,
"logps/ref_rejected": -79.89643096923828,
"logps/rejected": -94.45894622802734,
"loss": 0.9843,
"margin_dpo/margin_mean": 5.3184332847595215,
"margin_dpo/margin_std": 7.531092643737793,
"step": 212
},
{
"KL/chosen_KL_mean": -7.1361083984375,
"KL/mean": -10.112091064453125,
"KL/rejected_KL_mean": -13.088077545166016,
"KL/std": 6.956264495849609,
"epoch": 0.3219954648526077,
"fcm_dpo/beta": 0.15949341654777527,
"fcm_dpo/delta": -0.1670059859752655,
"fcm_dpo/margin": 5.951968193054199,
"fcm_dpo/q_t": 0.3250270485877991,
"grad_norm": 41.68220520019531,
"learning_rate": 4.3001854756006724e-07,
"logits/chosen": 0.29428863525390625,
"logits/rejected": 0.2708255648612976,
"logps/chosen": -72.35722351074219,
"logps/ref_chosen": -65.22111511230469,
"logps/ref_rejected": -80.1810302734375,
"logps/rejected": -93.26910400390625,
"loss": 0.9682,
"margin_dpo/margin_mean": 5.951968193054199,
"margin_dpo/margin_std": 8.158895492553711,
"step": 213
},
{
"KL/chosen_KL_mean": -8.329187393188477,
"KL/mean": -11.186603546142578,
"KL/rejected_KL_mean": -14.044015884399414,
"KL/std": 6.9872565269470215,
"epoch": 0.3235071806500378,
"fcm_dpo/beta": 0.15824972093105316,
"fcm_dpo/delta": -0.11610302329063416,
"fcm_dpo/margin": 5.714832305908203,
"fcm_dpo/q_t": 0.3369448184967041,
"grad_norm": 47.012935638427734,
"learning_rate": 4.290985500881143e-07,
"logits/chosen": 0.1531170755624771,
"logits/rejected": 0.1307399570941925,
"logps/chosen": -69.62152099609375,
"logps/ref_chosen": -61.292327880859375,
"logps/ref_rejected": -67.69841003417969,
"logps/rejected": -81.74242401123047,
"loss": 0.9658,
"margin_dpo/margin_mean": 5.714832305908203,
"margin_dpo/margin_std": 7.725873947143555,
"step": 214
},
{
"KL/chosen_KL_mean": -9.156429290771484,
"KL/mean": -12.22635555267334,
"KL/rejected_KL_mean": -15.296276092529297,
"KL/std": 7.057165145874023,
"epoch": 0.3250188964474679,
"fcm_dpo/beta": 0.15315330028533936,
"fcm_dpo/delta": -0.15492382645606995,
"fcm_dpo/margin": 6.139848709106445,
"fcm_dpo/q_t": 0.32713302969932556,
"grad_norm": 42.17342758178711,
"learning_rate": 4.281735428447157e-07,
"logits/chosen": 0.1951332986354828,
"logits/rejected": 0.10021235793828964,
"logps/chosen": -73.02556610107422,
"logps/ref_chosen": -63.869136810302734,
"logps/ref_rejected": -98.7657241821289,
"logps/rejected": -114.06199645996094,
"loss": 0.9299,
"margin_dpo/margin_mean": 6.139849662780762,
"margin_dpo/margin_std": 7.7090067863464355,
"step": 215
},
{
"KL/chosen_KL_mean": -8.983440399169922,
"KL/mean": -12.1254301071167,
"KL/rejected_KL_mean": -15.267414093017578,
"KL/std": 7.741682052612305,
"epoch": 0.32653061224489793,
"fcm_dpo/beta": 0.1480444371700287,
"fcm_dpo/delta": -0.14343297481536865,
"fcm_dpo/margin": 6.283971786499023,
"fcm_dpo/q_t": 0.3363327980041504,
"grad_norm": 37.66606903076172,
"learning_rate": 4.2724355170431247e-07,
"logits/chosen": 0.33266764879226685,
"logits/rejected": 0.2524465322494507,
"logps/chosen": -76.80839538574219,
"logps/ref_chosen": -67.824951171875,
"logps/ref_rejected": -96.40231323242188,
"logps/rejected": -111.66973114013672,
"loss": 0.9613,
"margin_dpo/margin_mean": 6.283972263336182,
"margin_dpo/margin_std": 9.071916580200195,
"step": 216
},
{
"KL/chosen_KL_mean": -9.532360076904297,
"KL/mean": -12.935258865356445,
"KL/rejected_KL_mean": -16.33816146850586,
"KL/std": 7.542253017425537,
"epoch": 0.328042328042328,
"fcm_dpo/beta": 0.14228859543800354,
"fcm_dpo/delta": -0.18680410087108612,
"fcm_dpo/margin": 6.8057990074157715,
"fcm_dpo/q_t": 0.3188336491584778,
"grad_norm": 36.55585861206055,
"learning_rate": 4.26308602680756e-07,
"logits/chosen": 0.24256332218647003,
"logits/rejected": 0.1458836793899536,
"logps/chosen": -70.037353515625,
"logps/ref_chosen": -60.5049934387207,
"logps/ref_rejected": -84.26618194580078,
"logps/rejected": -100.60433959960938,
"loss": 0.865,
"margin_dpo/margin_mean": 6.805798530578613,
"margin_dpo/margin_std": 7.959721565246582,
"step": 217
},
{
"KL/chosen_KL_mean": -9.821697235107422,
"KL/mean": -11.902130126953125,
"KL/rejected_KL_mean": -13.982563018798828,
"KL/std": 6.682396411895752,
"epoch": 0.3295540438397581,
"fcm_dpo/beta": 0.14719802141189575,
"fcm_dpo/delta": 0.19345054030418396,
"fcm_dpo/margin": 4.160863876342773,
"fcm_dpo/q_t": 0.3849286139011383,
"grad_norm": 44.22843551635742,
"learning_rate": 4.253687219265803e-07,
"logits/chosen": 0.16697943210601807,
"logits/rejected": 0.1599193960428238,
"logps/chosen": -80.416015625,
"logps/ref_chosen": -70.59431457519531,
"logps/ref_rejected": -73.89038848876953,
"logps/rejected": -87.87295532226562,
"loss": 1.2018,
"margin_dpo/margin_mean": 4.160863876342773,
"margin_dpo/margin_std": 8.526304244995117,
"step": 218
},
{
"KL/chosen_KL_mean": -9.010517120361328,
"KL/mean": -10.984228134155273,
"KL/rejected_KL_mean": -12.957944869995117,
"KL/std": 7.001385688781738,
"epoch": 0.3310657596371882,
"fcm_dpo/beta": 0.1508154571056366,
"fcm_dpo/delta": 0.2181931436061859,
"fcm_dpo/margin": 3.9474315643310547,
"fcm_dpo/q_t": 0.38393646478652954,
"grad_norm": 43.46605682373047,
"learning_rate": 4.2442393573227043e-07,
"logits/chosen": 0.21026837825775146,
"logits/rejected": 0.17155671119689941,
"logps/chosen": -69.50146484375,
"logps/ref_chosen": -60.490943908691406,
"logps/ref_rejected": -75.85001373291016,
"logps/rejected": -88.80795288085938,
"loss": 1.097,
"margin_dpo/margin_mean": 3.9474315643310547,
"margin_dpo/margin_std": 6.710477352142334,
"step": 219
},
{
"KL/chosen_KL_mean": -8.571613311767578,
"KL/mean": -11.141824722290039,
"KL/rejected_KL_mean": -13.7120361328125,
"KL/std": 7.094627380371094,
"epoch": 0.3325774754346183,
"fcm_dpo/beta": 0.1525796800851822,
"fcm_dpo/delta": 0.016179494559764862,
"fcm_dpo/margin": 5.140424728393555,
"fcm_dpo/q_t": 0.3578713536262512,
"grad_norm": 36.82176208496094,
"learning_rate": 4.234742705255272e-07,
"logits/chosen": 0.30773574113845825,
"logits/rejected": 0.2503643333911896,
"logps/chosen": -53.58501434326172,
"logps/ref_chosen": -45.013397216796875,
"logps/ref_rejected": -70.49369812011719,
"logps/rejected": -84.20573425292969,
"loss": 1.0369,
"margin_dpo/margin_mean": 5.140424728393555,
"margin_dpo/margin_std": 7.912476539611816,
"step": 220
},
{
"KL/chosen_KL_mean": -8.029104232788086,
"KL/mean": -10.780109405517578,
"KL/rejected_KL_mean": -13.531108856201172,
"KL/std": 6.923809051513672,
"epoch": 0.3340891912320484,
"fcm_dpo/beta": 0.15349453687667847,
"fcm_dpo/delta": -0.04903030022978783,
"fcm_dpo/margin": 5.502012252807617,
"fcm_dpo/q_t": 0.3516319692134857,
"grad_norm": 39.72418975830078,
"learning_rate": 4.22519752870528e-07,
"logits/chosen": 0.2654990255832672,
"logits/rejected": 0.20310282707214355,
"logps/chosen": -67.12493896484375,
"logps/ref_chosen": -59.09584045410156,
"logps/ref_rejected": -88.64388275146484,
"logps/rejected": -102.17498779296875,
"loss": 1.0059,
"margin_dpo/margin_mean": 5.502011775970459,
"margin_dpo/margin_std": 8.320975303649902,
"step": 221
},
{
"KL/chosen_KL_mean": -8.494781494140625,
"KL/mean": -12.023693084716797,
"KL/rejected_KL_mean": -15.552600860595703,
"KL/std": 8.078587532043457,
"epoch": 0.3356009070294785,
"fcm_dpo/beta": 0.14657087624073029,
"fcm_dpo/delta": -0.26139819622039795,
"fcm_dpo/margin": 7.057823181152344,
"fcm_dpo/q_t": 0.3049485385417938,
"grad_norm": 38.150306701660156,
"learning_rate": 4.2156040946718343e-07,
"logits/chosen": 0.3249172568321228,
"logits/rejected": 0.24639061093330383,
"logps/chosen": -64.49247741699219,
"logps/ref_chosen": -55.9976921081543,
"logps/ref_rejected": -111.94727325439453,
"logps/rejected": -127.4998779296875,
"loss": 0.8544,
"margin_dpo/margin_mean": 7.057823181152344,
"margin_dpo/margin_std": 7.885249137878418,
"step": 222
},
{
"KL/chosen_KL_mean": -8.748092651367188,
"KL/mean": -12.026680946350098,
"KL/rejected_KL_mean": -15.305274963378906,
"KL/std": 7.710515975952148,
"epoch": 0.3371126228269085,
"fcm_dpo/beta": 0.1403246521949768,
"fcm_dpo/delta": -0.13532818853855133,
"fcm_dpo/margin": 6.557183742523193,
"fcm_dpo/q_t": 0.3230859637260437,
"grad_norm": 33.81869125366211,
"learning_rate": 4.2059626715039065e-07,
"logits/chosen": 0.285654753446579,
"logits/rejected": 0.23378512263298035,
"logps/chosen": -68.63951110839844,
"logps/ref_chosen": -59.891422271728516,
"logps/ref_rejected": -86.28954315185547,
"logps/rejected": -101.59481811523438,
"loss": 0.8742,
"margin_dpo/margin_mean": 6.557183265686035,
"margin_dpo/margin_std": 7.321591377258301,
"step": 223
},
{
"KL/chosen_KL_mean": -9.40049934387207,
"KL/mean": -11.442483901977539,
"KL/rejected_KL_mean": -13.484466552734375,
"KL/std": 7.180830001831055,
"epoch": 0.3386243386243386,
"fcm_dpo/beta": 0.1448686271905899,
"fcm_dpo/delta": 0.22197052836418152,
"fcm_dpo/margin": 4.0839691162109375,
"fcm_dpo/q_t": 0.38782450556755066,
"grad_norm": 42.53331756591797,
"learning_rate": 4.1962735288928304e-07,
"logits/chosen": 0.3183774948120117,
"logits/rejected": 0.2993980050086975,
"logps/chosen": -73.44513702392578,
"logps/ref_chosen": -64.04463195800781,
"logps/ref_rejected": -75.05450439453125,
"logps/rejected": -88.53897094726562,
"loss": 1.081,
"margin_dpo/margin_mean": 4.0839691162109375,
"margin_dpo/margin_std": 6.880286693572998,
"step": 224
},
{
"KL/chosen_KL_mean": -9.292135238647461,
"KL/mean": -12.396595001220703,
"KL/rejected_KL_mean": -15.501056671142578,
"KL/std": 8.57298469543457,
"epoch": 0.3401360544217687,
"fcm_dpo/beta": 0.143642395734787,
"fcm_dpo/delta": -0.10299123823642731,
"fcm_dpo/margin": 6.20892333984375,
"fcm_dpo/q_t": 0.33599036931991577,
"grad_norm": 49.038352966308594,
"learning_rate": 4.186536937864752e-07,
"logits/chosen": 0.31163620948791504,
"logits/rejected": 0.20546850562095642,
"logps/chosen": -75.38795471191406,
"logps/ref_chosen": -66.0958251953125,
"logps/ref_rejected": -97.68675231933594,
"logps/rejected": -113.18780517578125,
"loss": 0.9474,
"margin_dpo/margin_mean": 6.20892333984375,
"margin_dpo/margin_std": 8.138298988342285,
"step": 225
},
{
"KL/chosen_KL_mean": -8.449029922485352,
"KL/mean": -11.095566749572754,
"KL/rejected_KL_mean": -13.742103576660156,
"KL/std": 7.645484447479248,
"epoch": 0.3416477702191988,
"fcm_dpo/beta": 0.144595205783844,
"fcm_dpo/delta": 0.03752833604812622,
"fcm_dpo/margin": 5.293073654174805,
"fcm_dpo/q_t": 0.35609138011932373,
"grad_norm": 34.39674377441406,
"learning_rate": 4.176753170773052e-07,
"logits/chosen": 0.3167535364627838,
"logits/rejected": 0.27241265773773193,
"logps/chosen": -59.865901947021484,
"logps/ref_chosen": -51.4168701171875,
"logps/ref_rejected": -66.30068969726562,
"logps/rejected": -80.04278564453125,
"loss": 1.043,
"margin_dpo/margin_mean": 5.293073654174805,
"margin_dpo/margin_std": 8.19736099243164,
"step": 226
},
{
"KL/chosen_KL_mean": -8.788469314575195,
"KL/mean": -11.457413673400879,
"KL/rejected_KL_mean": -14.126358032226562,
"KL/std": 8.207555770874023,
"epoch": 0.3431594860166289,
"fcm_dpo/beta": 0.1451629102230072,
"fcm_dpo/delta": 0.027083417400717735,
"fcm_dpo/margin": 5.337895393371582,
"fcm_dpo/q_t": 0.36309635639190674,
"grad_norm": 46.18406295776367,
"learning_rate": 4.166922501290729e-07,
"logits/chosen": 0.3289709687232971,
"logits/rejected": 0.29214709997177124,
"logps/chosen": -66.77824401855469,
"logps/ref_chosen": -57.989776611328125,
"logps/ref_rejected": -75.05464172363281,
"logps/rejected": -89.18099975585938,
"loss": 1.0797,
"margin_dpo/margin_mean": 5.337895393371582,
"margin_dpo/margin_std": 8.865333557128906,
"step": 227
},
{
"KL/chosen_KL_mean": -9.585563659667969,
"KL/mean": -12.280923843383789,
"KL/rejected_KL_mean": -14.97628402709961,
"KL/std": 7.654868125915527,
"epoch": 0.34467120181405897,
"fcm_dpo/beta": 0.14699329435825348,
"fcm_dpo/delta": 0.007324397563934326,
"fcm_dpo/margin": 5.390713214874268,
"fcm_dpo/q_t": 0.3541639447212219,
"grad_norm": 37.86101150512695,
"learning_rate": 4.1570452044027405e-07,
"logits/chosen": 0.3056115508079529,
"logits/rejected": 0.23478630185127258,
"logps/chosen": -65.14492797851562,
"logps/ref_chosen": -55.55936813354492,
"logps/ref_rejected": -77.02364349365234,
"logps/rejected": -91.99992370605469,
"loss": 1.0464,
"margin_dpo/margin_mean": 5.390713691711426,
"margin_dpo/margin_std": 8.415397644042969,
"step": 228
},
{
"KL/chosen_KL_mean": -8.10175895690918,
"KL/mean": -10.556568145751953,
"KL/rejected_KL_mean": -13.011375427246094,
"KL/std": 7.001654148101807,
"epoch": 0.34618291761148906,
"fcm_dpo/beta": 0.14782226085662842,
"fcm_dpo/delta": 0.08006762713193893,
"fcm_dpo/margin": 4.909615516662598,
"fcm_dpo/q_t": 0.35694581270217896,
"grad_norm": 102.48412322998047,
"learning_rate": 4.147121556398312e-07,
"logits/chosen": 0.38529714941978455,
"logits/rejected": 0.32762086391448975,
"logps/chosen": -58.89642333984375,
"logps/ref_chosen": -50.79466247558594,
"logps/ref_rejected": -78.4474105834961,
"logps/rejected": -91.45878601074219,
"loss": 1.0866,
"margin_dpo/margin_mean": 4.909615516662598,
"margin_dpo/margin_std": 8.180277824401855,
"step": 229
},
{
"KL/chosen_KL_mean": -8.855535507202148,
"KL/mean": -11.712057113647461,
"KL/rejected_KL_mean": -14.568580627441406,
"KL/std": 7.761747360229492,
"epoch": 0.3476946334089191,
"fcm_dpo/beta": 0.14939934015274048,
"fcm_dpo/delta": -0.0625389814376831,
"fcm_dpo/margin": 5.713042259216309,
"fcm_dpo/q_t": 0.3402790129184723,
"grad_norm": 41.512351989746094,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": 0.27887076139450073,
"logits/rejected": 0.2750973701477051,
"logps/chosen": -65.58476257324219,
"logps/ref_chosen": -56.729225158691406,
"logps/ref_rejected": -62.99180603027344,
"logps/rejected": -77.56038665771484,
"loss": 0.9828,
"margin_dpo/margin_mean": 5.713043212890625,
"margin_dpo/margin_std": 7.821670055389404,
"step": 230
},
{
"KL/chosen_KL_mean": -9.283416748046875,
"KL/mean": -13.417741775512695,
"KL/rejected_KL_mean": -17.552074432373047,
"KL/std": 7.7223920822143555,
"epoch": 0.3492063492063492,
"fcm_dpo/beta": 0.13743643462657928,
"fcm_dpo/delta": -0.384117990732193,
"fcm_dpo/margin": 8.268657684326172,
"fcm_dpo/q_t": 0.27989715337753296,
"grad_norm": 35.174171447753906,
"learning_rate": 4.1271363186719835e-07,
"logits/chosen": 0.22936806082725525,
"logits/rejected": 0.21267296373844147,
"logps/chosen": -81.8805160522461,
"logps/ref_chosen": -72.59709930419922,
"logps/ref_rejected": -86.2322998046875,
"logps/rejected": -103.78437805175781,
"loss": 0.7754,
"margin_dpo/margin_mean": 8.268656730651855,
"margin_dpo/margin_std": 7.923949241638184,
"step": 231
},
{
"KL/chosen_KL_mean": -9.224300384521484,
"KL/mean": -12.099236488342285,
"KL/rejected_KL_mean": -14.974166870117188,
"KL/std": 8.310755729675293,
"epoch": 0.3507180650037793,
"fcm_dpo/beta": 0.1356590837240219,
"fcm_dpo/delta": 0.021425606682896614,
"fcm_dpo/margin": 5.749870300292969,
"fcm_dpo/q_t": 0.3614855706691742,
"grad_norm": 41.390968322753906,
"learning_rate": 4.1170752879801436e-07,
"logits/chosen": 0.24424928426742554,
"logits/rejected": 0.21669036149978638,
"logps/chosen": -77.34283447265625,
"logps/ref_chosen": -68.1185302734375,
"logps/ref_rejected": -83.79415893554688,
"logps/rejected": -98.76832580566406,
"loss": 1.0644,
"margin_dpo/margin_mean": 5.749871253967285,
"margin_dpo/margin_std": 9.516833305358887,
"step": 232
},
{
"KL/chosen_KL_mean": -10.805501937866211,
"KL/mean": -13.204916954040527,
"KL/rejected_KL_mean": -15.604331970214844,
"KL/std": 8.392253875732422,
"epoch": 0.35222978080120937,
"fcm_dpo/beta": 0.1332673728466034,
"fcm_dpo/delta": -0.11425915360450745,
"fcm_dpo/margin": 4.798825740814209,
"fcm_dpo/q_t": 0.37997913360595703,
"grad_norm": 36.0398063659668,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": 0.28187963366508484,
"logits/rejected": 0.2293119728565216,
"logps/chosen": -65.87565612792969,
"logps/ref_chosen": -55.070152282714844,
"logps/ref_rejected": -66.61845397949219,
"logps/rejected": -82.2227783203125,
"loss": 1.0971,
"margin_dpo/margin_mean": 4.798825740814209,
"margin_dpo/margin_std": 7.930860996246338,
"step": 233
},
{
"KL/chosen_KL_mean": -9.701313018798828,
"KL/mean": -11.782073974609375,
"KL/rejected_KL_mean": -13.862836837768555,
"KL/std": 7.602666854858398,
"epoch": 0.35374149659863946,
"fcm_dpo/beta": 0.13307172060012817,
"fcm_dpo/delta": -0.014703216031193733,
"fcm_dpo/margin": 4.161526679992676,
"fcm_dpo/q_t": 0.3957321345806122,
"grad_norm": 38.414398193359375,
"learning_rate": 4.09681781007452e-07,
"logits/chosen": 0.22231775522232056,
"logits/rejected": 0.20927371084690094,
"logps/chosen": -65.62721252441406,
"logps/ref_chosen": -55.92589569091797,
"logps/ref_rejected": -51.11608123779297,
"logps/rejected": -64.97891998291016,
"loss": 1.1773,
"margin_dpo/margin_mean": 4.161526679992676,
"margin_dpo/margin_std": 8.233041763305664,
"step": 234
},
{
"KL/chosen_KL_mean": -8.079273223876953,
"KL/mean": -11.814519882202148,
"KL/rejected_KL_mean": -15.54977035522461,
"KL/std": 8.269691467285156,
"epoch": 0.35525321239606955,
"fcm_dpo/beta": 0.12929514050483704,
"fcm_dpo/delta": -0.18368813395500183,
"fcm_dpo/margin": 7.470497131347656,
"fcm_dpo/q_t": 0.3049233853816986,
"grad_norm": 32.90544509887695,
"learning_rate": 4.08662192950594e-07,
"logits/chosen": 0.31978702545166016,
"logits/rejected": 0.3014755845069885,
"logps/chosen": -72.6189956665039,
"logps/ref_chosen": -64.53972625732422,
"logps/ref_rejected": -77.69151306152344,
"logps/rejected": -93.24127960205078,
"loss": 0.8075,
"margin_dpo/margin_mean": 7.470496654510498,
"margin_dpo/margin_std": 7.160727500915527,
"step": 235
},
{
"KL/chosen_KL_mean": -11.760652542114258,
"KL/mean": -14.860483169555664,
"KL/rejected_KL_mean": -17.960315704345703,
"KL/std": 8.125932693481445,
"epoch": 0.35676492819349964,
"fcm_dpo/beta": 0.12746058404445648,
"fcm_dpo/delta": 0.009847693145275116,
"fcm_dpo/margin": 6.19966459274292,
"fcm_dpo/q_t": 0.3519732356071472,
"grad_norm": 36.77629470825195,
"learning_rate": 4.076381667711306e-07,
"logits/chosen": 0.2771919369697571,
"logits/rejected": 0.2631545066833496,
"logps/chosen": -82.91539001464844,
"logps/ref_chosen": -71.15473937988281,
"logps/ref_rejected": -84.88541412353516,
"logps/rejected": -102.84573364257812,
"loss": 1.0304,
"margin_dpo/margin_mean": 6.199664115905762,
"margin_dpo/margin_std": 9.351386070251465,
"step": 236
},
{
"KL/chosen_KL_mean": -9.832389831542969,
"KL/mean": -12.970930099487305,
"KL/rejected_KL_mean": -16.10947036743164,
"KL/std": 7.85981559753418,
"epoch": 0.35827664399092973,
"fcm_dpo/beta": 0.12796080112457275,
"fcm_dpo/delta": -0.003629550337791443,
"fcm_dpo/margin": 6.27708101272583,
"fcm_dpo/q_t": 0.34321698546409607,
"grad_norm": 39.69746398925781,
"learning_rate": 4.066097311132753e-07,
"logits/chosen": 0.3301270008087158,
"logits/rejected": 0.3169901371002197,
"logps/chosen": -85.97440338134766,
"logps/ref_chosen": -76.14201354980469,
"logps/ref_rejected": -80.88479614257812,
"logps/rejected": -96.9942626953125,
"loss": 1.0223,
"margin_dpo/margin_mean": 6.277080535888672,
"margin_dpo/margin_std": 9.050657272338867,
"step": 237
},
{
"KL/chosen_KL_mean": -9.327678680419922,
"KL/mean": -12.499244689941406,
"KL/rejected_KL_mean": -15.670822143554688,
"KL/std": 8.36613655090332,
"epoch": 0.35978835978835977,
"fcm_dpo/beta": 0.1276305615901947,
"fcm_dpo/delta": -0.010671408846974373,
"fcm_dpo/margin": 6.343140125274658,
"fcm_dpo/q_t": 0.3476073741912842,
"grad_norm": 53.07609939575195,
"learning_rate": 4.0557691474458414e-07,
"logits/chosen": 0.26444271206855774,
"logits/rejected": 0.24774247407913208,
"logps/chosen": -78.2125244140625,
"logps/ref_chosen": -68.88484954833984,
"logps/ref_rejected": -75.8946304321289,
"logps/rejected": -91.5654525756836,
"loss": 0.9909,
"margin_dpo/margin_mean": 6.343140602111816,
"margin_dpo/margin_std": 8.846379280090332,
"step": 238
},
{
"KL/chosen_KL_mean": -11.43686294555664,
"KL/mean": -14.855867385864258,
"KL/rejected_KL_mean": -18.27486801147461,
"KL/std": 9.134950637817383,
"epoch": 0.36130007558578986,
"fcm_dpo/beta": 0.1278074085712433,
"fcm_dpo/delta": -0.08258944004774094,
"fcm_dpo/margin": 6.838010311126709,
"fcm_dpo/q_t": 0.3380742073059082,
"grad_norm": 36.243316650390625,
"learning_rate": 4.045397465551513e-07,
"logits/chosen": 0.3806946277618408,
"logits/rejected": 0.2709374725818634,
"logps/chosen": -68.20869445800781,
"logps/ref_chosen": -56.771827697753906,
"logps/ref_rejected": -116.23050689697266,
"logps/rejected": -134.50537109375,
"loss": 0.9782,
"margin_dpo/margin_mean": 6.838012218475342,
"margin_dpo/margin_std": 9.391035079956055,
"step": 239
},
{
"KL/chosen_KL_mean": -10.77096939086914,
"KL/mean": -14.825616836547852,
"KL/rejected_KL_mean": -18.880260467529297,
"KL/std": 9.48426628112793,
"epoch": 0.36281179138321995,
"fcm_dpo/beta": 0.12276466935873032,
"fcm_dpo/delta": -0.21665102243423462,
"fcm_dpo/margin": 8.109292984008789,
"fcm_dpo/q_t": 0.30741313099861145,
"grad_norm": 27.501493453979492,
"learning_rate": 4.0349825555680045e-07,
"logits/chosen": 0.2991008460521698,
"logits/rejected": 0.21405178308486938,
"logps/chosen": -64.12507629394531,
"logps/ref_chosen": -53.35411071777344,
"logps/ref_rejected": -80.12019348144531,
"logps/rejected": -99.00045776367188,
"loss": 0.8584,
"margin_dpo/margin_mean": 8.109292984008789,
"margin_dpo/margin_std": 8.866523742675781,
"step": 240
},
{
"KL/chosen_KL_mean": -10.55604362487793,
"KL/mean": -12.819741249084473,
"KL/rejected_KL_mean": -15.083438873291016,
"KL/std": 7.995132923126221,
"epoch": 0.36432350718065004,
"fcm_dpo/beta": 0.12550224363803864,
"fcm_dpo/delta": 0.24444246292114258,
"fcm_dpo/margin": 4.5273942947387695,
"fcm_dpo/q_t": 0.39284324645996094,
"grad_norm": 40.22063064575195,
"learning_rate": 4.0245247088227377e-07,
"logits/chosen": 0.24775752425193787,
"logits/rejected": 0.2146153748035431,
"logps/chosen": -82.45145416259766,
"logps/ref_chosen": -71.89541625976562,
"logps/ref_rejected": -83.03492736816406,
"logps/rejected": -98.11836242675781,
"loss": 1.1212,
"margin_dpo/margin_mean": 4.527395248413086,
"margin_dpo/margin_std": 8.242197036743164,
"step": 241
},
{
"KL/chosen_KL_mean": -9.93331527709961,
"KL/mean": -13.826549530029297,
"KL/rejected_KL_mean": -17.71977996826172,
"KL/std": 9.657007217407227,
"epoch": 0.36583522297808013,
"fcm_dpo/beta": 0.12183021008968353,
"fcm_dpo/delta": -0.17005649209022522,
"fcm_dpo/margin": 7.786465644836426,
"fcm_dpo/q_t": 0.32561737298965454,
"grad_norm": 29.045923233032227,
"learning_rate": 4.0140242178441665e-07,
"logits/chosen": 0.26129892468452454,
"logits/rejected": 0.23901167511940002,
"logps/chosen": -67.86074829101562,
"logps/ref_chosen": -57.927433013916016,
"logps/ref_rejected": -67.838623046875,
"logps/rejected": -85.55839538574219,
"loss": 0.9366,
"margin_dpo/margin_mean": 7.786465167999268,
"margin_dpo/margin_std": 10.202951431274414,
"step": 242
},
{
"KL/chosen_KL_mean": -10.123703002929688,
"KL/mean": -13.311759948730469,
"KL/rejected_KL_mean": -16.49981689453125,
"KL/std": 8.933574676513672,
"epoch": 0.3673469387755102,
"fcm_dpo/beta": 0.12265647202730179,
"fcm_dpo/delta": 0.01945311576128006,
"fcm_dpo/margin": 6.376116752624512,
"fcm_dpo/q_t": 0.3523695170879364,
"grad_norm": 36.763248443603516,
"learning_rate": 4.003481376353596e-07,
"logits/chosen": 0.2812059819698334,
"logits/rejected": 0.2792655825614929,
"logps/chosen": -84.40037536621094,
"logps/ref_chosen": -74.27667236328125,
"logps/ref_rejected": -73.24340057373047,
"logps/rejected": -89.74321746826172,
"loss": 1.0002,
"margin_dpo/margin_mean": 6.376116752624512,
"margin_dpo/margin_std": 9.081279754638672,
"step": 243
},
{
"KL/chosen_KL_mean": -10.219358444213867,
"KL/mean": -14.52243423461914,
"KL/rejected_KL_mean": -18.825511932373047,
"KL/std": 8.50840950012207,
"epoch": 0.3688586545729403,
"fcm_dpo/beta": 0.11894027888774872,
"fcm_dpo/delta": -0.2484353482723236,
"fcm_dpo/margin": 8.606154441833496,
"fcm_dpo/q_t": 0.2950640320777893,
"grad_norm": 25.763652801513672,
"learning_rate": 3.9928964792569654e-07,
"logits/chosen": 0.3132631182670593,
"logits/rejected": 0.23436595499515533,
"logps/chosen": -63.5832633972168,
"logps/ref_chosen": -53.36390686035156,
"logps/ref_rejected": -71.10276794433594,
"logps/rejected": -89.92828369140625,
"loss": 0.7863,
"margin_dpo/margin_mean": 8.60615348815918,
"margin_dpo/margin_std": 8.056436538696289,
"step": 244
},
{
"KL/chosen_KL_mean": -10.508058547973633,
"KL/mean": -15.217109680175781,
"KL/rejected_KL_mean": -19.92615509033203,
"KL/std": 9.513158798217773,
"epoch": 0.37037037037037035,
"fcm_dpo/beta": 0.11189082264900208,
"fcm_dpo/delta": -0.28425049781799316,
"fcm_dpo/margin": 9.418100357055664,
"fcm_dpo/q_t": 0.29473158717155457,
"grad_norm": 77.52033996582031,
"learning_rate": 3.982269822636601e-07,
"logits/chosen": 0.33938145637512207,
"logits/rejected": 0.3116254210472107,
"logps/chosen": -81.70316314697266,
"logps/ref_chosen": -71.19510650634766,
"logps/ref_rejected": -80.76235961914062,
"logps/rejected": -100.68851470947266,
"loss": 0.8046,
"margin_dpo/margin_mean": 9.41810131072998,
"margin_dpo/margin_std": 9.551580429077148,
"step": 245
},
{
"KL/chosen_KL_mean": -12.594314575195312,
"KL/mean": -16.568037033081055,
"KL/rejected_KL_mean": -20.541751861572266,
"KL/std": 9.444729804992676,
"epoch": 0.37188208616780044,
"fcm_dpo/beta": 0.10943619906902313,
"fcm_dpo/delta": -0.07631853222846985,
"fcm_dpo/margin": 7.9474382400512695,
"fcm_dpo/q_t": 0.33763816952705383,
"grad_norm": 35.117340087890625,
"learning_rate": 3.971601703742932e-07,
"logits/chosen": 0.33962783217430115,
"logits/rejected": 0.2832232415676117,
"logps/chosen": -84.21536254882812,
"logps/ref_chosen": -71.62104797363281,
"logps/ref_rejected": -94.03392028808594,
"logps/rejected": -114.57567596435547,
"loss": 0.981,
"margin_dpo/margin_mean": 7.947439193725586,
"margin_dpo/margin_std": 11.065793991088867,
"step": 246
},
{
"KL/chosen_KL_mean": -13.099170684814453,
"KL/mean": -15.340646743774414,
"KL/rejected_KL_mean": -17.582115173339844,
"KL/std": 8.530660629272461,
"epoch": 0.37339380196523053,
"fcm_dpo/beta": 0.11499364674091339,
"fcm_dpo/delta": 0.29317426681518555,
"fcm_dpo/margin": 4.482947826385498,
"fcm_dpo/q_t": 0.39475196599960327,
"grad_norm": 39.144874572753906,
"learning_rate": 3.960892420986177e-07,
"logits/chosen": 0.35208380222320557,
"logits/rejected": 0.3406439423561096,
"logps/chosen": -93.12171936035156,
"logps/ref_chosen": -80.02254486083984,
"logps/ref_rejected": -89.22705841064453,
"logps/rejected": -106.80917358398438,
"loss": 1.1837,
"margin_dpo/margin_mean": 4.48294734954834,
"margin_dpo/margin_std": 9.121728897094727,
"step": 247
},
{
"KL/chosen_KL_mean": -11.430730819702148,
"KL/mean": -15.32693862915039,
"KL/rejected_KL_mean": -19.223148345947266,
"KL/std": 10.37348461151123,
"epoch": 0.3749055177626606,
"fcm_dpo/beta": 0.11418326944112778,
"fcm_dpo/delta": -0.09853696823120117,
"fcm_dpo/margin": 7.792417526245117,
"fcm_dpo/q_t": 0.343853235244751,
"grad_norm": 34.979244232177734,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": 0.31661760807037354,
"logits/rejected": 0.3491419553756714,
"logps/chosen": -76.80868530273438,
"logps/ref_chosen": -65.37796020507812,
"logps/ref_rejected": -61.365787506103516,
"logps/rejected": -80.58893585205078,
"loss": 1.0005,
"margin_dpo/margin_mean": 7.792417526245117,
"margin_dpo/margin_std": 11.333248138427734,
"step": 248
},
{
"KL/chosen_KL_mean": -14.106277465820312,
"KL/mean": -15.700241088867188,
"KL/rejected_KL_mean": -17.294208526611328,
"KL/std": 9.214258193969727,
"epoch": 0.3764172335600907,
"fcm_dpo/beta": 0.11448922753334045,
"fcm_dpo/delta": 0.12567032873630524,
"fcm_dpo/margin": 3.1879303455352783,
"fcm_dpo/q_t": 0.42352503538131714,
"grad_norm": 45.407470703125,
"learning_rate": 3.9393515632731094e-07,
"logits/chosen": 0.30869317054748535,
"logits/rejected": 0.33915185928344727,
"logps/chosen": -88.70773315429688,
"logps/ref_chosen": -74.60145568847656,
"logps/ref_rejected": -63.79338455200195,
"logps/rejected": -81.08759307861328,
"loss": 1.3556,
"margin_dpo/margin_mean": 3.1879305839538574,
"margin_dpo/margin_std": 10.071321487426758,
"step": 249
},
{
"KL/chosen_KL_mean": -11.981225967407227,
"KL/mean": -16.120920181274414,
"KL/rejected_KL_mean": -20.2606201171875,
"KL/std": 10.126691818237305,
"epoch": 0.3779289493575208,
"fcm_dpo/beta": 0.11403284966945648,
"fcm_dpo/delta": -0.15904656052589417,
"fcm_dpo/margin": 8.279390335083008,
"fcm_dpo/q_t": 0.32035911083221436,
"grad_norm": 31.749574661254883,
"learning_rate": 3.9285205908608934e-07,
"logits/chosen": 0.3941000699996948,
"logits/rejected": 0.3523021936416626,
"logps/chosen": -73.91943359375,
"logps/ref_chosen": -61.938209533691406,
"logps/ref_rejected": -72.21602630615234,
"logps/rejected": -92.47664642333984,
"loss": 0.9328,
"margin_dpo/margin_mean": 8.279389381408691,
"margin_dpo/margin_std": 10.527669906616211,
"step": 250
},
{
"KL/chosen_KL_mean": -12.728778839111328,
"KL/mean": -15.447917938232422,
"KL/rejected_KL_mean": -18.16705322265625,
"KL/std": 9.414742469787598,
"epoch": 0.3794406651549509,
"fcm_dpo/beta": 0.11536653339862823,
"fcm_dpo/delta": 0.1842348873615265,
"fcm_dpo/margin": 5.4382710456848145,
"fcm_dpo/q_t": 0.382129967212677,
"grad_norm": 40.527164459228516,
"learning_rate": 3.9176496596569265e-07,
"logits/chosen": 0.38748690485954285,
"logits/rejected": 0.34921795129776,
"logps/chosen": -79.58572387695312,
"logps/ref_chosen": -66.85694885253906,
"logps/ref_rejected": -84.83396911621094,
"logps/rejected": -103.00102233886719,
"loss": 1.1419,
"margin_dpo/margin_mean": 5.438270568847656,
"margin_dpo/margin_std": 10.37491512298584,
"step": 251
},
{
"KL/chosen_KL_mean": -12.064062118530273,
"KL/mean": -14.76373291015625,
"KL/rejected_KL_mean": -17.463409423828125,
"KL/std": 10.329211235046387,
"epoch": 0.38095238095238093,
"fcm_dpo/beta": 0.12140388786792755,
"fcm_dpo/delta": 0.1437467634677887,
"fcm_dpo/margin": 5.399339199066162,
"fcm_dpo/q_t": 0.37345564365386963,
"grad_norm": 39.94149398803711,
"learning_rate": 3.9067390737445254e-07,
"logits/chosen": 0.26361334323883057,
"logits/rejected": 0.21315331757068634,
"logps/chosen": -68.28799438476562,
"logps/ref_chosen": -56.22393035888672,
"logps/ref_rejected": -77.1136245727539,
"logps/rejected": -94.57703399658203,
"loss": 1.2079,
"margin_dpo/margin_mean": 5.399339199066162,
"margin_dpo/margin_std": 10.947092056274414,
"step": 252
},
{
"KL/chosen_KL_mean": -12.338104248046875,
"KL/mean": -15.188192367553711,
"KL/rejected_KL_mean": -18.03827667236328,
"KL/std": 9.178722381591797,
"epoch": 0.382464096749811,
"fcm_dpo/beta": 0.12051292508840561,
"fcm_dpo/delta": 0.11948665976524353,
"fcm_dpo/margin": 5.700176239013672,
"fcm_dpo/q_t": 0.37205415964126587,
"grad_norm": 34.4255256652832,
"learning_rate": 3.8957891383162304e-07,
"logits/chosen": 0.36925697326660156,
"logits/rejected": 0.32966768741607666,
"logps/chosen": -64.5481185913086,
"logps/ref_chosen": -52.21001434326172,
"logps/ref_rejected": -58.75764846801758,
"logps/rejected": -76.79592895507812,
"loss": 1.0874,
"margin_dpo/margin_mean": 5.700176239013672,
"margin_dpo/margin_std": 9.250205039978027,
"step": 253
},
{
"KL/chosen_KL_mean": -12.473712921142578,
"KL/mean": -15.742783546447754,
"KL/rejected_KL_mean": -19.01184844970703,
"KL/std": 10.278692245483398,
"epoch": 0.3839758125472411,
"fcm_dpo/beta": 0.12202567607164383,
"fcm_dpo/delta": 0.0014530569314956665,
"fcm_dpo/margin": 6.5381340980529785,
"fcm_dpo/q_t": 0.3580353856086731,
"grad_norm": 39.03754425048828,
"learning_rate": 3.884800159665276e-07,
"logits/chosen": 0.29129236936569214,
"logits/rejected": 0.24309565126895905,
"logps/chosen": -78.11003875732422,
"logps/ref_chosen": -65.63632202148438,
"logps/ref_rejected": -82.34425354003906,
"logps/rejected": -101.35609436035156,
"loss": 1.0392,
"margin_dpo/margin_mean": 6.5381340980529785,
"margin_dpo/margin_std": 10.117410659790039,
"step": 254
},
{
"KL/chosen_KL_mean": -10.72909164428711,
"KL/mean": -14.877052307128906,
"KL/rejected_KL_mean": -19.025009155273438,
"KL/std": 10.288159370422363,
"epoch": 0.3854875283446712,
"fcm_dpo/beta": 0.1184849888086319,
"fcm_dpo/delta": -0.20485825836658478,
"fcm_dpo/margin": 8.295927047729492,
"fcm_dpo/q_t": 0.32767289876937866,
"grad_norm": 32.1718635559082,
"learning_rate": 3.873772445177015e-07,
"logits/chosen": 0.2984389662742615,
"logits/rejected": 0.26697784662246704,
"logps/chosen": -78.64018249511719,
"logps/ref_chosen": -67.91108703613281,
"logps/ref_rejected": -83.89114379882812,
"logps/rejected": -102.91615295410156,
"loss": 0.9405,
"margin_dpo/margin_mean": 8.295927047729492,
"margin_dpo/margin_std": 11.091558456420898,
"step": 255
},
{
"KL/chosen_KL_mean": -12.830934524536133,
"KL/mean": -16.304931640625,
"KL/rejected_KL_mean": -19.778934478759766,
"KL/std": 9.357809066772461,
"epoch": 0.3869992441421013,
"fcm_dpo/beta": 0.11697204411029816,
"fcm_dpo/delta": -0.014655433595180511,
"fcm_dpo/margin": 6.947994709014893,
"fcm_dpo/q_t": 0.3568224012851715,
"grad_norm": 38.28756332397461,
"learning_rate": 3.862706303320329e-07,
"logits/chosen": 0.30309706926345825,
"logits/rejected": 0.24692294001579285,
"logps/chosen": -76.33091735839844,
"logps/ref_chosen": -63.49998474121094,
"logps/ref_rejected": -90.77104187011719,
"logps/rejected": -110.54997253417969,
"loss": 1.0638,
"margin_dpo/margin_mean": 6.947993755340576,
"margin_dpo/margin_std": 11.271989822387695,
"step": 256
},
{
"KL/chosen_KL_mean": -13.398094177246094,
"KL/mean": -18.00149917602539,
"KL/rejected_KL_mean": -22.604900360107422,
"KL/std": 11.196935653686523,
"epoch": 0.3885109599395314,
"fcm_dpo/beta": 0.11260446161031723,
"fcm_dpo/delta": -0.2655514180660248,
"fcm_dpo/margin": 9.206808090209961,
"fcm_dpo/q_t": 0.3245314955711365,
"grad_norm": 37.96772766113281,
"learning_rate": 3.851602043638994e-07,
"logits/chosen": 0.28178295493125916,
"logits/rejected": 0.22410434484481812,
"logps/chosen": -83.99874877929688,
"logps/ref_chosen": -70.60064697265625,
"logps/ref_rejected": -108.58313751220703,
"logps/rejected": -131.1880340576172,
"loss": 0.9396,
"margin_dpo/margin_mean": 9.206808090209961,
"margin_dpo/margin_std": 12.629020690917969,
"step": 257
},
{
"KL/chosen_KL_mean": -13.584089279174805,
"KL/mean": -17.121944427490234,
"KL/rejected_KL_mean": -20.659801483154297,
"KL/std": 9.215842247009277,
"epoch": 0.3900226757369615,
"fcm_dpo/beta": 0.11255334317684174,
"fcm_dpo/delta": 0.0021555423736572266,
"fcm_dpo/margin": 7.075709819793701,
"fcm_dpo/q_t": 0.3390353322029114,
"grad_norm": 34.61642074584961,
"learning_rate": 3.840459976743023e-07,
"logits/chosen": 0.3270367980003357,
"logits/rejected": 0.28003180027008057,
"logps/chosen": -72.8382568359375,
"logps/ref_chosen": -59.25416564941406,
"logps/ref_rejected": -85.58709716796875,
"logps/rejected": -106.24689483642578,
"loss": 0.9257,
"margin_dpo/margin_mean": 7.075709342956543,
"margin_dpo/margin_std": 8.351495742797852,
"step": 258
},
{
"KL/chosen_KL_mean": -10.683589935302734,
"KL/mean": -16.045759201049805,
"KL/rejected_KL_mean": -21.407928466796875,
"KL/std": 10.769359588623047,
"epoch": 0.3915343915343915,
"fcm_dpo/beta": 0.10471875965595245,
"fcm_dpo/delta": -0.36887070536613464,
"fcm_dpo/margin": 10.724337577819824,
"fcm_dpo/q_t": 0.288051962852478,
"grad_norm": 28.6914119720459,
"learning_rate": 3.8292804142999796e-07,
"logits/chosen": 0.2580224275588989,
"logits/rejected": 0.16532495617866516,
"logps/chosen": -76.11846923828125,
"logps/ref_chosen": -65.43487548828125,
"logps/ref_rejected": -95.41731262207031,
"logps/rejected": -116.82524108886719,
"loss": 0.8081,
"margin_dpo/margin_mean": 10.72433853149414,
"margin_dpo/margin_std": 11.031912803649902,
"step": 259
},
{
"KL/chosen_KL_mean": -11.84138298034668,
"KL/mean": -15.952880859375,
"KL/rejected_KL_mean": -20.064380645751953,
"KL/std": 9.865896224975586,
"epoch": 0.3930461073318216,
"fcm_dpo/beta": 0.10231013596057892,
"fcm_dpo/delta": -0.046053703874349594,
"fcm_dpo/margin": 8.22299861907959,
"fcm_dpo/q_t": 0.3451218008995056,
"grad_norm": 29.589677810668945,
"learning_rate": 3.818063669026256e-07,
"logits/chosen": 0.2971701920032501,
"logits/rejected": 0.21861127018928528,
"logps/chosen": -60.930965423583984,
"logps/ref_chosen": -49.08958435058594,
"logps/ref_rejected": -79.01708221435547,
"logps/rejected": -99.08146667480469,
"loss": 0.9952,
"margin_dpo/margin_mean": 8.22299861907959,
"margin_dpo/margin_std": 11.613203048706055,
"step": 260
},
{
"KL/chosen_KL_mean": -11.726337432861328,
"KL/mean": -15.044000625610352,
"KL/rejected_KL_mean": -18.361663818359375,
"KL/std": 9.771665573120117,
"epoch": 0.3945578231292517,
"fcm_dpo/beta": 0.10444238036870956,
"fcm_dpo/delta": 0.11493270099163055,
"fcm_dpo/margin": 6.635324001312256,
"fcm_dpo/q_t": 0.3719925880432129,
"grad_norm": 37.57694625854492,
"learning_rate": 3.806810054678331e-07,
"logits/chosen": 0.1974962055683136,
"logits/rejected": 0.21522969007492065,
"logps/chosen": -82.5987319946289,
"logps/ref_chosen": -70.87239074707031,
"logps/ref_rejected": -65.01522064208984,
"logps/rejected": -83.37688446044922,
"loss": 1.0653,
"margin_dpo/margin_mean": 6.635324478149414,
"margin_dpo/margin_std": 10.821256637573242,
"step": 261
},
{
"KL/chosen_KL_mean": -12.111078262329102,
"KL/mean": -15.972871780395508,
"KL/rejected_KL_mean": -19.834667205810547,
"KL/std": 9.611106872558594,
"epoch": 0.3960695389266818,
"fcm_dpo/beta": 0.1053507924079895,
"fcm_dpo/delta": -0.01538146659731865,
"fcm_dpo/margin": 7.723592758178711,
"fcm_dpo/q_t": 0.345390260219574,
"grad_norm": 32.13874053955078,
"learning_rate": 3.7955198860439887e-07,
"logits/chosen": 0.3609021306037903,
"logits/rejected": 0.3037142753601074,
"logps/chosen": -79.98170471191406,
"logps/ref_chosen": -67.8706283569336,
"logps/ref_rejected": -88.7205810546875,
"logps/rejected": -108.55525207519531,
"loss": 0.9575,
"margin_dpo/margin_mean": 7.723593235015869,
"margin_dpo/margin_std": 10.076356887817383,
"step": 262
},
{
"KL/chosen_KL_mean": -11.720417022705078,
"KL/mean": -14.867897033691406,
"KL/rejected_KL_mean": -18.015369415283203,
"KL/std": 9.436576843261719,
"epoch": 0.3975812547241119,
"fcm_dpo/beta": 0.10749039798974991,
"fcm_dpo/delta": 0.13047100603580475,
"fcm_dpo/margin": 6.294952392578125,
"fcm_dpo/q_t": 0.36887580156326294,
"grad_norm": 31.042261123657227,
"learning_rate": 3.784193478933516e-07,
"logits/chosen": 0.30105894804000854,
"logits/rejected": 0.20555788278579712,
"logps/chosen": -66.91500091552734,
"logps/ref_chosen": -55.194583892822266,
"logps/ref_rejected": -80.54048156738281,
"logps/rejected": -98.55585479736328,
"loss": 1.0718,
"margin_dpo/margin_mean": 6.294952392578125,
"margin_dpo/margin_std": 10.169485092163086,
"step": 263
},
{
"KL/chosen_KL_mean": -11.863348007202148,
"KL/mean": -15.760964393615723,
"KL/rejected_KL_mean": -19.658584594726562,
"KL/std": 10.724000930786133,
"epoch": 0.39909297052154197,
"fcm_dpo/beta": 0.10693207383155823,
"fcm_dpo/delta": -0.03660469502210617,
"fcm_dpo/margin": 7.795228958129883,
"fcm_dpo/q_t": 0.3510364890098572,
"grad_norm": 36.74233627319336,
"learning_rate": 3.7728311501708674e-07,
"logits/chosen": 0.23294666409492493,
"logits/rejected": 0.18829098343849182,
"logps/chosen": -95.0340347290039,
"logps/ref_chosen": -83.17068481445312,
"logps/ref_rejected": -88.33625793457031,
"logps/rejected": -107.99484252929688,
"loss": 1.0338,
"margin_dpo/margin_mean": 7.795228958129883,
"margin_dpo/margin_std": 12.017763137817383,
"step": 264
},
{
"KL/chosen_KL_mean": -11.657770156860352,
"KL/mean": -15.780095100402832,
"KL/rejected_KL_mean": -19.902423858642578,
"KL/std": 10.775840759277344,
"epoch": 0.40060468631897206,
"fcm_dpo/beta": 0.10615849494934082,
"fcm_dpo/delta": -0.0829143300652504,
"fcm_dpo/margin": 8.244649887084961,
"fcm_dpo/q_t": 0.34060657024383545,
"grad_norm": 37.233978271484375,
"learning_rate": 3.7614332175848027e-07,
"logits/chosen": 0.39187127351760864,
"logits/rejected": 0.3272865414619446,
"logps/chosen": -63.320621490478516,
"logps/ref_chosen": -51.66284942626953,
"logps/ref_rejected": -67.1720962524414,
"logps/rejected": -87.07452392578125,
"loss": 1.064,
"margin_dpo/margin_mean": 8.244649887084961,
"margin_dpo/margin_std": 12.866454124450684,
"step": 265
},
{
"KL/chosen_KL_mean": -10.886266708374023,
"KL/mean": -14.718881607055664,
"KL/rejected_KL_mean": -18.551498413085938,
"KL/std": 10.924835205078125,
"epoch": 0.4021164021164021,
"fcm_dpo/beta": 0.10467538237571716,
"fcm_dpo/delta": -0.002675635740160942,
"fcm_dpo/margin": 7.665229797363281,
"fcm_dpo/q_t": 0.3502148389816284,
"grad_norm": 33.46331787109375,
"learning_rate": 3.75e-07,
"logits/chosen": 0.31710296869277954,
"logits/rejected": 0.24927489459514618,
"logps/chosen": -68.33676147460938,
"logps/ref_chosen": -57.45049285888672,
"logps/ref_rejected": -77.60826110839844,
"logps/rejected": -96.15975952148438,
"loss": 1.0042,
"margin_dpo/margin_mean": 7.665229797363281,
"margin_dpo/margin_std": 11.188663482666016,
"step": 266
},
{
"KL/chosen_KL_mean": -9.593021392822266,
"KL/mean": -12.639689445495605,
"KL/rejected_KL_mean": -15.686355590820312,
"KL/std": 9.48430347442627,
"epoch": 0.4036281179138322,
"fcm_dpo/beta": 0.10875646024942398,
"fcm_dpo/delta": 0.14000800251960754,
"fcm_dpo/margin": 6.093325614929199,
"fcm_dpo/q_t": 0.37472468614578247,
"grad_norm": 31.011960983276367,
"learning_rate": 3.738531817228131e-07,
"logits/chosen": 0.31285083293914795,
"logits/rejected": 0.2954583168029785,
"logps/chosen": -64.62837219238281,
"logps/ref_chosen": -55.03535079956055,
"logps/ref_rejected": -66.0953369140625,
"logps/rejected": -81.78169250488281,
"loss": 1.1384,
"margin_dpo/margin_mean": 6.093325614929199,
"margin_dpo/margin_std": 11.069232940673828,
"step": 267
},
{
"KL/chosen_KL_mean": -9.607749938964844,
"KL/mean": -13.186819076538086,
"KL/rejected_KL_mean": -16.765892028808594,
"KL/std": 10.487470626831055,
"epoch": 0.4051398337112623,
"fcm_dpo/beta": 0.1089181900024414,
"fcm_dpo/delta": 0.021044310182332993,
"fcm_dpo/margin": 7.158140659332275,
"fcm_dpo/q_t": 0.3529961109161377,
"grad_norm": 29.234092712402344,
"learning_rate": 3.7270289900589204e-07,
"logits/chosen": 0.21093052625656128,
"logits/rejected": 0.1905559003353119,
"logps/chosen": -74.67948913574219,
"logps/ref_chosen": -65.07174682617188,
"logps/ref_rejected": -71.42485809326172,
"logps/rejected": -88.19075012207031,
"loss": 0.9735,
"margin_dpo/margin_mean": 7.158140182495117,
"margin_dpo/margin_std": 9.659717559814453,
"step": 268
},
{
"KL/chosen_KL_mean": -11.00994873046875,
"KL/mean": -15.063467025756836,
"KL/rejected_KL_mean": -19.116985321044922,
"KL/std": 10.769641876220703,
"epoch": 0.40665154950869237,
"fcm_dpo/beta": 0.10864944756031036,
"fcm_dpo/delta": -0.09277642518281937,
"fcm_dpo/margin": 8.107038497924805,
"fcm_dpo/q_t": 0.33289867639541626,
"grad_norm": 29.763572692871094,
"learning_rate": 3.7154918402511714e-07,
"logits/chosen": 0.44051095843315125,
"logits/rejected": 0.3936043679714203,
"logps/chosen": -78.14615631103516,
"logps/ref_chosen": -67.1362075805664,
"logps/ref_rejected": -82.55778503417969,
"logps/rejected": -101.67477416992188,
"loss": 0.9513,
"margin_dpo/margin_mean": 8.107038497924805,
"margin_dpo/margin_std": 10.162799835205078,
"step": 269
},
{
"KL/chosen_KL_mean": -10.442401885986328,
"KL/mean": -13.696825981140137,
"KL/rejected_KL_mean": -16.951248168945312,
"KL/std": 10.145946502685547,
"epoch": 0.40816326530612246,
"fcm_dpo/beta": 0.107734814286232,
"fcm_dpo/delta": 0.10635486245155334,
"fcm_dpo/margin": 6.508844375610352,
"fcm_dpo/q_t": 0.35935863852500916,
"grad_norm": 34.854164123535156,
"learning_rate": 3.7039206905237656e-07,
"logits/chosen": 0.3360714316368103,
"logits/rejected": 0.26730990409851074,
"logps/chosen": -77.131103515625,
"logps/ref_chosen": -66.6886978149414,
"logps/ref_rejected": -85.16129302978516,
"logps/rejected": -102.11254119873047,
"loss": 1.037,
"margin_dpo/margin_mean": 6.508844375610352,
"margin_dpo/margin_std": 9.949800491333008,
"step": 270
},
{
"KL/chosen_KL_mean": -9.98006820678711,
"KL/mean": -12.884943008422852,
"KL/rejected_KL_mean": -15.789817810058594,
"KL/std": 10.585667610168457,
"epoch": 0.40967498110355255,
"fcm_dpo/beta": 0.11065304279327393,
"fcm_dpo/delta": 0.16814447939395905,
"fcm_dpo/margin": 5.809749126434326,
"fcm_dpo/q_t": 0.4004424214363098,
"grad_norm": 39.59679412841797,
"learning_rate": 3.692315864546635e-07,
"logits/chosen": 0.33203697204589844,
"logits/rejected": 0.2761046290397644,
"logps/chosen": -82.38761901855469,
"logps/ref_chosen": -72.40754699707031,
"logps/ref_rejected": -92.06311798095703,
"logps/rejected": -107.85293579101562,
"loss": 1.2269,
"margin_dpo/margin_mean": 5.809748649597168,
"margin_dpo/margin_std": 12.749189376831055,
"step": 271
},
{
"KL/chosen_KL_mean": -8.188089370727539,
"KL/mean": -13.272232055664062,
"KL/rejected_KL_mean": -18.35637664794922,
"KL/std": 10.037698745727539,
"epoch": 0.41118669690098264,
"fcm_dpo/beta": 0.10786165297031403,
"fcm_dpo/delta": -0.332021564245224,
"fcm_dpo/margin": 10.168288230895996,
"fcm_dpo/q_t": 0.2953736186027527,
"grad_norm": 29.516803741455078,
"learning_rate": 3.6806776869317067e-07,
"logits/chosen": 0.33715561032295227,
"logits/rejected": 0.34234732389450073,
"logps/chosen": -74.78948974609375,
"logps/ref_chosen": -66.60140228271484,
"logps/ref_rejected": -67.74340057373047,
"logps/rejected": -86.09977722167969,
"loss": 0.7911,
"margin_dpo/margin_mean": 10.168288230895996,
"margin_dpo/margin_std": 10.265481948852539,
"step": 272
},
{
"KL/chosen_KL_mean": -11.362102508544922,
"KL/mean": -15.370185852050781,
"KL/rejected_KL_mean": -19.37826156616211,
"KL/std": 10.579803466796875,
"epoch": 0.4126984126984127,
"fcm_dpo/beta": 0.10470890998840332,
"fcm_dpo/delta": -0.04298901930451393,
"fcm_dpo/margin": 8.016157150268555,
"fcm_dpo/q_t": 0.3473791182041168,
"grad_norm": 33.51838302612305,
"learning_rate": 3.669006483223828e-07,
"logits/chosen": 0.339900940656662,
"logits/rejected": 0.27993035316467285,
"logps/chosen": -68.71697998046875,
"logps/ref_chosen": -57.35487747192383,
"logps/ref_rejected": -84.17168426513672,
"logps/rejected": -103.54994201660156,
"loss": 1.0409,
"margin_dpo/margin_mean": 8.016157150268555,
"margin_dpo/margin_std": 12.47188663482666,
"step": 273
},
{
"KL/chosen_KL_mean": -10.67816162109375,
"KL/mean": -14.641273498535156,
"KL/rejected_KL_mean": -18.604381561279297,
"KL/std": 10.35463809967041,
"epoch": 0.41421012849584277,
"fcm_dpo/beta": 0.10404108464717865,
"fcm_dpo/delta": -0.026914719492197037,
"fcm_dpo/margin": 7.9262237548828125,
"fcm_dpo/q_t": 0.34564095735549927,
"grad_norm": 28.05985450744629,
"learning_rate": 3.657302579891656e-07,
"logits/chosen": 0.20250210165977478,
"logits/rejected": 0.17951546609401703,
"logps/chosen": -70.31965637207031,
"logps/ref_chosen": -59.64149475097656,
"logps/ref_rejected": -68.29348754882812,
"logps/rejected": -86.89786529541016,
"loss": 1.0126,
"margin_dpo/margin_mean": 7.9262237548828125,
"margin_dpo/margin_std": 11.62763786315918,
"step": 274
},
{
"KL/chosen_KL_mean": -10.336505889892578,
"KL/mean": -14.75816535949707,
"KL/rejected_KL_mean": -19.179826736450195,
"KL/std": 10.844429016113281,
"epoch": 0.41572184429327286,
"fcm_dpo/beta": 0.1024307906627655,
"fcm_dpo/delta": -0.11632996052503586,
"fcm_dpo/margin": 8.843318939208984,
"fcm_dpo/q_t": 0.32907673716545105,
"grad_norm": 28.298322677612305,
"learning_rate": 3.645566304318526e-07,
"logits/chosen": 0.33164167404174805,
"logits/rejected": 0.25221920013427734,
"logps/chosen": -63.6031494140625,
"logps/ref_chosen": -53.26664352416992,
"logps/ref_rejected": -73.84062194824219,
"logps/rejected": -93.02044677734375,
"loss": 0.9391,
"margin_dpo/margin_mean": 8.843318939208984,
"margin_dpo/margin_std": 11.451114654541016,
"step": 275
},
{
"KL/chosen_KL_mean": -8.990222930908203,
"KL/mean": -13.361787796020508,
"KL/rejected_KL_mean": -17.733346939086914,
"KL/std": 9.985158920288086,
"epoch": 0.41723356009070295,
"fcm_dpo/beta": 0.09962226450443268,
"fcm_dpo/delta": -0.07865756750106812,
"fcm_dpo/margin": 8.743131637573242,
"fcm_dpo/q_t": 0.325851172208786,
"grad_norm": 27.074356079101562,
"learning_rate": 3.633797984793294e-07,
"logits/chosen": 0.26929759979248047,
"logits/rejected": 0.23500406742095947,
"logps/chosen": -62.011016845703125,
"logps/ref_chosen": -53.02079772949219,
"logps/ref_rejected": -61.56678771972656,
"logps/rejected": -79.30014038085938,
"loss": 0.8889,
"margin_dpo/margin_mean": 8.743131637573242,
"margin_dpo/margin_std": 9.81624984741211,
"step": 276
},
{
"KL/chosen_KL_mean": -11.36018180847168,
"KL/mean": -13.965312004089355,
"KL/rejected_KL_mean": -16.57044219970703,
"KL/std": 10.15247631072998,
"epoch": 0.41874527588813304,
"fcm_dpo/beta": 0.10431164503097534,
"fcm_dpo/delta": 0.26928621530532837,
"fcm_dpo/margin": 5.210268020629883,
"fcm_dpo/q_t": 0.3985578715801239,
"grad_norm": 35.48118209838867,
"learning_rate": 3.6219979505011555e-07,
"logits/chosen": 0.35306310653686523,
"logits/rejected": 0.37153375148773193,
"logps/chosen": -82.79316711425781,
"logps/ref_chosen": -71.43299102783203,
"logps/ref_rejected": -67.65852355957031,
"logps/rejected": -84.22897338867188,
"loss": 1.1703,
"margin_dpo/margin_mean": 5.210267066955566,
"margin_dpo/margin_std": 10.70171070098877,
"step": 277
},
{
"KL/chosen_KL_mean": -11.002376556396484,
"KL/mean": -15.127434730529785,
"KL/rejected_KL_mean": -19.252490997314453,
"KL/std": 11.776235580444336,
"epoch": 0.42025699168556313,
"fcm_dpo/beta": 0.1055298000574112,
"fcm_dpo/delta": -0.08074446022510529,
"fcm_dpo/margin": 8.25011157989502,
"fcm_dpo/q_t": 0.33413276076316833,
"grad_norm": 35.327354431152344,
"learning_rate": 3.6101665315144353e-07,
"logits/chosen": 0.23632648587226868,
"logits/rejected": 0.1905459463596344,
"logps/chosen": -78.11314392089844,
"logps/ref_chosen": -67.11076354980469,
"logps/ref_rejected": -88.74851989746094,
"logps/rejected": -108.00101470947266,
"loss": 0.9829,
"margin_dpo/margin_mean": 8.25011157989502,
"margin_dpo/margin_std": 11.206602096557617,
"step": 278
},
{
"KL/chosen_KL_mean": -7.801475524902344,
"KL/mean": -12.950601577758789,
"KL/rejected_KL_mean": -18.09972381591797,
"KL/std": 10.632495880126953,
"epoch": 0.4217687074829932,
"fcm_dpo/beta": 0.1002463549375534,
"fcm_dpo/delta": -0.2583245038986206,
"fcm_dpo/margin": 10.29825210571289,
"fcm_dpo/q_t": 0.3018980622291565,
"grad_norm": 25.455596923828125,
"learning_rate": 3.5983040587833563e-07,
"logits/chosen": 0.2795836925506592,
"logits/rejected": 0.23831382393836975,
"logps/chosen": -62.2989616394043,
"logps/ref_chosen": -54.49748611450195,
"logps/ref_rejected": -70.42373657226562,
"logps/rejected": -88.52345275878906,
"loss": 0.8223,
"margin_dpo/margin_mean": 10.29825210571289,
"margin_dpo/margin_std": 10.669867515563965,
"step": 279
},
{
"KL/chosen_KL_mean": -8.367753982543945,
"KL/mean": -13.79076862335205,
"KL/rejected_KL_mean": -19.213787078857422,
"KL/std": 11.13604736328125,
"epoch": 0.42328042328042326,
"fcm_dpo/beta": 0.09465308487415314,
"fcm_dpo/delta": -0.2527472972869873,
"fcm_dpo/margin": 10.846031188964844,
"fcm_dpo/q_t": 0.2982625365257263,
"grad_norm": 23.867778778076172,
"learning_rate": 3.586410864126781e-07,
"logits/chosen": 0.32292693853378296,
"logits/rejected": 0.2808569371700287,
"logps/chosen": -68.80056762695312,
"logps/ref_chosen": -60.43281173706055,
"logps/ref_rejected": -78.39051818847656,
"logps/rejected": -97.60430908203125,
"loss": 0.7738,
"margin_dpo/margin_mean": 10.846031188964844,
"margin_dpo/margin_std": 10.204471588134766,
"step": 280
},
{
"KL/chosen_KL_mean": -9.530132293701172,
"KL/mean": -14.347930908203125,
"KL/rejected_KL_mean": -19.165735244750977,
"KL/std": 11.033517837524414,
"epoch": 0.42479213907785335,
"fcm_dpo/beta": 0.09185758978128433,
"fcm_dpo/delta": -0.09395193308591843,
"fcm_dpo/margin": 9.635604858398438,
"fcm_dpo/q_t": 0.32688847184181213,
"grad_norm": 24.12879753112793,
"learning_rate": 3.574487280222929e-07,
"logits/chosen": 0.29109102487564087,
"logits/rejected": 0.29816192388534546,
"logps/chosen": -69.81222534179688,
"logps/ref_chosen": -60.2820930480957,
"logps/ref_rejected": -62.04009246826172,
"logps/rejected": -81.20582580566406,
"loss": 0.9115,
"margin_dpo/margin_mean": 9.635605812072754,
"margin_dpo/margin_std": 11.505289077758789,
"step": 281
},
{
"KL/chosen_KL_mean": -11.271791458129883,
"KL/mean": -15.761783599853516,
"KL/rejected_KL_mean": -20.25177764892578,
"KL/std": 11.111000061035156,
"epoch": 0.42630385487528344,
"fcm_dpo/beta": 0.09275418519973755,
"fcm_dpo/delta": -0.04129917919635773,
"fcm_dpo/margin": 8.979986190795898,
"fcm_dpo/q_t": 0.34636229276657104,
"grad_norm": 29.4520206451416,
"learning_rate": 3.562533640600075e-07,
"logits/chosen": 0.2616288959980011,
"logits/rejected": 0.21098898351192474,
"logps/chosen": -71.89571380615234,
"logps/ref_chosen": -60.623924255371094,
"logps/ref_rejected": -68.67400360107422,
"logps/rejected": -88.92578125,
"loss": 0.9787,
"margin_dpo/margin_mean": 8.979985237121582,
"margin_dpo/margin_std": 11.980823516845703,
"step": 282
},
{
"KL/chosen_KL_mean": -12.055740356445312,
"KL/mean": -15.898777961730957,
"KL/rejected_KL_mean": -19.741825103759766,
"KL/std": 11.119794845581055,
"epoch": 0.42781557067271353,
"fcm_dpo/beta": 0.0929352194070816,
"fcm_dpo/delta": 0.09024599939584732,
"fcm_dpo/margin": 7.6860809326171875,
"fcm_dpo/q_t": 0.3683665096759796,
"grad_norm": 34.17851638793945,
"learning_rate": 3.550550279627215e-07,
"logits/chosen": 0.29102981090545654,
"logits/rejected": 0.20298334956169128,
"logps/chosen": -79.7034912109375,
"logps/ref_chosen": -67.64775085449219,
"logps/ref_rejected": -99.96835327148438,
"logps/rejected": -119.7101821899414,
"loss": 1.0697,
"margin_dpo/margin_mean": 7.686081886291504,
"margin_dpo/margin_std": 12.399127960205078,
"step": 283
},
{
"KL/chosen_KL_mean": -11.307464599609375,
"KL/mean": -15.802129745483398,
"KL/rejected_KL_mean": -20.296794891357422,
"KL/std": 11.915395736694336,
"epoch": 0.4293272864701436,
"fcm_dpo/beta": 0.0916890799999237,
"fcm_dpo/delta": -0.027363691478967667,
"fcm_dpo/margin": 8.989330291748047,
"fcm_dpo/q_t": 0.33987635374069214,
"grad_norm": 25.31052589416504,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": 0.3455355763435364,
"logits/rejected": 0.285012423992157,
"logps/chosen": -68.27488708496094,
"logps/ref_chosen": -56.96742630004883,
"logps/ref_rejected": -86.36236572265625,
"logps/rejected": -106.65916442871094,
"loss": 0.9337,
"margin_dpo/margin_mean": 8.989330291748047,
"margin_dpo/margin_std": 11.496484756469727,
"step": 284
},
{
"KL/chosen_KL_mean": -14.233577728271484,
"KL/mean": -18.005645751953125,
"KL/rejected_KL_mean": -21.777708053588867,
"KL/std": 10.855697631835938,
"epoch": 0.4308390022675737,
"fcm_dpo/beta": 0.09401977807283401,
"fcm_dpo/delta": 0.09626507759094238,
"fcm_dpo/margin": 7.544127464294434,
"fcm_dpo/q_t": 0.36788657307624817,
"grad_norm": 32.50141525268555,
"learning_rate": 3.5264957352549375e-07,
"logits/chosen": 0.3715507984161377,
"logits/rejected": 0.34357473254203796,
"logps/chosen": -85.88968658447266,
"logps/ref_chosen": -71.65611267089844,
"logps/ref_rejected": -81.63829803466797,
"logps/rejected": -103.41600799560547,
"loss": 1.0366,
"margin_dpo/margin_mean": 7.544128894805908,
"margin_dpo/margin_std": 11.49722671508789,
"step": 285
},
{
"KL/chosen_KL_mean": -12.889640808105469,
"KL/mean": -18.95220947265625,
"KL/rejected_KL_mean": -25.01477813720703,
"KL/std": 12.186092376708984,
"epoch": 0.4323507180650038,
"fcm_dpo/beta": 0.08927808701992035,
"fcm_dpo/delta": -0.31835198402404785,
"fcm_dpo/margin": 12.125144958496094,
"fcm_dpo/q_t": 0.29214009642601013,
"grad_norm": 25.466272354125977,
"learning_rate": 3.514425224712835e-07,
"logits/chosen": 0.2750386595726013,
"logits/rejected": 0.18463309109210968,
"logps/chosen": -73.96916198730469,
"logps/ref_chosen": -61.07952117919922,
"logps/ref_rejected": -91.28128051757812,
"logps/rejected": -116.29605102539062,
"loss": 0.7992,
"margin_dpo/margin_mean": 12.125143051147461,
"margin_dpo/margin_std": 12.32309341430664,
"step": 286
},
{
"KL/chosen_KL_mean": -11.025419235229492,
"KL/mean": -16.85832977294922,
"KL/rejected_KL_mean": -22.691240310668945,
"KL/std": 13.327524185180664,
"epoch": 0.43386243386243384,
"fcm_dpo/beta": 0.08564598858356476,
"fcm_dpo/delta": -0.2206934094429016,
"fcm_dpo/margin": 11.665821075439453,
"fcm_dpo/q_t": 0.3125588297843933,
"grad_norm": 24.484149932861328,
"learning_rate": 3.502326338516534e-07,
"logits/chosen": 0.3468093276023865,
"logits/rejected": 0.30625537037849426,
"logps/chosen": -57.06121063232422,
"logps/ref_chosen": -46.035789489746094,
"logps/ref_rejected": -59.95293426513672,
"logps/rejected": -82.64418029785156,
"loss": 0.853,
"margin_dpo/margin_mean": 11.665821075439453,
"margin_dpo/margin_std": 13.080723762512207,
"step": 287
},
{
"KL/chosen_KL_mean": -14.967060089111328,
"KL/mean": -19.00350570678711,
"KL/rejected_KL_mean": -23.03994369506836,
"KL/std": 12.503591537475586,
"epoch": 0.43537414965986393,
"fcm_dpo/beta": 0.08595605194568634,
"fcm_dpo/delta": 0.11360116302967072,
"fcm_dpo/margin": 8.072890281677246,
"fcm_dpo/q_t": 0.36748266220092773,
"grad_norm": 31.567214965820312,
"learning_rate": 3.490199415097892e-07,
"logits/chosen": 0.21245820820331573,
"logits/rejected": 0.1599254459142685,
"logps/chosen": -80.35791015625,
"logps/ref_chosen": -65.3908462524414,
"logps/ref_rejected": -88.53607940673828,
"logps/rejected": -111.57601928710938,
"loss": 1.0563,
"margin_dpo/margin_mean": 8.072890281677246,
"margin_dpo/margin_std": 12.816825866699219,
"step": 288
},
{
"KL/chosen_KL_mean": -15.324104309082031,
"KL/mean": -19.525135040283203,
"KL/rejected_KL_mean": -23.72616195678711,
"KL/std": 11.964265823364258,
"epoch": 0.436885865457294,
"fcm_dpo/beta": 0.08751742541790009,
"fcm_dpo/delta": 0.06912409514188766,
"fcm_dpo/margin": 8.402060508728027,
"fcm_dpo/q_t": 0.3677643835544586,
"grad_norm": 27.802019119262695,
"learning_rate": 3.4780447936730247e-07,
"logits/chosen": 0.40906429290771484,
"logits/rejected": 0.3720993399620056,
"logps/chosen": -69.91773223876953,
"logps/ref_chosen": -54.5936279296875,
"logps/ref_rejected": -67.20855712890625,
"logps/rejected": -90.9347152709961,
"loss": 1.07,
"margin_dpo/margin_mean": 8.402060508728027,
"margin_dpo/margin_std": 13.60258674621582,
"step": 289
},
{
"KL/chosen_KL_mean": -17.48955535888672,
"KL/mean": -22.728565216064453,
"KL/rejected_KL_mean": -27.967571258544922,
"KL/std": 12.214452743530273,
"epoch": 0.4383975812547241,
"fcm_dpo/beta": 0.08549906313419342,
"fcm_dpo/delta": -0.10699837654829025,
"fcm_dpo/margin": 10.478013038635254,
"fcm_dpo/q_t": 0.33341556787490845,
"grad_norm": 32.633052825927734,
"learning_rate": 3.465862814232821e-07,
"logits/chosen": 0.4238796830177307,
"logits/rejected": 0.35533106327056885,
"logps/chosen": -78.87413024902344,
"logps/ref_chosen": -61.38457489013672,
"logps/ref_rejected": -91.92778015136719,
"logps/rejected": -119.89534759521484,
"loss": 0.9527,
"margin_dpo/margin_mean": 10.478012084960938,
"margin_dpo/margin_std": 13.828596115112305,
"step": 290
},
{
"KL/chosen_KL_mean": -16.39398765563965,
"KL/mean": -21.614397048950195,
"KL/rejected_KL_mean": -26.834808349609375,
"KL/std": 12.491828918457031,
"epoch": 0.4399092970521542,
"fcm_dpo/beta": 0.08563029021024704,
"fcm_dpo/delta": -0.10686700791120529,
"fcm_dpo/margin": 10.44081974029541,
"fcm_dpo/q_t": 0.33649247884750366,
"grad_norm": 30.15843963623047,
"learning_rate": 3.4536538175334343e-07,
"logits/chosen": 0.4574052095413208,
"logits/rejected": 0.3944157361984253,
"logps/chosen": -67.25701904296875,
"logps/ref_chosen": -50.863037109375,
"logps/ref_rejected": -82.20868682861328,
"logps/rejected": -109.04349517822266,
"loss": 0.9609,
"margin_dpo/margin_mean": 10.440818786621094,
"margin_dpo/margin_std": 13.68259048461914,
"step": 291
},
{
"KL/chosen_KL_mean": -15.907407760620117,
"KL/mean": -20.485567092895508,
"KL/rejected_KL_mean": -25.063724517822266,
"KL/std": 12.832338333129883,
"epoch": 0.4414210128495843,
"fcm_dpo/beta": 0.08440513163805008,
"fcm_dpo/delta": 0.02916642650961876,
"fcm_dpo/margin": 9.156310081481934,
"fcm_dpo/q_t": 0.3514280319213867,
"grad_norm": 32.388160705566406,
"learning_rate": 3.4414181450867465e-07,
"logits/chosen": 0.35157227516174316,
"logits/rejected": 0.3027421832084656,
"logps/chosen": -80.25629425048828,
"logps/ref_chosen": -64.34888458251953,
"logps/ref_rejected": -72.86434173583984,
"logps/rejected": -97.92807006835938,
"loss": 1.0183,
"margin_dpo/margin_mean": 9.156310081481934,
"margin_dpo/margin_std": 13.515422821044922,
"step": 292
},
{
"KL/chosen_KL_mean": -17.491422653198242,
"KL/mean": -23.502586364746094,
"KL/rejected_KL_mean": -29.513744354248047,
"KL/std": 13.59200668334961,
"epoch": 0.4429327286470144,
"fcm_dpo/beta": 0.08238425850868225,
"fcm_dpo/delta": -0.21106353402137756,
"fcm_dpo/margin": 12.022321701049805,
"fcm_dpo/q_t": 0.3155782222747803,
"grad_norm": 22.351655960083008,
"learning_rate": 3.4291561391508185e-07,
"logits/chosen": 0.4406129717826843,
"logits/rejected": 0.3572370707988739,
"logps/chosen": -72.36088562011719,
"logps/ref_chosen": -54.869468688964844,
"logps/ref_rejected": -81.858642578125,
"logps/rejected": -111.37239074707031,
"loss": 0.93,
"margin_dpo/margin_mean": 12.022321701049805,
"margin_dpo/margin_std": 15.044371604919434,
"step": 293
},
{
"KL/chosen_KL_mean": -16.280914306640625,
"KL/mean": -21.383319854736328,
"KL/rejected_KL_mean": -26.485719680786133,
"KL/std": 14.02008056640625,
"epoch": 0.4444444444444444,
"fcm_dpo/beta": 0.07986710965633392,
"fcm_dpo/delta": -0.01819680631160736,
"fcm_dpo/margin": 10.204802513122559,
"fcm_dpo/q_t": 0.3480180501937866,
"grad_norm": 23.034154891967773,
"learning_rate": 3.4168681427203153e-07,
"logits/chosen": 0.362077921628952,
"logits/rejected": 0.31854838132858276,
"logps/chosen": -72.95181274414062,
"logps/ref_chosen": -56.670902252197266,
"logps/ref_rejected": -70.32819366455078,
"logps/rejected": -96.81391143798828,
"loss": 0.952,
"margin_dpo/margin_mean": 10.204801559448242,
"margin_dpo/margin_std": 13.412900924682617,
"step": 294
},
{
"KL/chosen_KL_mean": -18.015230178833008,
"KL/mean": -22.486682891845703,
"KL/rejected_KL_mean": -26.958141326904297,
"KL/std": 13.785063743591309,
"epoch": 0.4459561602418745,
"fcm_dpo/beta": 0.08139024674892426,
"fcm_dpo/delta": 0.07783761620521545,
"fcm_dpo/margin": 8.942903518676758,
"fcm_dpo/q_t": 0.3671254515647888,
"grad_norm": 30.077556610107422,
"learning_rate": 3.4045544995169125e-07,
"logits/chosen": 0.3807584345340729,
"logits/rejected": 0.2897271513938904,
"logps/chosen": -68.41612243652344,
"logps/ref_chosen": -50.40088653564453,
"logps/ref_rejected": -83.43521881103516,
"logps/rejected": -110.39335632324219,
"loss": 1.0405,
"margin_dpo/margin_mean": 8.942903518676758,
"margin_dpo/margin_std": 13.93426513671875,
"step": 295
},
{
"KL/chosen_KL_mean": -18.11003875732422,
"KL/mean": -23.50696563720703,
"KL/rejected_KL_mean": -28.903888702392578,
"KL/std": 14.025440216064453,
"epoch": 0.4474678760393046,
"fcm_dpo/beta": 0.08038632571697235,
"fcm_dpo/delta": -0.07529260218143463,
"fcm_dpo/margin": 10.793851852416992,
"fcm_dpo/q_t": 0.33729755878448486,
"grad_norm": 28.054275512695312,
"learning_rate": 3.392215553979679e-07,
"logits/chosen": 0.3302781879901886,
"logits/rejected": 0.28872621059417725,
"logps/chosen": -87.26039123535156,
"logps/ref_chosen": -69.15034484863281,
"logps/ref_rejected": -89.60166931152344,
"logps/rejected": -118.50555419921875,
"loss": 0.9484,
"margin_dpo/margin_mean": 10.793851852416992,
"margin_dpo/margin_std": 14.17785930633545,
"step": 296
},
{
"KL/chosen_KL_mean": -19.260316848754883,
"KL/mean": -24.803028106689453,
"KL/rejected_KL_mean": -30.34573745727539,
"KL/std": 12.583738327026367,
"epoch": 0.4489795918367347,
"fcm_dpo/beta": 0.08010244369506836,
"fcm_dpo/delta": -0.09729193150997162,
"fcm_dpo/margin": 11.085417747497559,
"fcm_dpo/q_t": 0.32611143589019775,
"grad_norm": 24.371068954467773,
"learning_rate": 3.3798516512554485e-07,
"logits/chosen": 0.3475460112094879,
"logits/rejected": 0.28787142038345337,
"logps/chosen": -77.27662658691406,
"logps/ref_chosen": -58.01630401611328,
"logps/ref_rejected": -69.95780944824219,
"logps/rejected": -100.30354309082031,
"loss": 0.8852,
"margin_dpo/margin_mean": 11.085416793823242,
"margin_dpo/margin_std": 12.328832626342773,
"step": 297
},
{
"KL/chosen_KL_mean": -18.95709991455078,
"KL/mean": -23.594552993774414,
"KL/rejected_KL_mean": -28.232009887695312,
"KL/std": 13.951750755310059,
"epoch": 0.4504913076341648,
"fcm_dpo/beta": 0.07963920384645462,
"fcm_dpo/delta": 0.06629949063062668,
"fcm_dpo/margin": 9.274908065795898,
"fcm_dpo/q_t": 0.3650180995464325,
"grad_norm": 27.34107208251953,
"learning_rate": 3.367463137189156e-07,
"logits/chosen": 0.4564887285232544,
"logits/rejected": 0.39622586965560913,
"logps/chosen": -75.12641143798828,
"logps/ref_chosen": -56.1693115234375,
"logps/ref_rejected": -68.55052185058594,
"logps/rejected": -96.78253173828125,
"loss": 1.0754,
"margin_dpo/margin_mean": 9.274909973144531,
"margin_dpo/margin_std": 15.322766304016113,
"step": 298
},
{
"KL/chosen_KL_mean": -18.38543128967285,
"KL/mean": -22.410602569580078,
"KL/rejected_KL_mean": -26.435768127441406,
"KL/std": 13.180704116821289,
"epoch": 0.4520030234315949,
"fcm_dpo/beta": 0.08154933154582977,
"fcm_dpo/delta": 0.15386077761650085,
"fcm_dpo/margin": 8.050336837768555,
"fcm_dpo/q_t": 0.382235050201416,
"grad_norm": 29.40821075439453,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": 0.27602773904800415,
"logits/rejected": 0.24822445213794708,
"logps/chosen": -80.70323181152344,
"logps/ref_chosen": -62.31780242919922,
"logps/ref_rejected": -72.60028839111328,
"logps/rejected": -99.03605651855469,
"loss": 1.1539,
"margin_dpo/margin_mean": 8.050336837768555,
"margin_dpo/margin_std": 15.153702735900879,
"step": 299
},
{
"KL/chosen_KL_mean": -19.27455711364746,
"KL/mean": -24.168970108032227,
"KL/rejected_KL_mean": -29.06338119506836,
"KL/std": 13.379597663879395,
"epoch": 0.45351473922902497,
"fcm_dpo/beta": 0.08266250789165497,
"fcm_dpo/delta": -0.010115906596183777,
"fcm_dpo/margin": 9.788824081420898,
"fcm_dpo/q_t": 0.34636732935905457,
"grad_norm": 28.359783172607422,
"learning_rate": 3.3426136618426043e-07,
"logits/chosen": 0.3955562710762024,
"logits/rejected": 0.3320777416229248,
"logps/chosen": -79.65613555908203,
"logps/ref_chosen": -60.38157653808594,
"logps/ref_rejected": -75.45442199707031,
"logps/rejected": -104.51780700683594,
"loss": 1.0271,
"margin_dpo/margin_mean": 9.788824081420898,
"margin_dpo/margin_std": 14.559015274047852,
"step": 300
},
{
"KL/chosen_KL_mean": -18.379371643066406,
"KL/mean": -22.911306381225586,
"KL/rejected_KL_mean": -27.443241119384766,
"KL/std": 13.371219635009766,
"epoch": 0.455026455026455,
"fcm_dpo/beta": 0.08241571485996246,
"fcm_dpo/delta": 0.05638513341546059,
"fcm_dpo/margin": 9.063875198364258,
"fcm_dpo/q_t": 0.3640829026699066,
"grad_norm": 25.278783798217773,
"learning_rate": 3.3301533956555885e-07,
"logits/chosen": 0.42995303869247437,
"logits/rejected": 0.39872127771377563,
"logps/chosen": -71.23026275634766,
"logps/ref_chosen": -52.85089111328125,
"logps/ref_rejected": -69.97584533691406,
"logps/rejected": -97.4190902709961,
"loss": 1.0742,
"margin_dpo/margin_mean": 9.063875198364258,
"margin_dpo/margin_std": 14.9423828125,
"step": 301
},
{
"KL/chosen_KL_mean": -20.356815338134766,
"KL/mean": -23.603713989257812,
"KL/rejected_KL_mean": -26.850624084472656,
"KL/std": 12.972681045532227,
"epoch": 0.4565381708238851,
"fcm_dpo/beta": 0.08642345666885376,
"fcm_dpo/delta": 0.2532394230365753,
"fcm_dpo/margin": 6.493806838989258,
"fcm_dpo/q_t": 0.39695829153060913,
"grad_norm": 33.59251403808594,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": 0.26218295097351074,
"logits/rejected": 0.2100965827703476,
"logps/chosen": -87.32332611083984,
"logps/ref_chosen": -66.96650695800781,
"logps/ref_rejected": -88.09510803222656,
"logps/rejected": -114.94573974609375,
"loss": 1.1935,
"margin_dpo/margin_mean": 6.493806838989258,
"margin_dpo/margin_std": 13.955561637878418,
"step": 302
},
{
"KL/chosen_KL_mean": -17.155065536499023,
"KL/mean": -23.425048828125,
"KL/rejected_KL_mean": -29.695026397705078,
"KL/std": 13.855140686035156,
"epoch": 0.4580498866213152,
"fcm_dpo/beta": 0.08415515720844269,
"fcm_dpo/delta": -0.2851126790046692,
"fcm_dpo/margin": 12.539966583251953,
"fcm_dpo/q_t": 0.31551820039749146,
"grad_norm": 24.16552734375,
"learning_rate": 3.3051635489464793e-07,
"logits/chosen": 0.37548404932022095,
"logits/rejected": 0.3113349378108978,
"logps/chosen": -79.27659606933594,
"logps/ref_chosen": -62.12152862548828,
"logps/ref_rejected": -90.31204223632812,
"logps/rejected": -120.00707244873047,
"loss": 0.9423,
"margin_dpo/margin_mean": 12.539965629577637,
"margin_dpo/margin_std": 16.383039474487305,
"step": 303
},
{
"KL/chosen_KL_mean": -15.978334426879883,
"KL/mean": -22.127933502197266,
"KL/rejected_KL_mean": -28.277530670166016,
"KL/std": 14.6383638381958,
"epoch": 0.4595616024187453,
"fcm_dpo/beta": 0.08040931820869446,
"fcm_dpo/delta": -0.20942339301109314,
"fcm_dpo/margin": 12.299189567565918,
"fcm_dpo/q_t": 0.3040643036365509,
"grad_norm": 24.452234268188477,
"learning_rate": 3.292634667444117e-07,
"logits/chosen": 0.35874032974243164,
"logits/rejected": 0.3055311143398285,
"logps/chosen": -76.67342376708984,
"logps/ref_chosen": -60.695091247558594,
"logps/ref_rejected": -78.2525405883789,
"logps/rejected": -106.53007507324219,
"loss": 0.827,
"margin_dpo/margin_mean": 12.299190521240234,
"margin_dpo/margin_std": 12.52480697631836,
"step": 304
},
{
"KL/chosen_KL_mean": -19.23602294921875,
"KL/mean": -24.019302368164062,
"KL/rejected_KL_mean": -28.80258560180664,
"KL/std": 13.73517894744873,
"epoch": 0.46107331821617537,
"fcm_dpo/beta": 0.07919107377529144,
"fcm_dpo/delta": 0.0445355623960495,
"fcm_dpo/margin": 9.566566467285156,
"fcm_dpo/q_t": 0.35818392038345337,
"grad_norm": 30.306407928466797,
"learning_rate": 3.280083614246217e-07,
"logits/chosen": 0.3176822066307068,
"logits/rejected": 0.33436745405197144,
"logps/chosen": -91.93516540527344,
"logps/ref_chosen": -72.69914245605469,
"logps/ref_rejected": -65.65670776367188,
"logps/rejected": -94.45929718017578,
"loss": 1.0632,
"margin_dpo/margin_mean": 9.566566467285156,
"margin_dpo/margin_std": 15.191731452941895,
"step": 305
},
{
"KL/chosen_KL_mean": -17.9437198638916,
"KL/mean": -22.24250602722168,
"KL/rejected_KL_mean": -26.541290283203125,
"KL/std": 13.069320678710938,
"epoch": 0.46258503401360546,
"fcm_dpo/beta": 0.08063946664333344,
"fcm_dpo/delta": 0.11361439526081085,
"fcm_dpo/margin": 8.597566604614258,
"fcm_dpo/q_t": 0.3632877469062805,
"grad_norm": 24.9428653717041,
"learning_rate": 3.267510740432719e-07,
"logits/chosen": 0.41871410608291626,
"logits/rejected": 0.3094066083431244,
"logps/chosen": -71.91424560546875,
"logps/ref_chosen": -53.97052764892578,
"logps/ref_rejected": -71.02423095703125,
"logps/rejected": -97.56552124023438,
"loss": 1.0196,
"margin_dpo/margin_mean": 8.597567558288574,
"margin_dpo/margin_std": 12.287683486938477,
"step": 306
},
{
"KL/chosen_KL_mean": -15.186479568481445,
"KL/mean": -18.630168914794922,
"KL/rejected_KL_mean": -22.073871612548828,
"KL/std": 13.487642288208008,
"epoch": 0.46409674981103555,
"fcm_dpo/beta": 0.08535341918468475,
"fcm_dpo/delta": 0.22243700921535492,
"fcm_dpo/margin": 6.887389659881592,
"fcm_dpo/q_t": 0.40226835012435913,
"grad_norm": 32.934783935546875,
"learning_rate": 3.2549163976939285e-07,
"logits/chosen": 0.39580583572387695,
"logits/rejected": 0.35165637731552124,
"logps/chosen": -72.5995864868164,
"logps/ref_chosen": -57.413108825683594,
"logps/ref_rejected": -68.68010711669922,
"logps/rejected": -90.75398254394531,
"loss": 1.2401,
"margin_dpo/margin_mean": 6.887389183044434,
"margin_dpo/margin_std": 15.407466888427734,
"step": 307
},
{
"KL/chosen_KL_mean": -16.288928985595703,
"KL/mean": -20.996000289916992,
"KL/rejected_KL_mean": -25.703067779541016,
"KL/std": 13.363460540771484,
"epoch": 0.4656084656084656,
"fcm_dpo/beta": 0.08504834771156311,
"fcm_dpo/delta": -0.0017690509557724,
"fcm_dpo/margin": 9.414133071899414,
"fcm_dpo/q_t": 0.3468315005302429,
"grad_norm": 26.06618309020996,
"learning_rate": 3.2423009383206874e-07,
"logits/chosen": 0.36595430970191956,
"logits/rejected": 0.34875592589378357,
"logps/chosen": -82.88772583007812,
"logps/ref_chosen": -66.59879302978516,
"logps/ref_rejected": -74.337158203125,
"logps/rejected": -100.04022216796875,
"loss": 1.024,
"margin_dpo/margin_mean": 9.414133071899414,
"margin_dpo/margin_std": 13.817611694335938,
"step": 308
},
{
"KL/chosen_KL_mean": -19.097213745117188,
"KL/mean": -23.660863876342773,
"KL/rejected_KL_mean": -28.224517822265625,
"KL/std": 12.788629531860352,
"epoch": 0.4671201814058957,
"fcm_dpo/beta": 0.08512066304683685,
"fcm_dpo/delta": 0.023319870233535767,
"fcm_dpo/margin": 9.12730598449707,
"fcm_dpo/q_t": 0.3563210070133209,
"grad_norm": 34.374366760253906,
"learning_rate": 3.229664715194511e-07,
"logits/chosen": 0.4130929410457611,
"logits/rejected": 0.3572024703025818,
"logps/chosen": -84.49195861816406,
"logps/ref_chosen": -65.39474487304688,
"logps/ref_rejected": -75.70930480957031,
"logps/rejected": -103.93382263183594,
"loss": 1.0346,
"margin_dpo/margin_mean": 9.12730598449707,
"margin_dpo/margin_std": 13.821783065795898,
"step": 309
},
{
"KL/chosen_KL_mean": -19.24032974243164,
"KL/mean": -22.127826690673828,
"KL/rejected_KL_mean": -25.015323638916016,
"KL/std": 11.900168418884277,
"epoch": 0.46863189720332576,
"fcm_dpo/beta": 0.08966143429279327,
"fcm_dpo/delta": 0.2983810007572174,
"fcm_dpo/margin": 5.774989128112793,
"fcm_dpo/q_t": 0.41219472885131836,
"grad_norm": 35.78599548339844,
"learning_rate": 3.2170080817777257e-07,
"logits/chosen": 0.4417685568332672,
"logits/rejected": 0.429890513420105,
"logps/chosen": -93.90859985351562,
"logps/ref_chosen": -74.66827392578125,
"logps/ref_rejected": -80.5689697265625,
"logps/rejected": -105.58428955078125,
"loss": 1.2452,
"margin_dpo/margin_mean": 5.774989128112793,
"margin_dpo/margin_std": 13.688153266906738,
"step": 310
},
{
"KL/chosen_KL_mean": -15.007062911987305,
"KL/mean": -19.197357177734375,
"KL/rejected_KL_mean": -23.387653350830078,
"KL/std": 14.218526840209961,
"epoch": 0.47014361300075586,
"fcm_dpo/beta": 0.09115281701087952,
"fcm_dpo/delta": 0.03813778609037399,
"fcm_dpo/margin": 8.380594253540039,
"fcm_dpo/q_t": 0.3628461956977844,
"grad_norm": 31.881589889526367,
"learning_rate": 3.204331392103574e-07,
"logits/chosen": 0.31646156311035156,
"logits/rejected": 0.1944255530834198,
"logps/chosen": -74.7450942993164,
"logps/ref_chosen": -59.738033294677734,
"logps/ref_rejected": -93.60757446289062,
"logps/rejected": -116.99522399902344,
"loss": 1.0809,
"margin_dpo/margin_mean": 8.380593299865723,
"margin_dpo/margin_std": 14.251326560974121,
"step": 311
},
{
"KL/chosen_KL_mean": -15.454090118408203,
"KL/mean": -21.195587158203125,
"KL/rejected_KL_mean": -26.937084197998047,
"KL/std": 13.556852340698242,
"epoch": 0.47165532879818595,
"fcm_dpo/beta": 0.08916601538658142,
"fcm_dpo/delta": -0.24872109293937683,
"fcm_dpo/margin": 11.482992172241211,
"fcm_dpo/q_t": 0.30444252490997314,
"grad_norm": 24.69047737121582,
"learning_rate": 3.1916350007663176e-07,
"logits/chosen": 0.3721472918987274,
"logits/rejected": 0.27872756123542786,
"logps/chosen": -69.27052307128906,
"logps/ref_chosen": -53.816436767578125,
"logps/ref_rejected": -68.6575698852539,
"logps/rejected": -95.59465026855469,
"loss": 0.8523,
"margin_dpo/margin_mean": 11.482992172241211,
"margin_dpo/margin_std": 12.813817977905273,
"step": 312
},
{
"KL/chosen_KL_mean": -15.105289459228516,
"KL/mean": -18.892580032348633,
"KL/rejected_KL_mean": -22.679874420166016,
"KL/std": 13.558688163757324,
"epoch": 0.47316704459561604,
"fcm_dpo/beta": 0.08916016668081284,
"fcm_dpo/delta": 0.13385319709777832,
"fcm_dpo/margin": 7.574581146240234,
"fcm_dpo/q_t": 0.3830859661102295,
"grad_norm": 30.752466201782227,
"learning_rate": 3.178919262911314e-07,
"logits/chosen": 0.4537992477416992,
"logits/rejected": 0.431113064289093,
"logps/chosen": -75.06265258789062,
"logps/ref_chosen": -59.957359313964844,
"logps/ref_rejected": -69.31729888916016,
"logps/rejected": -91.99717712402344,
"loss": 1.1541,
"margin_dpo/margin_mean": 7.574580669403076,
"margin_dpo/margin_std": 14.770940780639648,
"step": 313
},
{
"KL/chosen_KL_mean": -14.3424072265625,
"KL/mean": -20.40030288696289,
"KL/rejected_KL_mean": -26.458194732666016,
"KL/std": 13.566604614257812,
"epoch": 0.47467876039304613,
"fcm_dpo/beta": 0.0859234556555748,
"fcm_dpo/delta": -0.27131664752960205,
"fcm_dpo/margin": 12.1157808303833,
"fcm_dpo/q_t": 0.30542221665382385,
"grad_norm": 24.523052215576172,
"learning_rate": 3.166184534225087e-07,
"logits/chosen": 0.37127453088760376,
"logits/rejected": 0.38916367292404175,
"logps/chosen": -84.61056518554688,
"logps/ref_chosen": -70.26815795898438,
"logps/ref_rejected": -69.23971557617188,
"logps/rejected": -95.69790649414062,
"loss": 0.8307,
"margin_dpo/margin_mean": 12.1157808303833,
"margin_dpo/margin_std": 13.054606437683105,
"step": 314
},
{
"KL/chosen_KL_mean": -15.060813903808594,
"KL/mean": -19.923267364501953,
"KL/rejected_KL_mean": -24.785724639892578,
"KL/std": 13.563258171081543,
"epoch": 0.47619047619047616,
"fcm_dpo/beta": 0.08582372963428497,
"fcm_dpo/delta": -0.040948450565338135,
"fcm_dpo/margin": 9.724916458129883,
"fcm_dpo/q_t": 0.3378736972808838,
"grad_norm": 25.47266960144043,
"learning_rate": 3.1534311709253723e-07,
"logits/chosen": 0.3101431727409363,
"logits/rejected": 0.2680996358394623,
"logps/chosen": -82.85550689697266,
"logps/ref_chosen": -67.79469299316406,
"logps/ref_rejected": -74.55148315429688,
"logps/rejected": -99.33720397949219,
"loss": 0.9679,
"margin_dpo/margin_mean": 9.724916458129883,
"margin_dpo/margin_std": 12.58622932434082,
"step": 315
},
{
"KL/chosen_KL_mean": -14.215065002441406,
"KL/mean": -20.012006759643555,
"KL/rejected_KL_mean": -25.80895233154297,
"KL/std": 13.516332626342773,
"epoch": 0.47770219198790626,
"fcm_dpo/beta": 0.08367159217596054,
"fcm_dpo/delta": -0.19175595045089722,
"fcm_dpo/margin": 11.593890190124512,
"fcm_dpo/q_t": 0.31763750314712524,
"grad_norm": 27.956092834472656,
"learning_rate": 3.1406595297511564e-07,
"logits/chosen": 0.2943209409713745,
"logits/rejected": 0.17805573344230652,
"logps/chosen": -69.50354766845703,
"logps/ref_chosen": -55.288482666015625,
"logps/ref_rejected": -96.15723419189453,
"logps/rejected": -121.9661865234375,
"loss": 0.8898,
"margin_dpo/margin_mean": 11.593889236450195,
"margin_dpo/margin_std": 12.889469146728516,
"step": 316
},
{
"KL/chosen_KL_mean": -12.102411270141602,
"KL/mean": -18.74686050415039,
"KL/rejected_KL_mean": -25.391311645507812,
"KL/std": 13.107463836669922,
"epoch": 0.47921390778533635,
"fcm_dpo/beta": 0.07773043215274811,
"fcm_dpo/delta": -0.26238229870796204,
"fcm_dpo/margin": 13.288894653320312,
"fcm_dpo/q_t": 0.2983187437057495,
"grad_norm": 23.87213134765625,
"learning_rate": 3.1278699679526975e-07,
"logits/chosen": 0.44091135263442993,
"logits/rejected": 0.39091646671295166,
"logps/chosen": -66.68379211425781,
"logps/ref_chosen": -54.58137512207031,
"logps/ref_rejected": -72.77232360839844,
"logps/rejected": -98.16363525390625,
"loss": 0.8,
"margin_dpo/margin_mean": 13.288894653320312,
"margin_dpo/margin_std": 13.080042839050293,
"step": 317
},
{
"KL/chosen_KL_mean": -14.92811393737793,
"KL/mean": -19.01508331298828,
"KL/rejected_KL_mean": -23.10205841064453,
"KL/std": 13.509125709533691,
"epoch": 0.48072562358276644,
"fcm_dpo/beta": 0.07906337827444077,
"fcm_dpo/delta": 0.16437119245529175,
"fcm_dpo/margin": 8.173942565917969,
"fcm_dpo/q_t": 0.38638216257095337,
"grad_norm": 29.862834930419922,
"learning_rate": 3.1150628432815336e-07,
"logits/chosen": 0.4298110008239746,
"logits/rejected": 0.36635473370552063,
"logps/chosen": -67.81634521484375,
"logps/ref_chosen": -52.88822937011719,
"logps/ref_rejected": -80.63988494873047,
"logps/rejected": -103.741943359375,
"loss": 1.223,
"margin_dpo/margin_mean": 8.173942565917969,
"margin_dpo/margin_std": 17.332202911376953,
"step": 318
},
{
"KL/chosen_KL_mean": -12.821718215942383,
"KL/mean": -18.56632423400879,
"KL/rejected_KL_mean": -24.310924530029297,
"KL/std": 13.857823371887207,
"epoch": 0.48223733938019653,
"fcm_dpo/beta": 0.07831594347953796,
"fcm_dpo/delta": -0.10991809517145157,
"fcm_dpo/margin": 11.489205360412598,
"fcm_dpo/q_t": 0.3347237706184387,
"grad_norm": 24.75864601135254,
"learning_rate": 3.1022385139804707e-07,
"logits/chosen": 0.35719725489616394,
"logits/rejected": 0.33642852306365967,
"logps/chosen": -77.18505096435547,
"logps/ref_chosen": -64.36333465576172,
"logps/ref_rejected": -79.47296142578125,
"logps/rejected": -103.78388214111328,
"loss": 0.9462,
"margin_dpo/margin_mean": 11.489205360412598,
"margin_dpo/margin_std": 15.297908782958984,
"step": 319
},
{
"KL/chosen_KL_mean": -12.259525299072266,
"KL/mean": -16.699268341064453,
"KL/rejected_KL_mean": -21.13901138305664,
"KL/std": 14.101001739501953,
"epoch": 0.4837490551776266,
"fcm_dpo/beta": 0.08037566393613815,
"fcm_dpo/delta": 0.08531537652015686,
"fcm_dpo/margin": 8.879487991333008,
"fcm_dpo/q_t": 0.3749288320541382,
"grad_norm": 27.196882247924805,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": 0.2817011773586273,
"logits/rejected": 0.23932918906211853,
"logps/chosen": -61.81827163696289,
"logps/ref_chosen": -49.558746337890625,
"logps/ref_rejected": -71.23444366455078,
"logps/rejected": -92.37345886230469,
"loss": 1.0895,
"margin_dpo/margin_mean": 8.879487037658691,
"margin_dpo/margin_std": 14.906489372253418,
"step": 320
},
{
"KL/chosen_KL_mean": -13.375057220458984,
"KL/mean": -18.363353729248047,
"KL/rejected_KL_mean": -23.351652145385742,
"KL/std": 13.216703414916992,
"epoch": 0.4852607709750567,
"fcm_dpo/beta": 0.07867099344730377,
"fcm_dpo/delta": 0.014172382652759552,
"fcm_dpo/margin": 9.976592063903809,
"fcm_dpo/q_t": 0.34726887941360474,
"grad_norm": 24.233501434326172,
"learning_rate": 3.0765396768561004e-07,
"logits/chosen": 0.3476206064224243,
"logits/rejected": 0.3286382555961609,
"logps/chosen": -65.4603271484375,
"logps/ref_chosen": -52.08526611328125,
"logps/ref_rejected": -55.58674621582031,
"logps/rejected": -78.93840026855469,
"loss": 1.0234,
"margin_dpo/margin_mean": 9.976592063903809,
"margin_dpo/margin_std": 14.353278160095215,
"step": 321
},
{
"KL/chosen_KL_mean": -14.237586975097656,
"KL/mean": -20.744802474975586,
"KL/rejected_KL_mean": -27.252017974853516,
"KL/std": 13.349346160888672,
"epoch": 0.48677248677248675,
"fcm_dpo/beta": 0.07750476151704788,
"fcm_dpo/delta": -0.23154690861701965,
"fcm_dpo/margin": 13.014429092407227,
"fcm_dpo/q_t": 0.2995946407318115,
"grad_norm": 22.012561798095703,
"learning_rate": 3.063665887884511e-07,
"logits/chosen": 0.4243776202201843,
"logits/rejected": 0.34563034772872925,
"logps/chosen": -61.64169692993164,
"logps/ref_chosen": -47.404109954833984,
"logps/ref_rejected": -73.4260025024414,
"logps/rejected": -100.67802429199219,
"loss": 0.8164,
"margin_dpo/margin_mean": 13.014429092407227,
"margin_dpo/margin_std": 12.93503189086914,
"step": 322
},
{
"KL/chosen_KL_mean": -14.6673583984375,
"KL/mean": -19.150596618652344,
"KL/rejected_KL_mean": -23.633834838867188,
"KL/std": 14.315862655639648,
"epoch": 0.48828420256991684,
"fcm_dpo/beta": 0.07800118625164032,
"fcm_dpo/delta": 0.10459380596876144,
"fcm_dpo/margin": 8.966480255126953,
"fcm_dpo/q_t": 0.38355547189712524,
"grad_norm": 29.142436981201172,
"learning_rate": 3.0507763319663517e-07,
"logits/chosen": 0.33733895421028137,
"logits/rejected": 0.2629064917564392,
"logps/chosen": -84.67366027832031,
"logps/ref_chosen": -70.00630187988281,
"logps/ref_rejected": -86.96690368652344,
"logps/rejected": -110.60073852539062,
"loss": 1.1505,
"margin_dpo/margin_mean": 8.966480255126953,
"margin_dpo/margin_std": 17.100955963134766,
"step": 323
},
{
"KL/chosen_KL_mean": -11.98857307434082,
"KL/mean": -17.415842056274414,
"KL/rejected_KL_mean": -22.84311294555664,
"KL/std": 14.033093452453613,
"epoch": 0.4897959183673469,
"fcm_dpo/beta": 0.0767996609210968,
"fcm_dpo/delta": -0.03729076683521271,
"fcm_dpo/margin": 10.854536056518555,
"fcm_dpo/q_t": 0.33308106660842896,
"grad_norm": 26.14889907836914,
"learning_rate": 3.0378713696502097e-07,
"logits/chosen": 0.4245554208755493,
"logits/rejected": 0.366682767868042,
"logps/chosen": -67.87739562988281,
"logps/ref_chosen": -55.88882064819336,
"logps/ref_rejected": -75.23088073730469,
"logps/rejected": -98.07398986816406,
"loss": 0.8972,
"margin_dpo/margin_mean": 10.854536056518555,
"margin_dpo/margin_std": 12.291630744934082,
"step": 324
},
{
"KL/chosen_KL_mean": -15.521261215209961,
"KL/mean": -20.76903533935547,
"KL/rejected_KL_mean": -26.016807556152344,
"KL/std": 13.765624046325684,
"epoch": 0.491307634164777,
"fcm_dpo/beta": 0.07600394636392593,
"fcm_dpo/delta": -0.001028984785079956,
"fcm_dpo/margin": 10.495555877685547,
"fcm_dpo/q_t": 0.34639471769332886,
"grad_norm": 24.565717697143555,
"learning_rate": 3.0249513619156206e-07,
"logits/chosen": 0.36406660079956055,
"logits/rejected": 0.3028401732444763,
"logps/chosen": -79.66827392578125,
"logps/ref_chosen": -64.14701843261719,
"logps/ref_rejected": -79.91143798828125,
"logps/rejected": -105.92823791503906,
"loss": 0.9933,
"margin_dpo/margin_mean": 10.495555877685547,
"margin_dpo/margin_std": 14.726129531860352,
"step": 325
},
{
"KL/chosen_KL_mean": -18.79259490966797,
"KL/mean": -21.617759704589844,
"KL/rejected_KL_mean": -24.44293212890625,
"KL/std": 13.5921630859375,
"epoch": 0.4928193499622071,
"fcm_dpo/beta": 0.08110679686069489,
"fcm_dpo/delta": 0.3585972487926483,
"fcm_dpo/margin": 5.650336265563965,
"fcm_dpo/q_t": 0.4131473898887634,
"grad_norm": 33.04014205932617,
"learning_rate": 3.012016670162977e-07,
"logits/chosen": 0.31773847341537476,
"logits/rejected": 0.3185911178588867,
"logps/chosen": -94.32390594482422,
"logps/ref_chosen": -75.53131103515625,
"logps/ref_rejected": -76.5898666381836,
"logps/rejected": -101.03279876708984,
"loss": 1.2653,
"margin_dpo/margin_mean": 5.650336265563965,
"margin_dpo/margin_std": 14.059412956237793,
"step": 326
},
{
"KL/chosen_KL_mean": -16.67436981201172,
"KL/mean": -21.07805633544922,
"KL/rejected_KL_mean": -25.481735229492188,
"KL/std": 14.634458541870117,
"epoch": 0.4943310657596372,
"fcm_dpo/beta": 0.08383779972791672,
"fcm_dpo/delta": 0.06597737222909927,
"fcm_dpo/margin": 8.807355880737305,
"fcm_dpo/q_t": 0.37003931403160095,
"grad_norm": 30.968721389770508,
"learning_rate": 2.99906765620341e-07,
"logits/chosen": 0.2856214940547943,
"logits/rejected": 0.24533884227275848,
"logps/chosen": -86.01153564453125,
"logps/ref_chosen": -69.33717346191406,
"logps/ref_rejected": -73.37751770019531,
"logps/rejected": -98.8592529296875,
"loss": 1.1417,
"margin_dpo/margin_mean": 8.807354927062988,
"margin_dpo/margin_std": 16.436473846435547,
"step": 327
},
{
"KL/chosen_KL_mean": -13.426244735717773,
"KL/mean": -18.46278953552246,
"KL/rejected_KL_mean": -23.49933624267578,
"KL/std": 13.932609558105469,
"epoch": 0.4958427815570673,
"fcm_dpo/beta": 0.08340919762849808,
"fcm_dpo/delta": -0.043860193341970444,
"fcm_dpo/margin": 10.073086738586426,
"fcm_dpo/q_t": 0.342735230922699,
"grad_norm": 26.51942253112793,
"learning_rate": 2.9861046822486766e-07,
"logits/chosen": 0.3326423764228821,
"logits/rejected": 0.3011217713356018,
"logps/chosen": -75.13247680664062,
"logps/ref_chosen": -61.70623016357422,
"logps/ref_rejected": -83.73808288574219,
"logps/rejected": -107.23741149902344,
"loss": 0.9377,
"margin_dpo/margin_mean": 10.073087692260742,
"margin_dpo/margin_std": 13.004785537719727,
"step": 328
},
{
"KL/chosen_KL_mean": -16.34695816040039,
"KL/mean": -21.78904151916504,
"KL/rejected_KL_mean": -27.231124877929688,
"KL/std": 14.933101654052734,
"epoch": 0.4973544973544973,
"fcm_dpo/beta": 0.08136263489723206,
"fcm_dpo/delta": -0.09547744691371918,
"fcm_dpo/margin": 10.884174346923828,
"fcm_dpo/q_t": 0.33907365798950195,
"grad_norm": 28.799869537353516,
"learning_rate": 2.9731281109010253e-07,
"logits/chosen": 0.40021979808807373,
"logits/rejected": 0.346312016248703,
"logps/chosen": -80.84536743164062,
"logps/ref_chosen": -64.4984130859375,
"logps/ref_rejected": -83.6591796875,
"logps/rejected": -110.89030456542969,
"loss": 0.9483,
"margin_dpo/margin_mean": 10.884174346923828,
"margin_dpo/margin_std": 14.5554780960083,
"step": 329
},
{
"KL/chosen_KL_mean": -13.03089714050293,
"KL/mean": -18.402721405029297,
"KL/rejected_KL_mean": -23.77454376220703,
"KL/std": 13.772836685180664,
"epoch": 0.4988662131519274,
"fcm_dpo/beta": 0.0811014473438263,
"fcm_dpo/delta": -0.07852261513471603,
"fcm_dpo/margin": 10.743646621704102,
"fcm_dpo/q_t": 0.33982962369918823,
"grad_norm": 25.62102699279785,
"learning_rate": 2.9601383051430505e-07,
"logits/chosen": 0.3539420962333679,
"logits/rejected": 0.2868247628211975,
"logps/chosen": -67.83554077148438,
"logps/ref_chosen": -54.80464172363281,
"logps/ref_rejected": -75.3194351196289,
"logps/rejected": -99.09397888183594,
"loss": 1.0445,
"margin_dpo/margin_mean": 10.743644714355469,
"margin_dpo/margin_std": 16.401023864746094,
"step": 330
},
{
"KL/chosen_KL_mean": -14.137678146362305,
"KL/mean": -21.697162628173828,
"KL/rejected_KL_mean": -29.256637573242188,
"KL/std": 15.32611083984375,
"epoch": 0.5003779289493575,
"fcm_dpo/beta": 0.07569363713264465,
"fcm_dpo/delta": -0.3894200921058655,
"fcm_dpo/margin": 15.118959426879883,
"fcm_dpo/q_t": 0.29206815361976624,
"grad_norm": 22.627124786376953,
"learning_rate": 2.947135628327544e-07,
"logits/chosen": 0.4677159786224365,
"logits/rejected": 0.4367586076259613,
"logps/chosen": -73.38026428222656,
"logps/ref_chosen": -59.242584228515625,
"logps/ref_rejected": -69.87483215332031,
"logps/rejected": -99.1314697265625,
"loss": 0.8435,
"margin_dpo/margin_mean": 15.118961334228516,
"margin_dpo/margin_std": 17.141651153564453,
"step": 331
},
{
"KL/chosen_KL_mean": -15.791358947753906,
"KL/mean": -21.50833511352539,
"KL/rejected_KL_mean": -27.22531509399414,
"KL/std": 14.802399635314941,
"epoch": 0.5018896447467877,
"fcm_dpo/beta": 0.07454757392406464,
"fcm_dpo/delta": -0.060058847069740295,
"fcm_dpo/margin": 11.433955192565918,
"fcm_dpo/q_t": 0.342519074678421,
"grad_norm": 24.499897003173828,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": 0.3224112391471863,
"logits/rejected": 0.2759397625923157,
"logps/chosen": -82.90111541748047,
"logps/ref_chosen": -67.10975646972656,
"logps/ref_rejected": -77.11839294433594,
"logps/rejected": -104.34370422363281,
"loss": 0.9353,
"margin_dpo/margin_mean": 11.433955192565918,
"margin_dpo/margin_std": 14.521880149841309,
"step": 332
},
{
"KL/chosen_KL_mean": -16.0902156829834,
"KL/mean": -23.16205596923828,
"KL/rejected_KL_mean": -30.23389434814453,
"KL/std": 15.021678924560547,
"epoch": 0.5034013605442177,
"fcm_dpo/beta": 0.07109569013118744,
"fcm_dpo/delta": -0.22809955477714539,
"fcm_dpo/margin": 14.1436767578125,
"fcm_dpo/q_t": 0.3082965314388275,
"grad_norm": 21.058923721313477,
"learning_rate": 2.921093116725076e-07,
"logits/chosen": 0.42060232162475586,
"logits/rejected": 0.3509610593318939,
"logps/chosen": -74.47134399414062,
"logps/ref_chosen": -58.381134033203125,
"logps/ref_rejected": -85.02839660644531,
"logps/rejected": -115.26228332519531,
"loss": 0.8292,
"margin_dpo/margin_mean": 14.1436767578125,
"margin_dpo/margin_std": 15.049093246459961,
"step": 333
},
{
"KL/chosen_KL_mean": -16.76015853881836,
"KL/mean": -21.550758361816406,
"KL/rejected_KL_mean": -26.341354370117188,
"KL/std": 15.843827247619629,
"epoch": 0.5049130763416477,
"fcm_dpo/beta": 0.0714268833398819,
"fcm_dpo/delta": 0.1240236908197403,
"fcm_dpo/margin": 9.581196784973145,
"fcm_dpo/q_t": 0.37355661392211914,
"grad_norm": 28.009719848632812,
"learning_rate": 2.9080540104031484e-07,
"logits/chosen": 0.4134102761745453,
"logits/rejected": 0.3671502470970154,
"logps/chosen": -83.65216064453125,
"logps/ref_chosen": -66.89199829101562,
"logps/ref_rejected": -91.83695220947266,
"logps/rejected": -118.17830657958984,
"loss": 1.1305,
"margin_dpo/margin_mean": 9.581195831298828,
"margin_dpo/margin_std": 17.586097717285156,
"step": 334
},
{
"KL/chosen_KL_mean": -16.416725158691406,
"KL/mean": -22.06592559814453,
"KL/rejected_KL_mean": -27.715133666992188,
"KL/std": 15.565815925598145,
"epoch": 0.5064247921390779,
"fcm_dpo/beta": 0.07200112193822861,
"fcm_dpo/delta": -0.01507379487156868,
"fcm_dpo/margin": 11.298408508300781,
"fcm_dpo/q_t": 0.3517000675201416,
"grad_norm": 25.294702529907227,
"learning_rate": 2.895003489933375e-07,
"logits/chosen": 0.395161509513855,
"logits/rejected": 0.3578339219093323,
"logps/chosen": -77.93118286132812,
"logps/ref_chosen": -61.51445770263672,
"logps/ref_rejected": -75.68916320800781,
"logps/rejected": -103.404296875,
"loss": 1.0078,
"margin_dpo/margin_mean": 11.298408508300781,
"margin_dpo/margin_std": 16.401405334472656,
"step": 335
},
{
"KL/chosen_KL_mean": -21.026308059692383,
"KL/mean": -26.18271255493164,
"KL/rejected_KL_mean": -31.33911895751953,
"KL/std": 15.847661972045898,
"epoch": 0.5079365079365079,
"fcm_dpo/beta": 0.07149013131856918,
"fcm_dpo/delta": 0.0657254010438919,
"fcm_dpo/margin": 10.312803268432617,
"fcm_dpo/q_t": 0.37415605783462524,
"grad_norm": 27.998376846313477,
"learning_rate": 2.8819419203668675e-07,
"logits/chosen": 0.32004469633102417,
"logits/rejected": 0.29973897337913513,
"logps/chosen": -89.87637329101562,
"logps/ref_chosen": -68.85006713867188,
"logps/ref_rejected": -92.99603271484375,
"logps/rejected": -124.33514404296875,
"loss": 1.0709,
"margin_dpo/margin_mean": 10.312803268432617,
"margin_dpo/margin_std": 17.34792709350586,
"step": 336
},
{
"KL/chosen_KL_mean": -20.296260833740234,
"KL/mean": -24.938064575195312,
"KL/rejected_KL_mean": -29.57986831665039,
"KL/std": 15.817164421081543,
"epoch": 0.509448223733938,
"fcm_dpo/beta": 0.07388242334127426,
"fcm_dpo/delta": 0.12251827120780945,
"fcm_dpo/margin": 9.283602714538574,
"fcm_dpo/q_t": 0.3708915710449219,
"grad_norm": 26.191547393798828,
"learning_rate": 2.8688696670638053e-07,
"logits/chosen": 0.2764531373977661,
"logits/rejected": 0.24225616455078125,
"logps/chosen": -93.48410034179688,
"logps/ref_chosen": -73.18783569335938,
"logps/ref_rejected": -86.89118957519531,
"logps/rejected": -116.47105407714844,
"loss": 1.0657,
"margin_dpo/margin_mean": 9.283601760864258,
"margin_dpo/margin_std": 15.305516242980957,
"step": 337
},
{
"KL/chosen_KL_mean": -20.28105926513672,
"KL/mean": -25.161582946777344,
"KL/rejected_KL_mean": -30.042102813720703,
"KL/std": 15.790666580200195,
"epoch": 0.5109599395313681,
"fcm_dpo/beta": 0.07535530626773834,
"fcm_dpo/delta": 0.06915910542011261,
"fcm_dpo/margin": 9.761046409606934,
"fcm_dpo/q_t": 0.36161673069000244,
"grad_norm": 27.036073684692383,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": 0.3518291711807251,
"logits/rejected": 0.3244253396987915,
"logps/chosen": -84.22067260742188,
"logps/ref_chosen": -63.939613342285156,
"logps/ref_rejected": -75.34243774414062,
"logps/rejected": -105.3845443725586,
"loss": 1.0544,
"margin_dpo/margin_mean": 9.761046409606934,
"margin_dpo/margin_std": 15.557717323303223,
"step": 338
},
{
"KL/chosen_KL_mean": -16.798437118530273,
"KL/mean": -22.42432975769043,
"KL/rejected_KL_mean": -28.050220489501953,
"KL/std": 14.957649230957031,
"epoch": 0.5124716553287982,
"fcm_dpo/beta": 0.07454653829336166,
"fcm_dpo/delta": -0.04305371642112732,
"fcm_dpo/margin": 11.251781463623047,
"fcm_dpo/q_t": 0.34308749437332153,
"grad_norm": 24.66077423095703,
"learning_rate": 2.842694572172736e-07,
"logits/chosen": 0.49047064781188965,
"logits/rejected": 0.40806323289871216,
"logps/chosen": -62.34756851196289,
"logps/ref_chosen": -45.54913330078125,
"logps/ref_rejected": -67.0482177734375,
"logps/rejected": -95.09843444824219,
"loss": 0.9566,
"margin_dpo/margin_mean": 11.25178337097168,
"margin_dpo/margin_std": 14.857077598571777,
"step": 339
},
{
"KL/chosen_KL_mean": -19.570297241210938,
"KL/mean": -25.74864959716797,
"KL/rejected_KL_mean": -31.9270076751709,
"KL/std": 17.383949279785156,
"epoch": 0.5139833711262283,
"fcm_dpo/beta": 0.07418019324541092,
"fcm_dpo/delta": -0.1296280026435852,
"fcm_dpo/margin": 12.356712341308594,
"fcm_dpo/q_t": 0.34674328565597534,
"grad_norm": 25.045578002929688,
"learning_rate": 2.8295924627584004e-07,
"logits/chosen": 0.381099671125412,
"logits/rejected": 0.35601145029067993,
"logps/chosen": -73.57594299316406,
"logps/ref_chosen": -54.00564956665039,
"logps/ref_rejected": -61.314430236816406,
"logps/rejected": -93.24143981933594,
"loss": 1.0518,
"margin_dpo/margin_mean": 12.356712341308594,
"margin_dpo/margin_std": 19.074668884277344,
"step": 340
},
{
"KL/chosen_KL_mean": -18.521739959716797,
"KL/mean": -24.781999588012695,
"KL/rejected_KL_mean": -31.042255401611328,
"KL/std": 15.980314254760742,
"epoch": 0.5154950869236583,
"fcm_dpo/beta": 0.07009994983673096,
"fcm_dpo/delta": -0.09934549033641815,
"fcm_dpo/margin": 12.520519256591797,
"fcm_dpo/q_t": 0.3412610590457916,
"grad_norm": 26.827455520629883,
"learning_rate": 2.816481133934373e-07,
"logits/chosen": 0.4074775278568268,
"logits/rejected": 0.3589476943016052,
"logps/chosen": -81.91683959960938,
"logps/ref_chosen": -63.39509582519531,
"logps/ref_rejected": -76.20973205566406,
"logps/rejected": -107.25199127197266,
"loss": 1.0172,
"margin_dpo/margin_mean": 12.520517349243164,
"margin_dpo/margin_std": 17.281801223754883,
"step": 341
},
{
"KL/chosen_KL_mean": -18.645898818969727,
"KL/mean": -25.731111526489258,
"KL/rejected_KL_mean": -32.81632995605469,
"KL/std": 17.429080963134766,
"epoch": 0.5170068027210885,
"fcm_dpo/beta": 0.06906507164239883,
"fcm_dpo/delta": -0.19917050004005432,
"fcm_dpo/margin": 14.170430183410645,
"fcm_dpo/q_t": 0.3246821165084839,
"grad_norm": 20.034122467041016,
"learning_rate": 2.8033609524527046e-07,
"logits/chosen": 0.3918822407722473,
"logits/rejected": 0.3495151698589325,
"logps/chosen": -71.69371032714844,
"logps/ref_chosen": -53.047813415527344,
"logps/ref_rejected": -68.2854232788086,
"logps/rejected": -101.10175323486328,
"loss": 0.8962,
"margin_dpo/margin_mean": 14.170430183410645,
"margin_dpo/margin_std": 17.488643646240234,
"step": 342
},
{
"KL/chosen_KL_mean": -17.64687728881836,
"KL/mean": -21.812679290771484,
"KL/rejected_KL_mean": -25.97848129272461,
"KL/std": 16.159303665161133,
"epoch": 0.5185185185185185,
"fcm_dpo/beta": 0.06995899975299835,
"fcm_dpo/delta": 0.2293698936700821,
"fcm_dpo/margin": 8.33160400390625,
"fcm_dpo/q_t": 0.38500741124153137,
"grad_norm": 24.00664710998535,
"learning_rate": 2.7902322853130753e-07,
"logits/chosen": 0.2996237874031067,
"logits/rejected": 0.2916584610939026,
"logps/chosen": -88.22540283203125,
"logps/ref_chosen": -70.57852935791016,
"logps/ref_rejected": -84.73873901367188,
"logps/rejected": -110.71722412109375,
"loss": 1.1325,
"margin_dpo/margin_mean": 8.33160400390625,
"margin_dpo/margin_std": 14.704826354980469,
"step": 343
},
{
"KL/chosen_KL_mean": -21.23578453063965,
"KL/mean": -27.21549415588379,
"KL/rejected_KL_mean": -33.19519805908203,
"KL/std": 16.221303939819336,
"epoch": 0.5200302343159486,
"fcm_dpo/beta": 0.07086661458015442,
"fcm_dpo/delta": -0.052916139364242554,
"fcm_dpo/margin": 11.959415435791016,
"fcm_dpo/q_t": 0.3419157862663269,
"grad_norm": 26.274948120117188,
"learning_rate": 2.7770954997525274e-07,
"logits/chosen": 0.39441317319869995,
"logits/rejected": 0.3343184292316437,
"logps/chosen": -77.04679107666016,
"logps/ref_chosen": -55.811004638671875,
"logps/ref_rejected": -84.77637481689453,
"logps/rejected": -117.97157287597656,
"loss": 0.9865,
"margin_dpo/margin_mean": 11.959415435791016,
"margin_dpo/margin_std": 16.728431701660156,
"step": 344
},
{
"KL/chosen_KL_mean": -15.331901550292969,
"KL/mean": -21.359270095825195,
"KL/rejected_KL_mean": -27.386638641357422,
"KL/std": 15.091092109680176,
"epoch": 0.5215419501133787,
"fcm_dpo/beta": 0.0703570544719696,
"fcm_dpo/delta": -0.05284047871828079,
"fcm_dpo/margin": 12.054736137390137,
"fcm_dpo/q_t": 0.34237420558929443,
"grad_norm": 22.506139755249023,
"learning_rate": 2.7639509632351927e-07,
"logits/chosen": 0.4815177321434021,
"logits/rejected": 0.4361530840396881,
"logps/chosen": -73.11799621582031,
"logps/ref_chosen": -57.78609848022461,
"logps/ref_rejected": -78.91847229003906,
"logps/rejected": -106.30511474609375,
"loss": 0.9907,
"margin_dpo/margin_mean": 12.054737091064453,
"margin_dpo/margin_std": 17.231456756591797,
"step": 345
},
{
"KL/chosen_KL_mean": -17.275279998779297,
"KL/mean": -24.348899841308594,
"KL/rejected_KL_mean": -31.422523498535156,
"KL/std": 17.06797218322754,
"epoch": 0.5230536659108088,
"fcm_dpo/beta": 0.0688161551952362,
"fcm_dpo/delta": -0.191911518573761,
"fcm_dpo/margin": 14.147241592407227,
"fcm_dpo/q_t": 0.32509636878967285,
"grad_norm": 24.509672164916992,
"learning_rate": 2.7507990434420123e-07,
"logits/chosen": 0.4331340193748474,
"logits/rejected": 0.36041998863220215,
"logps/chosen": -73.56040954589844,
"logps/ref_chosen": -56.285125732421875,
"logps/ref_rejected": -91.15303039550781,
"logps/rejected": -122.5755615234375,
"loss": 0.9438,
"margin_dpo/margin_mean": 14.147241592407227,
"margin_dpo/margin_std": 18.288009643554688,
"step": 346
},
{
"KL/chosen_KL_mean": -20.0422306060791,
"KL/mean": -25.513660430908203,
"KL/rejected_KL_mean": -30.98508071899414,
"KL/std": 17.32723617553711,
"epoch": 0.5245653817082389,
"fcm_dpo/beta": 0.06831781566143036,
"fcm_dpo/delta": 0.05656781792640686,
"fcm_dpo/margin": 10.942852973937988,
"fcm_dpo/q_t": 0.36600062251091003,
"grad_norm": 24.8229923248291,
"learning_rate": 2.737640108260456e-07,
"logits/chosen": 0.5241929292678833,
"logits/rejected": 0.47482144832611084,
"logps/chosen": -73.5417709350586,
"logps/ref_chosen": -53.499542236328125,
"logps/ref_rejected": -72.52565002441406,
"logps/rejected": -103.51073455810547,
"loss": 1.047,
"margin_dpo/margin_mean": 10.942852973937988,
"margin_dpo/margin_std": 17.520071029663086,
"step": 347
},
{
"KL/chosen_KL_mean": -16.621702194213867,
"KL/mean": -23.973649978637695,
"KL/rejected_KL_mean": -31.325599670410156,
"KL/std": 17.199548721313477,
"epoch": 0.5260770975056689,
"fcm_dpo/beta": 0.06601608544588089,
"fcm_dpo/delta": -0.1918950080871582,
"fcm_dpo/margin": 14.703897476196289,
"fcm_dpo/q_t": 0.3412366807460785,
"grad_norm": 21.577510833740234,
"learning_rate": 2.724474525774229e-07,
"logits/chosen": 0.5218849182128906,
"logits/rejected": 0.4909231662750244,
"logps/chosen": -67.40855407714844,
"logps/ref_chosen": -50.78684997558594,
"logps/ref_rejected": -68.63732147216797,
"logps/rejected": -99.96292114257812,
"loss": 0.9651,
"margin_dpo/margin_mean": 14.703897476196289,
"margin_dpo/margin_std": 20.968624114990234,
"step": 348
},
{
"KL/chosen_KL_mean": -17.335826873779297,
"KL/mean": -24.095809936523438,
"KL/rejected_KL_mean": -30.855792999267578,
"KL/std": 17.796833038330078,
"epoch": 0.527588813303099,
"fcm_dpo/beta": 0.06530401110649109,
"fcm_dpo/delta": -0.09097161144018173,
"fcm_dpo/margin": 13.519969940185547,
"fcm_dpo/q_t": 0.3345542550086975,
"grad_norm": 23.182514190673828,
"learning_rate": 2.711302664252973e-07,
"logits/chosen": 0.43678218126296997,
"logits/rejected": 0.3523603677749634,
"logps/chosen": -70.66083526611328,
"logps/ref_chosen": -53.325008392333984,
"logps/ref_rejected": -83.21236419677734,
"logps/rejected": -114.06816101074219,
"loss": 0.9442,
"margin_dpo/margin_mean": 13.519969940185547,
"margin_dpo/margin_std": 17.63075065612793,
"step": 349
},
{
"KL/chosen_KL_mean": -19.102840423583984,
"KL/mean": -27.495264053344727,
"KL/rejected_KL_mean": -35.88768768310547,
"KL/std": 18.53017807006836,
"epoch": 0.5291005291005291,
"fcm_dpo/beta": 0.06193181127309799,
"fcm_dpo/delta": -0.2687161862850189,
"fcm_dpo/margin": 16.784847259521484,
"fcm_dpo/q_t": 0.3046181797981262,
"grad_norm": 22.8939208984375,
"learning_rate": 2.698124892141971e-07,
"logits/chosen": 0.3871225416660309,
"logits/rejected": 0.31201279163360596,
"logps/chosen": -80.72860717773438,
"logps/ref_chosen": -61.625770568847656,
"logps/ref_rejected": -87.63627624511719,
"logps/rejected": -123.52395629882812,
"loss": 0.8411,
"margin_dpo/margin_mean": 16.784847259521484,
"margin_dpo/margin_std": 18.485410690307617,
"step": 350
},
{
"KL/chosen_KL_mean": -17.614654541015625,
"KL/mean": -24.530780792236328,
"KL/rejected_KL_mean": -31.4469051361084,
"KL/std": 16.754365921020508,
"epoch": 0.5306122448979592,
"fcm_dpo/beta": 0.06079188734292984,
"fcm_dpo/delta": -0.04478435963392258,
"fcm_dpo/margin": 13.832258224487305,
"fcm_dpo/q_t": 0.33148348331451416,
"grad_norm": 19.812429428100586,
"learning_rate": 2.6849415780518357e-07,
"logits/chosen": 0.36037567257881165,
"logits/rejected": 0.28710806369781494,
"logps/chosen": -73.87098693847656,
"logps/ref_chosen": -56.2563362121582,
"logps/ref_rejected": -79.11589813232422,
"logps/rejected": -110.56280517578125,
"loss": 1.0035,
"margin_dpo/margin_mean": 13.832258224487305,
"margin_dpo/margin_std": 19.36257553100586,
"step": 351
},
{
"KL/chosen_KL_mean": -17.411983489990234,
"KL/mean": -23.69668960571289,
"KL/rejected_KL_mean": -29.98139190673828,
"KL/std": 16.408573150634766,
"epoch": 0.5321239606953893,
"fcm_dpo/beta": 0.0607098788022995,
"fcm_dpo/delta": 0.039311863481998444,
"fcm_dpo/margin": 12.569415092468262,
"fcm_dpo/q_t": 0.3572866916656494,
"grad_norm": 21.984020233154297,
"learning_rate": 2.6717530907482024e-07,
"logits/chosen": 0.4072480797767639,
"logits/rejected": 0.35473206639289856,
"logps/chosen": -80.46393585205078,
"logps/ref_chosen": -63.05195236206055,
"logps/ref_rejected": -85.52035522460938,
"logps/rejected": -115.50174713134766,
"loss": 0.9818,
"margin_dpo/margin_mean": 12.569416046142578,
"margin_dpo/margin_std": 17.520357131958008,
"step": 352
},
{
"KL/chosen_KL_mean": -15.802751541137695,
"KL/mean": -22.807369232177734,
"KL/rejected_KL_mean": -29.811988830566406,
"KL/std": 17.35757064819336,
"epoch": 0.5336356764928194,
"fcm_dpo/beta": 0.061134468764066696,
"fcm_dpo/delta": -0.06244935095310211,
"fcm_dpo/margin": 14.009233474731445,
"fcm_dpo/q_t": 0.3378174304962158,
"grad_norm": 21.547151565551758,
"learning_rate": 2.658559799141411e-07,
"logits/chosen": 0.42149317264556885,
"logits/rejected": 0.42055612802505493,
"logps/chosen": -84.81193542480469,
"logps/ref_chosen": -69.00918579101562,
"logps/ref_rejected": -72.65840148925781,
"logps/rejected": -102.47039031982422,
"loss": 0.9734,
"margin_dpo/margin_mean": 14.009233474731445,
"margin_dpo/margin_std": 19.090660095214844,
"step": 353
},
{
"KL/chosen_KL_mean": -16.668289184570312,
"KL/mean": -24.297048568725586,
"KL/rejected_KL_mean": -31.925806045532227,
"KL/std": 16.85162353515625,
"epoch": 0.5351473922902494,
"fcm_dpo/beta": 0.05897783115506172,
"fcm_dpo/delta": -0.11220181733369827,
"fcm_dpo/margin": 15.25751781463623,
"fcm_dpo/q_t": 0.3294384777545929,
"grad_norm": 23.117361068725586,
"learning_rate": 2.6453620722761895e-07,
"logits/chosen": 0.47396335005760193,
"logits/rejected": 0.3462182283401489,
"logps/chosen": -56.456626892089844,
"logps/ref_chosen": -39.78833770751953,
"logps/ref_rejected": -69.56885528564453,
"logps/rejected": -101.49465942382812,
"loss": 0.9378,
"margin_dpo/margin_mean": 15.25751781463623,
"margin_dpo/margin_std": 19.276477813720703,
"step": 354
},
{
"KL/chosen_KL_mean": -19.132593154907227,
"KL/mean": -26.75240707397461,
"KL/rejected_KL_mean": -34.372222900390625,
"KL/std": 19.179889678955078,
"epoch": 0.5366591080876795,
"fcm_dpo/beta": 0.0584687814116478,
"fcm_dpo/delta": -0.09985160082578659,
"fcm_dpo/margin": 15.239628791809082,
"fcm_dpo/q_t": 0.33161279559135437,
"grad_norm": 22.381999969482422,
"learning_rate": 2.632160279321328e-07,
"logits/chosen": 0.4477384388446808,
"logits/rejected": 0.32458215951919556,
"logps/chosen": -65.38796997070312,
"logps/ref_chosen": -46.25537872314453,
"logps/ref_rejected": -78.20236206054688,
"logps/rejected": -112.5745849609375,
"loss": 0.9671,
"margin_dpo/margin_mean": 15.239627838134766,
"margin_dpo/margin_std": 20.278135299682617,
"step": 355
},
{
"KL/chosen_KL_mean": -17.08836555480957,
"KL/mean": -23.33837127685547,
"KL/rejected_KL_mean": -29.588375091552734,
"KL/std": 18.691139221191406,
"epoch": 0.5381708238851096,
"fcm_dpo/beta": 0.058297883719205856,
"fcm_dpo/delta": 0.076647087931633,
"fcm_dpo/margin": 12.500007629394531,
"fcm_dpo/q_t": 0.3649879992008209,
"grad_norm": 22.381298065185547,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": 0.4272801876068115,
"logits/rejected": 0.35394901037216187,
"logps/chosen": -64.99452209472656,
"logps/ref_chosen": -47.906158447265625,
"logps/ref_rejected": -74.29397583007812,
"logps/rejected": -103.8823471069336,
"loss": 1.1716,
"margin_dpo/margin_mean": 12.500007629394531,
"margin_dpo/margin_std": 23.757095336914062,
"step": 356
},
{
"KL/chosen_KL_mean": -20.82411766052246,
"KL/mean": -26.578861236572266,
"KL/rejected_KL_mean": -32.33361053466797,
"KL/std": 18.383647918701172,
"epoch": 0.5396825396825397,
"fcm_dpo/beta": 0.05899185687303543,
"fcm_dpo/delta": 0.12616032361984253,
"fcm_dpo/margin": 11.509489059448242,
"fcm_dpo/q_t": 0.37107419967651367,
"grad_norm": 23.38809585571289,
"learning_rate": 2.6057459723762076e-07,
"logits/chosen": 0.43112170696258545,
"logits/rejected": 0.40187525749206543,
"logps/chosen": -83.45912170410156,
"logps/ref_chosen": -62.63500213623047,
"logps/ref_rejected": -65.11399841308594,
"logps/rejected": -97.4476089477539,
"loss": 1.0963,
"margin_dpo/margin_mean": 11.509489059448242,
"margin_dpo/margin_std": 18.848796844482422,
"step": 357
},
{
"KL/chosen_KL_mean": -18.612560272216797,
"KL/mean": -26.671905517578125,
"KL/rejected_KL_mean": -34.73124694824219,
"KL/std": 18.668180465698242,
"epoch": 0.5411942554799698,
"fcm_dpo/beta": 0.059323687106370926,
"fcm_dpo/delta": -0.1726943552494049,
"fcm_dpo/margin": 16.118690490722656,
"fcm_dpo/q_t": 0.32745832204818726,
"grad_norm": 23.334945678710938,
"learning_rate": 2.5925341972508954e-07,
"logits/chosen": 0.3674103021621704,
"logits/rejected": 0.3703601658344269,
"logps/chosen": -85.82217407226562,
"logps/ref_chosen": -67.20960998535156,
"logps/ref_rejected": -69.34715270996094,
"logps/rejected": -104.07839965820312,
"loss": 0.9313,
"margin_dpo/margin_mean": 16.118690490722656,
"margin_dpo/margin_std": 20.610336303710938,
"step": 358
},
{
"KL/chosen_KL_mean": -20.406143188476562,
"KL/mean": -25.056005477905273,
"KL/rejected_KL_mean": -29.705869674682617,
"KL/std": 18.56830596923828,
"epoch": 0.5427059712773998,
"fcm_dpo/beta": 0.06133756786584854,
"fcm_dpo/delta": 0.23471681773662567,
"fcm_dpo/margin": 9.29973030090332,
"fcm_dpo/q_t": 0.39298108220100403,
"grad_norm": 25.49129867553711,
"learning_rate": 2.579319833745169e-07,
"logits/chosen": 0.3720915913581848,
"logits/rejected": 0.3445711135864258,
"logps/chosen": -82.93193054199219,
"logps/ref_chosen": -62.52578353881836,
"logps/ref_rejected": -76.63114929199219,
"logps/rejected": -106.33702087402344,
"loss": 1.1465,
"margin_dpo/margin_mean": 9.29973030090332,
"margin_dpo/margin_std": 17.169992446899414,
"step": 359
},
{
"KL/chosen_KL_mean": -20.363914489746094,
"KL/mean": -26.988189697265625,
"KL/rejected_KL_mean": -33.612464904785156,
"KL/std": 18.771831512451172,
"epoch": 0.54421768707483,
"fcm_dpo/beta": 0.060898374766111374,
"fcm_dpo/delta": -0.007558280602097511,
"fcm_dpo/margin": 13.248543739318848,
"fcm_dpo/q_t": 0.3549214005470276,
"grad_norm": 23.553117752075195,
"learning_rate": 2.5661032514931834e-07,
"logits/chosen": 0.33502113819122314,
"logits/rejected": 0.2443276047706604,
"logps/chosen": -83.85163879394531,
"logps/ref_chosen": -63.48772048950195,
"logps/ref_rejected": -90.6891098022461,
"logps/rejected": -124.30157470703125,
"loss": 0.9896,
"margin_dpo/margin_mean": 13.248543739318848,
"margin_dpo/margin_std": 19.227291107177734,
"step": 360
},
{
"KL/chosen_KL_mean": -19.366771697998047,
"KL/mean": -27.24359893798828,
"KL/rejected_KL_mean": -35.12042999267578,
"KL/std": 19.667043685913086,
"epoch": 0.54572940287226,
"fcm_dpo/beta": 0.059625256806612015,
"fcm_dpo/delta": -0.15384510159492493,
"fcm_dpo/margin": 15.753656387329102,
"fcm_dpo/q_t": 0.32015830278396606,
"grad_norm": 23.828569412231445,
"learning_rate": 2.552884820191154e-07,
"logits/chosen": 0.44018304347991943,
"logits/rejected": 0.38934749364852905,
"logps/chosen": -77.28392028808594,
"logps/ref_chosen": -57.917144775390625,
"logps/ref_rejected": -72.39089965820312,
"logps/rejected": -107.5113296508789,
"loss": 0.8998,
"margin_dpo/margin_mean": 15.753657341003418,
"margin_dpo/margin_std": 19.081100463867188,
"step": 361
},
{
"KL/chosen_KL_mean": -18.872962951660156,
"KL/mean": -26.762571334838867,
"KL/rejected_KL_mean": -34.65217590332031,
"KL/std": 18.649995803833008,
"epoch": 0.54724111866969,
"fcm_dpo/beta": 0.058759547770023346,
"fcm_dpo/delta": -0.14217320084571838,
"fcm_dpo/margin": 15.779216766357422,
"fcm_dpo/q_t": 0.32848185300827026,
"grad_norm": 21.277912139892578,
"learning_rate": 2.53966490958702e-07,
"logits/chosen": 0.49138182401657104,
"logits/rejected": 0.38976341485977173,
"logps/chosen": -82.31643676757812,
"logps/ref_chosen": -63.4434700012207,
"logps/ref_rejected": -103.45516967773438,
"logps/rejected": -138.1073455810547,
"loss": 0.9292,
"margin_dpo/margin_mean": 15.779217720031738,
"margin_dpo/margin_std": 19.549943923950195,
"step": 362
},
{
"KL/chosen_KL_mean": -22.18973159790039,
"KL/mean": -30.150291442871094,
"KL/rejected_KL_mean": -38.11085510253906,
"KL/std": 19.200645446777344,
"epoch": 0.5487528344671202,
"fcm_dpo/beta": 0.05654866620898247,
"fcm_dpo/delta": -0.11033031344413757,
"fcm_dpo/margin": 15.921126365661621,
"fcm_dpo/q_t": 0.3285680413246155,
"grad_norm": 20.080604553222656,
"learning_rate": 2.526443889470099e-07,
"logits/chosen": 0.45604056119918823,
"logits/rejected": 0.3346443176269531,
"logps/chosen": -70.84156036376953,
"logps/ref_chosen": -48.65182876586914,
"logps/ref_rejected": -88.65904235839844,
"logps/rejected": -126.7698974609375,
"loss": 0.9166,
"margin_dpo/margin_mean": 15.921126365661621,
"margin_dpo/margin_std": 19.751773834228516,
"step": 363
},
{
"KL/chosen_KL_mean": -18.949443817138672,
"KL/mean": -27.227487564086914,
"KL/rejected_KL_mean": -35.505531311035156,
"KL/std": 20.297557830810547,
"epoch": 0.5502645502645502,
"fcm_dpo/beta": 0.055115051567554474,
"fcm_dpo/delta": -0.12417322397232056,
"fcm_dpo/margin": 16.55609130859375,
"fcm_dpo/q_t": 0.33811190724372864,
"grad_norm": 18.80527114868164,
"learning_rate": 2.513222129660744e-07,
"logits/chosen": 0.3324137330055237,
"logits/rejected": 0.23981907963752747,
"logps/chosen": -76.82051849365234,
"logps/ref_chosen": -57.87107467651367,
"logps/ref_rejected": -80.95503234863281,
"logps/rejected": -116.46056365966797,
"loss": 1.0054,
"margin_dpo/margin_mean": 16.55609130859375,
"margin_dpo/margin_std": 24.270000457763672,
"step": 364
},
{
"KL/chosen_KL_mean": -16.02554702758789,
"KL/mean": -23.708436965942383,
"KL/rejected_KL_mean": -31.391326904296875,
"KL/std": 20.104900360107422,
"epoch": 0.5517762660619804,
"fcm_dpo/beta": 0.054510001093149185,
"fcm_dpo/delta": -0.04106505215167999,
"fcm_dpo/margin": 15.365779876708984,
"fcm_dpo/q_t": 0.33312904834747314,
"grad_norm": 18.545867919921875,
"learning_rate": 2.5e-07,
"logits/chosen": 0.44149670004844666,
"logits/rejected": 0.4310864508152008,
"logps/chosen": -80.96772766113281,
"logps/ref_chosen": -64.94217681884766,
"logps/ref_rejected": -74.8599853515625,
"logps/rejected": -106.25131225585938,
"loss": 0.9047,
"margin_dpo/margin_mean": 15.365779876708984,
"margin_dpo/margin_std": 17.665653228759766,
"step": 365
},
{
"KL/chosen_KL_mean": -14.832332611083984,
"KL/mean": -21.632671356201172,
"KL/rejected_KL_mean": -28.43301010131836,
"KL/std": 17.586444854736328,
"epoch": 0.5532879818594104,
"fcm_dpo/beta": 0.05539874732494354,
"fcm_dpo/delta": 0.047521963715553284,
"fcm_dpo/margin": 13.600683212280273,
"fcm_dpo/q_t": 0.3618229031562805,
"grad_norm": 21.13431739807129,
"learning_rate": 2.486777870339255e-07,
"logits/chosen": 0.37482506036758423,
"logits/rejected": 0.35465550422668457,
"logps/chosen": -69.9983139038086,
"logps/ref_chosen": -55.16598129272461,
"logps/ref_rejected": -65.26121520996094,
"logps/rejected": -93.69422149658203,
"loss": 1.0611,
"margin_dpo/margin_mean": 13.600683212280273,
"margin_dpo/margin_std": 21.480377197265625,
"step": 366
},
{
"KL/chosen_KL_mean": -19.378307342529297,
"KL/mean": -25.97715187072754,
"KL/rejected_KL_mean": -32.575992584228516,
"KL/std": 19.300640106201172,
"epoch": 0.5547996976568406,
"fcm_dpo/beta": 0.055359289050102234,
"fcm_dpo/delta": 0.07484433799982071,
"fcm_dpo/margin": 13.197684288024902,
"fcm_dpo/q_t": 0.35306018590927124,
"grad_norm": 21.098054885864258,
"learning_rate": 2.4735561105299014e-07,
"logits/chosen": 0.3957292139530182,
"logits/rejected": 0.29518669843673706,
"logps/chosen": -75.3887710571289,
"logps/ref_chosen": -56.01046371459961,
"logps/ref_rejected": -77.31010437011719,
"logps/rejected": -109.88610076904297,
"loss": 1.0177,
"margin_dpo/margin_mean": 13.197683334350586,
"margin_dpo/margin_std": 19.19463539123535,
"step": 367
},
{
"KL/chosen_KL_mean": -21.07410430908203,
"KL/mean": -27.410612106323242,
"KL/rejected_KL_mean": -33.74711990356445,
"KL/std": 18.693862915039062,
"epoch": 0.5563114134542706,
"fcm_dpo/beta": 0.056563593447208405,
"fcm_dpo/delta": 0.0895220935344696,
"fcm_dpo/margin": 12.673014640808105,
"fcm_dpo/q_t": 0.36096638441085815,
"grad_norm": 22.17424964904785,
"learning_rate": 2.46033509041298e-07,
"logits/chosen": 0.2558997869491577,
"logits/rejected": 0.24736079573631287,
"logps/chosen": -95.90338134765625,
"logps/ref_chosen": -74.82927703857422,
"logps/ref_rejected": -76.11680603027344,
"logps/rejected": -109.86392211914062,
"loss": 1.0518,
"margin_dpo/margin_mean": 12.673015594482422,
"margin_dpo/margin_std": 19.6419620513916,
"step": 368
},
{
"KL/chosen_KL_mean": -20.30316162109375,
"KL/mean": -26.592472076416016,
"KL/rejected_KL_mean": -32.88178253173828,
"KL/std": 19.905651092529297,
"epoch": 0.5578231292517006,
"fcm_dpo/beta": 0.057878073304891586,
"fcm_dpo/delta": 0.07577980309724808,
"fcm_dpo/margin": 12.578624725341797,
"fcm_dpo/q_t": 0.36213284730911255,
"grad_norm": 21.1649112701416,
"learning_rate": 2.447115179808846e-07,
"logits/chosen": 0.4168482720851898,
"logits/rejected": 0.3676788806915283,
"logps/chosen": -78.62937927246094,
"logps/ref_chosen": -58.32621765136719,
"logps/ref_rejected": -80.92183685302734,
"logps/rejected": -113.80361938476562,
"loss": 1.052,
"margin_dpo/margin_mean": 12.578624725341797,
"margin_dpo/margin_std": 19.676862716674805,
"step": 369
},
{
"KL/chosen_KL_mean": -19.603811264038086,
"KL/mean": -27.947715759277344,
"KL/rejected_KL_mean": -36.29161834716797,
"KL/std": 21.021804809570312,
"epoch": 0.5593348450491308,
"fcm_dpo/beta": 0.05625259131193161,
"fcm_dpo/delta": -0.15349817276000977,
"fcm_dpo/margin": 16.687808990478516,
"fcm_dpo/q_t": 0.3311444818973541,
"grad_norm": 21.677335739135742,
"learning_rate": 2.4338967485068164e-07,
"logits/chosen": 0.49989569187164307,
"logits/rejected": 0.4341312050819397,
"logps/chosen": -72.48753356933594,
"logps/ref_chosen": -52.88372039794922,
"logps/ref_rejected": -79.43692016601562,
"logps/rejected": -115.7285385131836,
"loss": 1.0079,
"margin_dpo/margin_mean": 16.687808990478516,
"margin_dpo/margin_std": 24.34197235107422,
"step": 370
},
{
"KL/chosen_KL_mean": -19.531940460205078,
"KL/mean": -27.286758422851562,
"KL/rejected_KL_mean": -35.04158020019531,
"KL/std": 20.940555572509766,
"epoch": 0.5608465608465608,
"fcm_dpo/beta": 0.055904123932123184,
"fcm_dpo/delta": -0.07506079971790314,
"fcm_dpo/margin": 15.509639739990234,
"fcm_dpo/q_t": 0.34340009093284607,
"grad_norm": 21.58955192565918,
"learning_rate": 2.420680166254831e-07,
"logits/chosen": 0.5270491242408752,
"logits/rejected": 0.49259671568870544,
"logps/chosen": -68.75614929199219,
"logps/ref_chosen": -49.224212646484375,
"logps/ref_rejected": -63.348472595214844,
"logps/rejected": -98.39005279541016,
"loss": 0.986,
"margin_dpo/margin_mean": 15.509639739990234,
"margin_dpo/margin_std": 21.508487701416016,
"step": 371
},
{
"KL/chosen_KL_mean": -21.663251876831055,
"KL/mean": -26.70006561279297,
"KL/rejected_KL_mean": -31.736879348754883,
"KL/std": 19.95144271850586,
"epoch": 0.562358276643991,
"fcm_dpo/beta": 0.057659346610307693,
"fcm_dpo/delta": 0.22692981362342834,
"fcm_dpo/margin": 10.073625564575195,
"fcm_dpo/q_t": 0.3950161337852478,
"grad_norm": 28.074542999267578,
"learning_rate": 2.4074658027491044e-07,
"logits/chosen": 0.4539080858230591,
"logits/rejected": 0.3669462203979492,
"logps/chosen": -73.93280029296875,
"logps/ref_chosen": -52.269554138183594,
"logps/ref_rejected": -72.99522399902344,
"logps/rejected": -104.73210144042969,
"loss": 1.3033,
"margin_dpo/margin_mean": 10.073625564575195,
"margin_dpo/margin_std": 24.8012752532959,
"step": 372
},
{
"KL/chosen_KL_mean": -24.116884231567383,
"KL/mean": -30.572702407836914,
"KL/rejected_KL_mean": -37.02852249145508,
"KL/std": 20.55548858642578,
"epoch": 0.563869992441421,
"fcm_dpo/beta": 0.058202799409627914,
"fcm_dpo/delta": 0.052256401628255844,
"fcm_dpo/margin": 12.911628723144531,
"fcm_dpo/q_t": 0.37231653928756714,
"grad_norm": 31.09714126586914,
"learning_rate": 2.394254027623792e-07,
"logits/chosen": 0.4642970860004425,
"logits/rejected": 0.3941759765148163,
"logps/chosen": -85.22988891601562,
"logps/ref_chosen": -61.112998962402344,
"logps/ref_rejected": -76.24851989746094,
"logps/rejected": -113.27703857421875,
"loss": 1.1794,
"margin_dpo/margin_mean": 12.911629676818848,
"margin_dpo/margin_std": 25.102832794189453,
"step": 373
},
{
"KL/chosen_KL_mean": -20.470600128173828,
"KL/mean": -30.454559326171875,
"KL/rejected_KL_mean": -40.43851852416992,
"KL/std": 20.557178497314453,
"epoch": 0.5653817082388511,
"fcm_dpo/beta": 0.05505363643169403,
"fcm_dpo/delta": -0.3393453061580658,
"fcm_dpo/margin": 19.967914581298828,
"fcm_dpo/q_t": 0.2875816822052002,
"grad_norm": 24.58673858642578,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": 0.33565571904182434,
"logits/rejected": 0.32568567991256714,
"logps/chosen": -93.13980102539062,
"logps/ref_chosen": -72.66920471191406,
"logps/ref_rejected": -76.83158874511719,
"logps/rejected": -117.27011108398438,
"loss": 0.787,
"margin_dpo/margin_mean": 19.967914581298828,
"margin_dpo/margin_std": 19.655311584472656,
"step": 374
},
{
"KL/chosen_KL_mean": -19.648855209350586,
"KL/mean": -25.847253799438477,
"KL/rejected_KL_mean": -32.04564666748047,
"KL/std": 20.406822204589844,
"epoch": 0.5668934240362812,
"fcm_dpo/beta": 0.055172353982925415,
"fcm_dpo/delta": 0.12467709928750992,
"fcm_dpo/margin": 12.396795272827148,
"fcm_dpo/q_t": 0.3717153072357178,
"grad_norm": 24.862714767456055,
"learning_rate": 2.3678397206786715e-07,
"logits/chosen": 0.44463562965393066,
"logits/rejected": 0.3905963599681854,
"logps/chosen": -77.33216094970703,
"logps/ref_chosen": -57.68330383300781,
"logps/ref_rejected": -79.34097290039062,
"logps/rejected": -111.3866195678711,
"loss": 1.0856,
"margin_dpo/margin_mean": 12.396795272827148,
"margin_dpo/margin_std": 20.809398651123047,
"step": 375
},
{
"KL/chosen_KL_mean": -21.62276268005371,
"KL/mean": -31.52696418762207,
"KL/rejected_KL_mean": -41.43116760253906,
"KL/std": 21.306678771972656,
"epoch": 0.5684051398337112,
"fcm_dpo/beta": 0.053182363510131836,
"fcm_dpo/delta": -0.285739004611969,
"fcm_dpo/margin": 19.808406829833984,
"fcm_dpo/q_t": 0.3129886984825134,
"grad_norm": 21.40363883972168,
"learning_rate": 2.3546379277238103e-07,
"logits/chosen": 0.47032493352890015,
"logits/rejected": 0.40646880865097046,
"logps/chosen": -73.29682922363281,
"logps/ref_chosen": -51.674072265625,
"logps/ref_rejected": -75.69713592529297,
"logps/rejected": -117.12830352783203,
"loss": 0.9135,
"margin_dpo/margin_mean": 19.808406829833984,
"margin_dpo/margin_std": 24.855318069458008,
"step": 376
},
{
"KL/chosen_KL_mean": -22.585697174072266,
"KL/mean": -29.564029693603516,
"KL/rejected_KL_mean": -36.54236602783203,
"KL/std": 20.325942993164062,
"epoch": 0.5699168556311414,
"fcm_dpo/beta": 0.053586531430482864,
"fcm_dpo/delta": 0.054271575063467026,
"fcm_dpo/margin": 13.956663131713867,
"fcm_dpo/q_t": 0.35530292987823486,
"grad_norm": 20.946025848388672,
"learning_rate": 2.3414402008585886e-07,
"logits/chosen": 0.41215571761131287,
"logits/rejected": 0.3891144394874573,
"logps/chosen": -68.76422882080078,
"logps/ref_chosen": -46.17853546142578,
"logps/ref_rejected": -57.756500244140625,
"logps/rejected": -94.29886627197266,
"loss": 1.0308,
"margin_dpo/margin_mean": 13.956663131713867,
"margin_dpo/margin_std": 20.671688079833984,
"step": 377
},
{
"KL/chosen_KL_mean": -21.790258407592773,
"KL/mean": -27.98929214477539,
"KL/rejected_KL_mean": -34.188323974609375,
"KL/std": 21.171764373779297,
"epoch": 0.5714285714285714,
"fcm_dpo/beta": 0.054639630019664764,
"fcm_dpo/delta": 0.12991301715373993,
"fcm_dpo/margin": 12.398064613342285,
"fcm_dpo/q_t": 0.3740558624267578,
"grad_norm": 23.512828826904297,
"learning_rate": 2.3282469092517977e-07,
"logits/chosen": 0.46384197473526,
"logits/rejected": 0.41567403078079224,
"logps/chosen": -81.00914001464844,
"logps/ref_chosen": -59.21887969970703,
"logps/ref_rejected": -71.24818420410156,
"logps/rejected": -105.43650817871094,
"loss": 1.1043,
"margin_dpo/margin_mean": 12.398064613342285,
"margin_dpo/margin_std": 21.77179718017578,
"step": 378
},
{
"KL/chosen_KL_mean": -19.731494903564453,
"KL/mean": -28.210784912109375,
"KL/rejected_KL_mean": -36.690086364746094,
"KL/std": 20.615100860595703,
"epoch": 0.5729402872260015,
"fcm_dpo/beta": 0.05373363196849823,
"fcm_dpo/delta": -0.12232452630996704,
"fcm_dpo/margin": 16.95858383178711,
"fcm_dpo/q_t": 0.332636296749115,
"grad_norm": 23.33953285217285,
"learning_rate": 2.3150584219481643e-07,
"logits/chosen": 0.46484893560409546,
"logits/rejected": 0.39884310960769653,
"logps/chosen": -96.04808044433594,
"logps/ref_chosen": -76.31658935546875,
"logps/ref_rejected": -104.26200103759766,
"logps/rejected": -140.95208740234375,
"loss": 0.9318,
"margin_dpo/margin_mean": 16.95858383178711,
"margin_dpo/margin_std": 21.930681228637695,
"step": 379
},
{
"KL/chosen_KL_mean": -18.223163604736328,
"KL/mean": -27.74345588684082,
"KL/rejected_KL_mean": -37.26375198364258,
"KL/std": 19.516929626464844,
"epoch": 0.5744520030234316,
"fcm_dpo/beta": 0.05176670104265213,
"fcm_dpo/delta": -0.20594152808189392,
"fcm_dpo/margin": 19.04058837890625,
"fcm_dpo/q_t": 0.3121461868286133,
"grad_norm": 20.619007110595703,
"learning_rate": 2.3018751078580283e-07,
"logits/chosen": 0.454281210899353,
"logits/rejected": 0.4146312475204468,
"logps/chosen": -79.50633239746094,
"logps/ref_chosen": -61.283164978027344,
"logps/ref_rejected": -72.38892364501953,
"logps/rejected": -109.65267944335938,
"loss": 0.9093,
"margin_dpo/margin_mean": 19.04058837890625,
"margin_dpo/margin_std": 23.003185272216797,
"step": 380
},
{
"KL/chosen_KL_mean": -23.842905044555664,
"KL/mean": -27.962167739868164,
"KL/rejected_KL_mean": -32.081424713134766,
"KL/std": 20.596736907958984,
"epoch": 0.5759637188208617,
"fcm_dpo/beta": 0.051557689905166626,
"fcm_dpo/delta": 0.06958503276109695,
"fcm_dpo/margin": 8.238516807556152,
"fcm_dpo/q_t": 0.4186873435974121,
"grad_norm": 24.15900421142578,
"learning_rate": 2.288697335747027e-07,
"logits/chosen": 0.41163092851638794,
"logits/rejected": 0.385709285736084,
"logps/chosen": -82.05690002441406,
"logps/ref_chosen": -58.2139892578125,
"logps/ref_rejected": -60.78669357299805,
"logps/rejected": -92.86811828613281,
"loss": 1.2613,
"margin_dpo/margin_mean": 8.238516807556152,
"margin_dpo/margin_std": 20.93825912475586,
"step": 381
},
{
"KL/chosen_KL_mean": -24.551029205322266,
"KL/mean": -31.066709518432617,
"KL/rejected_KL_mean": -37.58238983154297,
"KL/std": 21.11056137084961,
"epoch": 0.5774754346182918,
"fcm_dpo/beta": 0.05340856313705444,
"fcm_dpo/delta": 0.1072634756565094,
"fcm_dpo/margin": 13.031362533569336,
"fcm_dpo/q_t": 0.3651576638221741,
"grad_norm": 22.464122772216797,
"learning_rate": 2.2755254742257706e-07,
"logits/chosen": 0.4638483226299286,
"logits/rejected": 0.4158809781074524,
"logps/chosen": -86.37635803222656,
"logps/ref_chosen": -61.82532501220703,
"logps/ref_rejected": -83.0452880859375,
"logps/rejected": -120.62767028808594,
"loss": 1.0163,
"margin_dpo/margin_mean": 13.031362533569336,
"margin_dpo/margin_std": 18.759021759033203,
"step": 382
},
{
"KL/chosen_KL_mean": -22.348957061767578,
"KL/mean": -30.717674255371094,
"KL/rejected_KL_mean": -39.086395263671875,
"KL/std": 21.498252868652344,
"epoch": 0.5789871504157218,
"fcm_dpo/beta": 0.05201186239719391,
"fcm_dpo/delta": -0.07921411097049713,
"fcm_dpo/margin": 16.737438201904297,
"fcm_dpo/q_t": 0.34389805793762207,
"grad_norm": 23.397573471069336,
"learning_rate": 2.2623598917395436e-07,
"logits/chosen": 0.33132052421569824,
"logits/rejected": 0.3495492935180664,
"logps/chosen": -102.91221618652344,
"logps/ref_chosen": -80.56326293945312,
"logps/ref_rejected": -74.62922668457031,
"logps/rejected": -113.71562194824219,
"loss": 1.0053,
"margin_dpo/margin_mean": 16.737438201904297,
"margin_dpo/margin_std": 24.382492065429688,
"step": 383
},
{
"KL/chosen_KL_mean": -23.70047950744629,
"KL/mean": -31.56975555419922,
"KL/rejected_KL_mean": -39.43903350830078,
"KL/std": 21.286376953125,
"epoch": 0.5804988662131519,
"fcm_dpo/beta": 0.05222079157829285,
"fcm_dpo/delta": -0.02404698356986046,
"fcm_dpo/margin": 15.738565444946289,
"fcm_dpo/q_t": 0.34051215648651123,
"grad_norm": 21.495080947875977,
"learning_rate": 2.2492009565579875e-07,
"logits/chosen": 0.44171950221061707,
"logits/rejected": 0.39739423990249634,
"logps/chosen": -89.17562866210938,
"logps/ref_chosen": -65.47514343261719,
"logps/ref_rejected": -79.67378234863281,
"logps/rejected": -119.11282348632812,
"loss": 0.9563,
"margin_dpo/margin_mean": 15.738564491271973,
"margin_dpo/margin_std": 20.684423446655273,
"step": 384
},
{
"KL/chosen_KL_mean": -22.995277404785156,
"KL/mean": -31.722030639648438,
"KL/rejected_KL_mean": -40.44878005981445,
"KL/std": 20.493146896362305,
"epoch": 0.582010582010582,
"fcm_dpo/beta": 0.05136442184448242,
"fcm_dpo/delta": -0.10590949654579163,
"fcm_dpo/margin": 17.45350456237793,
"fcm_dpo/q_t": 0.3273235857486725,
"grad_norm": 21.761558532714844,
"learning_rate": 2.2360490367648084e-07,
"logits/chosen": 0.382984459400177,
"logits/rejected": 0.3436170220375061,
"logps/chosen": -89.05178833007812,
"logps/ref_chosen": -66.0565185546875,
"logps/ref_rejected": -86.68023681640625,
"logps/rejected": -127.12901306152344,
"loss": 0.9035,
"margin_dpo/margin_mean": 17.453502655029297,
"margin_dpo/margin_std": 20.69217300415039,
"step": 385
},
{
"KL/chosen_KL_mean": -26.571060180664062,
"KL/mean": -32.992767333984375,
"KL/rejected_KL_mean": -39.41448211669922,
"KL/std": 21.929290771484375,
"epoch": 0.5835222978080121,
"fcm_dpo/beta": 0.05175807327032089,
"fcm_dpo/delta": 0.14512205123901367,
"fcm_dpo/margin": 12.843421936035156,
"fcm_dpo/q_t": 0.36995601654052734,
"grad_norm": 23.099040985107422,
"learning_rate": 2.2229045002474724e-07,
"logits/chosen": 0.38640153408050537,
"logits/rejected": 0.32749783992767334,
"logps/chosen": -102.19471740722656,
"logps/ref_chosen": -75.6236572265625,
"logps/ref_rejected": -92.62330627441406,
"logps/rejected": -132.0377960205078,
"loss": 1.0665,
"margin_dpo/margin_mean": 12.843421936035156,
"margin_dpo/margin_std": 20.646764755249023,
"step": 386
},
{
"KL/chosen_KL_mean": -23.750308990478516,
"KL/mean": -32.67223358154297,
"KL/rejected_KL_mean": -41.59415817260742,
"KL/std": 20.869110107421875,
"epoch": 0.5850340136054422,
"fcm_dpo/beta": 0.0514523983001709,
"fcm_dpo/delta": -0.12989971041679382,
"fcm_dpo/margin": 17.843847274780273,
"fcm_dpo/q_t": 0.3237932324409485,
"grad_norm": 20.114234924316406,
"learning_rate": 2.209767714686924e-07,
"logits/chosen": 0.4548864960670471,
"logits/rejected": 0.3586340546607971,
"logps/chosen": -70.97201538085938,
"logps/ref_chosen": -47.22170639038086,
"logps/ref_rejected": -87.338134765625,
"logps/rejected": -128.9322967529297,
"loss": 0.8648,
"margin_dpo/margin_mean": 17.84384536743164,
"margin_dpo/margin_std": 20.037647247314453,
"step": 387
},
{
"KL/chosen_KL_mean": -23.332077026367188,
"KL/mean": -29.797340393066406,
"KL/rejected_KL_mean": -36.26259994506836,
"KL/std": 20.760181427001953,
"epoch": 0.5865457294028723,
"fcm_dpo/beta": 0.051743343472480774,
"fcm_dpo/delta": 0.1403004229068756,
"fcm_dpo/margin": 12.930517196655273,
"fcm_dpo/q_t": 0.3804495930671692,
"grad_norm": 22.458322525024414,
"learning_rate": 2.1966390475472954e-07,
"logits/chosen": 0.46087580919265747,
"logits/rejected": 0.4473402202129364,
"logps/chosen": -97.91154479980469,
"logps/ref_chosen": -74.5794677734375,
"logps/ref_rejected": -79.92558288574219,
"logps/rejected": -116.18818664550781,
"loss": 1.1276,
"margin_dpo/margin_mean": 12.93051815032959,
"margin_dpo/margin_std": 23.506805419921875,
"step": 388
},
{
"KL/chosen_KL_mean": -25.07407569885254,
"KL/mean": -33.77997970581055,
"KL/rejected_KL_mean": -42.48588943481445,
"KL/std": 21.998149871826172,
"epoch": 0.5880574452003023,
"fcm_dpo/beta": 0.051715172827243805,
"fcm_dpo/delta": -0.11027075350284576,
"fcm_dpo/margin": 17.41181182861328,
"fcm_dpo/q_t": 0.33060115575790405,
"grad_norm": 39.576087951660156,
"learning_rate": 2.1835188660656265e-07,
"logits/chosen": 0.435343861579895,
"logits/rejected": 0.398581862449646,
"logps/chosen": -86.69844055175781,
"logps/ref_chosen": -61.624366760253906,
"logps/ref_rejected": -76.50978088378906,
"logps/rejected": -118.99566650390625,
"loss": 0.9666,
"margin_dpo/margin_mean": 17.41181182861328,
"margin_dpo/margin_std": 23.3957576751709,
"step": 389
},
{
"KL/chosen_KL_mean": -22.420507431030273,
"KL/mean": -30.25091552734375,
"KL/rejected_KL_mean": -38.081329345703125,
"KL/std": 21.462738037109375,
"epoch": 0.5895691609977324,
"fcm_dpo/beta": 0.051210470497608185,
"fcm_dpo/delta": -0.002262681722640991,
"fcm_dpo/margin": 15.660820960998535,
"fcm_dpo/q_t": 0.34443798661231995,
"grad_norm": 20.065223693847656,
"learning_rate": 2.170407537241599e-07,
"logits/chosen": 0.4916721284389496,
"logits/rejected": 0.42790883779525757,
"logps/chosen": -68.29237365722656,
"logps/ref_chosen": -45.871864318847656,
"logps/ref_rejected": -61.305999755859375,
"logps/rejected": -99.3873291015625,
"loss": 0.9572,
"margin_dpo/margin_mean": 15.660821914672852,
"margin_dpo/margin_std": 20.54749298095703,
"step": 390
},
{
"KL/chosen_KL_mean": -23.46485710144043,
"KL/mean": -32.447296142578125,
"KL/rejected_KL_mean": -41.42973709106445,
"KL/std": 22.090721130371094,
"epoch": 0.5910808767951625,
"fcm_dpo/beta": 0.0502469539642334,
"fcm_dpo/delta": -0.11369304358959198,
"fcm_dpo/margin": 17.964881896972656,
"fcm_dpo/q_t": 0.3362467288970947,
"grad_norm": 21.207117080688477,
"learning_rate": 2.1573054278272636e-07,
"logits/chosen": 0.4358006715774536,
"logits/rejected": 0.3706232011318207,
"logps/chosen": -81.65187072753906,
"logps/ref_chosen": -58.18701171875,
"logps/ref_rejected": -83.63442993164062,
"logps/rejected": -125.06417083740234,
"loss": 1.0037,
"margin_dpo/margin_mean": 17.964881896972656,
"margin_dpo/margin_std": 26.00237274169922,
"step": 391
},
{
"KL/chosen_KL_mean": -19.514892578125,
"KL/mean": -29.6951904296875,
"KL/rejected_KL_mean": -39.87548065185547,
"KL/std": 22.76531219482422,
"epoch": 0.5925925925925926,
"fcm_dpo/beta": 0.04908373951911926,
"fcm_dpo/delta": -0.22158034145832062,
"fcm_dpo/margin": 20.360595703125,
"fcm_dpo/q_t": 0.3249374032020569,
"grad_norm": 21.660350799560547,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": 0.5273096561431885,
"logits/rejected": 0.4682093858718872,
"logps/chosen": -89.2594223022461,
"logps/ref_chosen": -69.7445297241211,
"logps/ref_rejected": -94.05877685546875,
"logps/rejected": -133.93426513671875,
"loss": 0.9433,
"margin_dpo/margin_mean": 20.360595703125,
"margin_dpo/margin_std": 27.020687103271484,
"step": 392
},
{
"KL/chosen_KL_mean": -25.08833122253418,
"KL/mean": -34.988651275634766,
"KL/rejected_KL_mean": -44.888973236083984,
"KL/std": 22.98666763305664,
"epoch": 0.5941043083900227,
"fcm_dpo/beta": 0.046630434691905975,
"fcm_dpo/delta": -0.13836631178855896,
"fcm_dpo/margin": 19.800640106201172,
"fcm_dpo/q_t": 0.3229396343231201,
"grad_norm": 18.535551071166992,
"learning_rate": 2.131130332936195e-07,
"logits/chosen": 0.48956263065338135,
"logits/rejected": 0.44990265369415283,
"logps/chosen": -77.42323303222656,
"logps/ref_chosen": -52.33489990234375,
"logps/ref_rejected": -74.33809661865234,
"logps/rejected": -119.22706604003906,
"loss": 0.8709,
"margin_dpo/margin_mean": 19.800640106201172,
"margin_dpo/margin_std": 22.033645629882812,
"step": 393
},
{
"KL/chosen_KL_mean": -23.047691345214844,
"KL/mean": -30.93512725830078,
"KL/rejected_KL_mean": -38.82255935668945,
"KL/std": 19.23456573486328,
"epoch": 0.5956160241874527,
"fcm_dpo/beta": 0.04730773717164993,
"fcm_dpo/delta": 0.0572335422039032,
"fcm_dpo/margin": 15.77486801147461,
"fcm_dpo/q_t": 0.34974604845046997,
"grad_norm": 24.01777458190918,
"learning_rate": 2.1180580796331323e-07,
"logits/chosen": 0.5058435201644897,
"logits/rejected": 0.4730924963951111,
"logps/chosen": -83.72383117675781,
"logps/ref_chosen": -60.6761360168457,
"logps/ref_rejected": -71.36074829101562,
"logps/rejected": -110.18331146240234,
"loss": 0.956,
"margin_dpo/margin_mean": 15.77486801147461,
"margin_dpo/margin_std": 19.874174118041992,
"step": 394
},
{
"KL/chosen_KL_mean": -24.339027404785156,
"KL/mean": -31.597896575927734,
"KL/rejected_KL_mean": -38.85675811767578,
"KL/std": 21.322864532470703,
"epoch": 0.5971277399848829,
"fcm_dpo/beta": 0.04836595058441162,
"fcm_dpo/delta": 0.10261309146881104,
"fcm_dpo/margin": 14.517733573913574,
"fcm_dpo/q_t": 0.3664752244949341,
"grad_norm": 23.358461380004883,
"learning_rate": 2.104996510066625e-07,
"logits/chosen": 0.46007978916168213,
"logits/rejected": 0.3733363747596741,
"logps/chosen": -74.94335174560547,
"logps/ref_chosen": -50.60432434082031,
"logps/ref_rejected": -77.08731079101562,
"logps/rejected": -115.94407653808594,
"loss": 1.0173,
"margin_dpo/margin_mean": 14.517733573913574,
"margin_dpo/margin_std": 21.095766067504883,
"step": 395
},
{
"KL/chosen_KL_mean": -23.455059051513672,
"KL/mean": -32.36639404296875,
"KL/rejected_KL_mean": -41.2777214050293,
"KL/std": 24.375507354736328,
"epoch": 0.5986394557823129,
"fcm_dpo/beta": 0.046926774084568024,
"fcm_dpo/delta": -0.048550158739089966,
"fcm_dpo/margin": 17.822664260864258,
"fcm_dpo/q_t": 0.3385160267353058,
"grad_norm": 20.81109619140625,
"learning_rate": 2.0919459895968517e-07,
"logits/chosen": 0.4549301564693451,
"logits/rejected": 0.36139392852783203,
"logps/chosen": -74.81466674804688,
"logps/ref_chosen": -51.35961151123047,
"logps/ref_rejected": -79.89360046386719,
"logps/rejected": -121.17131805419922,
"loss": 0.9418,
"margin_dpo/margin_mean": 17.822662353515625,
"margin_dpo/margin_std": 21.45787811279297,
"step": 396
},
{
"KL/chosen_KL_mean": -28.059925079345703,
"KL/mean": -32.58983612060547,
"KL/rejected_KL_mean": -37.11973190307617,
"KL/std": 23.16779899597168,
"epoch": 0.600151171579743,
"fcm_dpo/beta": 0.050632819533348083,
"fcm_dpo/delta": 0.3545893132686615,
"fcm_dpo/margin": 9.059812545776367,
"fcm_dpo/q_t": 0.4069092869758606,
"grad_norm": 33.780357360839844,
"learning_rate": 2.078906883274924e-07,
"logits/chosen": 0.3834320902824402,
"logits/rejected": 0.33937060832977295,
"logps/chosen": -94.51615142822266,
"logps/ref_chosen": -66.45622253417969,
"logps/ref_rejected": -85.74736785888672,
"logps/rejected": -122.86709594726562,
"loss": 1.3316,
"margin_dpo/margin_mean": 9.059812545776367,
"margin_dpo/margin_std": 25.34778594970703,
"step": 397
},
{
"KL/chosen_KL_mean": -23.686159133911133,
"KL/mean": -33.28020095825195,
"KL/rejected_KL_mean": -42.874244689941406,
"KL/std": 23.769163131713867,
"epoch": 0.6016628873771731,
"fcm_dpo/beta": 0.0496966615319252,
"fcm_dpo/delta": -0.1711999624967575,
"fcm_dpo/margin": 19.18807601928711,
"fcm_dpo/q_t": 0.3260038495063782,
"grad_norm": 21.69061851501465,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": 0.4456123113632202,
"logits/rejected": 0.3802449703216553,
"logps/chosen": -72.93040466308594,
"logps/ref_chosen": -49.244239807128906,
"logps/ref_rejected": -75.18949127197266,
"logps/rejected": -118.06373596191406,
"loss": 0.8871,
"margin_dpo/margin_mean": 19.18807601928711,
"margin_dpo/margin_std": 23.326528549194336,
"step": 398
},
{
"KL/chosen_KL_mean": -26.581912994384766,
"KL/mean": -37.71871566772461,
"KL/rejected_KL_mean": -48.85551452636719,
"KL/std": 23.6285457611084,
"epoch": 0.6031746031746031,
"fcm_dpo/beta": 0.047361068427562714,
"fcm_dpo/delta": -0.2857217788696289,
"fcm_dpo/margin": 22.273609161376953,
"fcm_dpo/q_t": 0.30849629640579224,
"grad_norm": 20.774028778076172,
"learning_rate": 2.052864371672457e-07,
"logits/chosen": 0.39033693075180054,
"logits/rejected": 0.2585032880306244,
"logps/chosen": -94.88870239257812,
"logps/ref_chosen": -68.30679321289062,
"logps/ref_rejected": -113.2708511352539,
"logps/rejected": -162.12637329101562,
"loss": 0.8394,
"margin_dpo/margin_mean": 22.273609161376953,
"margin_dpo/margin_std": 25.607967376708984,
"step": 399
},
{
"KL/chosen_KL_mean": -31.611953735351562,
"KL/mean": -38.24412536621094,
"KL/rejected_KL_mean": -44.876312255859375,
"KL/std": 23.857276916503906,
"epoch": 0.6046863189720333,
"fcm_dpo/beta": 0.04748568683862686,
"fcm_dpo/delta": 0.1808999478816986,
"fcm_dpo/margin": 13.264352798461914,
"fcm_dpo/q_t": 0.38564884662628174,
"grad_norm": 29.400917053222656,
"learning_rate": 2.0398616948569493e-07,
"logits/chosen": 0.4609530568122864,
"logits/rejected": 0.4057099223136902,
"logps/chosen": -103.23844909667969,
"logps/ref_chosen": -71.62649536132812,
"logps/ref_rejected": -90.98765563964844,
"logps/rejected": -135.8639678955078,
"loss": 1.1172,
"margin_dpo/margin_mean": 13.264352798461914,
"margin_dpo/margin_std": 23.702287673950195,
"step": 400
},
{
"KL/chosen_KL_mean": -22.307126998901367,
"KL/mean": -33.3577766418457,
"KL/rejected_KL_mean": -44.408424377441406,
"KL/std": 25.831157684326172,
"epoch": 0.6061980347694633,
"fcm_dpo/beta": 0.04660568758845329,
"fcm_dpo/delta": -0.25692829489707947,
"fcm_dpo/margin": 22.101306915283203,
"fcm_dpo/q_t": 0.30949389934539795,
"grad_norm": 15.676539421081543,
"learning_rate": 2.0268718890989752e-07,
"logits/chosen": 0.4863562285900116,
"logits/rejected": 0.3901751935482025,
"logps/chosen": -76.0320816040039,
"logps/ref_chosen": -53.72495651245117,
"logps/ref_rejected": -75.06304931640625,
"logps/rejected": -119.47147369384766,
"loss": 0.8297,
"margin_dpo/margin_mean": 22.101306915283203,
"margin_dpo/margin_std": 24.05353546142578,
"step": 401
},
{
"KL/chosen_KL_mean": -25.090410232543945,
"KL/mean": -33.568870544433594,
"KL/rejected_KL_mean": -42.04733657836914,
"KL/std": 23.15046501159668,
"epoch": 0.6077097505668935,
"fcm_dpo/beta": 0.04576147347688675,
"fcm_dpo/delta": 0.02423737943172455,
"fcm_dpo/margin": 16.956928253173828,
"fcm_dpo/q_t": 0.35193073749542236,
"grad_norm": 20.219722747802734,
"learning_rate": 2.013895317751323e-07,
"logits/chosen": 0.4610044062137604,
"logits/rejected": 0.42455703020095825,
"logps/chosen": -86.96434020996094,
"logps/ref_chosen": -61.873931884765625,
"logps/ref_rejected": -66.15198516845703,
"logps/rejected": -108.19932556152344,
"loss": 1.0099,
"margin_dpo/margin_mean": 16.956926345825195,
"margin_dpo/margin_std": 24.181861877441406,
"step": 402
},
{
"KL/chosen_KL_mean": -26.66484260559082,
"KL/mean": -37.880027770996094,
"KL/rejected_KL_mean": -49.0952262878418,
"KL/std": 26.00891876220703,
"epoch": 0.6092214663643235,
"fcm_dpo/beta": 0.04488415643572807,
"fcm_dpo/delta": -0.2293957620859146,
"fcm_dpo/margin": 22.430383682250977,
"fcm_dpo/q_t": 0.3173214793205261,
"grad_norm": 19.93121910095215,
"learning_rate": 2.0009323437965898e-07,
"logits/chosen": 0.5491383075714111,
"logits/rejected": 0.4719066619873047,
"logps/chosen": -77.98634338378906,
"logps/ref_chosen": -51.321502685546875,
"logps/ref_rejected": -86.54010772705078,
"logps/rejected": -135.6353302001953,
"loss": 0.8867,
"margin_dpo/margin_mean": 22.430381774902344,
"margin_dpo/margin_std": 26.761947631835938,
"step": 403
},
{
"KL/chosen_KL_mean": -24.51099967956543,
"KL/mean": -34.40475082397461,
"KL/rejected_KL_mean": -44.298500061035156,
"KL/std": 23.84885025024414,
"epoch": 0.6107331821617535,
"fcm_dpo/beta": 0.04325392469763756,
"fcm_dpo/delta": -0.06493037939071655,
"fcm_dpo/margin": 19.787498474121094,
"fcm_dpo/q_t": 0.33856746554374695,
"grad_norm": 20.945728302001953,
"learning_rate": 1.9879833298370237e-07,
"logits/chosen": 0.4318276643753052,
"logits/rejected": 0.34396785497665405,
"logps/chosen": -86.77388763427734,
"logps/ref_chosen": -62.26288604736328,
"logps/ref_rejected": -95.19029998779297,
"logps/rejected": -139.48880004882812,
"loss": 0.9584,
"margin_dpo/margin_mean": 19.787500381469727,
"margin_dpo/margin_std": 25.836761474609375,
"step": 404
},
{
"KL/chosen_KL_mean": -26.306303024291992,
"KL/mean": -34.85593795776367,
"KL/rejected_KL_mean": -43.405582427978516,
"KL/std": 23.573284149169922,
"epoch": 0.6122448979591837,
"fcm_dpo/beta": 0.043290168046951294,
"fcm_dpo/delta": 0.05967658758163452,
"fcm_dpo/margin": 17.099279403686523,
"fcm_dpo/q_t": 0.358862042427063,
"grad_norm": 17.597557067871094,
"learning_rate": 1.975048638084379e-07,
"logits/chosen": 0.5193575620651245,
"logits/rejected": 0.4726630747318268,
"logps/chosen": -76.89064025878906,
"logps/ref_chosen": -50.5843391418457,
"logps/ref_rejected": -65.43156433105469,
"logps/rejected": -108.83714294433594,
"loss": 0.9932,
"margin_dpo/margin_mean": 17.099279403686523,
"margin_dpo/margin_std": 22.427947998046875,
"step": 405
},
{
"KL/chosen_KL_mean": -25.730045318603516,
"KL/mean": -36.04084777832031,
"KL/rejected_KL_mean": -46.35163879394531,
"KL/std": 24.698978424072266,
"epoch": 0.6137566137566137,
"fcm_dpo/beta": 0.04326792061328888,
"fcm_dpo/delta": -0.10235971957445145,
"fcm_dpo/margin": 20.621601104736328,
"fcm_dpo/q_t": 0.33060896396636963,
"grad_norm": 18.64668846130371,
"learning_rate": 1.9621286303497914e-07,
"logits/chosen": 0.5034130811691284,
"logits/rejected": 0.35505062341690063,
"logps/chosen": -74.72564697265625,
"logps/ref_chosen": -48.99560546875,
"logps/ref_rejected": -92.47774505615234,
"logps/rejected": -138.82937622070312,
"loss": 0.9493,
"margin_dpo/margin_mean": 20.62160301208496,
"margin_dpo/margin_std": 26.697372436523438,
"step": 406
},
{
"KL/chosen_KL_mean": -32.66217803955078,
"KL/mean": -40.6044921875,
"KL/rejected_KL_mean": -48.546810150146484,
"KL/std": 26.861190795898438,
"epoch": 0.6152683295540439,
"fcm_dpo/beta": 0.04399215802550316,
"fcm_dpo/delta": 0.10871283710002899,
"fcm_dpo/margin": 15.884638786315918,
"fcm_dpo/q_t": 0.36650407314300537,
"grad_norm": 22.695158004760742,
"learning_rate": 1.9492236680336483e-07,
"logits/chosen": 0.3715978264808655,
"logits/rejected": 0.29767975211143494,
"logps/chosen": -122.062744140625,
"logps/ref_chosen": -89.40056610107422,
"logps/ref_rejected": -99.28775024414062,
"logps/rejected": -147.83456420898438,
"loss": 1.0453,
"margin_dpo/margin_mean": 15.884637832641602,
"margin_dpo/margin_std": 24.79953384399414,
"step": 407
},
{
"KL/chosen_KL_mean": -23.589235305786133,
"KL/mean": -35.32756805419922,
"KL/rejected_KL_mean": -47.06589126586914,
"KL/std": 24.08920669555664,
"epoch": 0.6167800453514739,
"fcm_dpo/beta": 0.043174490332603455,
"fcm_dpo/delta": -0.23764048516750336,
"fcm_dpo/margin": 23.47665023803711,
"fcm_dpo/q_t": 0.30589696764945984,
"grad_norm": 16.45615577697754,
"learning_rate": 1.9363341121154895e-07,
"logits/chosen": 0.46635252237319946,
"logits/rejected": 0.387617826461792,
"logps/chosen": -78.29315185546875,
"logps/ref_chosen": -54.70391845703125,
"logps/ref_rejected": -73.98648834228516,
"logps/rejected": -121.05238342285156,
"loss": 0.8415,
"margin_dpo/margin_mean": 23.47665023803711,
"margin_dpo/margin_std": 24.754615783691406,
"step": 408
},
{
"KL/chosen_KL_mean": -29.324310302734375,
"KL/mean": -36.13206481933594,
"KL/rejected_KL_mean": -42.93982696533203,
"KL/std": 22.537269592285156,
"epoch": 0.618291761148904,
"fcm_dpo/beta": 0.04347747936844826,
"fcm_dpo/delta": 0.22149579226970673,
"fcm_dpo/margin": 13.61551284790039,
"fcm_dpo/q_t": 0.3939046263694763,
"grad_norm": 21.601776123046875,
"learning_rate": 1.9234603231438994e-07,
"logits/chosen": 0.4784550368785858,
"logits/rejected": 0.4812003970146179,
"logps/chosen": -91.44253540039062,
"logps/ref_chosen": -62.11822509765625,
"logps/ref_rejected": -61.933509826660156,
"logps/rejected": -104.87333679199219,
"loss": 1.1333,
"margin_dpo/margin_mean": 13.615513801574707,
"margin_dpo/margin_std": 25.808116912841797,
"step": 409
},
{
"KL/chosen_KL_mean": -27.31926727294922,
"KL/mean": -36.74012756347656,
"KL/rejected_KL_mean": -46.16097640991211,
"KL/std": 23.640226364135742,
"epoch": 0.6198034769463341,
"fcm_dpo/beta": 0.043405093252658844,
"fcm_dpo/delta": -0.02259385585784912,
"fcm_dpo/margin": 18.84171485900879,
"fcm_dpo/q_t": 0.333656370639801,
"grad_norm": 19.895095825195312,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": 0.47311586141586304,
"logits/rejected": 0.44732558727264404,
"logps/chosen": -89.12193298339844,
"logps/ref_chosen": -61.80266189575195,
"logps/ref_rejected": -76.60002136230469,
"logps/rejected": -122.76099395751953,
"loss": 0.8784,
"margin_dpo/margin_mean": 18.84171485900879,
"margin_dpo/margin_std": 19.36935806274414,
"step": 410
},
{
"KL/chosen_KL_mean": -32.32759475708008,
"KL/mean": -42.412288665771484,
"KL/rejected_KL_mean": -52.496978759765625,
"KL/std": 28.044313430786133,
"epoch": 0.6213151927437641,
"fcm_dpo/beta": 0.04318157210946083,
"fcm_dpo/delta": -0.07840226590633392,
"fcm_dpo/margin": 20.169395446777344,
"fcm_dpo/q_t": 0.3437616527080536,
"grad_norm": 24.5039119720459,
"learning_rate": 1.8977614860195296e-07,
"logits/chosen": 0.47114986181259155,
"logits/rejected": 0.4128589630126953,
"logps/chosen": -86.77298736572266,
"logps/ref_chosen": -54.44539260864258,
"logps/ref_rejected": -74.5650863647461,
"logps/rejected": -127.06206512451172,
"loss": 1.0059,
"margin_dpo/margin_mean": 20.169395446777344,
"margin_dpo/margin_std": 29.603900909423828,
"step": 411
},
{
"KL/chosen_KL_mean": -34.439205169677734,
"KL/mean": -44.22332763671875,
"KL/rejected_KL_mean": -54.00743865966797,
"KL/std": 24.429527282714844,
"epoch": 0.6228269085411943,
"fcm_dpo/beta": 0.042693983763456345,
"fcm_dpo/delta": -0.0395471453666687,
"fcm_dpo/margin": 19.5682373046875,
"fcm_dpo/q_t": 0.34421515464782715,
"grad_norm": 19.786897659301758,
"learning_rate": 1.8849371567184662e-07,
"logits/chosen": 0.4648180603981018,
"logits/rejected": 0.39858123660087585,
"logps/chosen": -89.68728637695312,
"logps/ref_chosen": -55.248085021972656,
"logps/ref_rejected": -68.96623229980469,
"logps/rejected": -122.97366333007812,
"loss": 0.9329,
"margin_dpo/margin_mean": 19.5682373046875,
"margin_dpo/margin_std": 24.99486541748047,
"step": 412
},
{
"KL/chosen_KL_mean": -35.61238098144531,
"KL/mean": -44.424903869628906,
"KL/rejected_KL_mean": -53.237422943115234,
"KL/std": 24.641502380371094,
"epoch": 0.6243386243386243,
"fcm_dpo/beta": 0.04312215372920036,
"fcm_dpo/delta": 0.04325508326292038,
"fcm_dpo/margin": 17.625036239624023,
"fcm_dpo/q_t": 0.35896366834640503,
"grad_norm": 25.902362823486328,
"learning_rate": 1.872130032047302e-07,
"logits/chosen": 0.3176858425140381,
"logits/rejected": 0.284635990858078,
"logps/chosen": -104.3331298828125,
"logps/ref_chosen": -68.72074890136719,
"logps/ref_rejected": -78.76539611816406,
"logps/rejected": -132.00282287597656,
"loss": 1.0748,
"margin_dpo/margin_mean": 17.62503433227539,
"margin_dpo/margin_std": 29.25439453125,
"step": 413
},
{
"KL/chosen_KL_mean": -32.86102294921875,
"KL/mean": -43.88529968261719,
"KL/rejected_KL_mean": -54.909568786621094,
"KL/std": 29.22039222717285,
"epoch": 0.6258503401360545,
"fcm_dpo/beta": 0.04218984767794609,
"fcm_dpo/delta": -0.14414259791374207,
"fcm_dpo/margin": 22.048547744750977,
"fcm_dpo/q_t": 0.324825644493103,
"grad_norm": 18.57297706604004,
"learning_rate": 1.8593404702488436e-07,
"logits/chosen": 0.4745393991470337,
"logits/rejected": 0.4136067032814026,
"logps/chosen": -86.99923706054688,
"logps/ref_chosen": -54.138214111328125,
"logps/ref_rejected": -74.65741729736328,
"logps/rejected": -129.56698608398438,
"loss": 0.91,
"margin_dpo/margin_mean": 22.04854965209961,
"margin_dpo/margin_std": 27.189411163330078,
"step": 414
},
{
"KL/chosen_KL_mean": -32.201171875,
"KL/mean": -42.07051467895508,
"KL/rejected_KL_mean": -51.939857482910156,
"KL/std": 28.312644958496094,
"epoch": 0.6273620559334845,
"fcm_dpo/beta": 0.04169802740216255,
"fcm_dpo/delta": -0.025423936545848846,
"fcm_dpo/margin": 19.738685607910156,
"fcm_dpo/q_t": 0.3469679355621338,
"grad_norm": 21.86366844177246,
"learning_rate": 1.846568829074628e-07,
"logits/chosen": 0.47773563861846924,
"logits/rejected": 0.45856568217277527,
"logps/chosen": -88.11973571777344,
"logps/ref_chosen": -55.91856002807617,
"logps/ref_rejected": -61.747703552246094,
"logps/rejected": -113.68756103515625,
"loss": 1.0134,
"margin_dpo/margin_mean": 19.738685607910156,
"margin_dpo/margin_std": 29.190603256225586,
"step": 415
},
{
"KL/chosen_KL_mean": -35.82605743408203,
"KL/mean": -43.756290435791016,
"KL/rejected_KL_mean": -51.686519622802734,
"KL/std": 29.21381378173828,
"epoch": 0.6288737717309146,
"fcm_dpo/beta": 0.04106555134057999,
"fcm_dpo/delta": -0.1727903038263321,
"fcm_dpo/margin": 15.860455513000488,
"fcm_dpo/q_t": 0.37967050075531006,
"grad_norm": 23.822437286376953,
"learning_rate": 1.8338154657749128e-07,
"logits/chosen": 0.4701812267303467,
"logits/rejected": 0.419062077999115,
"logps/chosen": -90.54914093017578,
"logps/ref_chosen": -54.72308349609375,
"logps/ref_rejected": -69.17388916015625,
"logps/rejected": -120.86041259765625,
"loss": 1.1331,
"margin_dpo/margin_mean": 15.860454559326172,
"margin_dpo/margin_std": 27.254606246948242,
"step": 416
},
{
"KL/chosen_KL_mean": -38.493438720703125,
"KL/mean": -49.2604866027832,
"KL/rejected_KL_mean": -60.02753448486328,
"KL/std": 26.403274536132812,
"epoch": 0.6303854875283447,
"fcm_dpo/beta": 0.040380626916885376,
"fcm_dpo/delta": -0.07827968895435333,
"fcm_dpo/margin": 21.53409194946289,
"fcm_dpo/q_t": 0.3280333876609802,
"grad_norm": 22.19675064086914,
"learning_rate": 1.8210807370886849e-07,
"logits/chosen": 0.5746136903762817,
"logits/rejected": 0.5063204169273376,
"logps/chosen": -95.28469848632812,
"logps/ref_chosen": -56.791259765625,
"logps/ref_rejected": -68.7791748046875,
"logps/rejected": -128.80670166015625,
"loss": 0.967,
"margin_dpo/margin_mean": 21.534093856811523,
"margin_dpo/margin_std": 28.17495346069336,
"step": 417
},
{
"KL/chosen_KL_mean": -41.96894454956055,
"KL/mean": -49.41185760498047,
"KL/rejected_KL_mean": -56.854766845703125,
"KL/std": 28.295509338378906,
"epoch": 0.6318972033257747,
"fcm_dpo/beta": 0.040416378527879715,
"fcm_dpo/delta": 0.20983785390853882,
"fcm_dpo/margin": 14.885822296142578,
"fcm_dpo/q_t": 0.3924134373664856,
"grad_norm": 26.25454330444336,
"learning_rate": 1.8083649992336825e-07,
"logits/chosen": 0.5161227583885193,
"logits/rejected": 0.522834062576294,
"logps/chosen": -111.07693481445312,
"logps/ref_chosen": -69.10798645019531,
"logps/ref_rejected": -75.09132385253906,
"logps/rejected": -131.9460906982422,
"loss": 1.1556,
"margin_dpo/margin_mean": 14.885823249816895,
"margin_dpo/margin_std": 28.37472915649414,
"step": 418
},
{
"KL/chosen_KL_mean": -30.667163848876953,
"KL/mean": -42.797237396240234,
"KL/rejected_KL_mean": -54.92730712890625,
"KL/std": 27.933753967285156,
"epoch": 0.6334089191232048,
"fcm_dpo/beta": 0.040009621530771255,
"fcm_dpo/delta": -0.19122833013534546,
"fcm_dpo/margin": 24.260149002075195,
"fcm_dpo/q_t": 0.32048365473747253,
"grad_norm": 19.607967376708984,
"learning_rate": 1.7956686078964255e-07,
"logits/chosen": 0.3947601318359375,
"logits/rejected": 0.3380334973335266,
"logps/chosen": -88.83892822265625,
"logps/ref_chosen": -58.1717643737793,
"logps/ref_rejected": -71.67066955566406,
"logps/rejected": -126.59797668457031,
"loss": 0.8978,
"margin_dpo/margin_mean": 24.260147094726562,
"margin_dpo/margin_std": 29.379589080810547,
"step": 419
},
{
"KL/chosen_KL_mean": -42.81511688232422,
"KL/mean": -49.22902297973633,
"KL/rejected_KL_mean": -55.642921447753906,
"KL/std": 28.587371826171875,
"epoch": 0.6349206349206349,
"fcm_dpo/beta": 0.041276998817920685,
"fcm_dpo/delta": 0.2865866422653198,
"fcm_dpo/margin": 12.827811241149902,
"fcm_dpo/q_t": 0.4050375819206238,
"grad_norm": 25.256057739257812,
"learning_rate": 1.782991918222275e-07,
"logits/chosen": 0.4635976552963257,
"logits/rejected": 0.4160757064819336,
"logps/chosen": -99.86862182617188,
"logps/ref_chosen": -57.05351257324219,
"logps/ref_rejected": -62.670982360839844,
"logps/rejected": -118.31390380859375,
"loss": 1.2747,
"margin_dpo/margin_mean": 12.827811241149902,
"margin_dpo/margin_std": 31.175682067871094,
"step": 420
},
{
"KL/chosen_KL_mean": -40.56446075439453,
"KL/mean": -49.28892135620117,
"KL/rejected_KL_mean": -58.01338577270508,
"KL/std": 28.834896087646484,
"epoch": 0.636432350718065,
"fcm_dpo/beta": 0.04242285341024399,
"fcm_dpo/delta": 0.06434623152017593,
"fcm_dpo/margin": 17.448923110961914,
"fcm_dpo/q_t": 0.3707146644592285,
"grad_norm": 26.102148056030273,
"learning_rate": 1.7703352848054887e-07,
"logits/chosen": 0.43699386715888977,
"logits/rejected": 0.3786683678627014,
"logps/chosen": -97.88771057128906,
"logps/ref_chosen": -57.32324981689453,
"logps/ref_rejected": -75.33782958984375,
"logps/rejected": -133.35121154785156,
"loss": 1.1907,
"margin_dpo/margin_mean": 17.448925018310547,
"margin_dpo/margin_std": 34.14581298828125,
"step": 421
},
{
"KL/chosen_KL_mean": -33.99824142456055,
"KL/mean": -44.69304275512695,
"KL/rejected_KL_mean": -55.387847900390625,
"KL/std": 26.90380859375,
"epoch": 0.6379440665154951,
"fcm_dpo/beta": 0.042237088084220886,
"fcm_dpo/delta": -0.11359019577503204,
"fcm_dpo/margin": 21.389610290527344,
"fcm_dpo/q_t": 0.3315642476081848,
"grad_norm": 22.26106834411621,
"learning_rate": 1.7576990616793137e-07,
"logits/chosen": 0.4836987257003784,
"logits/rejected": 0.4552809000015259,
"logps/chosen": -101.05581665039062,
"logps/ref_chosen": -67.05757141113281,
"logps/ref_rejected": -72.12803649902344,
"logps/rejected": -127.51588439941406,
"loss": 0.93,
"margin_dpo/margin_mean": 21.389610290527344,
"margin_dpo/margin_std": 27.35793685913086,
"step": 422
},
{
"KL/chosen_KL_mean": -36.689964294433594,
"KL/mean": -48.771995544433594,
"KL/rejected_KL_mean": -60.854026794433594,
"KL/std": 30.184703826904297,
"epoch": 0.6394557823129252,
"fcm_dpo/beta": 0.04042931646108627,
"fcm_dpo/delta": -0.19745704531669617,
"fcm_dpo/margin": 24.1640625,
"fcm_dpo/q_t": 0.32379740476608276,
"grad_norm": 20.41207504272461,
"learning_rate": 1.745083602306071e-07,
"logits/chosen": 0.5219099521636963,
"logits/rejected": 0.45282691717147827,
"logps/chosen": -90.75163269042969,
"logps/ref_chosen": -54.06167221069336,
"logps/ref_rejected": -76.64092254638672,
"logps/rejected": -137.4949493408203,
"loss": 0.9104,
"margin_dpo/margin_mean": 24.1640625,
"margin_dpo/margin_std": 30.572715759277344,
"step": 423
},
{
"KL/chosen_KL_mean": -36.551551818847656,
"KL/mean": -49.043670654296875,
"KL/rejected_KL_mean": -61.53579330444336,
"KL/std": 27.892314910888672,
"epoch": 0.6409674981103552,
"fcm_dpo/beta": 0.03874318301677704,
"fcm_dpo/delta": -0.1893288791179657,
"fcm_dpo/margin": 24.984237670898438,
"fcm_dpo/q_t": 0.3210356831550598,
"grad_norm": 19.895740509033203,
"learning_rate": 1.7324892595672804e-07,
"logits/chosen": 0.41898566484451294,
"logits/rejected": 0.38174790143966675,
"logps/chosen": -90.16041564941406,
"logps/ref_chosen": -53.60887145996094,
"logps/ref_rejected": -79.2139892578125,
"logps/rejected": -140.74978637695312,
"loss": 0.8929,
"margin_dpo/margin_mean": 24.984237670898438,
"margin_dpo/margin_std": 29.843223571777344,
"step": 424
},
{
"KL/chosen_KL_mean": -36.94670867919922,
"KL/mean": -46.86540985107422,
"KL/rejected_KL_mean": -56.78410339355469,
"KL/std": 28.853939056396484,
"epoch": 0.6424792139077853,
"fcm_dpo/beta": 0.038760796189308167,
"fcm_dpo/delta": 0.033539630472660065,
"fcm_dpo/margin": 19.837398529052734,
"fcm_dpo/q_t": 0.35705694556236267,
"grad_norm": 19.53350830078125,
"learning_rate": 1.7199163857537824e-07,
"logits/chosen": 0.5291003584861755,
"logits/rejected": 0.4982011318206787,
"logps/chosen": -95.36138916015625,
"logps/ref_chosen": -58.41468048095703,
"logps/ref_rejected": -66.59054565429688,
"logps/rejected": -123.37464904785156,
"loss": 1.0102,
"margin_dpo/margin_mean": 19.837398529052734,
"margin_dpo/margin_std": 29.011436462402344,
"step": 425
},
{
"KL/chosen_KL_mean": -46.007965087890625,
"KL/mean": -52.275394439697266,
"KL/rejected_KL_mean": -58.54281997680664,
"KL/std": 27.631093978881836,
"epoch": 0.6439909297052154,
"fcm_dpo/beta": 0.040759261697530746,
"fcm_dpo/delta": 0.3047676682472229,
"fcm_dpo/margin": 12.534847259521484,
"fcm_dpo/q_t": 0.40703439712524414,
"grad_norm": 30.417728424072266,
"learning_rate": 1.7073653325558828e-07,
"logits/chosen": 0.4371623992919922,
"logits/rejected": 0.4418582022190094,
"logps/chosen": -117.7161865234375,
"logps/ref_chosen": -71.70822143554688,
"logps/ref_rejected": -73.57725524902344,
"logps/rejected": -132.12008666992188,
"loss": 1.2719,
"margin_dpo/margin_mean": 12.534847259521484,
"margin_dpo/margin_std": 31.50692367553711,
"step": 426
},
{
"KL/chosen_KL_mean": -42.995643615722656,
"KL/mean": -52.96397018432617,
"KL/rejected_KL_mean": -62.93229675292969,
"KL/std": 30.741945266723633,
"epoch": 0.6455026455026455,
"fcm_dpo/beta": 0.041296087205410004,
"fcm_dpo/delta": -0.02539144642651081,
"fcm_dpo/margin": 19.936656951904297,
"fcm_dpo/q_t": 0.3556680679321289,
"grad_norm": 25.805469512939453,
"learning_rate": 1.6948364510535218e-07,
"logits/chosen": 0.5277816653251648,
"logits/rejected": 0.47356265783309937,
"logps/chosen": -101.63841247558594,
"logps/ref_chosen": -58.64276885986328,
"logps/ref_rejected": -86.25437927246094,
"logps/rejected": -149.18667602539062,
"loss": 1.0654,
"margin_dpo/margin_mean": 19.936656951904297,
"margin_dpo/margin_std": 32.75372314453125,
"step": 427
},
{
"KL/chosen_KL_mean": -40.44948959350586,
"KL/mean": -52.776580810546875,
"KL/rejected_KL_mean": -65.10366821289062,
"KL/std": 31.763904571533203,
"epoch": 0.6470143613000756,
"fcm_dpo/beta": 0.040205471217632294,
"fcm_dpo/delta": -0.2117908000946045,
"fcm_dpo/margin": 24.6541748046875,
"fcm_dpo/q_t": 0.32707393169403076,
"grad_norm": 22.02398109436035,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": 0.4488622546195984,
"logits/rejected": 0.39951950311660767,
"logps/chosen": -107.0455322265625,
"logps/ref_chosen": -66.5960464477539,
"logps/ref_rejected": -82.3941650390625,
"logps/rejected": -147.49783325195312,
"loss": 0.9509,
"margin_dpo/margin_mean": 24.6541748046875,
"margin_dpo/margin_std": 33.260841369628906,
"step": 428
},
{
"KL/chosen_KL_mean": -43.406707763671875,
"KL/mean": -52.90144348144531,
"KL/rejected_KL_mean": -62.39617919921875,
"KL/std": 29.375505447387695,
"epoch": 0.6485260770975056,
"fcm_dpo/beta": 0.040120624005794525,
"fcm_dpo/delta": 0.038818396627902985,
"fcm_dpo/margin": 18.98946762084961,
"fcm_dpo/q_t": 0.3599487543106079,
"grad_norm": 23.097915649414062,
"learning_rate": 1.669846604344412e-07,
"logits/chosen": 0.4337460398674011,
"logits/rejected": 0.44470861554145813,
"logps/chosen": -100.41641235351562,
"logps/ref_chosen": -57.00970458984375,
"logps/ref_rejected": -59.86549377441406,
"logps/rejected": -122.26167297363281,
"loss": 1.0651,
"margin_dpo/margin_mean": 18.98946762084961,
"margin_dpo/margin_std": 30.113819122314453,
"step": 429
},
{
"KL/chosen_KL_mean": -39.49730682373047,
"KL/mean": -52.07994842529297,
"KL/rejected_KL_mean": -64.66258239746094,
"KL/std": 29.181060791015625,
"epoch": 0.6500377928949358,
"fcm_dpo/beta": 0.0386689230799675,
"fcm_dpo/delta": -0.1919030398130417,
"fcm_dpo/margin": 25.165279388427734,
"fcm_dpo/q_t": 0.317873477935791,
"grad_norm": 19.91645622253418,
"learning_rate": 1.6573863381573954e-07,
"logits/chosen": 0.40141820907592773,
"logits/rejected": 0.3949132263660431,
"logps/chosen": -99.06050109863281,
"logps/ref_chosen": -59.563194274902344,
"logps/ref_rejected": -70.52289581298828,
"logps/rejected": -135.18548583984375,
"loss": 0.8937,
"margin_dpo/margin_mean": 25.165279388427734,
"margin_dpo/margin_std": 30.14092254638672,
"step": 430
},
{
"KL/chosen_KL_mean": -35.84678649902344,
"KL/mean": -45.385589599609375,
"KL/rejected_KL_mean": -54.92439270019531,
"KL/std": 29.223169326782227,
"epoch": 0.6515495086923658,
"fcm_dpo/beta": 0.03864824026823044,
"fcm_dpo/delta": 0.06772233545780182,
"fcm_dpo/margin": 19.077606201171875,
"fcm_dpo/q_t": 0.35975968837738037,
"grad_norm": 20.980796813964844,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": 0.46574872732162476,
"logits/rejected": 0.41989612579345703,
"logps/chosen": -86.04711151123047,
"logps/ref_chosen": -50.20032501220703,
"logps/ref_rejected": -77.81680297851562,
"logps/rejected": -132.74119567871094,
"loss": 1.0254,
"margin_dpo/margin_mean": 19.077606201171875,
"margin_dpo/margin_std": 28.919309616088867,
"step": 431
},
{
"KL/chosen_KL_mean": -37.896148681640625,
"KL/mean": -47.94352722167969,
"KL/rejected_KL_mean": -57.990901947021484,
"KL/std": 27.537246704101562,
"epoch": 0.6530612244897959,
"fcm_dpo/beta": 0.038857653737068176,
"fcm_dpo/delta": 0.020730186253786087,
"fcm_dpo/margin": 20.09475326538086,
"fcm_dpo/q_t": 0.35490578413009644,
"grad_norm": 21.475805282592773,
"learning_rate": 1.632536862810844e-07,
"logits/chosen": 0.49139007925987244,
"logits/rejected": 0.44433116912841797,
"logps/chosen": -99.55890655517578,
"logps/ref_chosen": -61.662757873535156,
"logps/ref_rejected": -83.94496154785156,
"logps/rejected": -141.93585205078125,
"loss": 1.0392,
"margin_dpo/margin_mean": 20.09475326538086,
"margin_dpo/margin_std": 31.069698333740234,
"step": 432
},
{
"KL/chosen_KL_mean": -37.729713439941406,
"KL/mean": -50.83671188354492,
"KL/rejected_KL_mean": -63.9437141418457,
"KL/std": 30.168743133544922,
"epoch": 0.654572940287226,
"fcm_dpo/beta": 0.038006819784641266,
"fcm_dpo/delta": -0.21750611066818237,
"fcm_dpo/margin": 26.213993072509766,
"fcm_dpo/q_t": 0.31744182109832764,
"grad_norm": 18.48863983154297,
"learning_rate": 1.6201483487445515e-07,
"logits/chosen": 0.5908124446868896,
"logits/rejected": 0.5816738605499268,
"logps/chosen": -101.45889282226562,
"logps/ref_chosen": -63.72917938232422,
"logps/ref_rejected": -65.8391342163086,
"logps/rejected": -129.7828369140625,
"loss": 0.9122,
"margin_dpo/margin_mean": 26.213993072509766,
"margin_dpo/margin_std": 32.15654754638672,
"step": 433
},
{
"KL/chosen_KL_mean": -33.527801513671875,
"KL/mean": -45.83747100830078,
"KL/rejected_KL_mean": -58.14714813232422,
"KL/std": 28.78369903564453,
"epoch": 0.656084656084656,
"fcm_dpo/beta": 0.03632465749979019,
"fcm_dpo/delta": -0.10747957974672318,
"fcm_dpo/margin": 24.619342803955078,
"fcm_dpo/q_t": 0.32917672395706177,
"grad_norm": 18.348094940185547,
"learning_rate": 1.6077844460203204e-07,
"logits/chosen": 0.5969550609588623,
"logits/rejected": 0.5314214825630188,
"logps/chosen": -81.50111389160156,
"logps/ref_chosen": -47.97331619262695,
"logps/ref_rejected": -72.51132202148438,
"logps/rejected": -130.65847778320312,
"loss": 0.9834,
"margin_dpo/margin_mean": 24.619342803955078,
"margin_dpo/margin_std": 33.49932098388672,
"step": 434
},
{
"KL/chosen_KL_mean": -39.075477600097656,
"KL/mean": -49.97040557861328,
"KL/rejected_KL_mean": -60.865325927734375,
"KL/std": 29.0299015045166,
"epoch": 0.6575963718820862,
"fcm_dpo/beta": 0.03686121478676796,
"fcm_dpo/delta": -0.005386091768741608,
"fcm_dpo/margin": 21.789846420288086,
"fcm_dpo/q_t": 0.3473183214664459,
"grad_norm": 20.230436325073242,
"learning_rate": 1.5954455004830878e-07,
"logits/chosen": 0.6023925542831421,
"logits/rejected": 0.5621955990791321,
"logps/chosen": -96.13572692871094,
"logps/ref_chosen": -57.06024932861328,
"logps/ref_rejected": -71.69146728515625,
"logps/rejected": -132.55679321289062,
"loss": 0.9939,
"margin_dpo/margin_mean": 21.789846420288086,
"margin_dpo/margin_std": 30.15741729736328,
"step": 435
},
{
"KL/chosen_KL_mean": -37.826961517333984,
"KL/mean": -47.93717575073242,
"KL/rejected_KL_mean": -58.04738998413086,
"KL/std": 29.83072280883789,
"epoch": 0.6591080876795162,
"fcm_dpo/beta": 0.0369018018245697,
"fcm_dpo/delta": 0.05788743868470192,
"fcm_dpo/margin": 20.220436096191406,
"fcm_dpo/q_t": 0.3603705167770386,
"grad_norm": 19.659523010253906,
"learning_rate": 1.5831318572796847e-07,
"logits/chosen": 0.4917636513710022,
"logits/rejected": 0.4353067874908447,
"logps/chosen": -93.98501586914062,
"logps/ref_chosen": -56.158050537109375,
"logps/ref_rejected": -67.63787841796875,
"logps/rejected": -125.68527221679688,
"loss": 1.0435,
"margin_dpo/margin_mean": 20.220436096191406,
"margin_dpo/margin_std": 31.14856719970703,
"step": 436
},
{
"KL/chosen_KL_mean": -41.31220245361328,
"KL/mean": -51.047908782958984,
"KL/rejected_KL_mean": -60.78361892700195,
"KL/std": 29.29422950744629,
"epoch": 0.6606198034769464,
"fcm_dpo/beta": 0.03676586598157883,
"fcm_dpo/delta": 0.08598443865776062,
"fcm_dpo/margin": 19.471410751342773,
"fcm_dpo/q_t": 0.37089112401008606,
"grad_norm": 21.479209899902344,
"learning_rate": 1.5708438608491815e-07,
"logits/chosen": 0.4810078740119934,
"logits/rejected": 0.35320615768432617,
"logps/chosen": -98.29798889160156,
"logps/ref_chosen": -56.98578643798828,
"logps/ref_rejected": -85.61524963378906,
"logps/rejected": -146.39886474609375,
"loss": 1.1268,
"margin_dpo/margin_mean": 19.471410751342773,
"margin_dpo/margin_std": 33.92582702636719,
"step": 437
},
{
"KL/chosen_KL_mean": -36.553382873535156,
"KL/mean": -49.979061126708984,
"KL/rejected_KL_mean": -63.40473175048828,
"KL/std": 32.274356842041016,
"epoch": 0.6621315192743764,
"fcm_dpo/beta": 0.036649350076913834,
"fcm_dpo/delta": -0.20369064807891846,
"fcm_dpo/margin": 26.85135841369629,
"fcm_dpo/q_t": 0.32231590151786804,
"grad_norm": 20.170085906982422,
"learning_rate": 1.558581854913253e-07,
"logits/chosen": 0.5478121042251587,
"logits/rejected": 0.48098286986351013,
"logps/chosen": -77.83116149902344,
"logps/ref_chosen": -41.27777862548828,
"logps/ref_rejected": -65.33840942382812,
"logps/rejected": -128.74314880371094,
"loss": 0.907,
"margin_dpo/margin_mean": 26.851356506347656,
"margin_dpo/margin_std": 33.0677490234375,
"step": 438
},
{
"KL/chosen_KL_mean": -37.44815444946289,
"KL/mean": -48.96977996826172,
"KL/rejected_KL_mean": -60.49140930175781,
"KL/std": 30.81436538696289,
"epoch": 0.6636432350718064,
"fcm_dpo/beta": 0.036244139075279236,
"fcm_dpo/delta": -0.040471963584423065,
"fcm_dpo/margin": 23.043258666992188,
"fcm_dpo/q_t": 0.33878690004348755,
"grad_norm": 19.450157165527344,
"learning_rate": 1.5463461824665658e-07,
"logits/chosen": 0.43274006247520447,
"logits/rejected": 0.3965134620666504,
"logps/chosen": -118.86580657958984,
"logps/ref_chosen": -81.41764831542969,
"logps/ref_rejected": -94.72309875488281,
"logps/rejected": -155.21450805664062,
"loss": 0.9298,
"margin_dpo/margin_mean": 23.043258666992188,
"margin_dpo/margin_std": 28.11056137084961,
"step": 439
},
{
"KL/chosen_KL_mean": -35.14704132080078,
"KL/mean": -47.27345657348633,
"KL/rejected_KL_mean": -59.399871826171875,
"KL/std": 28.511383056640625,
"epoch": 0.6651549508692366,
"fcm_dpo/beta": 0.03566228598356247,
"fcm_dpo/delta": -0.07170647382736206,
"fcm_dpo/margin": 24.252822875976562,
"fcm_dpo/q_t": 0.33973562717437744,
"grad_norm": 26.302644729614258,
"learning_rate": 1.534137185767178e-07,
"logits/chosen": 0.4645977020263672,
"logits/rejected": 0.3686174154281616,
"logps/chosen": -77.68522644042969,
"logps/ref_chosen": -42.538185119628906,
"logps/ref_rejected": -69.78813934326172,
"logps/rejected": -129.18801879882812,
"loss": 0.9728,
"margin_dpo/margin_mean": 24.252822875976562,
"margin_dpo/margin_std": 32.987892150878906,
"step": 440
},
{
"KL/chosen_KL_mean": -32.36697006225586,
"KL/mean": -45.41593551635742,
"KL/rejected_KL_mean": -58.46489715576172,
"KL/std": 29.915069580078125,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.0345030352473259,
"fcm_dpo/delta": -0.11155462265014648,
"fcm_dpo/margin": 26.097930908203125,
"fcm_dpo/q_t": 0.320126473903656,
"grad_norm": 17.972639083862305,
"learning_rate": 1.521955206326976e-07,
"logits/chosen": 0.457683265209198,
"logits/rejected": 0.36754998564720154,
"logps/chosen": -89.96018981933594,
"logps/ref_chosen": -57.593223571777344,
"logps/ref_rejected": -84.82878875732422,
"logps/rejected": -143.29368591308594,
"loss": 0.8487,
"margin_dpo/margin_mean": 26.097930908203125,
"margin_dpo/margin_std": 26.978858947753906,
"step": 441
},
{
"KL/chosen_KL_mean": -42.26836395263672,
"KL/mean": -54.38083267211914,
"KL/rejected_KL_mean": -66.49330139160156,
"KL/std": 29.323583602905273,
"epoch": 0.6681783824640968,
"fcm_dpo/beta": 0.034237202256917953,
"fcm_dpo/delta": -0.03224414587020874,
"fcm_dpo/margin": 24.224939346313477,
"fcm_dpo/q_t": 0.34052807092666626,
"grad_norm": 19.867328643798828,
"learning_rate": 1.5098005849021078e-07,
"logits/chosen": 0.4998372793197632,
"logits/rejected": 0.45130839943885803,
"logps/chosen": -109.72958374023438,
"logps/ref_chosen": -67.46121978759766,
"logps/ref_rejected": -89.0693588256836,
"logps/rejected": -155.56265258789062,
"loss": 0.9555,
"margin_dpo/margin_mean": 24.224939346313477,
"margin_dpo/margin_std": 31.437952041625977,
"step": 442
},
{
"KL/chosen_KL_mean": -34.84177780151367,
"KL/mean": -49.455894470214844,
"KL/rejected_KL_mean": -64.07000732421875,
"KL/std": 32.635169982910156,
"epoch": 0.6696900982615268,
"fcm_dpo/beta": 0.033378299325704575,
"fcm_dpo/delta": -0.19421806931495667,
"fcm_dpo/margin": 29.228227615356445,
"fcm_dpo/q_t": 0.31669682264328003,
"grad_norm": 19.16834259033203,
"learning_rate": 1.4976736614834662e-07,
"logits/chosen": 0.5136935710906982,
"logits/rejected": 0.44100815057754517,
"logps/chosen": -89.63787841796875,
"logps/ref_chosen": -54.79610061645508,
"logps/ref_rejected": -77.80781555175781,
"logps/rejected": -141.87783813476562,
"loss": 0.9044,
"margin_dpo/margin_mean": 29.228227615356445,
"margin_dpo/margin_std": 35.64894485473633,
"step": 443
},
{
"KL/chosen_KL_mean": -44.902381896972656,
"KL/mean": -50.10022735595703,
"KL/rejected_KL_mean": -55.29807662963867,
"KL/std": 30.544994354248047,
"epoch": 0.671201814058957,
"fcm_dpo/beta": 0.033911701291799545,
"fcm_dpo/delta": 0.14310048520565033,
"fcm_dpo/margin": 10.39570426940918,
"fcm_dpo/q_t": 0.4301198720932007,
"grad_norm": 23.90824317932129,
"learning_rate": 1.4855747752871654e-07,
"logits/chosen": 0.5034670829772949,
"logits/rejected": 0.4187578558921814,
"logps/chosen": -103.65144348144531,
"logps/ref_chosen": -58.749061584472656,
"logps/ref_rejected": -86.87396240234375,
"logps/rejected": -142.1720428466797,
"loss": 1.3012,
"margin_dpo/margin_mean": 10.39570426940918,
"margin_dpo/margin_std": 29.98647689819336,
"step": 444
},
{
"KL/chosen_KL_mean": -43.239959716796875,
"KL/mean": -55.58454132080078,
"KL/rejected_KL_mean": -67.92911529541016,
"KL/std": 31.177345275878906,
"epoch": 0.672713529856387,
"fcm_dpo/beta": 0.033827681094408035,
"fcm_dpo/delta": -0.038472820073366165,
"fcm_dpo/margin": 24.68915557861328,
"fcm_dpo/q_t": 0.3407127261161804,
"grad_norm": 19.97569465637207,
"learning_rate": 1.473504264745062e-07,
"logits/chosen": 0.48346036672592163,
"logits/rejected": 0.466796875,
"logps/chosen": -104.15739440917969,
"logps/ref_chosen": -60.91743850708008,
"logps/ref_rejected": -71.5637435913086,
"logps/rejected": -139.49285888671875,
"loss": 0.956,
"margin_dpo/margin_mean": 24.68915557861328,
"margin_dpo/margin_std": 32.249595642089844,
"step": 445
},
{
"KL/chosen_KL_mean": -35.729644775390625,
"KL/mean": -51.252044677734375,
"KL/rejected_KL_mean": -66.77444458007812,
"KL/std": 31.810598373413086,
"epoch": 0.674225245653817,
"fcm_dpo/beta": 0.03296538069844246,
"fcm_dpo/delta": -0.2501997649669647,
"fcm_dpo/margin": 31.0447998046875,
"fcm_dpo/q_t": 0.2978588044643402,
"grad_norm": 14.429303169250488,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": 0.5177422761917114,
"logits/rejected": 0.4415864944458008,
"logps/chosen": -84.52889251708984,
"logps/ref_chosen": -48.79924774169922,
"logps/ref_rejected": -71.8719482421875,
"logps/rejected": -138.64639282226562,
"loss": 0.7828,
"margin_dpo/margin_mean": 31.0447998046875,
"margin_dpo/margin_std": 27.450092315673828,
"step": 446
},
{
"KL/chosen_KL_mean": -34.26579284667969,
"KL/mean": -49.93895721435547,
"KL/rejected_KL_mean": -65.61212158203125,
"KL/std": 31.172748565673828,
"epoch": 0.6757369614512472,
"fcm_dpo/beta": 0.031016748398542404,
"fcm_dpo/delta": -0.19166170060634613,
"fcm_dpo/margin": 31.346324920654297,
"fcm_dpo/q_t": 0.3062456548213959,
"grad_norm": 18.319395065307617,
"learning_rate": 1.4494497203727843e-07,
"logits/chosen": 0.43708741664886475,
"logits/rejected": 0.3397822082042694,
"logps/chosen": -87.9485092163086,
"logps/ref_chosen": -53.682716369628906,
"logps/ref_rejected": -88.17315673828125,
"logps/rejected": -153.7852783203125,
"loss": 0.8674,
"margin_dpo/margin_mean": 31.346324920654297,
"margin_dpo/margin_std": 34.39900207519531,
"step": 447
},
{
"KL/chosen_KL_mean": -38.89901351928711,
"KL/mean": -51.21916198730469,
"KL/rejected_KL_mean": -63.53931427001953,
"KL/std": 31.395763397216797,
"epoch": 0.6772486772486772,
"fcm_dpo/beta": 0.030965670943260193,
"fcm_dpo/delta": 0.04007108882069588,
"fcm_dpo/margin": 24.640295028686523,
"fcm_dpo/q_t": 0.3454161286354065,
"grad_norm": 15.666644096374512,
"learning_rate": 1.4374663593999256e-07,
"logits/chosen": 0.5203387141227722,
"logits/rejected": 0.46833568811416626,
"logps/chosen": -92.6502685546875,
"logps/ref_chosen": -53.75125503540039,
"logps/ref_rejected": -77.17623901367188,
"logps/rejected": -140.71554565429688,
"loss": 0.9618,
"margin_dpo/margin_mean": 24.64029312133789,
"margin_dpo/margin_std": 31.69839859008789,
"step": 448
},
{
"KL/chosen_KL_mean": -48.00468063354492,
"KL/mean": -54.875057220458984,
"KL/rejected_KL_mean": -61.74543380737305,
"KL/std": 31.669912338256836,
"epoch": 0.6787603930461074,
"fcm_dpo/beta": 0.03255101665854454,
"fcm_dpo/delta": 0.37090978026390076,
"fcm_dpo/margin": 13.740757942199707,
"fcm_dpo/q_t": 0.41214174032211304,
"grad_norm": 24.92876625061035,
"learning_rate": 1.4255127197770707e-07,
"logits/chosen": 0.36804431676864624,
"logits/rejected": 0.35851621627807617,
"logps/chosen": -123.83206176757812,
"logps/ref_chosen": -75.82737731933594,
"logps/ref_rejected": -82.20687866210938,
"logps/rejected": -143.95230102539062,
"loss": 1.1675,
"margin_dpo/margin_mean": 13.740757942199707,
"margin_dpo/margin_std": 27.162803649902344,
"step": 449
},
{
"KL/chosen_KL_mean": -37.56608581542969,
"KL/mean": -46.34649658203125,
"KL/rejected_KL_mean": -55.12691116333008,
"KL/std": 29.79147720336914,
"epoch": 0.6802721088435374,
"fcm_dpo/beta": 0.03442070633172989,
"fcm_dpo/delta": 0.20848983526229858,
"fcm_dpo/margin": 17.560827255249023,
"fcm_dpo/q_t": 0.3890036344528198,
"grad_norm": 21.428207397460938,
"learning_rate": 1.4135891358732205e-07,
"logits/chosen": 0.5896279215812683,
"logits/rejected": 0.4792223274707794,
"logps/chosen": -84.68180847167969,
"logps/ref_chosen": -47.11572265625,
"logps/ref_rejected": -78.7546615600586,
"logps/rejected": -133.88157653808594,
"loss": 1.114,
"margin_dpo/margin_mean": 17.56082534790039,
"margin_dpo/margin_std": 31.81939697265625,
"step": 450
},
{
"KL/chosen_KL_mean": -38.557586669921875,
"KL/mean": -48.040714263916016,
"KL/rejected_KL_mean": -57.52384567260742,
"KL/std": 30.844379425048828,
"epoch": 0.6817838246409675,
"fcm_dpo/beta": 0.03564320504665375,
"fcm_dpo/delta": 0.13255426287651062,
"fcm_dpo/margin": 18.966249465942383,
"fcm_dpo/q_t": 0.3727077841758728,
"grad_norm": 20.818710327148438,
"learning_rate": 1.4016959412166437e-07,
"logits/chosen": 0.46854376792907715,
"logits/rejected": 0.4182296395301819,
"logps/chosen": -101.90803527832031,
"logps/ref_chosen": -63.350440979003906,
"logps/ref_rejected": -76.28530883789062,
"logps/rejected": -133.80914306640625,
"loss": 1.0651,
"margin_dpo/margin_mean": 18.966251373291016,
"margin_dpo/margin_std": 30.914825439453125,
"step": 451
},
{
"KL/chosen_KL_mean": -38.73877716064453,
"KL/mean": -49.515777587890625,
"KL/rejected_KL_mean": -60.29277801513672,
"KL/std": 30.167449951171875,
"epoch": 0.6832955404383976,
"fcm_dpo/beta": 0.035997722297906876,
"fcm_dpo/delta": 0.02604661136865616,
"fcm_dpo/margin": 21.553997039794922,
"fcm_dpo/q_t": 0.3517000377178192,
"grad_norm": 20.391658782958984,
"learning_rate": 1.3898334684855645e-07,
"logits/chosen": 0.47314220666885376,
"logits/rejected": 0.3956415057182312,
"logps/chosen": -94.32461547851562,
"logps/ref_chosen": -55.58583450317383,
"logps/ref_rejected": -77.68738555908203,
"logps/rejected": -137.98016357421875,
"loss": 1.0526,
"margin_dpo/margin_mean": 21.553997039794922,
"margin_dpo/margin_std": 33.468746185302734,
"step": 452
},
{
"KL/chosen_KL_mean": -36.91946029663086,
"KL/mean": -48.99592590332031,
"KL/rejected_KL_mean": -61.0723876953125,
"KL/std": 30.896400451660156,
"epoch": 0.6848072562358276,
"fcm_dpo/beta": 0.03558473661541939,
"fcm_dpo/delta": -0.06524658203125,
"fcm_dpo/margin": 24.152931213378906,
"fcm_dpo/q_t": 0.3470444679260254,
"grad_norm": 26.403024673461914,
"learning_rate": 1.3780020494988445e-07,
"logits/chosen": 0.46417856216430664,
"logits/rejected": 0.43202823400497437,
"logps/chosen": -98.69766235351562,
"logps/ref_chosen": -61.778202056884766,
"logps/ref_rejected": -71.51403045654297,
"logps/rejected": -132.58642578125,
"loss": 1.0065,
"margin_dpo/margin_mean": 24.152931213378906,
"margin_dpo/margin_std": 35.1126708984375,
"step": 453
},
{
"KL/chosen_KL_mean": -34.68466567993164,
"KL/mean": -47.10640335083008,
"KL/rejected_KL_mean": -59.52813720703125,
"KL/std": 32.204864501953125,
"epoch": 0.6863189720332578,
"fcm_dpo/beta": 0.035109348595142365,
"fcm_dpo/delta": -0.07914970070123672,
"fcm_dpo/margin": 24.843469619750977,
"fcm_dpo/q_t": 0.3328770399093628,
"grad_norm": 18.879844665527344,
"learning_rate": 1.366202015206706e-07,
"logits/chosen": 0.5144060850143433,
"logits/rejected": 0.47107866406440735,
"logps/chosen": -86.27981567382812,
"logps/ref_chosen": -51.59515380859375,
"logps/ref_rejected": -63.96732711791992,
"logps/rejected": -123.49546813964844,
"loss": 0.9862,
"margin_dpo/margin_mean": 24.843469619750977,
"margin_dpo/margin_std": 34.712310791015625,
"step": 454
},
{
"KL/chosen_KL_mean": -40.27165985107422,
"KL/mean": -52.858604431152344,
"KL/rejected_KL_mean": -65.44554901123047,
"KL/std": 31.356714248657227,
"epoch": 0.6878306878306878,
"fcm_dpo/beta": 0.034394294023513794,
"fcm_dpo/delta": -0.07318489253520966,
"fcm_dpo/margin": 25.173885345458984,
"fcm_dpo/q_t": 0.33834701776504517,
"grad_norm": 19.06486701965332,
"learning_rate": 1.354433695681474e-07,
"logits/chosen": 0.36979854106903076,
"logits/rejected": 0.3327193856239319,
"logps/chosen": -110.92337036132812,
"logps/ref_chosen": -70.65170288085938,
"logps/ref_rejected": -77.44276428222656,
"logps/rejected": -142.8883056640625,
"loss": 0.93,
"margin_dpo/margin_mean": 25.173885345458984,
"margin_dpo/margin_std": 31.99268913269043,
"step": 455
},
{
"KL/chosen_KL_mean": -39.89326477050781,
"KL/mean": -50.87599182128906,
"KL/rejected_KL_mean": -61.858734130859375,
"KL/std": 31.083784103393555,
"epoch": 0.6893424036281179,
"fcm_dpo/beta": 0.03456852585077286,
"fcm_dpo/delta": 0.043941106647253036,
"fcm_dpo/margin": 21.965469360351562,
"fcm_dpo/q_t": 0.35616612434387207,
"grad_norm": 22.54862403869629,
"learning_rate": 1.3426974201083439e-07,
"logits/chosen": 0.41186925768852234,
"logits/rejected": 0.3542647957801819,
"logps/chosen": -96.29154968261719,
"logps/ref_chosen": -56.398284912109375,
"logps/ref_rejected": -82.61642456054688,
"logps/rejected": -144.47515869140625,
"loss": 1.0011,
"margin_dpo/margin_mean": 21.96546745300293,
"margin_dpo/margin_std": 31.476985931396484,
"step": 456
},
{
"KL/chosen_KL_mean": -41.337158203125,
"KL/mean": -52.13238525390625,
"KL/rejected_KL_mean": -62.927616119384766,
"KL/std": 32.70546340942383,
"epoch": 0.690854119425548,
"fcm_dpo/beta": 0.03470756113529205,
"fcm_dpo/delta": 0.053345732390880585,
"fcm_dpo/margin": 21.590456008911133,
"fcm_dpo/q_t": 0.3532480001449585,
"grad_norm": 17.304533004760742,
"learning_rate": 1.3309935167761717e-07,
"logits/chosen": 0.5655590295791626,
"logits/rejected": 0.493374228477478,
"logps/chosen": -86.0577392578125,
"logps/ref_chosen": -44.72057342529297,
"logps/ref_rejected": -68.1158676147461,
"logps/rejected": -131.04348754882812,
"loss": 0.9677,
"margin_dpo/margin_mean": 21.590456008911133,
"margin_dpo/margin_std": 27.787086486816406,
"step": 457
},
{
"KL/chosen_KL_mean": -36.466941833496094,
"KL/mean": -48.640602111816406,
"KL/rejected_KL_mean": -60.81425476074219,
"KL/std": 33.46665954589844,
"epoch": 0.6923658352229781,
"fcm_dpo/beta": 0.034741275012493134,
"fcm_dpo/delta": -0.05059751495718956,
"fcm_dpo/margin": 24.34731674194336,
"fcm_dpo/q_t": 0.3377965986728668,
"grad_norm": 18.70404815673828,
"learning_rate": 1.3193223130682936e-07,
"logits/chosen": 0.4883077144622803,
"logits/rejected": 0.37608468532562256,
"logps/chosen": -86.4726333618164,
"logps/ref_chosen": -50.00569152832031,
"logps/ref_rejected": -87.50015258789062,
"logps/rejected": -148.3144073486328,
"loss": 0.9674,
"margin_dpo/margin_mean": 24.347320556640625,
"margin_dpo/margin_std": 32.958587646484375,
"step": 458
},
{
"KL/chosen_KL_mean": -35.83440017700195,
"KL/mean": -50.071266174316406,
"KL/rejected_KL_mean": -64.30813598632812,
"KL/std": 33.272953033447266,
"epoch": 0.6938775510204082,
"fcm_dpo/beta": 0.034444473683834076,
"fcm_dpo/delta": -0.20470258593559265,
"fcm_dpo/margin": 28.473735809326172,
"fcm_dpo/q_t": 0.3152352571487427,
"grad_norm": 18.777652740478516,
"learning_rate": 1.3076841354533658e-07,
"logits/chosen": 0.5320541858673096,
"logits/rejected": 0.4936879873275757,
"logps/chosen": -101.21234893798828,
"logps/ref_chosen": -65.37794494628906,
"logps/ref_rejected": -88.19244384765625,
"logps/rejected": -152.50057983398438,
"loss": 0.8944,
"margin_dpo/margin_mean": 28.473735809326172,
"margin_dpo/margin_std": 31.597108840942383,
"step": 459
},
{
"KL/chosen_KL_mean": -39.52485656738281,
"KL/mean": -54.53339385986328,
"KL/rejected_KL_mean": -69.54193115234375,
"KL/std": 32.83936309814453,
"epoch": 0.6953892668178382,
"fcm_dpo/beta": 0.032258301973342896,
"fcm_dpo/delta": -0.18851391971111298,
"fcm_dpo/margin": 30.017070770263672,
"fcm_dpo/q_t": 0.3154947757720947,
"grad_norm": 17.20231056213379,
"learning_rate": 1.2960793094762345e-07,
"logits/chosen": 0.5572216510772705,
"logits/rejected": 0.4307131767272949,
"logps/chosen": -104.0865249633789,
"logps/ref_chosen": -64.5616683959961,
"logps/ref_rejected": -88.67890167236328,
"logps/rejected": -158.2208251953125,
"loss": 0.8574,
"margin_dpo/margin_mean": 30.017070770263672,
"margin_dpo/margin_std": 33.085060119628906,
"step": 460
},
{
"KL/chosen_KL_mean": -32.57419204711914,
"KL/mean": -46.81694030761719,
"KL/rejected_KL_mean": -61.0596923828125,
"KL/std": 30.406360626220703,
"epoch": 0.6969009826152683,
"fcm_dpo/beta": 0.031209370121359825,
"fcm_dpo/delta": -0.10361548513174057,
"fcm_dpo/margin": 28.48550033569336,
"fcm_dpo/q_t": 0.3270339369773865,
"grad_norm": 16.9658203125,
"learning_rate": 1.2845081597488286e-07,
"logits/chosen": 0.6307059526443481,
"logits/rejected": 0.5390516519546509,
"logps/chosen": -82.05210876464844,
"logps/ref_chosen": -49.4779167175293,
"logps/ref_rejected": -72.65262603759766,
"logps/rejected": -133.71231079101562,
"loss": 0.8926,
"margin_dpo/margin_mean": 28.48550033569336,
"margin_dpo/margin_std": 31.559621810913086,
"step": 461
},
{
"KL/chosen_KL_mean": -36.72931671142578,
"KL/mean": -52.78752899169922,
"KL/rejected_KL_mean": -68.84573364257812,
"KL/std": 33.4709587097168,
"epoch": 0.6984126984126984,
"fcm_dpo/beta": 0.030348652973771095,
"fcm_dpo/delta": -0.19649741053581238,
"fcm_dpo/margin": 32.11642074584961,
"fcm_dpo/q_t": 0.3061380684375763,
"grad_norm": 16.422710418701172,
"learning_rate": 1.27297100994108e-07,
"logits/chosen": 0.49276840686798096,
"logits/rejected": 0.43394631147384644,
"logps/chosen": -97.22443389892578,
"logps/ref_chosen": -60.4951171875,
"logps/ref_rejected": -74.82136535644531,
"logps/rejected": -143.6671142578125,
"loss": 0.8381,
"margin_dpo/margin_mean": 32.11642074584961,
"margin_dpo/margin_std": 33.13414001464844,
"step": 462
},
{
"KL/chosen_KL_mean": -41.92523956298828,
"KL/mean": -52.19264602661133,
"KL/rejected_KL_mean": -62.46005630493164,
"KL/std": 30.781005859375,
"epoch": 0.6999244142101285,
"fcm_dpo/beta": 0.030946975573897362,
"fcm_dpo/delta": 0.1760467141866684,
"fcm_dpo/margin": 20.53483009338379,
"fcm_dpo/q_t": 0.3690449595451355,
"grad_norm": 20.61341667175293,
"learning_rate": 1.2614681827718695e-07,
"logits/chosen": 0.4970882833003998,
"logits/rejected": 0.4902383089065552,
"logps/chosen": -109.6103515625,
"logps/ref_chosen": -67.68511962890625,
"logps/ref_rejected": -71.32196044921875,
"logps/rejected": -133.78201293945312,
"loss": 1.025,
"margin_dpo/margin_mean": 20.53483009338379,
"margin_dpo/margin_std": 28.81637954711914,
"step": 463
},
{
"KL/chosen_KL_mean": -41.53732681274414,
"KL/mean": -54.971805572509766,
"KL/rejected_KL_mean": -68.40628051757812,
"KL/std": 33.168521881103516,
"epoch": 0.7014361300075586,
"fcm_dpo/beta": 0.031339820474386215,
"fcm_dpo/delta": -0.0464196652173996,
"fcm_dpo/margin": 26.86896514892578,
"fcm_dpo/q_t": 0.3392544388771057,
"grad_norm": 20.619489669799805,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": 0.5063756704330444,
"logits/rejected": 0.4772282838821411,
"logps/chosen": -100.70297241210938,
"logps/ref_chosen": -59.16564178466797,
"logps/ref_rejected": -69.56146240234375,
"logps/rejected": -137.96774291992188,
"loss": 0.9753,
"margin_dpo/margin_mean": 26.86896514892578,
"margin_dpo/margin_std": 35.64836883544922,
"step": 464
},
{
"KL/chosen_KL_mean": -44.17216110229492,
"KL/mean": -55.57133102416992,
"KL/rejected_KL_mean": -66.97048950195312,
"KL/std": 32.37845230102539,
"epoch": 0.7029478458049887,
"fcm_dpo/beta": 0.031462736427783966,
"fcm_dpo/delta": 0.08915092796087265,
"fcm_dpo/margin": 22.798336029052734,
"fcm_dpo/q_t": 0.3652403652667999,
"grad_norm": 22.905973434448242,
"learning_rate": 1.238566782415197e-07,
"logits/chosen": 0.5705190896987915,
"logits/rejected": 0.507891833782196,
"logps/chosen": -102.68583679199219,
"logps/ref_chosen": -58.513671875,
"logps/ref_rejected": -84.31745910644531,
"logps/rejected": -151.28794860839844,
"loss": 1.0512,
"margin_dpo/margin_mean": 22.7983341217041,
"margin_dpo/margin_std": 35.545631408691406,
"step": 465
},
{
"KL/chosen_KL_mean": -45.907772064208984,
"KL/mean": -53.66548156738281,
"KL/rejected_KL_mean": -61.42319869995117,
"KL/std": 33.30298614501953,
"epoch": 0.7044595616024187,
"fcm_dpo/beta": 0.03294553607702255,
"fcm_dpo/delta": 0.3054059147834778,
"fcm_dpo/margin": 15.515429496765137,
"fcm_dpo/q_t": 0.4019228518009186,
"grad_norm": 25.403379440307617,
"learning_rate": 1.2271688498291334e-07,
"logits/chosen": 0.5198469161987305,
"logits/rejected": 0.5233859419822693,
"logps/chosen": -119.173583984375,
"logps/ref_chosen": -73.26580810546875,
"logps/ref_rejected": -74.83621215820312,
"logps/rejected": -136.25941467285156,
"loss": 1.1349,
"margin_dpo/margin_mean": 15.51543140411377,
"margin_dpo/margin_std": 29.112449645996094,
"step": 466
},
{
"KL/chosen_KL_mean": -43.1234130859375,
"KL/mean": -54.358455657958984,
"KL/rejected_KL_mean": -65.593505859375,
"KL/std": 34.307823181152344,
"epoch": 0.7059712773998488,
"fcm_dpo/beta": 0.0338030569255352,
"fcm_dpo/delta": 0.04378930479288101,
"fcm_dpo/margin": 22.47007942199707,
"fcm_dpo/q_t": 0.3505485951900482,
"grad_norm": 18.757200241088867,
"learning_rate": 1.2158065210664848e-07,
"logits/chosen": 0.5469769835472107,
"logits/rejected": 0.3987196385860443,
"logps/chosen": -90.702880859375,
"logps/ref_chosen": -47.57947540283203,
"logps/ref_rejected": -78.68522644042969,
"logps/rejected": -144.27871704101562,
"loss": 0.9565,
"margin_dpo/margin_mean": 22.470081329345703,
"margin_dpo/margin_std": 29.331863403320312,
"step": 467
},
{
"KL/chosen_KL_mean": -36.90409851074219,
"KL/mean": -53.89661407470703,
"KL/rejected_KL_mean": -70.88912963867188,
"KL/std": 35.14104080200195,
"epoch": 0.7074829931972789,
"fcm_dpo/beta": 0.032390519976615906,
"fcm_dpo/delta": -0.33713847398757935,
"fcm_dpo/margin": 33.98503494262695,
"fcm_dpo/q_t": 0.29361268877983093,
"grad_norm": 18.30913734436035,
"learning_rate": 1.204480113956011e-07,
"logits/chosen": 0.4965260624885559,
"logits/rejected": 0.48857080936431885,
"logps/chosen": -100.8318862915039,
"logps/ref_chosen": -63.92778778076172,
"logps/ref_rejected": -76.51626586914062,
"logps/rejected": -147.4053955078125,
"loss": 0.8048,
"margin_dpo/margin_mean": 33.98503112792969,
"margin_dpo/margin_std": 35.08293914794922,
"step": 468
},
{
"KL/chosen_KL_mean": -38.55724334716797,
"KL/mean": -52.420753479003906,
"KL/rejected_KL_mean": -66.28427124023438,
"KL/std": 32.45834732055664,
"epoch": 0.708994708994709,
"fcm_dpo/beta": 0.030920889228582382,
"fcm_dpo/delta": -0.06855427473783493,
"fcm_dpo/margin": 27.72703742980957,
"fcm_dpo/q_t": 0.33362048864364624,
"grad_norm": 19.86760902404785,
"learning_rate": 1.1931899453216697e-07,
"logits/chosen": 0.5974500179290771,
"logits/rejected": 0.5843087434768677,
"logps/chosen": -97.61542510986328,
"logps/ref_chosen": -59.05818176269531,
"logps/ref_rejected": -75.67672729492188,
"logps/rejected": -141.96099853515625,
"loss": 0.9071,
"margin_dpo/margin_mean": 27.727035522460938,
"margin_dpo/margin_std": 31.478986740112305,
"step": 469
},
{
"KL/chosen_KL_mean": -40.4164924621582,
"KL/mean": -53.52194595336914,
"KL/rejected_KL_mean": -66.62739562988281,
"KL/std": 32.51350402832031,
"epoch": 0.7105064247921391,
"fcm_dpo/beta": 0.03135482966899872,
"fcm_dpo/delta": -0.02440682053565979,
"fcm_dpo/margin": 26.21091079711914,
"fcm_dpo/q_t": 0.3376317322254181,
"grad_norm": 16.981752395629883,
"learning_rate": 1.1819363309737438e-07,
"logits/chosen": 0.5298024415969849,
"logits/rejected": 0.460299551486969,
"logps/chosen": -88.283935546875,
"logps/ref_chosen": -47.86743927001953,
"logps/ref_rejected": -65.96859741210938,
"logps/rejected": -132.5959930419922,
"loss": 0.9647,
"margin_dpo/margin_mean": 26.21091079711914,
"margin_dpo/margin_std": 34.34257507324219,
"step": 470
},
{
"KL/chosen_KL_mean": -35.6857795715332,
"KL/mean": -51.856781005859375,
"KL/rejected_KL_mean": -68.02778625488281,
"KL/std": 33.62290573120117,
"epoch": 0.7120181405895691,
"fcm_dpo/beta": 0.030277177691459656,
"fcm_dpo/delta": -0.19842389225959778,
"fcm_dpo/margin": 32.342002868652344,
"fcm_dpo/q_t": 0.3140113055706024,
"grad_norm": 17.59916877746582,
"learning_rate": 1.1707195857000215e-07,
"logits/chosen": 0.5253442525863647,
"logits/rejected": 0.4637981951236725,
"logps/chosen": -93.46363830566406,
"logps/ref_chosen": -57.777854919433594,
"logps/ref_rejected": -73.81172180175781,
"logps/rejected": -141.83950805664062,
"loss": 0.8804,
"margin_dpo/margin_mean": 32.342002868652344,
"margin_dpo/margin_std": 37.22600555419922,
"step": 471
},
{
"KL/chosen_KL_mean": -37.23526382446289,
"KL/mean": -50.47922897338867,
"KL/rejected_KL_mean": -63.72319030761719,
"KL/std": 33.24669647216797,
"epoch": 0.7135298563869993,
"fcm_dpo/beta": 0.03014422580599785,
"fcm_dpo/delta": 0.00048603862524032593,
"fcm_dpo/margin": 26.48792266845703,
"fcm_dpo/q_t": 0.35318523645401,
"grad_norm": 19.13536834716797,
"learning_rate": 1.1595400232569768e-07,
"logits/chosen": 0.5265265703201294,
"logits/rejected": 0.47909194231033325,
"logps/chosen": -93.14393615722656,
"logps/ref_chosen": -55.908668518066406,
"logps/ref_rejected": -74.70294189453125,
"logps/rejected": -138.42613220214844,
"loss": 1.019,
"margin_dpo/margin_mean": 26.48792266845703,
"margin_dpo/margin_std": 38.939327239990234,
"step": 472
},
{
"KL/chosen_KL_mean": -37.54869842529297,
"KL/mean": -50.015480041503906,
"KL/rejected_KL_mean": -62.48226547241211,
"KL/std": 35.00571823120117,
"epoch": 0.7150415721844293,
"fcm_dpo/beta": 0.030113372951745987,
"fcm_dpo/delta": 0.0531911626458168,
"fcm_dpo/margin": 24.933568954467773,
"fcm_dpo/q_t": 0.3640349209308624,
"grad_norm": 21.23849105834961,
"learning_rate": 1.1483979563610069e-07,
"logits/chosen": 0.612478494644165,
"logits/rejected": 0.5012402534484863,
"logps/chosen": -91.70957946777344,
"logps/ref_chosen": -54.16088104248047,
"logps/ref_rejected": -92.76789855957031,
"logps/rejected": -155.25015258789062,
"loss": 1.0737,
"margin_dpo/margin_mean": 24.933568954467773,
"margin_dpo/margin_std": 40.785545349121094,
"step": 473
},
{
"KL/chosen_KL_mean": -43.27687072753906,
"KL/mean": -54.43352508544922,
"KL/rejected_KL_mean": -65.59016418457031,
"KL/std": 34.479217529296875,
"epoch": 0.7165532879818595,
"fcm_dpo/beta": 0.030809249728918076,
"fcm_dpo/delta": 0.12073996663093567,
"fcm_dpo/margin": 22.31329345703125,
"fcm_dpo/q_t": 0.3735297918319702,
"grad_norm": 23.4538631439209,
"learning_rate": 1.1372936966796709e-07,
"logits/chosen": 0.627903401851654,
"logits/rejected": 0.5501044988632202,
"logps/chosen": -89.96257781982422,
"logps/ref_chosen": -46.685707092285156,
"logps/ref_rejected": -71.44731903076172,
"logps/rejected": -137.0374755859375,
"loss": 1.09,
"margin_dpo/margin_mean": 22.31329345703125,
"margin_dpo/margin_std": 37.7652587890625,
"step": 474
},
{
"KL/chosen_KL_mean": -40.59794616699219,
"KL/mean": -58.311309814453125,
"KL/rejected_KL_mean": -76.02467346191406,
"KL/std": 35.56901550292969,
"epoch": 0.7180650037792895,
"fcm_dpo/beta": 0.029465150088071823,
"fcm_dpo/delta": -0.2766759693622589,
"fcm_dpo/margin": 35.42672348022461,
"fcm_dpo/q_t": 0.2970644235610962,
"grad_norm": 15.178966522216797,
"learning_rate": 1.126227554822985e-07,
"logits/chosen": 0.5457562208175659,
"logits/rejected": 0.4907382130622864,
"logps/chosen": -99.08525085449219,
"logps/ref_chosen": -58.4873046875,
"logps/ref_rejected": -87.00187683105469,
"logps/rejected": -163.02655029296875,
"loss": 0.7885,
"margin_dpo/margin_mean": 35.42672348022461,
"margin_dpo/margin_std": 33.91914367675781,
"step": 475
},
{
"KL/chosen_KL_mean": -48.61393356323242,
"KL/mean": -60.248931884765625,
"KL/rejected_KL_mean": -71.8839340209961,
"KL/std": 35.561439514160156,
"epoch": 0.7195767195767195,
"fcm_dpo/beta": 0.029918601736426353,
"fcm_dpo/delta": 0.11073946952819824,
"fcm_dpo/margin": 23.26999855041504,
"fcm_dpo/q_t": 0.3694406747817993,
"grad_norm": 21.122638702392578,
"learning_rate": 1.1151998403347243e-07,
"logits/chosen": 0.4469614028930664,
"logits/rejected": 0.43910348415374756,
"logps/chosen": -123.99555969238281,
"logps/ref_chosen": -75.38162231445312,
"logps/ref_rejected": -76.99822235107422,
"logps/rejected": -148.8821563720703,
"loss": 1.0497,
"margin_dpo/margin_mean": 23.270000457763672,
"margin_dpo/margin_std": 36.53971481323242,
"step": 476
},
{
"KL/chosen_KL_mean": -49.38127899169922,
"KL/mean": -61.15679931640625,
"KL/rejected_KL_mean": -72.93232727050781,
"KL/std": 35.085174560546875,
"epoch": 0.7210884353741497,
"fcm_dpo/beta": 0.03036920353770256,
"fcm_dpo/delta": 0.09131810069084167,
"fcm_dpo/margin": 23.55103874206543,
"fcm_dpo/q_t": 0.36580678820610046,
"grad_norm": 24.693286895751953,
"learning_rate": 1.1042108616837692e-07,
"logits/chosen": 0.5518933534622192,
"logits/rejected": 0.5037527084350586,
"logps/chosen": -110.45466613769531,
"logps/ref_chosen": -61.073387145996094,
"logps/ref_rejected": -81.34375,
"logps/rejected": -154.27606201171875,
"loss": 1.1101,
"margin_dpo/margin_mean": 23.551036834716797,
"margin_dpo/margin_std": 40.80988311767578,
"step": 477
},
{
"KL/chosen_KL_mean": -41.85948181152344,
"KL/mean": -52.576988220214844,
"KL/rejected_KL_mean": -63.29448699951172,
"KL/std": 33.8995475769043,
"epoch": 0.7226001511715797,
"fcm_dpo/beta": 0.031108807772397995,
"fcm_dpo/delta": 0.14290378987789154,
"fcm_dpo/margin": 21.435009002685547,
"fcm_dpo/q_t": 0.380104124546051,
"grad_norm": 25.761295318603516,
"learning_rate": 1.0932609262554746e-07,
"logits/chosen": 0.49377888441085815,
"logits/rejected": 0.4970715641975403,
"logps/chosen": -99.02679443359375,
"logps/ref_chosen": -57.16731643676758,
"logps/ref_rejected": -53.30917739868164,
"logps/rejected": -116.60366821289062,
"loss": 1.1518,
"margin_dpo/margin_mean": 21.435009002685547,
"margin_dpo/margin_std": 40.712486267089844,
"step": 478
},
{
"KL/chosen_KL_mean": -44.9219856262207,
"KL/mean": -54.529151916503906,
"KL/rejected_KL_mean": -64.13630676269531,
"KL/std": 31.78256607055664,
"epoch": 0.7241118669690099,
"fcm_dpo/beta": 0.032088786363601685,
"fcm_dpo/delta": 0.19578224420547485,
"fcm_dpo/margin": 19.21432876586914,
"fcm_dpo/q_t": 0.3787573575973511,
"grad_norm": 22.831226348876953,
"learning_rate": 1.0823503403430734e-07,
"logits/chosen": 0.45101386308670044,
"logits/rejected": 0.39903637766838074,
"logps/chosen": -103.83529663085938,
"logps/ref_chosen": -58.91331481933594,
"logps/ref_rejected": -63.7403450012207,
"logps/rejected": -127.87665557861328,
"loss": 1.123,
"margin_dpo/margin_mean": 19.21432876586914,
"margin_dpo/margin_std": 34.718929290771484,
"step": 479
},
{
"KL/chosen_KL_mean": -43.14649200439453,
"KL/mean": -56.787742614746094,
"KL/rejected_KL_mean": -70.42898559570312,
"KL/std": 32.69535827636719,
"epoch": 0.7256235827664399,
"fcm_dpo/beta": 0.03261556476354599,
"fcm_dpo/delta": -0.10105873644351959,
"fcm_dpo/margin": 27.282489776611328,
"fcm_dpo/q_t": 0.3315708041191101,
"grad_norm": 24.251625061035156,
"learning_rate": 1.0714794091391072e-07,
"logits/chosen": 0.4233095645904541,
"logits/rejected": 0.41327613592147827,
"logps/chosen": -105.94709777832031,
"logps/ref_chosen": -62.80061340332031,
"logps/ref_rejected": -67.58859252929688,
"logps/rejected": -138.017578125,
"loss": 1.0316,
"margin_dpo/margin_mean": 27.282489776611328,
"margin_dpo/margin_std": 39.7613525390625,
"step": 480
},
{
"KL/chosen_KL_mean": -42.957916259765625,
"KL/mean": -57.020362854003906,
"KL/rejected_KL_mean": -71.08280944824219,
"KL/std": 36.26377868652344,
"epoch": 0.72713529856387,
"fcm_dpo/beta": 0.03143627941608429,
"fcm_dpo/delta": -0.09304574131965637,
"fcm_dpo/margin": 28.12488555908203,
"fcm_dpo/q_t": 0.3399431109428406,
"grad_norm": 21.250574111938477,
"learning_rate": 1.0606484367268906e-07,
"logits/chosen": 0.4657706618309021,
"logits/rejected": 0.4617427885532379,
"logps/chosen": -108.2444076538086,
"logps/ref_chosen": -65.28649139404297,
"logps/ref_rejected": -70.78668212890625,
"logps/rejected": -141.86949157714844,
"loss": 0.9748,
"margin_dpo/margin_mean": 28.12488555908203,
"margin_dpo/margin_std": 38.75299072265625,
"step": 481
},
{
"KL/chosen_KL_mean": -48.609527587890625,
"KL/mean": -60.3476676940918,
"KL/rejected_KL_mean": -72.08580780029297,
"KL/std": 34.90166473388672,
"epoch": 0.7286470143613001,
"fcm_dpo/beta": 0.031867966055870056,
"fcm_dpo/delta": 0.054899971932172775,
"fcm_dpo/margin": 23.47627830505371,
"fcm_dpo/q_t": 0.36690202355384827,
"grad_norm": 25.70149040222168,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": 0.4665817618370056,
"logits/rejected": 0.3242020606994629,
"logps/chosen": -109.51571655273438,
"logps/ref_chosen": -60.906185150146484,
"logps/ref_rejected": -103.44656372070312,
"logps/rejected": -175.53237915039062,
"loss": 1.1398,
"margin_dpo/margin_mean": 23.47627830505371,
"margin_dpo/margin_std": 42.882415771484375,
"step": 482
},
{
"KL/chosen_KL_mean": -41.326690673828125,
"KL/mean": -55.978431701660156,
"KL/rejected_KL_mean": -70.63018035888672,
"KL/std": 33.76634979248047,
"epoch": 0.7301587301587301,
"fcm_dpo/beta": 0.031191810965538025,
"fcm_dpo/delta": -0.12544870376586914,
"fcm_dpo/margin": 29.303483963012695,
"fcm_dpo/q_t": 0.32259464263916016,
"grad_norm": 20.508411407470703,
"learning_rate": 1.0391075790138232e-07,
"logits/chosen": 0.5879669189453125,
"logits/rejected": 0.4857994616031647,
"logps/chosen": -94.51870727539062,
"logps/ref_chosen": -53.192012786865234,
"logps/ref_rejected": -81.83927154541016,
"logps/rejected": -152.46945190429688,
"loss": 0.919,
"margin_dpo/margin_mean": 29.303482055664062,
"margin_dpo/margin_std": 36.168243408203125,
"step": 483
},
{
"KL/chosen_KL_mean": -41.7369499206543,
"KL/mean": -54.1743049621582,
"KL/rejected_KL_mean": -66.61164855957031,
"KL/std": 32.98217010498047,
"epoch": 0.7316704459561603,
"fcm_dpo/beta": 0.031263504177331924,
"fcm_dpo/delta": 0.023358818143606186,
"fcm_dpo/margin": 24.87470245361328,
"fcm_dpo/q_t": 0.344391405582428,
"grad_norm": 21.53194808959961,
"learning_rate": 1.0283982962570681e-07,
"logits/chosen": 0.5854651927947998,
"logits/rejected": 0.5462943911552429,
"logps/chosen": -99.50640869140625,
"logps/ref_chosen": -57.76945877075195,
"logps/ref_rejected": -71.6829833984375,
"logps/rejected": -138.29464721679688,
"loss": 0.9359,
"margin_dpo/margin_mean": 24.87470245361328,
"margin_dpo/margin_std": 30.0977783203125,
"step": 484
},
{
"KL/chosen_KL_mean": -41.71012878417969,
"KL/mean": -54.20851516723633,
"KL/rejected_KL_mean": -66.70690155029297,
"KL/std": 35.231590270996094,
"epoch": 0.7331821617535903,
"fcm_dpo/beta": 0.030793113633990288,
"fcm_dpo/delta": 0.02861456573009491,
"fcm_dpo/margin": 24.996782302856445,
"fcm_dpo/q_t": 0.3545028567314148,
"grad_norm": 19.133703231811523,
"learning_rate": 1.0177301773633992e-07,
"logits/chosen": 0.5398764610290527,
"logits/rejected": 0.5143481492996216,
"logps/chosen": -98.34597778320312,
"logps/ref_chosen": -56.63584899902344,
"logps/ref_rejected": -70.85614013671875,
"logps/rejected": -137.56304931640625,
"loss": 1.004,
"margin_dpo/margin_mean": 24.996782302856445,
"margin_dpo/margin_std": 35.169715881347656,
"step": 485
},
{
"KL/chosen_KL_mean": -52.103946685791016,
"KL/mean": -62.98194885253906,
"KL/rejected_KL_mean": -73.85993957519531,
"KL/std": 37.7567138671875,
"epoch": 0.7346938775510204,
"fcm_dpo/beta": 0.03189978003501892,
"fcm_dpo/delta": 0.113531693816185,
"fcm_dpo/margin": 21.755998611450195,
"fcm_dpo/q_t": 0.3762109875679016,
"grad_norm": 22.65691375732422,
"learning_rate": 1.007103520743035e-07,
"logits/chosen": 0.5532824993133545,
"logits/rejected": 0.4355977177619934,
"logps/chosen": -108.45097351074219,
"logps/ref_chosen": -56.347023010253906,
"logps/ref_rejected": -85.97221374511719,
"logps/rejected": -159.8321533203125,
"loss": 1.11,
"margin_dpo/margin_mean": 21.756000518798828,
"margin_dpo/margin_std": 39.52480697631836,
"step": 486
},
{
"KL/chosen_KL_mean": -46.250057220458984,
"KL/mean": -58.29290771484375,
"KL/rejected_KL_mean": -70.33575439453125,
"KL/std": 31.51739501953125,
"epoch": 0.7362055933484505,
"fcm_dpo/beta": 0.03211412951350212,
"fcm_dpo/delta": 0.028735807165503502,
"fcm_dpo/margin": 24.085702896118164,
"fcm_dpo/q_t": 0.3533576428890228,
"grad_norm": 25.153072357177734,
"learning_rate": 9.965186236464046e-08,
"logits/chosen": 0.6023333668708801,
"logits/rejected": 0.5408201217651367,
"logps/chosen": -106.86727905273438,
"logps/ref_chosen": -60.617218017578125,
"logps/ref_rejected": -82.50975036621094,
"logps/rejected": -152.8455047607422,
"loss": 1.0122,
"margin_dpo/margin_mean": 24.085702896118164,
"margin_dpo/margin_std": 35.66450500488281,
"step": 487
},
{
"KL/chosen_KL_mean": -41.968387603759766,
"KL/mean": -56.97295379638672,
"KL/rejected_KL_mean": -71.97752380371094,
"KL/std": 33.929840087890625,
"epoch": 0.7377173091458806,
"fcm_dpo/beta": 0.031626880168914795,
"fcm_dpo/delta": -0.16451723873615265,
"fcm_dpo/margin": 30.00912857055664,
"fcm_dpo/q_t": 0.3240189850330353,
"grad_norm": 22.58987045288086,
"learning_rate": 9.859757821558337e-08,
"logits/chosen": 0.5339430570602417,
"logits/rejected": 0.46293365955352783,
"logps/chosen": -105.07743835449219,
"logps/ref_chosen": -63.10905075073242,
"logps/ref_rejected": -82.49348449707031,
"logps/rejected": -154.47100830078125,
"loss": 0.888,
"margin_dpo/margin_mean": 30.009132385253906,
"margin_dpo/margin_std": 35.635520935058594,
"step": 488
},
{
"KL/chosen_KL_mean": -48.408485412597656,
"KL/mean": -56.583763122558594,
"KL/rejected_KL_mean": -64.75904846191406,
"KL/std": 34.571693420410156,
"epoch": 0.7392290249433107,
"fcm_dpo/beta": 0.03243595361709595,
"fcm_dpo/delta": 0.2853114604949951,
"fcm_dpo/margin": 16.350563049316406,
"fcm_dpo/q_t": 0.40409570932388306,
"grad_norm": 25.71051025390625,
"learning_rate": 9.754752911772615e-08,
"logits/chosen": 0.49547716975212097,
"logits/rejected": 0.4507462978363037,
"logps/chosen": -113.39744567871094,
"logps/ref_chosen": -64.98896026611328,
"logps/ref_rejected": -84.39607238769531,
"logps/rejected": -149.15512084960938,
"loss": 1.2059,
"margin_dpo/margin_mean": 16.350563049316406,
"margin_dpo/margin_std": 35.75886154174805,
"step": 489
},
{
"KL/chosen_KL_mean": -42.95598602294922,
"KL/mean": -53.87427520751953,
"KL/rejected_KL_mean": -64.79255676269531,
"KL/std": 36.25221633911133,
"epoch": 0.7407407407407407,
"fcm_dpo/beta": 0.03329470753669739,
"fcm_dpo/delta": 0.07874666899442673,
"fcm_dpo/margin": 21.836578369140625,
"fcm_dpo/q_t": 0.3706052899360657,
"grad_norm": 23.770259857177734,
"learning_rate": 9.650174444319956e-08,
"logits/chosen": 0.5940006375312805,
"logits/rejected": 0.567367434501648,
"logps/chosen": -104.86473083496094,
"logps/ref_chosen": -61.90874481201172,
"logps/ref_rejected": -70.58566284179688,
"logps/rejected": -135.3782196044922,
"loss": 1.2193,
"margin_dpo/margin_mean": 21.836578369140625,
"margin_dpo/margin_std": 44.323036193847656,
"step": 490
},
{
"KL/chosen_KL_mean": -42.657981872558594,
"KL/mean": -54.71141052246094,
"KL/rejected_KL_mean": -66.76484680175781,
"KL/std": 33.00274658203125,
"epoch": 0.7422524565381708,
"fcm_dpo/beta": 0.03311682492494583,
"fcm_dpo/delta": -0.0010885968804359436,
"fcm_dpo/margin": 24.10685920715332,
"fcm_dpo/q_t": 0.3481886386871338,
"grad_norm": 19.895963668823242,
"learning_rate": 9.546025344484868e-08,
"logits/chosen": 0.4401285648345947,
"logits/rejected": 0.38554367423057556,
"logps/chosen": -98.13368225097656,
"logps/ref_chosen": -55.47570037841797,
"logps/ref_rejected": -78.70318603515625,
"logps/rejected": -145.468017578125,
"loss": 0.9984,
"margin_dpo/margin_mean": 24.10685920715332,
"margin_dpo/margin_std": 33.24077606201172,
"step": 491
},
{
"KL/chosen_KL_mean": -53.54511260986328,
"KL/mean": -63.428951263427734,
"KL/rejected_KL_mean": -73.31278991699219,
"KL/std": 37.33125305175781,
"epoch": 0.7437641723356009,
"fcm_dpo/beta": 0.03454633057117462,
"fcm_dpo/delta": 0.12079239636659622,
"fcm_dpo/margin": 19.767681121826172,
"fcm_dpo/q_t": 0.38081228733062744,
"grad_norm": 26.302976608276367,
"learning_rate": 9.442308525541589e-08,
"logits/chosen": 0.49925363063812256,
"logits/rejected": 0.42450058460235596,
"logps/chosen": -120.83149719238281,
"logps/ref_chosen": -67.28638458251953,
"logps/ref_rejected": -82.78628540039062,
"logps/rejected": -156.09909057617188,
"loss": 1.1787,
"margin_dpo/margin_mean": 19.767677307128906,
"margin_dpo/margin_std": 38.34576416015625,
"step": 492
},
{
"KL/chosen_KL_mean": -41.301639556884766,
"KL/mean": -55.703651428222656,
"KL/rejected_KL_mean": -70.10565948486328,
"KL/std": 35.31755447387695,
"epoch": 0.745275888133031,
"fcm_dpo/beta": 0.03364936634898186,
"fcm_dpo/delta": -0.18719664216041565,
"fcm_dpo/margin": 28.804014205932617,
"fcm_dpo/q_t": 0.3174014687538147,
"grad_norm": 20.67091178894043,
"learning_rate": 9.339026888672468e-08,
"logits/chosen": 0.48015329241752625,
"logits/rejected": 0.3986372947692871,
"logps/chosen": -97.22914123535156,
"logps/ref_chosen": -55.92750549316406,
"logps/ref_rejected": -79.12149810791016,
"logps/rejected": -149.22715759277344,
"loss": 0.9349,
"margin_dpo/margin_mean": 28.80401611328125,
"margin_dpo/margin_std": 37.06830596923828,
"step": 493
},
{
"KL/chosen_KL_mean": -43.42155838012695,
"KL/mean": -53.814109802246094,
"KL/rejected_KL_mean": -64.2066650390625,
"KL/std": 34.440528869628906,
"epoch": 0.7467876039304611,
"fcm_dpo/beta": 0.033497944474220276,
"fcm_dpo/delta": 0.1116424947977066,
"fcm_dpo/margin": 20.78510284423828,
"fcm_dpo/q_t": 0.37312787771224976,
"grad_norm": 27.04566764831543,
"learning_rate": 9.236183322886945e-08,
"logits/chosen": 0.4167022109031677,
"logits/rejected": 0.36079585552215576,
"logps/chosen": -111.37567138671875,
"logps/ref_chosen": -67.95410919189453,
"logps/ref_rejected": -90.50865173339844,
"logps/rejected": -154.71531677246094,
"loss": 1.1999,
"margin_dpo/margin_mean": 20.78510284423828,
"margin_dpo/margin_std": 42.49012756347656,
"step": 494
},
{
"KL/chosen_KL_mean": -41.72477722167969,
"KL/mean": -51.252037048339844,
"KL/rejected_KL_mean": -60.779300689697266,
"KL/std": 33.50157165527344,
"epoch": 0.7482993197278912,
"fcm_dpo/beta": 0.03483235090970993,
"fcm_dpo/delta": 0.14320899546146393,
"fcm_dpo/margin": 19.054513931274414,
"fcm_dpo/q_t": 0.3821936249732971,
"grad_norm": 25.66999626159668,
"learning_rate": 9.133780704940594e-08,
"logits/chosen": 0.6148525476455688,
"logits/rejected": 0.5430424213409424,
"logps/chosen": -94.35023498535156,
"logps/ref_chosen": -52.62546157836914,
"logps/ref_rejected": -72.06781005859375,
"logps/rejected": -132.84710693359375,
"loss": 1.1185,
"margin_dpo/margin_mean": 19.054515838623047,
"margin_dpo/margin_std": 34.809410095214844,
"step": 495
},
{
"KL/chosen_KL_mean": -46.68023681640625,
"KL/mean": -58.58380889892578,
"KL/rejected_KL_mean": -70.48738098144531,
"KL/std": 34.729984283447266,
"epoch": 0.7498110355253212,
"fcm_dpo/beta": 0.034354642033576965,
"fcm_dpo/delta": -0.021552205085754395,
"fcm_dpo/margin": 23.80715560913086,
"fcm_dpo/q_t": 0.3606174886226654,
"grad_norm": 22.564855575561523,
"learning_rate": 9.031821899254797e-08,
"logits/chosen": 0.5376992225646973,
"logits/rejected": 0.42464008927345276,
"logps/chosen": -104.27755737304688,
"logps/ref_chosen": -57.597320556640625,
"logps/ref_rejected": -94.36127471923828,
"logps/rejected": -164.84866333007812,
"loss": 1.0669,
"margin_dpo/margin_mean": 23.80715560913086,
"margin_dpo/margin_std": 39.395347595214844,
"step": 496
},
{
"KL/chosen_KL_mean": -46.8216552734375,
"KL/mean": -61.867698669433594,
"KL/rejected_KL_mean": -76.91375732421875,
"KL/std": 36.08230972290039,
"epoch": 0.7513227513227513,
"fcm_dpo/beta": 0.03335585445165634,
"fcm_dpo/delta": -0.22783339023590088,
"fcm_dpo/margin": 30.09210205078125,
"fcm_dpo/q_t": 0.31446051597595215,
"grad_norm": 19.67770004272461,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": 0.5464165210723877,
"logits/rejected": 0.5131070613861084,
"logps/chosen": -119.61160278320312,
"logps/ref_chosen": -72.78994750976562,
"logps/ref_rejected": -89.48483276367188,
"logps/rejected": -166.39859008789062,
"loss": 0.9049,
"margin_dpo/margin_mean": 30.09210205078125,
"margin_dpo/margin_std": 37.14356231689453,
"step": 497
},
{
"KL/chosen_KL_mean": -42.5738525390625,
"KL/mean": -56.43513870239258,
"KL/rejected_KL_mean": -70.29643249511719,
"KL/std": 35.45441436767578,
"epoch": 0.7528344671201814,
"fcm_dpo/beta": 0.03254364803433418,
"fcm_dpo/delta": -0.11236497759819031,
"fcm_dpo/margin": 27.72256851196289,
"fcm_dpo/q_t": 0.3303517699241638,
"grad_norm": 22.639673233032227,
"learning_rate": 8.829247120198563e-08,
"logits/chosen": 0.5322725176811218,
"logits/rejected": 0.49616390466690063,
"logps/chosen": -110.9395751953125,
"logps/ref_chosen": -68.36572265625,
"logps/ref_rejected": -71.28846740722656,
"logps/rejected": -141.58489990234375,
"loss": 0.9052,
"margin_dpo/margin_mean": 27.722570419311523,
"margin_dpo/margin_std": 33.93904113769531,
"step": 498
},
{
"KL/chosen_KL_mean": -40.05500411987305,
"KL/mean": -53.02268981933594,
"KL/rejected_KL_mean": -65.9903793334961,
"KL/std": 34.889076232910156,
"epoch": 0.7543461829176115,
"fcm_dpo/beta": 0.03228309750556946,
"fcm_dpo/delta": -0.040800243616104126,
"fcm_dpo/margin": 25.935379028320312,
"fcm_dpo/q_t": 0.3481753468513489,
"grad_norm": 23.10430145263672,
"learning_rate": 8.728636813280163e-08,
"logits/chosen": 0.5184221267700195,
"logits/rejected": 0.4499463438987732,
"logps/chosen": -101.96383666992188,
"logps/ref_chosen": -61.90882873535156,
"logps/ref_rejected": -91.9411392211914,
"logps/rejected": -157.9315185546875,
"loss": 1.0679,
"margin_dpo/margin_mean": 25.93537712097168,
"margin_dpo/margin_std": 41.14997100830078,
"step": 499
},
{
"KL/chosen_KL_mean": -43.87847137451172,
"KL/mean": -56.561927795410156,
"KL/rejected_KL_mean": -69.24537658691406,
"KL/std": 33.54810333251953,
"epoch": 0.7558578987150416,
"fcm_dpo/beta": 0.03199248015880585,
"fcm_dpo/delta": -0.012633852660655975,
"fcm_dpo/margin": 25.366901397705078,
"fcm_dpo/q_t": 0.34740620851516724,
"grad_norm": 24.847909927368164,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": 0.4715893566608429,
"logits/rejected": 0.46398016810417175,
"logps/chosen": -114.10430145263672,
"logps/ref_chosen": -70.225830078125,
"logps/ref_rejected": -71.72203063964844,
"logps/rejected": -140.9674072265625,
"loss": 1.0277,
"margin_dpo/margin_mean": 25.366901397705078,
"margin_dpo/margin_std": 37.98773956298828,
"step": 500
},
{
"KL/chosen_KL_mean": -43.072383880615234,
"KL/mean": -54.34223175048828,
"KL/rejected_KL_mean": -65.61207580566406,
"KL/std": 34.32850646972656,
"epoch": 0.7573696145124716,
"fcm_dpo/beta": 0.032621338963508606,
"fcm_dpo/delta": 0.06773628294467926,
"fcm_dpo/margin": 22.539688110351562,
"fcm_dpo/q_t": 0.35141244530677795,
"grad_norm": 17.035171508789062,
"learning_rate": 8.528784436016878e-08,
"logits/chosen": 0.5009163618087769,
"logits/rejected": 0.494549036026001,
"logps/chosen": -107.67118835449219,
"logps/ref_chosen": -64.59880828857422,
"logps/ref_rejected": -70.59329223632812,
"logps/rejected": -136.20535278320312,
"loss": 0.9471,
"margin_dpo/margin_mean": 22.539688110351562,
"margin_dpo/margin_std": 26.915557861328125,
"step": 501
},
{
"KL/chosen_KL_mean": -41.572818756103516,
"KL/mean": -53.240657806396484,
"KL/rejected_KL_mean": -64.90849304199219,
"KL/std": 33.89706039428711,
"epoch": 0.7588813303099018,
"fcm_dpo/beta": 0.03277207911014557,
"fcm_dpo/delta": 0.03764596953988075,
"fcm_dpo/margin": 23.335674285888672,
"fcm_dpo/q_t": 0.35313090682029724,
"grad_norm": 23.165653228759766,
"learning_rate": 8.4295479559726e-08,
"logits/chosen": 0.5166738033294678,
"logits/rejected": 0.46954381465911865,
"logps/chosen": -107.0394515991211,
"logps/ref_chosen": -65.46662902832031,
"logps/ref_rejected": -90.22233581542969,
"logps/rejected": -155.13082885742188,
"loss": 0.9621,
"margin_dpo/margin_mean": 23.335674285888672,
"margin_dpo/margin_std": 31.047378540039062,
"step": 502
},
{
"KL/chosen_KL_mean": -38.187461853027344,
"KL/mean": -50.40232467651367,
"KL/rejected_KL_mean": -62.617183685302734,
"KL/std": 34.579429626464844,
"epoch": 0.7603930461073318,
"fcm_dpo/beta": 0.032632194459438324,
"fcm_dpo/delta": 0.0029222238808870316,
"fcm_dpo/margin": 24.42971420288086,
"fcm_dpo/q_t": 0.3498132824897766,
"grad_norm": 19.567941665649414,
"learning_rate": 8.330774987092712e-08,
"logits/chosen": 0.5233071446418762,
"logits/rejected": 0.5128970146179199,
"logps/chosen": -90.02223205566406,
"logps/ref_chosen": -51.83476257324219,
"logps/ref_rejected": -57.62522506713867,
"logps/rejected": -120.2424087524414,
"loss": 1.0475,
"margin_dpo/margin_mean": 24.42971420288086,
"margin_dpo/margin_std": 37.409400939941406,
"step": 503
},
{
"KL/chosen_KL_mean": -39.26737976074219,
"KL/mean": -55.71110534667969,
"KL/rejected_KL_mean": -72.15483093261719,
"KL/std": 33.35061264038086,
"epoch": 0.7619047619047619,
"fcm_dpo/beta": 0.03167734667658806,
"fcm_dpo/delta": -0.2690557837486267,
"fcm_dpo/margin": 32.88744354248047,
"fcm_dpo/q_t": 0.29890376329421997,
"grad_norm": 23.10883140563965,
"learning_rate": 8.232468292269479e-08,
"logits/chosen": 0.4726182222366333,
"logits/rejected": 0.44822973012924194,
"logps/chosen": -107.9185791015625,
"logps/ref_chosen": -68.65119934082031,
"logps/ref_rejected": -77.91394805908203,
"logps/rejected": -150.06878662109375,
"loss": 0.7965,
"margin_dpo/margin_mean": 32.88744354248047,
"margin_dpo/margin_std": 32.372901916503906,
"step": 504
},
{
"KL/chosen_KL_mean": -41.26458740234375,
"KL/mean": -51.810516357421875,
"KL/rejected_KL_mean": -62.35645294189453,
"KL/std": 36.557350158691406,
"epoch": 0.763416477702192,
"fcm_dpo/beta": 0.03131503239274025,
"fcm_dpo/delta": 0.14808888733386993,
"fcm_dpo/margin": 21.09187126159668,
"fcm_dpo/q_t": 0.3847939372062683,
"grad_norm": 23.190839767456055,
"learning_rate": 8.134630621352483e-08,
"logits/chosen": 0.5082249641418457,
"logits/rejected": 0.4687017798423767,
"logps/chosen": -101.26343536376953,
"logps/ref_chosen": -59.99884796142578,
"logps/ref_rejected": -76.88048553466797,
"logps/rejected": -139.2369384765625,
"loss": 1.1594,
"margin_dpo/margin_mean": 21.091869354248047,
"margin_dpo/margin_std": 40.211585998535156,
"step": 505
},
{
"KL/chosen_KL_mean": -38.800819396972656,
"KL/mean": -49.737892150878906,
"KL/rejected_KL_mean": -60.674964904785156,
"KL/std": 32.124107360839844,
"epoch": 0.764928193499622,
"fcm_dpo/beta": 0.03223157674074173,
"fcm_dpo/delta": 0.10202737152576447,
"fcm_dpo/margin": 21.874143600463867,
"fcm_dpo/q_t": 0.3677794635295868,
"grad_norm": 22.604772567749023,
"learning_rate": 8.037264711071698e-08,
"logits/chosen": 0.5274264812469482,
"logits/rejected": 0.5046772956848145,
"logps/chosen": -108.87211608886719,
"logps/ref_chosen": -70.07130432128906,
"logps/ref_rejected": -82.03775024414062,
"logps/rejected": -142.71270751953125,
"loss": 1.1034,
"margin_dpo/margin_mean": 21.874143600463867,
"margin_dpo/margin_std": 38.15148162841797,
"step": 506
},
{
"KL/chosen_KL_mean": -43.51123046875,
"KL/mean": -56.59437942504883,
"KL/rejected_KL_mean": -69.67752075195312,
"KL/std": 34.81589126586914,
"epoch": 0.7664399092970522,
"fcm_dpo/beta": 0.032264385372400284,
"fcm_dpo/delta": -0.04865808039903641,
"fcm_dpo/margin": 26.166301727294922,
"fcm_dpo/q_t": 0.35244131088256836,
"grad_norm": 20.77465057373047,
"learning_rate": 7.940373284960933e-08,
"logits/chosen": 0.5034874081611633,
"logits/rejected": 0.45192790031433105,
"logps/chosen": -115.51826477050781,
"logps/ref_chosen": -72.00703430175781,
"logps/ref_rejected": -93.94987487792969,
"logps/rejected": -163.6273956298828,
"loss": 1.0071,
"margin_dpo/margin_mean": 26.166301727294922,
"margin_dpo/margin_std": 38.84321594238281,
"step": 507
},
{
"KL/chosen_KL_mean": -39.335166931152344,
"KL/mean": -53.35478591918945,
"KL/rejected_KL_mean": -67.37440490722656,
"KL/std": 35.52327346801758,
"epoch": 0.7679516250944822,
"fcm_dpo/beta": 0.032064348459243774,
"fcm_dpo/delta": -0.10965946316719055,
"fcm_dpo/margin": 28.039234161376953,
"fcm_dpo/q_t": 0.33421868085861206,
"grad_norm": 20.46658706665039,
"learning_rate": 7.843959053281663e-08,
"logits/chosen": 0.4993664622306824,
"logits/rejected": 0.372380793094635,
"logps/chosen": -99.55509185791016,
"logps/ref_chosen": -60.21992492675781,
"logps/ref_rejected": -95.9200668334961,
"logps/rejected": -163.29446411132812,
"loss": 0.9376,
"margin_dpo/margin_mean": 28.039234161376953,
"margin_dpo/margin_std": 35.876731872558594,
"step": 508
},
{
"KL/chosen_KL_mean": -42.130619049072266,
"KL/mean": -54.088356018066406,
"KL/rejected_KL_mean": -66.04609680175781,
"KL/std": 34.5006217956543,
"epoch": 0.7694633408919124,
"fcm_dpo/beta": 0.03165077418088913,
"fcm_dpo/delta": 0.04634883999824524,
"fcm_dpo/margin": 23.91547966003418,
"fcm_dpo/q_t": 0.35058048367500305,
"grad_norm": 21.03251838684082,
"learning_rate": 7.748024712947204e-08,
"logits/chosen": 0.46801474690437317,
"logits/rejected": 0.44395408034324646,
"logps/chosen": -108.40078735351562,
"logps/ref_chosen": -66.27017211914062,
"logps/ref_rejected": -71.73065185546875,
"logps/rejected": -137.77674865722656,
"loss": 0.9953,
"margin_dpo/margin_mean": 23.915477752685547,
"margin_dpo/margin_std": 33.073978424072266,
"step": 509
},
{
"KL/chosen_KL_mean": -41.646202087402344,
"KL/mean": -55.19512176513672,
"KL/rejected_KL_mean": -68.74403381347656,
"KL/std": 33.61712646484375,
"epoch": 0.7709750566893424,
"fcm_dpo/beta": 0.03142780438065529,
"fcm_dpo/delta": -0.05741361528635025,
"fcm_dpo/margin": 27.09783172607422,
"fcm_dpo/q_t": 0.34271568059921265,
"grad_norm": 19.332889556884766,
"learning_rate": 7.652572947447272e-08,
"logits/chosen": 0.6219863891601562,
"logits/rejected": 0.5252499580383301,
"logps/chosen": -95.19107818603516,
"logps/ref_chosen": -53.54487609863281,
"logps/ref_rejected": -91.36648559570312,
"logps/rejected": -160.11053466796875,
"loss": 0.9856,
"margin_dpo/margin_mean": 27.09783172607422,
"margin_dpo/margin_std": 38.01484680175781,
"step": 510
},
{
"KL/chosen_KL_mean": -42.845680236816406,
"KL/mean": -58.7375373840332,
"KL/rejected_KL_mean": -74.62939453125,
"KL/std": 35.452720642089844,
"epoch": 0.7724867724867724,
"fcm_dpo/beta": 0.030704926699399948,
"fcm_dpo/delta": -0.19456720352172852,
"fcm_dpo/margin": 31.78371238708496,
"fcm_dpo/q_t": 0.3209506869316101,
"grad_norm": 19.716928482055664,
"learning_rate": 7.557606426772961e-08,
"logits/chosen": 0.5421375036239624,
"logits/rejected": 0.48624932765960693,
"logps/chosen": -98.6900634765625,
"logps/ref_chosen": -55.844383239746094,
"logps/ref_rejected": -86.49819946289062,
"logps/rejected": -161.12759399414062,
"loss": 0.9377,
"margin_dpo/margin_mean": 31.783714294433594,
"margin_dpo/margin_std": 40.96788787841797,
"step": 511
},
{
"KL/chosen_KL_mean": -40.811737060546875,
"KL/mean": -52.81915283203125,
"KL/rejected_KL_mean": -64.82656860351562,
"KL/std": 32.51373291015625,
"epoch": 0.7739984882842026,
"fcm_dpo/beta": 0.030653230845928192,
"fcm_dpo/delta": 0.06883647292852402,
"fcm_dpo/margin": 24.014835357666016,
"fcm_dpo/q_t": 0.3677193522453308,
"grad_norm": 19.665395736694336,
"learning_rate": 7.463127807341966e-08,
"logits/chosen": 0.4046107530593872,
"logits/rejected": 0.3887116312980652,
"logps/chosen": -102.46477508544922,
"logps/ref_chosen": -61.653038024902344,
"logps/ref_rejected": -72.83148193359375,
"logps/rejected": -137.65805053710938,
"loss": 1.0805,
"margin_dpo/margin_mean": 24.014835357666016,
"margin_dpo/margin_std": 40.261390686035156,
"step": 512
},
{
"KL/chosen_KL_mean": -37.0535888671875,
"KL/mean": -50.049591064453125,
"KL/rejected_KL_mean": -63.045597076416016,
"KL/std": 34.30278015136719,
"epoch": 0.7755102040816326,
"fcm_dpo/beta": 0.030527032911777496,
"fcm_dpo/delta": 0.005987279117107391,
"fcm_dpo/margin": 25.99200439453125,
"fcm_dpo/q_t": 0.3471200466156006,
"grad_norm": 17.770233154296875,
"learning_rate": 7.369139731924401e-08,
"logits/chosen": 0.6464536786079407,
"logits/rejected": 0.5946289300918579,
"logps/chosen": -87.90615844726562,
"logps/ref_chosen": -50.85256576538086,
"logps/ref_rejected": -69.21754455566406,
"logps/rejected": -132.2631378173828,
"loss": 0.9392,
"margin_dpo/margin_mean": 25.99200439453125,
"margin_dpo/margin_std": 33.05658721923828,
"step": 513
},
{
"KL/chosen_KL_mean": -41.696224212646484,
"KL/mean": -56.08380889892578,
"KL/rejected_KL_mean": -70.47138977050781,
"KL/std": 33.583072662353516,
"epoch": 0.7770219198790628,
"fcm_dpo/beta": 0.03049997240304947,
"fcm_dpo/delta": -0.08509883284568787,
"fcm_dpo/margin": 28.775178909301758,
"fcm_dpo/q_t": 0.32821977138519287,
"grad_norm": 18.91166877746582,
"learning_rate": 7.275644829568747e-08,
"logits/chosen": 0.5680727362632751,
"logits/rejected": 0.5272949934005737,
"logps/chosen": -111.08116149902344,
"logps/ref_chosen": -69.38493347167969,
"logps/ref_rejected": -83.32447814941406,
"logps/rejected": -153.79586791992188,
"loss": 0.922,
"margin_dpo/margin_mean": 28.77518081665039,
"margin_dpo/margin_std": 35.041664123535156,
"step": 514
},
{
"KL/chosen_KL_mean": -47.496299743652344,
"KL/mean": -60.2870979309082,
"KL/rejected_KL_mean": -73.07791137695312,
"KL/std": 33.58501434326172,
"epoch": 0.7785336356764928,
"fcm_dpo/beta": 0.030235309153795242,
"fcm_dpo/delta": 0.028453219681978226,
"fcm_dpo/margin": 25.581607818603516,
"fcm_dpo/q_t": 0.35126420855522156,
"grad_norm": 21.126333236694336,
"learning_rate": 7.182645715528435e-08,
"logits/chosen": 0.5368679761886597,
"logits/rejected": 0.4550856649875641,
"logps/chosen": -101.18333435058594,
"logps/ref_chosen": -53.687034606933594,
"logps/ref_rejected": -83.59614562988281,
"logps/rejected": -156.67404174804688,
"loss": 0.9922,
"margin_dpo/margin_mean": 25.581605911254883,
"margin_dpo/margin_std": 35.4997444152832,
"step": 515
},
{
"KL/chosen_KL_mean": -39.445045471191406,
"KL/mean": -50.16015625,
"KL/rejected_KL_mean": -60.875267028808594,
"KL/std": 32.9947395324707,
"epoch": 0.780045351473923,
"fcm_dpo/beta": 0.031198721379041672,
"fcm_dpo/delta": 0.139729306101799,
"fcm_dpo/margin": 21.430225372314453,
"fcm_dpo/q_t": 0.36643847823143005,
"grad_norm": 22.579246520996094,
"learning_rate": 7.090144991188568e-08,
"logits/chosen": 0.48671144247055054,
"logits/rejected": 0.43809783458709717,
"logps/chosen": -96.34677124023438,
"logps/ref_chosen": -56.9017219543457,
"logps/ref_rejected": -67.83477783203125,
"logps/rejected": -128.71005249023438,
"loss": 1.0848,
"margin_dpo/margin_mean": 21.430227279663086,
"margin_dpo/margin_std": 35.83974838256836,
"step": 516
},
{
"KL/chosen_KL_mean": -48.5676155090332,
"KL/mean": -58.601890563964844,
"KL/rejected_KL_mean": -68.63616943359375,
"KL/std": 33.302223205566406,
"epoch": 0.781557067271353,
"fcm_dpo/beta": 0.03218144178390503,
"fcm_dpo/delta": 0.16378606855869293,
"fcm_dpo/margin": 20.068546295166016,
"fcm_dpo/q_t": 0.3800460696220398,
"grad_norm": 23.016653060913086,
"learning_rate": 6.998145243993284e-08,
"logits/chosen": 0.5580442547798157,
"logits/rejected": 0.5505795478820801,
"logps/chosen": -110.34275817871094,
"logps/ref_chosen": -61.775142669677734,
"logps/ref_rejected": -62.88270950317383,
"logps/rejected": -131.51889038085938,
"loss": 1.0889,
"margin_dpo/margin_mean": 20.068546295166016,
"margin_dpo/margin_std": 34.71467590332031,
"step": 517
},
{
"KL/chosen_KL_mean": -38.425636291503906,
"KL/mean": -49.03041458129883,
"KL/rejected_KL_mean": -59.635189056396484,
"KL/std": 33.62690353393555,
"epoch": 0.783068783068783,
"fcm_dpo/beta": 0.032912157475948334,
"fcm_dpo/delta": 0.10931538045406342,
"fcm_dpo/margin": 21.209548950195312,
"fcm_dpo/q_t": 0.3719986081123352,
"grad_norm": 20.80961036682129,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": 0.5091816186904907,
"logits/rejected": 0.4563264846801758,
"logps/chosen": -100.45086669921875,
"logps/ref_chosen": -62.02523422241211,
"logps/ref_rejected": -79.06085205078125,
"logps/rejected": -138.696044921875,
"loss": 1.0609,
"margin_dpo/margin_mean": 21.209548950195312,
"margin_dpo/margin_std": 34.63611602783203,
"step": 518
},
{
"KL/chosen_KL_mean": -47.89688491821289,
"KL/mean": -56.045166015625,
"KL/rejected_KL_mean": -64.19346618652344,
"KL/std": 33.93360900878906,
"epoch": 0.7845804988662132,
"fcm_dpo/beta": 0.03449524939060211,
"fcm_dpo/delta": 0.24971872568130493,
"fcm_dpo/margin": 16.296571731567383,
"fcm_dpo/q_t": 0.4011024832725525,
"grad_norm": 32.703617095947266,
"learning_rate": 6.815658960673781e-08,
"logits/chosen": 0.5393502712249756,
"logits/rejected": 0.4814758896827698,
"logps/chosen": -109.50325012207031,
"logps/ref_chosen": -61.60636901855469,
"logps/ref_rejected": -74.50727844238281,
"logps/rejected": -138.70074462890625,
"loss": 1.3142,
"margin_dpo/margin_mean": 16.29656982421875,
"margin_dpo/margin_std": 41.276824951171875,
"step": 519
},
{
"KL/chosen_KL_mean": -42.39877700805664,
"KL/mean": -52.96178436279297,
"KL/rejected_KL_mean": -63.5247917175293,
"KL/std": 33.05720138549805,
"epoch": 0.7860922146636432,
"fcm_dpo/beta": 0.03477100655436516,
"fcm_dpo/delta": 0.06951478123664856,
"fcm_dpo/margin": 21.126018524169922,
"fcm_dpo/q_t": 0.3603675365447998,
"grad_norm": 26.019474029541016,
"learning_rate": 6.725177529083209e-08,
"logits/chosen": 0.5982969999313354,
"logits/rejected": 0.5375903248786926,
"logps/chosen": -105.27220916748047,
"logps/ref_chosen": -62.87343215942383,
"logps/ref_rejected": -76.505615234375,
"logps/rejected": -140.0303955078125,
"loss": 1.036,
"margin_dpo/margin_mean": 21.126018524169922,
"margin_dpo/margin_std": 31.89874839782715,
"step": 520
},
{
"KL/chosen_KL_mean": -43.5118522644043,
"KL/mean": -56.493614196777344,
"KL/rejected_KL_mean": -69.47537994384766,
"KL/std": 32.319976806640625,
"epoch": 0.7876039304610734,
"fcm_dpo/beta": 0.03475981950759888,
"fcm_dpo/delta": -0.11252060532569885,
"fcm_dpo/margin": 25.96352767944336,
"fcm_dpo/q_t": 0.3285518288612366,
"grad_norm": 26.865089416503906,
"learning_rate": 6.63520728356167e-08,
"logits/chosen": 0.42334693670272827,
"logits/rejected": 0.3446962237358093,
"logps/chosen": -107.7185287475586,
"logps/ref_chosen": -64.20668029785156,
"logps/ref_rejected": -92.28083038330078,
"logps/rejected": -161.75621032714844,
"loss": 0.9379,
"margin_dpo/margin_mean": 25.96352767944336,
"margin_dpo/margin_std": 33.507266998291016,
"step": 521
},
{
"KL/chosen_KL_mean": -41.547828674316406,
"KL/mean": -52.55769348144531,
"KL/rejected_KL_mean": -63.56755828857422,
"KL/std": 35.37586975097656,
"epoch": 0.7891156462585034,
"fcm_dpo/beta": 0.03454647958278656,
"fcm_dpo/delta": 0.04247668385505676,
"fcm_dpo/margin": 22.01972770690918,
"fcm_dpo/q_t": 0.366945743560791,
"grad_norm": 27.949600219726562,
"learning_rate": 6.545750740770336e-08,
"logits/chosen": 0.5129572749137878,
"logits/rejected": 0.5026385188102722,
"logps/chosen": -99.91755676269531,
"logps/ref_chosen": -58.369720458984375,
"logps/ref_rejected": -68.79248046875,
"logps/rejected": -132.36004638671875,
"loss": 1.1764,
"margin_dpo/margin_mean": 22.019729614257812,
"margin_dpo/margin_std": 42.28102111816406,
"step": 522
},
{
"KL/chosen_KL_mean": -45.6678581237793,
"KL/mean": -58.220497131347656,
"KL/rejected_KL_mean": -70.77313232421875,
"KL/std": 33.746395111083984,
"epoch": 0.7906273620559335,
"fcm_dpo/beta": 0.03418704867362976,
"fcm_dpo/delta": -0.0648900717496872,
"fcm_dpo/margin": 25.105276107788086,
"fcm_dpo/q_t": 0.3337825834751129,
"grad_norm": 30.4615478515625,
"learning_rate": 6.456810403001012e-08,
"logits/chosen": 0.5436546802520752,
"logits/rejected": 0.42364761233329773,
"logps/chosen": -111.38109588623047,
"logps/ref_chosen": -65.71324157714844,
"logps/ref_rejected": -91.98896789550781,
"logps/rejected": -162.76211547851562,
"loss": 1.0339,
"margin_dpo/margin_mean": 25.105276107788086,
"margin_dpo/margin_std": 37.20182800292969,
"step": 523
},
{
"KL/chosen_KL_mean": -36.952911376953125,
"KL/mean": -47.42578887939453,
"KL/rejected_KL_mean": -57.89865493774414,
"KL/std": 30.665462493896484,
"epoch": 0.7921390778533636,
"fcm_dpo/beta": 0.03484393283724785,
"fcm_dpo/delta": 0.07469457387924194,
"fcm_dpo/margin": 20.945743560791016,
"fcm_dpo/q_t": 0.35503047704696655,
"grad_norm": 23.227760314941406,
"learning_rate": 6.368388758106134e-08,
"logits/chosen": 0.4536820650100708,
"logits/rejected": 0.4314419627189636,
"logps/chosen": -113.30416107177734,
"logps/ref_chosen": -76.35124969482422,
"logps/ref_rejected": -89.96072387695312,
"logps/rejected": -147.859375,
"loss": 1.0179,
"margin_dpo/margin_mean": 20.945743560791016,
"margin_dpo/margin_std": 30.672534942626953,
"step": 524
},
{
"KL/chosen_KL_mean": -41.9482307434082,
"KL/mean": -50.95294189453125,
"KL/rejected_KL_mean": -59.95765686035156,
"KL/std": 30.69168472290039,
"epoch": 0.7936507936507936,
"fcm_dpo/beta": 0.035696301609277725,
"fcm_dpo/delta": 0.16779160499572754,
"fcm_dpo/margin": 18.009424209594727,
"fcm_dpo/q_t": 0.3807103633880615,
"grad_norm": 26.58245277404785,
"learning_rate": 6.280488279429185e-08,
"logits/chosen": 0.35608696937561035,
"logits/rejected": 0.34583795070648193,
"logps/chosen": -117.44401550292969,
"logps/ref_chosen": -75.49578857421875,
"logps/ref_rejected": -84.04852294921875,
"logps/rejected": -144.0061798095703,
"loss": 1.1024,
"margin_dpo/margin_mean": 18.00942611694336,
"margin_dpo/margin_std": 31.897258758544922,
"step": 525
},
{
"KL/chosen_KL_mean": -44.732521057128906,
"KL/mean": -54.399269104003906,
"KL/rejected_KL_mean": -64.06602478027344,
"KL/std": 31.663698196411133,
"epoch": 0.7951625094482238,
"fcm_dpo/beta": 0.03591318428516388,
"fcm_dpo/delta": 0.10863810777664185,
"fcm_dpo/margin": 19.333499908447266,
"fcm_dpo/q_t": 0.3647363781929016,
"grad_norm": 24.61683464050293,
"learning_rate": 6.193111425735515e-08,
"logits/chosen": 0.5265854597091675,
"logits/rejected": 0.4591507315635681,
"logps/chosen": -106.02494049072266,
"logps/ref_chosen": -61.29241943359375,
"logps/ref_rejected": -82.47763061523438,
"logps/rejected": -146.54364013671875,
"loss": 1.0761,
"margin_dpo/margin_mean": 19.3335018157959,
"margin_dpo/margin_std": 30.015331268310547,
"step": 526
},
{
"KL/chosen_KL_mean": -46.88569641113281,
"KL/mean": -54.68010711669922,
"KL/rejected_KL_mean": -62.474517822265625,
"KL/std": 31.084400177001953,
"epoch": 0.7966742252456538,
"fcm_dpo/beta": 0.03820966184139252,
"fcm_dpo/delta": 0.21504317224025726,
"fcm_dpo/margin": 15.588825225830078,
"fcm_dpo/q_t": 0.3955618739128113,
"grad_norm": 30.495433807373047,
"learning_rate": 6.106260641143546e-08,
"logits/chosen": 0.6129889488220215,
"logits/rejected": 0.5357133746147156,
"logps/chosen": -108.35832214355469,
"logps/ref_chosen": -61.472625732421875,
"logps/ref_rejected": -90.52831268310547,
"logps/rejected": -153.00283813476562,
"loss": 1.2257,
"margin_dpo/margin_mean": 15.588825225830078,
"margin_dpo/margin_std": 34.68670654296875,
"step": 527
},
{
"KL/chosen_KL_mean": -43.2332763671875,
"KL/mean": -51.08110046386719,
"KL/rejected_KL_mean": -58.92891311645508,
"KL/std": 31.543315887451172,
"epoch": 0.7981859410430839,
"fcm_dpo/beta": 0.0391681045293808,
"fcm_dpo/delta": 0.19740989804267883,
"fcm_dpo/margin": 15.695642471313477,
"fcm_dpo/q_t": 0.3858627676963806,
"grad_norm": 34.73171615600586,
"learning_rate": 6.019938355056422e-08,
"logits/chosen": 0.43448758125305176,
"logits/rejected": 0.35089200735092163,
"logps/chosen": -102.0252914428711,
"logps/ref_chosen": -58.792015075683594,
"logps/ref_rejected": -71.82516479492188,
"logps/rejected": -130.7540740966797,
"loss": 1.2577,
"margin_dpo/margin_mean": 15.695640563964844,
"margin_dpo/margin_std": 35.330623626708984,
"step": 528
},
{
"KL/chosen_KL_mean": -40.24607849121094,
"KL/mean": -58.42034912109375,
"KL/rejected_KL_mean": -76.59461975097656,
"KL/std": 35.38734436035156,
"epoch": 0.799697656840514,
"fcm_dpo/beta": 0.03599797189235687,
"fcm_dpo/delta": -0.5999960899353027,
"fcm_dpo/margin": 36.348541259765625,
"fcm_dpo/q_t": 0.2665252983570099,
"grad_norm": 20.099990844726562,
"learning_rate": 5.934146982094049e-08,
"logits/chosen": 0.4503468871116638,
"logits/rejected": 0.39226827025413513,
"logps/chosen": -95.3170394897461,
"logps/ref_chosen": -55.070960998535156,
"logps/ref_rejected": -75.44007873535156,
"logps/rejected": -152.03469848632812,
"loss": 0.7776,
"margin_dpo/margin_mean": 36.348541259765625,
"margin_dpo/margin_std": 36.78180694580078,
"step": 529
},
{
"KL/chosen_KL_mean": -41.45820999145508,
"KL/mean": -52.497615814208984,
"KL/rejected_KL_mean": -63.537017822265625,
"KL/std": 33.97320556640625,
"epoch": 0.8012093726379441,
"fcm_dpo/beta": 0.03562724217772484,
"fcm_dpo/delta": 0.014175062999129295,
"fcm_dpo/margin": 22.078805923461914,
"fcm_dpo/q_t": 0.3531653881072998,
"grad_norm": 22.5502986907959,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": 0.5277613401412964,
"logits/rejected": 0.485904723405838,
"logps/chosen": -98.2020263671875,
"logps/ref_chosen": -56.743812561035156,
"logps/ref_rejected": -76.6692123413086,
"logps/rejected": -140.20623779296875,
"loss": 0.9715,
"margin_dpo/margin_mean": 22.078805923461914,
"margin_dpo/margin_std": 29.655553817749023,
"step": 530
},
{
"KL/chosen_KL_mean": -42.525909423828125,
"KL/mean": -52.817569732666016,
"KL/rejected_KL_mean": -63.109230041503906,
"KL/std": 33.2888069152832,
"epoch": 0.8027210884353742,
"fcm_dpo/beta": 0.03640486299991608,
"fcm_dpo/delta": 0.04753082990646362,
"fcm_dpo/margin": 20.583328247070312,
"fcm_dpo/q_t": 0.3651937246322632,
"grad_norm": 27.369985580444336,
"learning_rate": 5.7641665597021435e-08,
"logits/chosen": 0.5008730888366699,
"logits/rejected": 0.4344239830970764,
"logps/chosen": -93.64236450195312,
"logps/ref_chosen": -51.116455078125,
"logps/ref_rejected": -79.52884674072266,
"logps/rejected": -142.63807678222656,
"loss": 1.1001,
"margin_dpo/margin_mean": 20.58332633972168,
"margin_dpo/margin_std": 34.31003189086914,
"step": 531
},
{
"KL/chosen_KL_mean": -48.05036926269531,
"KL/mean": -60.802268981933594,
"KL/rejected_KL_mean": -73.55416107177734,
"KL/std": 31.71292495727539,
"epoch": 0.8042328042328042,
"fcm_dpo/beta": 0.03519435226917267,
"fcm_dpo/delta": -0.10766403377056122,
"fcm_dpo/margin": 25.503793716430664,
"fcm_dpo/q_t": 0.33816105127334595,
"grad_norm": 22.831806182861328,
"learning_rate": 5.679982264990424e-08,
"logits/chosen": 0.45231497287750244,
"logits/rejected": 0.3992037773132324,
"logps/chosen": -106.330322265625,
"logps/ref_chosen": -58.279945373535156,
"logps/ref_rejected": -78.05426788330078,
"logps/rejected": -151.60842895507812,
"loss": 1.0079,
"margin_dpo/margin_mean": 25.503793716430664,
"margin_dpo/margin_std": 37.307273864746094,
"step": 532
},
{
"KL/chosen_KL_mean": -34.005062103271484,
"KL/mean": -45.2939567565918,
"KL/rejected_KL_mean": -56.582855224609375,
"KL/std": 33.391746520996094,
"epoch": 0.8057445200302343,
"fcm_dpo/beta": 0.03524235635995865,
"fcm_dpo/delta": 0.0043890513479709625,
"fcm_dpo/margin": 22.57779884338379,
"fcm_dpo/q_t": 0.3489721715450287,
"grad_norm": 18.338743209838867,
"learning_rate": 5.596338392706076e-08,
"logits/chosen": 0.6012002229690552,
"logits/rejected": 0.5376021265983582,
"logps/chosen": -90.4230728149414,
"logps/ref_chosen": -56.41801071166992,
"logps/ref_rejected": -73.89324951171875,
"logps/rejected": -130.47610473632812,
"loss": 1.0184,
"margin_dpo/margin_mean": 22.577800750732422,
"margin_dpo/margin_std": 33.613582611083984,
"step": 533
},
{
"KL/chosen_KL_mean": -38.655921936035156,
"KL/mean": -51.250999450683594,
"KL/rejected_KL_mean": -63.84608840942383,
"KL/std": 32.82971954345703,
"epoch": 0.8072562358276644,
"fcm_dpo/beta": 0.03448425233364105,
"fcm_dpo/delta": -0.07676675170660019,
"fcm_dpo/margin": 25.190170288085938,
"fcm_dpo/q_t": 0.3423423171043396,
"grad_norm": 20.127193450927734,
"learning_rate": 5.513237282548033e-08,
"logits/chosen": 0.5142478942871094,
"logits/rejected": 0.4719163179397583,
"logps/chosen": -99.40460968017578,
"logps/ref_chosen": -60.748687744140625,
"logps/ref_rejected": -73.8623046875,
"logps/rejected": -137.70838928222656,
"loss": 1.0206,
"margin_dpo/margin_mean": 25.190170288085938,
"margin_dpo/margin_std": 37.62242126464844,
"step": 534
},
{
"KL/chosen_KL_mean": -46.664085388183594,
"KL/mean": -57.11603546142578,
"KL/rejected_KL_mean": -67.56797790527344,
"KL/std": 35.46537780761719,
"epoch": 0.8087679516250945,
"fcm_dpo/beta": 0.035117894411087036,
"fcm_dpo/delta": 0.07030545920133591,
"fcm_dpo/margin": 20.903888702392578,
"fcm_dpo/q_t": 0.3673914074897766,
"grad_norm": 25.055700302124023,
"learning_rate": 5.430681259032957e-08,
"logits/chosen": 0.4137476086616516,
"logits/rejected": 0.3489595055580139,
"logps/chosen": -108.30149841308594,
"logps/ref_chosen": -61.637413024902344,
"logps/ref_rejected": -80.93138885498047,
"logps/rejected": -148.49935913085938,
"loss": 1.1134,
"margin_dpo/margin_mean": 20.90389060974121,
"margin_dpo/margin_std": 37.192806243896484,
"step": 535
},
{
"KL/chosen_KL_mean": -39.921539306640625,
"KL/mean": -54.335689544677734,
"KL/rejected_KL_mean": -68.74983978271484,
"KL/std": 34.82649612426758,
"epoch": 0.8102796674225246,
"fcm_dpo/beta": 0.033950429409742355,
"fcm_dpo/delta": -0.19902583956718445,
"fcm_dpo/margin": 28.82830047607422,
"fcm_dpo/q_t": 0.31275475025177,
"grad_norm": 17.977373123168945,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": 0.5412774682044983,
"logits/rejected": 0.45443296432495117,
"logps/chosen": -91.81051635742188,
"logps/ref_chosen": -51.88897705078125,
"logps/ref_rejected": -73.34864044189453,
"logps/rejected": -142.09848022460938,
"loss": 0.8262,
"margin_dpo/margin_mean": 28.828296661376953,
"margin_dpo/margin_std": 30.391464233398438,
"step": 536
},
{
"KL/chosen_KL_mean": -45.69548416137695,
"KL/mean": -57.746681213378906,
"KL/rejected_KL_mean": -69.79786682128906,
"KL/std": 36.305625915527344,
"epoch": 0.8117913832199547,
"fcm_dpo/beta": 0.03342650458216667,
"fcm_dpo/delta": -0.007336266338825226,
"fcm_dpo/margin": 24.102388381958008,
"fcm_dpo/q_t": 0.35287484526634216,
"grad_norm": 20.5838680267334,
"learning_rate": 5.267213693697695e-08,
"logits/chosen": 0.6057338714599609,
"logits/rejected": 0.5138819217681885,
"logps/chosen": -99.94410705566406,
"logps/ref_chosen": -54.248619079589844,
"logps/ref_rejected": -94.94343566894531,
"logps/rejected": -164.74130249023438,
"loss": 1.0201,
"margin_dpo/margin_mean": 24.102388381958008,
"margin_dpo/margin_std": 35.75040817260742,
"step": 537
},
{
"KL/chosen_KL_mean": -43.729248046875,
"KL/mean": -57.41365051269531,
"KL/rejected_KL_mean": -71.09806060791016,
"KL/std": 34.0536003112793,
"epoch": 0.8133030990173847,
"fcm_dpo/beta": 0.033294469118118286,
"fcm_dpo/delta": -0.12256969511508942,
"fcm_dpo/margin": 27.368816375732422,
"fcm_dpo/q_t": 0.3270268440246582,
"grad_norm": 21.047393798828125,
"learning_rate": 5.1863067244167144e-08,
"logits/chosen": 0.5291279554367065,
"logits/rejected": 0.49264344573020935,
"logps/chosen": -113.82278442382812,
"logps/ref_chosen": -70.09353637695312,
"logps/ref_rejected": -79.49833679199219,
"logps/rejected": -150.59640502929688,
"loss": 0.9168,
"margin_dpo/margin_mean": 27.368816375732422,
"margin_dpo/margin_std": 33.95469284057617,
"step": 538
},
{
"KL/chosen_KL_mean": -44.2559700012207,
"KL/mean": -55.49917221069336,
"KL/rejected_KL_mean": -66.74237060546875,
"KL/std": 33.12880325317383,
"epoch": 0.8148148148148148,
"fcm_dpo/beta": 0.032977961003780365,
"fcm_dpo/delta": 0.06287629902362823,
"fcm_dpo/margin": 22.486400604248047,
"fcm_dpo/q_t": 0.3576567769050598,
"grad_norm": 21.476024627685547,
"learning_rate": 5.105953986729195e-08,
"logits/chosen": 0.4770214557647705,
"logits/rejected": 0.3988742530345917,
"logps/chosen": -106.18766784667969,
"logps/ref_chosen": -61.93169403076172,
"logps/ref_rejected": -84.08946228027344,
"logps/rejected": -150.8318328857422,
"loss": 0.9873,
"margin_dpo/margin_mean": 22.486400604248047,
"margin_dpo/margin_std": 31.093643188476562,
"step": 539
},
{
"KL/chosen_KL_mean": -41.77684783935547,
"KL/mean": -56.266788482666016,
"KL/rejected_KL_mean": -70.75672149658203,
"KL/std": 35.936214447021484,
"epoch": 0.8163265306122449,
"fcm_dpo/beta": 0.03290818631649017,
"fcm_dpo/delta": -0.1717756688594818,
"fcm_dpo/margin": 28.979869842529297,
"fcm_dpo/q_t": 0.3199213445186615,
"grad_norm": 20.107343673706055,
"learning_rate": 5.026157728273966e-08,
"logits/chosen": 0.5539939403533936,
"logits/rejected": 0.45686525106430054,
"logps/chosen": -104.4811019897461,
"logps/ref_chosen": -62.704254150390625,
"logps/ref_rejected": -95.63597106933594,
"logps/rejected": -166.3927001953125,
"loss": 0.8828,
"margin_dpo/margin_mean": 28.979869842529297,
"margin_dpo/margin_std": 32.712791442871094,
"step": 540
},
{
"KL/chosen_KL_mean": -39.33304214477539,
"KL/mean": -53.905635833740234,
"KL/rejected_KL_mean": -68.47822570800781,
"KL/std": 35.508766174316406,
"epoch": 0.817838246409675,
"fcm_dpo/beta": 0.031236249953508377,
"fcm_dpo/delta": -0.12479140609502792,
"fcm_dpo/margin": 29.145185470581055,
"fcm_dpo/q_t": 0.3244413137435913,
"grad_norm": 19.948898315429688,
"learning_rate": 4.9469201811239035e-08,
"logits/chosen": 0.5305638909339905,
"logits/rejected": 0.5417298078536987,
"logps/chosen": -101.81388854980469,
"logps/ref_chosen": -62.48084259033203,
"logps/ref_rejected": -57.55541229248047,
"logps/rejected": -126.03364562988281,
"loss": 0.8897,
"margin_dpo/margin_mean": 29.145185470581055,
"margin_dpo/margin_std": 33.119834899902344,
"step": 541
},
{
"KL/chosen_KL_mean": -34.64030838012695,
"KL/mean": -49.422584533691406,
"KL/rejected_KL_mean": -64.20486450195312,
"KL/std": 33.06791687011719,
"epoch": 0.8193499622071051,
"fcm_dpo/beta": 0.030691038817167282,
"fcm_dpo/delta": -0.11876482516527176,
"fcm_dpo/margin": 29.564552307128906,
"fcm_dpo/q_t": 0.32889920473098755,
"grad_norm": 17.267976760864258,
"learning_rate": 4.868243561723534e-08,
"logits/chosen": 0.5545158982276917,
"logits/rejected": 0.5000336170196533,
"logps/chosen": -84.09519958496094,
"logps/ref_chosen": -49.454891204833984,
"logps/ref_rejected": -65.33275604248047,
"logps/rejected": -129.53762817382812,
"loss": 0.9402,
"margin_dpo/margin_mean": 29.564552307128906,
"margin_dpo/margin_std": 38.110191345214844,
"step": 542
},
{
"KL/chosen_KL_mean": -39.595008850097656,
"KL/mean": -52.76807403564453,
"KL/rejected_KL_mean": -65.94114685058594,
"KL/std": 33.115394592285156,
"epoch": 0.8208616780045351,
"fcm_dpo/beta": 0.03072110377252102,
"fcm_dpo/delta": -0.01072387769818306,
"fcm_dpo/margin": 26.34613800048828,
"fcm_dpo/q_t": 0.3363792300224304,
"grad_norm": 15.949999809265137,
"learning_rate": 4.790130070827028e-08,
"logits/chosen": 0.527794599533081,
"logits/rejected": 0.4371853172779083,
"logps/chosen": -90.69586181640625,
"logps/ref_chosen": -51.100860595703125,
"logps/ref_rejected": -76.06130981445312,
"logps/rejected": -142.00245666503906,
"loss": 0.9566,
"margin_dpo/margin_mean": 26.34613800048828,
"margin_dpo/margin_std": 34.06919860839844,
"step": 543
},
{
"KL/chosen_KL_mean": -41.76414108276367,
"KL/mean": -57.745872497558594,
"KL/rejected_KL_mean": -73.72760009765625,
"KL/std": 35.78254699707031,
"epoch": 0.8223733938019653,
"fcm_dpo/beta": 0.029825232923030853,
"fcm_dpo/delta": -0.16930848360061646,
"fcm_dpo/margin": 31.963459014892578,
"fcm_dpo/q_t": 0.32053062319755554,
"grad_norm": 16.513900756835938,
"learning_rate": 4.7125818934366454e-08,
"logits/chosen": 0.5192776322364807,
"logits/rejected": 0.4438924193382263,
"logps/chosen": -102.04137420654297,
"logps/ref_chosen": -60.2772331237793,
"logps/ref_rejected": -88.40553283691406,
"logps/rejected": -162.13314819335938,
"loss": 0.8972,
"margin_dpo/margin_mean": 31.96346092224121,
"margin_dpo/margin_std": 38.4505729675293,
"step": 544
},
{
"KL/chosen_KL_mean": -44.36931610107422,
"KL/mean": -55.0029296875,
"KL/rejected_KL_mean": -65.63653564453125,
"KL/std": 35.26418685913086,
"epoch": 0.8238851095993953,
"fcm_dpo/beta": 0.03026110678911209,
"fcm_dpo/delta": 0.16723273694515228,
"fcm_dpo/margin": 21.267223358154297,
"fcm_dpo/q_t": 0.374667763710022,
"grad_norm": 18.300811767578125,
"learning_rate": 4.635601198741607e-08,
"logits/chosen": 0.4809110462665558,
"logits/rejected": 0.42077383399009705,
"logps/chosen": -105.98455810546875,
"logps/ref_chosen": -61.61524963378906,
"logps/ref_rejected": -78.71266174316406,
"logps/rejected": -144.3491973876953,
"loss": 1.0727,
"margin_dpo/margin_mean": 21.267223358154297,
"margin_dpo/margin_std": 34.66839599609375,
"step": 545
},
{
"KL/chosen_KL_mean": -41.47944641113281,
"KL/mean": -53.87554931640625,
"KL/rejected_KL_mean": -66.27165222167969,
"KL/std": 31.500242233276367,
"epoch": 0.8253968253968254,
"fcm_dpo/beta": 0.030640142038464546,
"fcm_dpo/delta": 0.043633583933115005,
"fcm_dpo/margin": 24.79220962524414,
"fcm_dpo/q_t": 0.34807056188583374,
"grad_norm": 21.26259994506836,
"learning_rate": 4.559190140057428e-08,
"logits/chosen": 0.5719273090362549,
"logits/rejected": 0.5586786270141602,
"logps/chosen": -100.79270935058594,
"logps/ref_chosen": -59.313262939453125,
"logps/ref_rejected": -64.73631286621094,
"logps/rejected": -131.00796508789062,
"loss": 0.9787,
"margin_dpo/margin_mean": 24.79220962524414,
"margin_dpo/margin_std": 32.89813232421875,
"step": 546
},
{
"KL/chosen_KL_mean": -37.5168342590332,
"KL/mean": -51.37678527832031,
"KL/rejected_KL_mean": -65.23674011230469,
"KL/std": 32.473785400390625,
"epoch": 0.8269085411942555,
"fcm_dpo/beta": 0.030482735484838486,
"fcm_dpo/delta": -0.049464091658592224,
"fcm_dpo/margin": 27.719913482666016,
"fcm_dpo/q_t": 0.32883119583129883,
"grad_norm": 19.5365047454834,
"learning_rate": 4.483350854765672e-08,
"logits/chosen": 0.45498591661453247,
"logits/rejected": 0.3874082565307617,
"logps/chosen": -92.49357604980469,
"logps/ref_chosen": -54.97674560546875,
"logps/ref_rejected": -75.35922241210938,
"logps/rejected": -140.59596252441406,
"loss": 0.9384,
"margin_dpo/margin_mean": 27.719913482666016,
"margin_dpo/margin_std": 34.43286895751953,
"step": 547
},
{
"KL/chosen_KL_mean": -44.438194274902344,
"KL/mean": -54.13751220703125,
"KL/rejected_KL_mean": -63.836830139160156,
"KL/std": 34.09819030761719,
"epoch": 0.8284202569916855,
"fcm_dpo/beta": 0.03167761117219925,
"fcm_dpo/delta": 0.19408327341079712,
"fcm_dpo/margin": 19.398635864257812,
"fcm_dpo/q_t": 0.3884551227092743,
"grad_norm": 22.586746215820312,
"learning_rate": 4.4080854642541826e-08,
"logits/chosen": 0.44461339712142944,
"logits/rejected": 0.37233829498291016,
"logps/chosen": -107.64886474609375,
"logps/ref_chosen": -63.21067428588867,
"logps/ref_rejected": -81.23347473144531,
"logps/rejected": -145.0703125,
"loss": 1.1137,
"margin_dpo/margin_mean": 19.398635864257812,
"margin_dpo/margin_std": 34.943870544433594,
"step": 548
},
{
"KL/chosen_KL_mean": -42.7745475769043,
"KL/mean": -53.077823638916016,
"KL/rejected_KL_mean": -63.381103515625,
"KL/std": 33.2050666809082,
"epoch": 0.8299319727891157,
"fcm_dpo/beta": 0.03237488865852356,
"fcm_dpo/delta": 0.1423811912536621,
"fcm_dpo/margin": 20.60655975341797,
"fcm_dpo/q_t": 0.3701988160610199,
"grad_norm": 21.782363891601562,
"learning_rate": 4.333396073857723e-08,
"logits/chosen": 0.5990781784057617,
"logits/rejected": 0.5306206345558167,
"logps/chosen": -107.04806518554688,
"logps/ref_chosen": -64.27351379394531,
"logps/ref_rejected": -92.31663513183594,
"logps/rejected": -155.69773864746094,
"loss": 1.0858,
"margin_dpo/margin_mean": 20.60655975341797,
"margin_dpo/margin_std": 34.06036376953125,
"step": 549
},
{
"KL/chosen_KL_mean": -47.24556350708008,
"KL/mean": -55.89653778076172,
"KL/rejected_KL_mean": -64.54751586914062,
"KL/std": 33.23480987548828,
"epoch": 0.8314436885865457,
"fcm_dpo/beta": 0.03347238153219223,
"fcm_dpo/delta": 0.23484791815280914,
"fcm_dpo/margin": 17.301950454711914,
"fcm_dpo/q_t": 0.3901774287223816,
"grad_norm": 22.462038040161133,
"learning_rate": 4.259284772799099e-08,
"logits/chosen": 0.5305013060569763,
"logits/rejected": 0.4978870749473572,
"logps/chosen": -103.47599792480469,
"logps/ref_chosen": -56.230438232421875,
"logps/ref_rejected": -62.59788513183594,
"logps/rejected": -127.14540100097656,
"loss": 1.1365,
"margin_dpo/margin_mean": 17.301950454711914,
"margin_dpo/margin_std": 31.971088409423828,
"step": 550
},
{
"KL/chosen_KL_mean": -43.95768737792969,
"KL/mean": -54.62355422973633,
"KL/rejected_KL_mean": -65.2894287109375,
"KL/std": 32.458740234375,
"epoch": 0.8329554043839759,
"fcm_dpo/beta": 0.03470988571643829,
"fcm_dpo/delta": 0.0632665827870369,
"fcm_dpo/margin": 21.33173179626465,
"fcm_dpo/q_t": 0.360112726688385,
"grad_norm": 21.560270309448242,
"learning_rate": 4.1857536341307176e-08,
"logits/chosen": 0.5501875877380371,
"logits/rejected": 0.5143569707870483,
"logps/chosen": -111.70489501953125,
"logps/ref_chosen": -67.74720764160156,
"logps/ref_rejected": -87.04285430908203,
"logps/rejected": -152.332275390625,
"loss": 0.9942,
"margin_dpo/margin_mean": 21.331729888916016,
"margin_dpo/margin_std": 30.471147537231445,
"step": 551
},
{
"KL/chosen_KL_mean": -41.49971008300781,
"KL/mean": -54.09961700439453,
"KL/rejected_KL_mean": -66.69952392578125,
"KL/std": 33.441932678222656,
"epoch": 0.8344671201814059,
"fcm_dpo/beta": 0.034507181495428085,
"fcm_dpo/delta": -0.07692129164934158,
"fcm_dpo/margin": 25.199825286865234,
"fcm_dpo/q_t": 0.3218590021133423,
"grad_norm": 26.227811813354492,
"learning_rate": 4.112804714676593e-08,
"logits/chosen": 0.4940846860408783,
"logits/rejected": 0.4388379454612732,
"logps/chosen": -104.42596435546875,
"logps/ref_chosen": -62.92625427246094,
"logps/ref_rejected": -82.98365783691406,
"logps/rejected": -149.6831817626953,
"loss": 0.9485,
"margin_dpo/margin_mean": 25.199825286865234,
"margin_dpo/margin_std": 31.810752868652344,
"step": 552
},
{
"KL/chosen_KL_mean": -47.06233215332031,
"KL/mean": -58.25977325439453,
"KL/rejected_KL_mean": -69.45719909667969,
"KL/std": 34.09125518798828,
"epoch": 0.8359788359788359,
"fcm_dpo/beta": 0.034190136939287186,
"fcm_dpo/delta": 0.03711305931210518,
"fcm_dpo/margin": 22.39487075805664,
"fcm_dpo/q_t": 0.37020933628082275,
"grad_norm": 24.34133529663086,
"learning_rate": 4.0404400549748144e-08,
"logits/chosen": 0.49757149815559387,
"logits/rejected": 0.3874713182449341,
"logps/chosen": -103.100830078125,
"logps/ref_chosen": -56.038490295410156,
"logps/ref_rejected": -84.48454284667969,
"logps/rejected": -153.94174194335938,
"loss": 1.1121,
"margin_dpo/margin_mean": 22.39487075805664,
"margin_dpo/margin_std": 40.02998352050781,
"step": 553
},
{
"KL/chosen_KL_mean": -43.54690933227539,
"KL/mean": -55.97633361816406,
"KL/rejected_KL_mean": -68.40576934814453,
"KL/std": 33.090389251708984,
"epoch": 0.8374905517762661,
"fcm_dpo/beta": 0.03405106067657471,
"fcm_dpo/delta": -0.05085303634405136,
"fcm_dpo/margin": 24.858858108520508,
"fcm_dpo/q_t": 0.34414488077163696,
"grad_norm": 25.071014404296875,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": 0.4864945411682129,
"logits/rejected": 0.46010738611221313,
"logps/chosen": -108.07749938964844,
"logps/ref_chosen": -64.53059387207031,
"logps/ref_rejected": -71.2155990600586,
"logps/rejected": -139.62136840820312,
"loss": 0.9963,
"margin_dpo/margin_mean": 24.858858108520508,
"margin_dpo/margin_std": 36.17631530761719,
"step": 554
},
{
"KL/chosen_KL_mean": -48.42340850830078,
"KL/mean": -60.23335647583008,
"KL/rejected_KL_mean": -72.04330444335938,
"KL/std": 32.82422637939453,
"epoch": 0.8390022675736961,
"fcm_dpo/beta": 0.03450681269168854,
"fcm_dpo/delta": -0.022731244564056396,
"fcm_dpo/margin": 23.619890213012695,
"fcm_dpo/q_t": 0.3422093391418457,
"grad_norm": 25.595046997070312,
"learning_rate": 3.89747159520904e-08,
"logits/chosen": 0.5255084037780762,
"logits/rejected": 0.4931866526603699,
"logps/chosen": -115.07533264160156,
"logps/ref_chosen": -66.65191650390625,
"logps/ref_rejected": -68.6667251586914,
"logps/rejected": -140.71002197265625,
"loss": 1.0571,
"margin_dpo/margin_mean": 23.619890213012695,
"margin_dpo/margin_std": 35.09877014160156,
"step": 555
},
{
"KL/chosen_KL_mean": -48.8804931640625,
"KL/mean": -59.77089309692383,
"KL/rejected_KL_mean": -70.66129302978516,
"KL/std": 36.48758316040039,
"epoch": 0.8405139833711263,
"fcm_dpo/beta": 0.034058939665555954,
"fcm_dpo/delta": 0.06282925605773926,
"fcm_dpo/margin": 21.78080177307129,
"fcm_dpo/q_t": 0.373182475566864,
"grad_norm": 22.0106201171875,
"learning_rate": 3.826871794280192e-08,
"logits/chosen": 0.5408731698989868,
"logits/rejected": 0.486266165971756,
"logps/chosen": -101.71286010742188,
"logps/ref_chosen": -52.832366943359375,
"logps/ref_rejected": -64.49044036865234,
"logps/rejected": -135.1517333984375,
"loss": 1.1129,
"margin_dpo/margin_mean": 21.780799865722656,
"margin_dpo/margin_std": 38.57257843017578,
"step": 556
},
{
"KL/chosen_KL_mean": -47.91828536987305,
"KL/mean": -62.190040588378906,
"KL/rejected_KL_mean": -76.4617919921875,
"KL/std": 34.56977081298828,
"epoch": 0.8420256991685563,
"fcm_dpo/beta": 0.03313036635518074,
"fcm_dpo/delta": -0.16441971063613892,
"fcm_dpo/margin": 28.54350471496582,
"fcm_dpo/q_t": 0.3250656723976135,
"grad_norm": 22.4103946685791,
"learning_rate": 3.756864251262143e-08,
"logits/chosen": 0.626792848110199,
"logits/rejected": 0.5437628626823425,
"logps/chosen": -102.95426940917969,
"logps/ref_chosen": -55.03598403930664,
"logps/ref_rejected": -75.80644989013672,
"logps/rejected": -152.26824951171875,
"loss": 0.8931,
"margin_dpo/margin_mean": 28.543506622314453,
"margin_dpo/margin_std": 33.947593688964844,
"step": 557
},
{
"KL/chosen_KL_mean": -40.828495025634766,
"KL/mean": -55.58545684814453,
"KL/rejected_KL_mean": -70.34242248535156,
"KL/std": 35.937992095947266,
"epoch": 0.8435374149659864,
"fcm_dpo/beta": 0.03220053017139435,
"fcm_dpo/delta": -0.16744528710842133,
"fcm_dpo/margin": 29.5139217376709,
"fcm_dpo/q_t": 0.3242967128753662,
"grad_norm": 18.342342376708984,
"learning_rate": 3.687450924416341e-08,
"logits/chosen": 0.6076118350028992,
"logits/rejected": 0.5475571751594543,
"logps/chosen": -104.05484008789062,
"logps/ref_chosen": -63.226348876953125,
"logps/ref_rejected": -91.46881866455078,
"logps/rejected": -161.81124877929688,
"loss": 0.8853,
"margin_dpo/margin_mean": 29.5139217376709,
"margin_dpo/margin_std": 34.866798400878906,
"step": 558
},
{
"KL/chosen_KL_mean": -45.766387939453125,
"KL/mean": -59.00743103027344,
"KL/rejected_KL_mean": -72.24846649169922,
"KL/std": 34.96254348754883,
"epoch": 0.8450491307634165,
"fcm_dpo/beta": 0.031507570296525955,
"fcm_dpo/delta": -0.040891341865062714,
"fcm_dpo/margin": 26.482074737548828,
"fcm_dpo/q_t": 0.34787797927856445,
"grad_norm": 19.380739212036133,
"learning_rate": 3.6186337553827743e-08,
"logits/chosen": 0.4963953197002411,
"logits/rejected": 0.4245949387550354,
"logps/chosen": -107.28803253173828,
"logps/ref_chosen": -61.521644592285156,
"logps/ref_rejected": -82.83859252929688,
"logps/rejected": -155.08706665039062,
"loss": 0.9999,
"margin_dpo/margin_mean": 26.482074737548828,
"margin_dpo/margin_std": 37.74245834350586,
"step": 559
},
{
"KL/chosen_KL_mean": -46.50340270996094,
"KL/mean": -58.90251159667969,
"KL/rejected_KL_mean": -71.3016128540039,
"KL/std": 33.09571075439453,
"epoch": 0.8465608465608465,
"fcm_dpo/beta": 0.032199226319789886,
"fcm_dpo/delta": -0.0014753341674804688,
"fcm_dpo/margin": 24.79821014404297,
"fcm_dpo/q_t": 0.3513341546058655,
"grad_norm": 21.060949325561523,
"learning_rate": 3.550414669125573e-08,
"logits/chosen": 0.5204602479934692,
"logits/rejected": 0.4805315136909485,
"logps/chosen": -107.14462280273438,
"logps/ref_chosen": -60.64122009277344,
"logps/ref_rejected": -78.75474548339844,
"logps/rejected": -150.05636596679688,
"loss": 0.9854,
"margin_dpo/margin_mean": 24.798206329345703,
"margin_dpo/margin_std": 33.596336364746094,
"step": 560
},
{
"KL/chosen_KL_mean": -40.62968444824219,
"KL/mean": -54.02823257446289,
"KL/rejected_KL_mean": -67.42678833007812,
"KL/std": 33.786094665527344,
"epoch": 0.8480725623582767,
"fcm_dpo/beta": 0.03154933452606201,
"fcm_dpo/delta": -0.04966564476490021,
"fcm_dpo/margin": 26.797100067138672,
"fcm_dpo/q_t": 0.34592384099960327,
"grad_norm": 17.998380661010742,
"learning_rate": 3.482795573879241e-08,
"logits/chosen": 0.510515570640564,
"logits/rejected": 0.4774494171142578,
"logps/chosen": -103.12828063964844,
"logps/ref_chosen": -62.49859619140625,
"logps/ref_rejected": -78.72064208984375,
"logps/rejected": -146.14743041992188,
"loss": 0.9637,
"margin_dpo/margin_mean": 26.797100067138672,
"margin_dpo/margin_std": 36.19969177246094,
"step": 561
},
{
"KL/chosen_KL_mean": -45.762332916259766,
"KL/mean": -60.15574264526367,
"KL/rejected_KL_mean": -74.54913330078125,
"KL/std": 35.69789123535156,
"epoch": 0.8495842781557067,
"fcm_dpo/beta": 0.030817052349448204,
"fcm_dpo/delta": -0.09736991673707962,
"fcm_dpo/margin": 28.78680992126465,
"fcm_dpo/q_t": 0.33300113677978516,
"grad_norm": 21.175722122192383,
"learning_rate": 3.415778361095226e-08,
"logits/chosen": 0.5440159440040588,
"logits/rejected": 0.5005948543548584,
"logps/chosen": -120.5440673828125,
"logps/ref_chosen": -74.78173828125,
"logps/ref_rejected": -92.63499450683594,
"logps/rejected": -167.18414306640625,
"loss": 0.9237,
"margin_dpo/margin_mean": 28.786806106567383,
"margin_dpo/margin_std": 35.51923370361328,
"step": 562
},
{
"KL/chosen_KL_mean": -40.64170837402344,
"KL/mean": -53.593021392822266,
"KL/rejected_KL_mean": -66.54432678222656,
"KL/std": 33.413021087646484,
"epoch": 0.8510959939531368,
"fcm_dpo/beta": 0.030957389622926712,
"fcm_dpo/delta": -0.0022011548280715942,
"fcm_dpo/margin": 25.90262222290039,
"fcm_dpo/q_t": 0.34764784574508667,
"grad_norm": 23.740327835083008,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": 0.5938107967376709,
"logits/rejected": 0.5400828123092651,
"logps/chosen": -90.8402099609375,
"logps/ref_chosen": -50.19850158691406,
"logps/ref_rejected": -66.76687622070312,
"logps/rejected": -133.3112030029297,
"loss": 1.0339,
"margin_dpo/margin_mean": 25.90262222290039,
"margin_dpo/margin_std": 38.862754821777344,
"step": 563
},
{
"KL/chosen_KL_mean": -39.63123321533203,
"KL/mean": -55.32246398925781,
"KL/rejected_KL_mean": -71.01370239257812,
"KL/std": 33.93794250488281,
"epoch": 0.8526077097505669,
"fcm_dpo/beta": 0.030182164162397385,
"fcm_dpo/delta": -0.16239605844020844,
"fcm_dpo/margin": 31.38246726989746,
"fcm_dpo/q_t": 0.31941738724708557,
"grad_norm": 17.085735321044922,
"learning_rate": 3.283557064487785e-08,
"logits/chosen": 0.4971418082714081,
"logits/rejected": 0.46422910690307617,
"logps/chosen": -95.37208557128906,
"logps/ref_chosen": -55.7408447265625,
"logps/ref_rejected": -74.82323455810547,
"logps/rejected": -145.83694458007812,
"loss": 0.9018,
"margin_dpo/margin_mean": 31.38246726989746,
"margin_dpo/margin_std": 37.687286376953125,
"step": 564
},
{
"KL/chosen_KL_mean": -51.22956848144531,
"KL/mean": -62.0440673828125,
"KL/rejected_KL_mean": -72.85856628417969,
"KL/std": 36.59965515136719,
"epoch": 0.854119425547997,
"fcm_dpo/beta": 0.030782857909798622,
"fcm_dpo/delta": 0.14088793098926544,
"fcm_dpo/margin": 21.62899398803711,
"fcm_dpo/q_t": 0.3720448613166809,
"grad_norm": 19.96152687072754,
"learning_rate": 3.218356679178252e-08,
"logits/chosen": 0.563768744468689,
"logits/rejected": 0.5072394013404846,
"logps/chosen": -109.56695556640625,
"logps/ref_chosen": -58.33738327026367,
"logps/ref_rejected": -78.31776428222656,
"logps/rejected": -151.17633056640625,
"loss": 1.06,
"margin_dpo/margin_mean": 21.628995895385742,
"margin_dpo/margin_std": 34.05863571166992,
"step": 565
},
{
"KL/chosen_KL_mean": -44.82415008544922,
"KL/mean": -56.39575958251953,
"KL/rejected_KL_mean": -67.96736145019531,
"KL/std": 34.829559326171875,
"epoch": 0.8556311413454271,
"fcm_dpo/beta": 0.03129101172089577,
"fcm_dpo/delta": 0.08050192892551422,
"fcm_dpo/margin": 23.143218994140625,
"fcm_dpo/q_t": 0.3658328950405121,
"grad_norm": 26.74419593811035,
"learning_rate": 3.1537655732553764e-08,
"logits/chosen": 0.536413311958313,
"logits/rejected": 0.5143975615501404,
"logps/chosen": -116.04789733886719,
"logps/ref_chosen": -71.22373962402344,
"logps/ref_rejected": -71.11601257324219,
"logps/rejected": -139.0833740234375,
"loss": 1.1239,
"margin_dpo/margin_mean": 23.143218994140625,
"margin_dpo/margin_std": 40.9520149230957,
"step": 566
},
{
"KL/chosen_KL_mean": -44.22145080566406,
"KL/mean": -57.35389709472656,
"KL/rejected_KL_mean": -70.48634338378906,
"KL/std": 35.569793701171875,
"epoch": 0.8571428571428571,
"fcm_dpo/beta": 0.03084486722946167,
"fcm_dpo/delta": -0.013498269021511078,
"fcm_dpo/margin": 26.26490020751953,
"fcm_dpo/q_t": 0.3430580496788025,
"grad_norm": 16.386756896972656,
"learning_rate": 3.089785553471233e-08,
"logits/chosen": 0.5396600365638733,
"logits/rejected": 0.4434698820114136,
"logps/chosen": -96.8907241821289,
"logps/ref_chosen": -52.669273376464844,
"logps/ref_rejected": -74.34785461425781,
"logps/rejected": -144.83419799804688,
"loss": 0.9424,
"margin_dpo/margin_mean": 26.2648983001709,
"margin_dpo/margin_std": 32.46692657470703,
"step": 567
},
{
"KL/chosen_KL_mean": -41.46443176269531,
"KL/mean": -56.34814453125,
"KL/rejected_KL_mean": -71.23185729980469,
"KL/std": 33.74592208862305,
"epoch": 0.8586545729402872,
"fcm_dpo/beta": 0.03083047643303871,
"fcm_dpo/delta": -0.13012617826461792,
"fcm_dpo/margin": 29.767419815063477,
"fcm_dpo/q_t": 0.3229033350944519,
"grad_norm": 19.885181427001953,
"learning_rate": 3.026418409484513e-08,
"logits/chosen": 0.5687329173088074,
"logits/rejected": 0.4785163998603821,
"logps/chosen": -93.6424331665039,
"logps/ref_chosen": -52.178001403808594,
"logps/ref_rejected": -85.8277587890625,
"logps/rejected": -157.05963134765625,
"loss": 0.8686,
"margin_dpo/margin_mean": 29.767419815063477,
"margin_dpo/margin_std": 32.10868835449219,
"step": 568
},
{
"KL/chosen_KL_mean": -46.65216827392578,
"KL/mean": -57.41333770751953,
"KL/rejected_KL_mean": -68.17449951171875,
"KL/std": 34.74163818359375,
"epoch": 0.8601662887377173,
"fcm_dpo/beta": 0.030656516551971436,
"fcm_dpo/delta": 0.14884671568870544,
"fcm_dpo/margin": 21.522335052490234,
"fcm_dpo/q_t": 0.37315359711647034,
"grad_norm": 20.921825408935547,
"learning_rate": 2.963665913810451e-08,
"logits/chosen": 0.46112173795700073,
"logits/rejected": 0.4297167658805847,
"logps/chosen": -109.30142211914062,
"logps/ref_chosen": -62.649261474609375,
"logps/ref_rejected": -75.4298324584961,
"logps/rejected": -143.60433959960938,
"loss": 1.0718,
"margin_dpo/margin_mean": 21.522335052490234,
"margin_dpo/margin_std": 34.24241256713867,
"step": 569
},
{
"KL/chosen_KL_mean": -40.823997497558594,
"KL/mean": -58.369590759277344,
"KL/rejected_KL_mean": -75.9151840209961,
"KL/std": 33.9276123046875,
"epoch": 0.8616780045351474,
"fcm_dpo/beta": 0.030290057882666588,
"fcm_dpo/delta": -0.2935383915901184,
"fcm_dpo/margin": 35.091190338134766,
"fcm_dpo/q_t": 0.2969120740890503,
"grad_norm": 17.00665283203125,
"learning_rate": 2.9015298217712453e-08,
"logits/chosen": 0.4849631190299988,
"logits/rejected": 0.39886677265167236,
"logps/chosen": -90.86579132080078,
"logps/ref_chosen": -50.04179382324219,
"logps/ref_rejected": -78.27146911621094,
"logps/rejected": -154.1866455078125,
"loss": 0.8096,
"margin_dpo/margin_mean": 35.091190338134766,
"margin_dpo/margin_std": 35.0093994140625,
"step": 570
},
{
"KL/chosen_KL_mean": -45.955867767333984,
"KL/mean": -55.82860565185547,
"KL/rejected_KL_mean": -65.70135498046875,
"KL/std": 32.06074142456055,
"epoch": 0.8631897203325775,
"fcm_dpo/beta": 0.030671749264001846,
"fcm_dpo/delta": 0.20318175852298737,
"fcm_dpo/margin": 19.74549102783203,
"fcm_dpo/q_t": 0.375748872756958,
"grad_norm": 21.15029525756836,
"learning_rate": 2.840011871446962e-08,
"logits/chosen": 0.5245357751846313,
"logits/rejected": 0.487672358751297,
"logps/chosen": -99.61268615722656,
"logps/ref_chosen": -53.65681457519531,
"logps/ref_rejected": -66.13298034667969,
"logps/rejected": -131.83433532714844,
"loss": 1.0978,
"margin_dpo/margin_mean": 19.74549102783203,
"margin_dpo/margin_std": 32.583404541015625,
"step": 571
},
{
"KL/chosen_KL_mean": -44.28086853027344,
"KL/mean": -56.86238098144531,
"KL/rejected_KL_mean": -69.44389343261719,
"KL/std": 33.44170379638672,
"epoch": 0.8647014361300076,
"fcm_dpo/beta": 0.030863357707858086,
"fcm_dpo/delta": 0.025278106331825256,
"fcm_dpo/margin": 25.163036346435547,
"fcm_dpo/q_t": 0.346957266330719,
"grad_norm": 21.679777145385742,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": 0.5327567458152771,
"logits/rejected": 0.5728912353515625,
"logps/chosen": -119.09879302978516,
"logps/ref_chosen": -74.81792449951172,
"logps/ref_rejected": -65.88681030273438,
"logps/rejected": -135.33070373535156,
"loss": 0.9486,
"margin_dpo/margin_mean": 25.163036346435547,
"margin_dpo/margin_std": 31.716094970703125,
"step": 572
},
{
"KL/chosen_KL_mean": -50.17157745361328,
"KL/mean": -61.936317443847656,
"KL/rejected_KL_mean": -73.7010498046875,
"KL/std": 36.69175338745117,
"epoch": 0.8662131519274376,
"fcm_dpo/beta": 0.03104744479060173,
"fcm_dpo/delta": 0.074785977602005,
"fcm_dpo/margin": 23.529462814331055,
"fcm_dpo/q_t": 0.3677501380443573,
"grad_norm": 26.699995040893555,
"learning_rate": 2.718837261761528e-08,
"logits/chosen": 0.519621729850769,
"logits/rejected": 0.4729001522064209,
"logps/chosen": -118.89722442626953,
"logps/ref_chosen": -68.72564697265625,
"logps/ref_rejected": -88.16201782226562,
"logps/rejected": -161.86306762695312,
"loss": 1.1149,
"margin_dpo/margin_mean": 23.529464721679688,
"margin_dpo/margin_std": 41.90831756591797,
"step": 573
},
{
"KL/chosen_KL_mean": -44.603668212890625,
"KL/mean": -59.30667495727539,
"KL/rejected_KL_mean": -74.00968170166016,
"KL/std": 35.03868103027344,
"epoch": 0.8677248677248677,
"fcm_dpo/beta": 0.031052954494953156,
"fcm_dpo/delta": -0.12524369359016418,
"fcm_dpo/margin": 29.406003952026367,
"fcm_dpo/q_t": 0.32094401121139526,
"grad_norm": 16.99439811706543,
"learning_rate": 2.659183991914696e-08,
"logits/chosen": 0.5790042877197266,
"logits/rejected": 0.5058803558349609,
"logps/chosen": -100.91706848144531,
"logps/ref_chosen": -56.31340026855469,
"logps/ref_rejected": -83.91553497314453,
"logps/rejected": -157.9252166748047,
"loss": 0.8578,
"margin_dpo/margin_mean": 29.406003952026367,
"margin_dpo/margin_std": 31.216388702392578,
"step": 574
},
{
"KL/chosen_KL_mean": -44.00529479980469,
"KL/mean": -53.75147247314453,
"KL/rejected_KL_mean": -63.49766540527344,
"KL/std": 32.914180755615234,
"epoch": 0.8692365835222978,
"fcm_dpo/beta": 0.03111717477440834,
"fcm_dpo/delta": 0.20531663298606873,
"fcm_dpo/margin": 19.49237060546875,
"fcm_dpo/q_t": 0.3899354934692383,
"grad_norm": 22.751699447631836,
"learning_rate": 2.600155642716606e-08,
"logits/chosen": 0.5909614562988281,
"logits/rejected": 0.5080176591873169,
"logps/chosen": -108.58941650390625,
"logps/ref_chosen": -64.5841293334961,
"logps/ref_rejected": -93.47034454345703,
"logps/rejected": -156.968017578125,
"loss": 1.159,
"margin_dpo/margin_mean": 19.492368698120117,
"margin_dpo/margin_std": 37.30479049682617,
"step": 575
},
{
"KL/chosen_KL_mean": -39.35127258300781,
"KL/mean": -53.455238342285156,
"KL/rejected_KL_mean": -67.5592041015625,
"KL/std": 34.92966079711914,
"epoch": 0.8707482993197279,
"fcm_dpo/beta": 0.03119221143424511,
"fcm_dpo/delta": -0.08927100151777267,
"fcm_dpo/margin": 28.207935333251953,
"fcm_dpo/q_t": 0.3355104923248291,
"grad_norm": 18.904682159423828,
"learning_rate": 2.5417538653170754e-08,
"logits/chosen": 0.558881402015686,
"logits/rejected": 0.4506233334541321,
"logps/chosen": -92.63179779052734,
"logps/ref_chosen": -53.28052520751953,
"logps/ref_rejected": -84.2000503540039,
"logps/rejected": -151.75924682617188,
"loss": 0.9491,
"margin_dpo/margin_mean": 28.20793342590332,
"margin_dpo/margin_std": 36.800270080566406,
"step": 576
},
{
"KL/chosen_KL_mean": -43.53594207763672,
"KL/mean": -54.1453857421875,
"KL/rejected_KL_mean": -64.75482940673828,
"KL/std": 32.73902130126953,
"epoch": 0.872260015117158,
"fcm_dpo/beta": 0.03182173892855644,
"fcm_dpo/delta": 0.13401471078395844,
"fcm_dpo/margin": 21.218887329101562,
"fcm_dpo/q_t": 0.3719717860221863,
"grad_norm": 21.348833084106445,
"learning_rate": 2.4839802933393607e-08,
"logits/chosen": 0.5364026427268982,
"logits/rejected": 0.5159157514572144,
"logps/chosen": -105.86062622070312,
"logps/ref_chosen": -62.32468795776367,
"logps/ref_rejected": -67.300537109375,
"logps/rejected": -132.05535888671875,
"loss": 1.1081,
"margin_dpo/margin_mean": 21.218889236450195,
"margin_dpo/margin_std": 37.124244689941406,
"step": 577
},
{
"KL/chosen_KL_mean": -43.72724151611328,
"KL/mean": -52.62541961669922,
"KL/rejected_KL_mean": -61.52360534667969,
"KL/std": 33.30746078491211,
"epoch": 0.873771730914588,
"fcm_dpo/beta": 0.033036790788173676,
"fcm_dpo/delta": 0.22592979669570923,
"fcm_dpo/margin": 17.796363830566406,
"fcm_dpo/q_t": 0.3949415981769562,
"grad_norm": 21.883607864379883,
"learning_rate": 2.4268365428344733e-08,
"logits/chosen": 0.5855566263198853,
"logits/rejected": 0.5576690435409546,
"logps/chosen": -100.3828125,
"logps/ref_chosen": -56.65557861328125,
"logps/ref_rejected": -68.21835327148438,
"logps/rejected": -129.74195861816406,
"loss": 1.1512,
"margin_dpo/margin_mean": 17.796363830566406,
"margin_dpo/margin_std": 34.408897399902344,
"step": 578
},
{
"KL/chosen_KL_mean": -44.7183837890625,
"KL/mean": -60.99937438964844,
"KL/rejected_KL_mean": -77.28036499023438,
"KL/std": 35.42094421386719,
"epoch": 0.8752834467120182,
"fcm_dpo/beta": 0.03220862150192261,
"fcm_dpo/delta": -0.2783782482147217,
"fcm_dpo/margin": 32.561981201171875,
"fcm_dpo/q_t": 0.30090928077697754,
"grad_norm": 17.46132469177246,
"learning_rate": 2.3703242122359357e-08,
"logits/chosen": 0.4664205312728882,
"logits/rejected": 0.43535664677619934,
"logps/chosen": -101.52804565429688,
"logps/ref_chosen": -56.809661865234375,
"logps/ref_rejected": -68.09613037109375,
"logps/rejected": -145.37649536132812,
"loss": 0.825,
"margin_dpo/margin_mean": 32.56198501586914,
"margin_dpo/margin_std": 34.768150329589844,
"step": 579
},
{
"KL/chosen_KL_mean": -44.27873992919922,
"KL/mean": -55.907466888427734,
"KL/rejected_KL_mean": -67.53619384765625,
"KL/std": 36.50806427001953,
"epoch": 0.8767951625094482,
"fcm_dpo/beta": 0.032273322343826294,
"fcm_dpo/delta": 0.05180711671710014,
"fcm_dpo/margin": 23.257455825805664,
"fcm_dpo/q_t": 0.3628014028072357,
"grad_norm": 23.451509475708008,
"learning_rate": 2.3144448823151392e-08,
"logits/chosen": 0.49390000104904175,
"logits/rejected": 0.43802976608276367,
"logps/chosen": -101.97885131835938,
"logps/ref_chosen": -57.70011520385742,
"logps/ref_rejected": -77.90664672851562,
"logps/rejected": -145.44284057617188,
"loss": 1.0788,
"margin_dpo/margin_mean": 23.25745391845703,
"margin_dpo/margin_std": 38.16685485839844,
"step": 580
},
{
"KL/chosen_KL_mean": -50.066375732421875,
"KL/mean": -60.091793060302734,
"KL/rejected_KL_mean": -70.11721801757812,
"KL/std": 32.76508331298828,
"epoch": 0.8783068783068783,
"fcm_dpo/beta": 0.03265048936009407,
"fcm_dpo/delta": 0.1556333750486374,
"fcm_dpo/margin": 20.05084228515625,
"fcm_dpo/q_t": 0.37884268164634705,
"grad_norm": 22.65587043762207,
"learning_rate": 2.259200116137039e-08,
"logits/chosen": 0.5524417161941528,
"logits/rejected": 0.48380810022354126,
"logps/chosen": -109.39872741699219,
"logps/ref_chosen": -59.332359313964844,
"logps/ref_rejected": -83.64482116699219,
"logps/rejected": -153.7620391845703,
"loss": 1.1162,
"margin_dpo/margin_mean": 20.05084228515625,
"margin_dpo/margin_std": 35.896873474121094,
"step": 581
},
{
"KL/chosen_KL_mean": -44.435325622558594,
"KL/mean": -57.18058776855469,
"KL/rejected_KL_mean": -69.92584228515625,
"KL/std": 34.72920227050781,
"epoch": 0.8798185941043084,
"fcm_dpo/beta": 0.032961271703243256,
"fcm_dpo/delta": -0.0438670888543129,
"fcm_dpo/margin": 25.49052619934082,
"fcm_dpo/q_t": 0.341183066368103,
"grad_norm": 22.326169967651367,
"learning_rate": 2.204591459016525e-08,
"logits/chosen": 0.5303821563720703,
"logits/rejected": 0.557191014289856,
"logps/chosen": -108.59818267822266,
"logps/ref_chosen": -64.16285705566406,
"logps/ref_rejected": -58.632896423339844,
"logps/rejected": -128.55874633789062,
"loss": 0.9808,
"margin_dpo/margin_mean": 25.490524291992188,
"margin_dpo/margin_std": 35.270545959472656,
"step": 582
},
{
"KL/chosen_KL_mean": -44.09051513671875,
"KL/mean": -56.927040100097656,
"KL/rejected_KL_mean": -69.76355743408203,
"KL/std": 34.18093490600586,
"epoch": 0.8813303099017384,
"fcm_dpo/beta": 0.032621078193187714,
"fcm_dpo/delta": -0.04094298556447029,
"fcm_dpo/margin": 25.67304801940918,
"fcm_dpo/q_t": 0.3495996296405792,
"grad_norm": 24.98419761657715,
"learning_rate": 2.1506204384751064e-08,
"logits/chosen": 0.6439502239227295,
"logits/rejected": 0.5294203162193298,
"logps/chosen": -95.96290588378906,
"logps/ref_chosen": -51.87239456176758,
"logps/ref_rejected": -83.86331176757812,
"logps/rejected": -153.62686157226562,
"loss": 1.0603,
"margin_dpo/margin_mean": 25.673049926757812,
"margin_dpo/margin_std": 40.889869689941406,
"step": 583
},
{
"KL/chosen_KL_mean": -39.34344482421875,
"KL/mean": -51.7713508605957,
"KL/rejected_KL_mean": -64.19924926757812,
"KL/std": 32.398521423339844,
"epoch": 0.8828420256991686,
"fcm_dpo/beta": 0.03228888660669327,
"fcm_dpo/delta": -0.004207976162433624,
"fcm_dpo/margin": 24.855804443359375,
"fcm_dpo/q_t": 0.3564729690551758,
"grad_norm": 23.26299285888672,
"learning_rate": 2.09728856419826e-08,
"logits/chosen": 0.6478234529495239,
"logits/rejected": 0.5402784943580627,
"logps/chosen": -85.91484069824219,
"logps/ref_chosen": -46.571388244628906,
"logps/ref_rejected": -80.67969512939453,
"logps/rejected": -144.87893676757812,
"loss": 1.0805,
"margin_dpo/margin_mean": 24.855804443359375,
"margin_dpo/margin_std": 40.67819595336914,
"step": 584
},
{
"KL/chosen_KL_mean": -45.52735900878906,
"KL/mean": -55.15117645263672,
"KL/rejected_KL_mean": -64.77499389648438,
"KL/std": 35.69919967651367,
"epoch": 0.8843537414965986,
"fcm_dpo/beta": 0.033551305532455444,
"fcm_dpo/delta": 0.1630236804485321,
"fcm_dpo/margin": 19.247638702392578,
"fcm_dpo/q_t": 0.3736453950405121,
"grad_norm": 20.46152114868164,
"learning_rate": 2.044597327993153e-08,
"logits/chosen": 0.4899832606315613,
"logits/rejected": 0.44771748781204224,
"logps/chosen": -103.65189361572266,
"logps/ref_chosen": -58.124534606933594,
"logps/ref_rejected": -79.00538635253906,
"logps/rejected": -143.78038024902344,
"loss": 1.1302,
"margin_dpo/margin_mean": 19.247638702392578,
"margin_dpo/margin_std": 35.1866569519043,
"step": 585
},
{
"KL/chosen_KL_mean": -44.563514709472656,
"KL/mean": -58.280799865722656,
"KL/rejected_KL_mean": -71.99807739257812,
"KL/std": 34.64778518676758,
"epoch": 0.8858654572940288,
"fcm_dpo/beta": 0.03286353871226311,
"fcm_dpo/delta": -0.11315355449914932,
"fcm_dpo/margin": 27.4345703125,
"fcm_dpo/q_t": 0.32458969950675964,
"grad_norm": 23.63680648803711,
"learning_rate": 1.9925482037469187e-08,
"logits/chosen": 0.5588226914405823,
"logits/rejected": 0.5063307881355286,
"logps/chosen": -98.66515350341797,
"logps/ref_chosen": -54.10163879394531,
"logps/ref_rejected": -63.72113037109375,
"logps/rejected": -135.71920776367188,
"loss": 0.8879,
"margin_dpo/margin_mean": 27.4345703125,
"margin_dpo/margin_std": 31.724838256835938,
"step": 586
},
{
"KL/chosen_KL_mean": -42.331520080566406,
"KL/mean": -55.22560119628906,
"KL/rejected_KL_mean": -68.11968994140625,
"KL/std": 35.381317138671875,
"epoch": 0.8873771730914588,
"fcm_dpo/beta": 0.03263702988624573,
"fcm_dpo/delta": -0.04549926146864891,
"fcm_dpo/margin": 25.788166046142578,
"fcm_dpo/q_t": 0.3424980342388153,
"grad_norm": 22.418487548828125,
"learning_rate": 1.9411426473854687e-08,
"logits/chosen": 0.5717095136642456,
"logits/rejected": 0.5568786859512329,
"logps/chosen": -105.74871826171875,
"logps/ref_chosen": -63.41719436645508,
"logps/ref_rejected": -63.47003936767578,
"logps/rejected": -131.5897216796875,
"loss": 1.0842,
"margin_dpo/margin_mean": 25.788166046142578,
"margin_dpo/margin_std": 42.68872833251953,
"step": 587
},
{
"KL/chosen_KL_mean": -45.627567291259766,
"KL/mean": -58.562164306640625,
"KL/rejected_KL_mean": -71.49676513671875,
"KL/std": 34.04606628417969,
"epoch": 0.8888888888888888,
"fcm_dpo/beta": 0.03227134793996811,
"fcm_dpo/delta": -0.038421668112277985,
"fcm_dpo/margin": 25.86919403076172,
"fcm_dpo/q_t": 0.3437250256538391,
"grad_norm": 23.81206703186035,
"learning_rate": 1.890382096832699e-08,
"logits/chosen": 0.574599027633667,
"logits/rejected": 0.5247448682785034,
"logps/chosen": -107.82859802246094,
"logps/ref_chosen": -62.20103454589844,
"logps/ref_rejected": -82.10249328613281,
"logps/rejected": -153.59925842285156,
"loss": 0.9862,
"margin_dpo/margin_mean": 25.869192123413086,
"margin_dpo/margin_std": 36.37947082519531,
"step": 588
},
{
"KL/chosen_KL_mean": -39.77646255493164,
"KL/mean": -54.80766677856445,
"KL/rejected_KL_mean": -69.8388671875,
"KL/std": 35.71990966796875,
"epoch": 0.890400604686319,
"fcm_dpo/beta": 0.03161867707967758,
"fcm_dpo/delta": -0.1660689115524292,
"fcm_dpo/margin": 30.06241226196289,
"fcm_dpo/q_t": 0.3181713819503784,
"grad_norm": 20.573936462402344,
"learning_rate": 1.840267971970344e-08,
"logits/chosen": 0.5346547961235046,
"logits/rejected": 0.5033892393112183,
"logps/chosen": -96.49008178710938,
"logps/ref_chosen": -56.71361541748047,
"logps/ref_rejected": -76.7366943359375,
"logps/rejected": -146.5755615234375,
"loss": 0.8598,
"margin_dpo/margin_mean": 30.06241226196289,
"margin_dpo/margin_std": 33.88391876220703,
"step": 589
},
{
"KL/chosen_KL_mean": -46.1741828918457,
"KL/mean": -61.5664176940918,
"KL/rejected_KL_mean": -76.95864868164062,
"KL/std": 34.06959533691406,
"epoch": 0.891912320483749,
"fcm_dpo/beta": 0.030643416568636894,
"fcm_dpo/delta": -0.15798437595367432,
"fcm_dpo/margin": 30.784465789794922,
"fcm_dpo/q_t": 0.32272064685821533,
"grad_norm": 19.49903678894043,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": 0.46632716059684753,
"logits/rejected": 0.4379855990409851,
"logps/chosen": -112.68800354003906,
"logps/ref_chosen": -66.5138168334961,
"logps/ref_rejected": -85.70820617675781,
"logps/rejected": -162.66685485839844,
"loss": 0.9171,
"margin_dpo/margin_mean": 30.784465789794922,
"margin_dpo/margin_std": 38.08427047729492,
"step": 590
},
{
"KL/chosen_KL_mean": -38.659080505371094,
"KL/mean": -54.095245361328125,
"KL/rejected_KL_mean": -69.53141784667969,
"KL/std": 35.19484329223633,
"epoch": 0.8934240362811792,
"fcm_dpo/beta": 0.02937409281730652,
"fcm_dpo/delta": -0.12251757830381393,
"fcm_dpo/margin": 30.872339248657227,
"fcm_dpo/q_t": 0.328896701335907,
"grad_norm": 19.901981353759766,
"learning_rate": 1.7419845883949098e-08,
"logits/chosen": 0.62455815076828,
"logits/rejected": 0.5571376085281372,
"logps/chosen": -99.35626220703125,
"logps/ref_chosen": -60.697181701660156,
"logps/ref_rejected": -86.12278747558594,
"logps/rejected": -155.65420532226562,
"loss": 0.9763,
"margin_dpo/margin_mean": 30.872339248657227,
"margin_dpo/margin_std": 40.98619079589844,
"step": 591
},
{
"KL/chosen_KL_mean": -39.370513916015625,
"KL/mean": -52.374717712402344,
"KL/rejected_KL_mean": -65.37891387939453,
"KL/std": 33.04375457763672,
"epoch": 0.8949357520786092,
"fcm_dpo/beta": 0.029393857344985008,
"fcm_dpo/delta": 0.036264337599277496,
"fcm_dpo/margin": 26.008399963378906,
"fcm_dpo/q_t": 0.35765397548675537,
"grad_norm": 18.51282501220703,
"learning_rate": 1.6938180788793556e-08,
"logits/chosen": 0.5873167514801025,
"logits/rejected": 0.4715259373188019,
"logps/chosen": -90.60784912109375,
"logps/ref_chosen": -51.237327575683594,
"logps/ref_rejected": -81.60242462158203,
"logps/rejected": -146.98133850097656,
"loss": 0.9725,
"margin_dpo/margin_mean": 26.008399963378906,
"margin_dpo/margin_std": 34.70623779296875,
"step": 592
},
{
"KL/chosen_KL_mean": -38.42186737060547,
"KL/mean": -50.738250732421875,
"KL/rejected_KL_mean": -63.05463790893555,
"KL/std": 35.53569793701172,
"epoch": 0.8964474678760394,
"fcm_dpo/beta": 0.03002101369202137,
"fcm_dpo/delta": 0.06538180261850357,
"fcm_dpo/margin": 24.632770538330078,
"fcm_dpo/q_t": 0.3631042242050171,
"grad_norm": 21.09285545349121,
"learning_rate": 1.6463034933723336e-08,
"logits/chosen": 0.5370926856994629,
"logits/rejected": 0.4419124722480774,
"logps/chosen": -80.50186920166016,
"logps/ref_chosen": -42.08000183105469,
"logps/ref_rejected": -68.47499084472656,
"logps/rejected": -131.52963256835938,
"loss": 1.0906,
"margin_dpo/margin_mean": 24.632770538330078,
"margin_dpo/margin_std": 41.283905029296875,
"step": 593
},
{
"KL/chosen_KL_mean": -46.77622985839844,
"KL/mean": -57.62593078613281,
"KL/rejected_KL_mean": -68.47563171386719,
"KL/std": 34.27396011352539,
"epoch": 0.8979591836734694,
"fcm_dpo/beta": 0.03068956546485424,
"fcm_dpo/delta": 0.14385400712490082,
"fcm_dpo/margin": 21.699413299560547,
"fcm_dpo/q_t": 0.3688500225543976,
"grad_norm": 18.898942947387695,
"learning_rate": 1.5994421609589385e-08,
"logits/chosen": 0.45002150535583496,
"logits/rejected": 0.43023163080215454,
"logps/chosen": -110.43489837646484,
"logps/ref_chosen": -63.658668518066406,
"logps/ref_rejected": -70.35597229003906,
"logps/rejected": -138.83160400390625,
"loss": 1.0382,
"margin_dpo/margin_mean": 21.699413299560547,
"margin_dpo/margin_std": 32.530616760253906,
"step": 594
},
{
"KL/chosen_KL_mean": -42.06338119506836,
"KL/mean": -58.380043029785156,
"KL/rejected_KL_mean": -74.69670104980469,
"KL/std": 35.81193542480469,
"epoch": 0.8994708994708994,
"fcm_dpo/beta": 0.030249230563640594,
"fcm_dpo/delta": -0.20712698996067047,
"fcm_dpo/margin": 32.63332748413086,
"fcm_dpo/q_t": 0.31433260440826416,
"grad_norm": 17.953968048095703,
"learning_rate": 1.553235392451377e-08,
"logits/chosen": 0.5894097089767456,
"logits/rejected": 0.4952569007873535,
"logps/chosen": -98.28213500976562,
"logps/ref_chosen": -56.21875762939453,
"logps/ref_rejected": -83.95773315429688,
"logps/rejected": -158.65444946289062,
"loss": 0.912,
"margin_dpo/margin_mean": 32.633331298828125,
"margin_dpo/margin_std": 40.300350189208984,
"step": 595
},
{
"KL/chosen_KL_mean": -47.10646057128906,
"KL/mean": -55.662208557128906,
"KL/rejected_KL_mean": -64.21795654296875,
"KL/std": 32.69890594482422,
"epoch": 0.9009826152683296,
"fcm_dpo/beta": 0.0309823639690876,
"fcm_dpo/delta": 0.2856805920600891,
"fcm_dpo/margin": 17.111492156982422,
"fcm_dpo/q_t": 0.4075329303741455,
"grad_norm": 21.329530715942383,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": 0.45342981815338135,
"logits/rejected": 0.46392911672592163,
"logps/chosen": -115.58734130859375,
"logps/ref_chosen": -68.48088073730469,
"logps/ref_rejected": -61.732967376708984,
"logps/rejected": -125.950927734375,
"loss": 1.1914,
"margin_dpo/margin_mean": 17.111494064331055,
"margin_dpo/margin_std": 37.079368591308594,
"step": 596
},
{
"KL/chosen_KL_mean": -37.21601867675781,
"KL/mean": -49.4849853515625,
"KL/rejected_KL_mean": -61.75395202636719,
"KL/std": 31.241012573242188,
"epoch": 0.9024943310657596,
"fcm_dpo/beta": 0.03160978853702545,
"fcm_dpo/delta": 0.026392444968223572,
"fcm_dpo/margin": 24.53792381286621,
"fcm_dpo/q_t": 0.36042332649230957,
"grad_norm": 18.138704299926758,
"learning_rate": 1.4627906988186111e-08,
"logits/chosen": 0.5056836605072021,
"logits/rejected": 0.4817969799041748,
"logps/chosen": -86.07353210449219,
"logps/ref_chosen": -48.85750961303711,
"logps/ref_rejected": -55.068084716796875,
"logps/rejected": -116.82203674316406,
"loss": 1.0104,
"margin_dpo/margin_mean": 24.537925720214844,
"margin_dpo/margin_std": 36.30146026611328,
"step": 597
},
{
"KL/chosen_KL_mean": -49.368431091308594,
"KL/mean": -57.680335998535156,
"KL/rejected_KL_mean": -65.99224090576172,
"KL/std": 34.18096160888672,
"epoch": 0.9040060468631897,
"fcm_dpo/beta": 0.03309684246778488,
"fcm_dpo/delta": 0.2630840837955475,
"fcm_dpo/margin": 16.62381362915039,
"fcm_dpo/q_t": 0.40100929141044617,
"grad_norm": 39.50839614868164,
"learning_rate": 1.4185553036259095e-08,
"logits/chosen": 0.5584957599639893,
"logits/rejected": 0.4789044260978699,
"logps/chosen": -108.25558471679688,
"logps/ref_chosen": -58.88715362548828,
"logps/ref_rejected": -81.43145751953125,
"logps/rejected": -147.4237060546875,
"loss": 1.204,
"margin_dpo/margin_mean": 16.62381362915039,
"margin_dpo/margin_std": 36.20482635498047,
"step": 598
},
{
"KL/chosen_KL_mean": -50.590545654296875,
"KL/mean": -60.70787048339844,
"KL/rejected_KL_mean": -70.8251953125,
"KL/std": 31.916885375976562,
"epoch": 0.9055177626606198,
"fcm_dpo/beta": 0.03424448519945145,
"fcm_dpo/delta": 0.11299724876880646,
"fcm_dpo/margin": 20.234647750854492,
"fcm_dpo/q_t": 0.3777085244655609,
"grad_norm": 25.235319137573242,
"learning_rate": 1.3749795321332885e-08,
"logits/chosen": 0.619079053401947,
"logits/rejected": 0.5716946721076965,
"logps/chosen": -108.19773864746094,
"logps/ref_chosen": -57.60719299316406,
"logps/ref_rejected": -71.80469512939453,
"logps/rejected": -142.6298828125,
"loss": 1.1176,
"margin_dpo/margin_mean": 20.234647750854492,
"margin_dpo/margin_std": 36.37825012207031,
"step": 599
},
{
"KL/chosen_KL_mean": -46.038330078125,
"KL/mean": -56.578067779541016,
"KL/rejected_KL_mean": -67.11781311035156,
"KL/std": 35.091941833496094,
"epoch": 0.9070294784580499,
"fcm_dpo/beta": 0.03507785126566887,
"fcm_dpo/delta": 0.0583866685628891,
"fcm_dpo/margin": 21.0794734954834,
"fcm_dpo/q_t": 0.36688366532325745,
"grad_norm": 26.915437698364258,
"learning_rate": 1.3320646032487393e-08,
"logits/chosen": 0.591224193572998,
"logits/rejected": 0.5398635864257812,
"logps/chosen": -104.48063659667969,
"logps/ref_chosen": -58.44231414794922,
"logps/ref_rejected": -83.64639282226562,
"logps/rejected": -150.76419067382812,
"loss": 1.1057,
"margin_dpo/margin_mean": 21.0794734954834,
"margin_dpo/margin_std": 35.62135314941406,
"step": 600
},
{
"KL/chosen_KL_mean": -38.50769805908203,
"KL/mean": -52.836692810058594,
"KL/rejected_KL_mean": -67.16569519042969,
"KL/std": 36.2598991394043,
"epoch": 0.90854119425548,
"fcm_dpo/beta": 0.033360324800014496,
"fcm_dpo/delta": -0.17645078897476196,
"fcm_dpo/margin": 28.657991409301758,
"fcm_dpo/q_t": 0.3295706808567047,
"grad_norm": 18.96521759033203,
"learning_rate": 1.2898117173950868e-08,
"logits/chosen": 0.5417522192001343,
"logits/rejected": 0.4645710289478302,
"logps/chosen": -94.10202026367188,
"logps/ref_chosen": -55.59432601928711,
"logps/ref_rejected": -83.68630981445312,
"logps/rejected": -150.8520050048828,
"loss": 0.9661,
"margin_dpo/margin_mean": 28.65799331665039,
"margin_dpo/margin_std": 38.92144012451172,
"step": 601
},
{
"KL/chosen_KL_mean": -34.5461311340332,
"KL/mean": -48.17620086669922,
"KL/rejected_KL_mean": -61.8062629699707,
"KL/std": 32.06867218017578,
"epoch": 0.91005291005291,
"fcm_dpo/beta": 0.033003613352775574,
"fcm_dpo/delta": -0.10947325825691223,
"fcm_dpo/margin": 27.260128021240234,
"fcm_dpo/q_t": 0.32961416244506836,
"grad_norm": 19.130779266357422,
"learning_rate": 1.2482220564763667e-08,
"logits/chosen": 0.5375270843505859,
"logits/rejected": 0.5018444061279297,
"logps/chosen": -90.89531707763672,
"logps/ref_chosen": -56.349185943603516,
"logps/ref_rejected": -71.9959716796875,
"logps/rejected": -133.80223083496094,
"loss": 0.9007,
"margin_dpo/margin_mean": 27.260128021240234,
"margin_dpo/margin_std": 32.607025146484375,
"step": 602
},
{
"KL/chosen_KL_mean": -38.932960510253906,
"KL/mean": -52.24224853515625,
"KL/rejected_KL_mean": -65.55152893066406,
"KL/std": 31.548992156982422,
"epoch": 0.9115646258503401,
"fcm_dpo/beta": 0.03229742497205734,
"fcm_dpo/delta": -0.06548966467380524,
"fcm_dpo/margin": 26.618579864501953,
"fcm_dpo/q_t": 0.34004509449005127,
"grad_norm": 21.881919860839844,
"learning_rate": 1.2072967838448051e-08,
"logits/chosen": 0.49233826994895935,
"logits/rejected": 0.4304012656211853,
"logps/chosen": -92.10134887695312,
"logps/ref_chosen": -53.16838836669922,
"logps/ref_rejected": -73.8604736328125,
"logps/rejected": -139.41201782226562,
"loss": 0.9581,
"margin_dpo/margin_mean": 26.618579864501953,
"margin_dpo/margin_std": 35.89923095703125,
"step": 603
},
{
"KL/chosen_KL_mean": -38.09632873535156,
"KL/mean": -50.37153244018555,
"KL/rejected_KL_mean": -62.64673614501953,
"KL/std": 33.445926666259766,
"epoch": 0.9130763416477702,
"fcm_dpo/beta": 0.0323166623711586,
"fcm_dpo/delta": 0.0071356454864144325,
"fcm_dpo/margin": 24.550411224365234,
"fcm_dpo/q_t": 0.35812100768089294,
"grad_norm": 20.806612014770508,
"learning_rate": 1.1670370442682459e-08,
"logits/chosen": 0.4992177486419678,
"logits/rejected": 0.500224232673645,
"logps/chosen": -110.74574279785156,
"logps/ref_chosen": -72.64942169189453,
"logps/ref_rejected": -69.8792724609375,
"logps/rejected": -132.5260009765625,
"loss": 1.0781,
"margin_dpo/margin_mean": 24.550411224365234,
"margin_dpo/margin_std": 41.21388244628906,
"step": 604
},
{
"KL/chosen_KL_mean": -45.63953399658203,
"KL/mean": -57.18891906738281,
"KL/rejected_KL_mean": -68.7383041381836,
"KL/std": 32.24746322631836,
"epoch": 0.9145880574452003,
"fcm_dpo/beta": 0.032349593937397,
"fcm_dpo/delta": 0.05652875080704689,
"fcm_dpo/margin": 23.098766326904297,
"fcm_dpo/q_t": 0.3580436110496521,
"grad_norm": 22.0025577545166,
"learning_rate": 1.1274439638981532e-08,
"logits/chosen": 0.5846168398857117,
"logits/rejected": 0.522821843624115,
"logps/chosen": -107.25238037109375,
"logps/ref_chosen": -61.61284637451172,
"logps/ref_rejected": -79.34398651123047,
"logps/rejected": -148.08229064941406,
"loss": 1.0632,
"margin_dpo/margin_mean": 23.098766326904297,
"margin_dpo/margin_std": 36.80876922607422,
"step": 605
},
{
"KL/chosen_KL_mean": -39.366722106933594,
"KL/mean": -52.45832824707031,
"KL/rejected_KL_mean": -65.54991912841797,
"KL/std": 31.701202392578125,
"epoch": 0.9160997732426304,
"fcm_dpo/beta": 0.03257810324430466,
"fcm_dpo/delta": -0.05840463563799858,
"fcm_dpo/margin": 26.183189392089844,
"fcm_dpo/q_t": 0.3414803743362427,
"grad_norm": 19.46663475036621,
"learning_rate": 1.0885186502381016e-08,
"logits/chosen": 0.5263932943344116,
"logits/rejected": 0.45905801653862,
"logps/chosen": -93.83096313476562,
"logps/ref_chosen": -54.46424102783203,
"logps/ref_rejected": -79.62708282470703,
"logps/rejected": -145.177001953125,
"loss": 0.9523,
"margin_dpo/margin_mean": 26.183189392089844,
"margin_dpo/margin_std": 34.3382568359375,
"step": 606
},
{
"KL/chosen_KL_mean": -44.537776947021484,
"KL/mean": -58.091129302978516,
"KL/rejected_KL_mean": -71.64448547363281,
"KL/std": 35.20539855957031,
"epoch": 0.9176114890400605,
"fcm_dpo/beta": 0.03151046857237816,
"fcm_dpo/delta": -0.06450414657592773,
"fcm_dpo/margin": 27.106712341308594,
"fcm_dpo/q_t": 0.3402349054813385,
"grad_norm": 20.594440460205078,
"learning_rate": 1.0502621921127774e-08,
"logits/chosen": 0.4834175109863281,
"logits/rejected": 0.4526243209838867,
"logps/chosen": -107.39864349365234,
"logps/ref_chosen": -62.86086654663086,
"logps/ref_rejected": -72.5501937866211,
"logps/rejected": -144.19467163085938,
"loss": 0.9645,
"margin_dpo/margin_mean": 27.106712341308594,
"margin_dpo/margin_std": 35.469757080078125,
"step": 607
},
{
"KL/chosen_KL_mean": -46.28819274902344,
"KL/mean": -56.1732177734375,
"KL/rejected_KL_mean": -66.05824279785156,
"KL/std": 33.02288055419922,
"epoch": 0.9191232048374905,
"fcm_dpo/beta": 0.032674677670001984,
"fcm_dpo/delta": 0.16450561583042145,
"fcm_dpo/margin": 19.770050048828125,
"fcm_dpo/q_t": 0.3787013590335846,
"grad_norm": 23.87982177734375,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": 0.5198084115982056,
"logits/rejected": 0.4453343152999878,
"logps/chosen": -109.46891784667969,
"logps/ref_chosen": -63.18071746826172,
"logps/ref_rejected": -99.15888214111328,
"logps/rejected": -165.21713256835938,
"loss": 1.0811,
"margin_dpo/margin_mean": 19.77005386352539,
"margin_dpo/margin_std": 33.525691986083984,
"step": 608
},
{
"KL/chosen_KL_mean": -38.11963653564453,
"KL/mean": -53.10977554321289,
"KL/rejected_KL_mean": -68.09991455078125,
"KL/std": 33.28239440917969,
"epoch": 0.9206349206349206,
"fcm_dpo/beta": 0.03173757344484329,
"fcm_dpo/delta": -0.1720995157957077,
"fcm_dpo/margin": 29.980270385742188,
"fcm_dpo/q_t": 0.30832743644714355,
"grad_norm": 18.805438995361328,
"learning_rate": 9.757601041885694e-09,
"logits/chosen": 0.6107975244522095,
"logits/rejected": 0.5735124349594116,
"logps/chosen": -86.74285888671875,
"logps/ref_chosen": -48.62322235107422,
"logps/ref_rejected": -68.28271484375,
"logps/rejected": -136.38262939453125,
"loss": 0.8461,
"margin_dpo/margin_mean": 29.980270385742188,
"margin_dpo/margin_std": 30.857749938964844,
"step": 609
},
{
"KL/chosen_KL_mean": -44.18012237548828,
"KL/mean": -58.19615936279297,
"KL/rejected_KL_mean": -72.21221160888672,
"KL/std": 35.15657043457031,
"epoch": 0.9221466364323507,
"fcm_dpo/beta": 0.03127070516347885,
"fcm_dpo/delta": -0.08477940410375595,
"fcm_dpo/margin": 28.032087326049805,
"fcm_dpo/q_t": 0.33978596329689026,
"grad_norm": 20.88776969909668,
"learning_rate": 9.395165583732379e-09,
"logits/chosen": 0.5239899754524231,
"logits/rejected": 0.5249122381210327,
"logps/chosen": -116.84524536132812,
"logps/ref_chosen": -72.66513061523438,
"logps/ref_rejected": -87.15310668945312,
"logps/rejected": -159.36532592773438,
"loss": 0.9586,
"margin_dpo/margin_mean": 28.032085418701172,
"margin_dpo/margin_std": 37.857051849365234,
"step": 610
},
{
"KL/chosen_KL_mean": -41.32282257080078,
"KL/mean": -51.38006591796875,
"KL/rejected_KL_mean": -61.43731689453125,
"KL/std": 32.634307861328125,
"epoch": 0.9236583522297808,
"fcm_dpo/beta": 0.031811684370040894,
"fcm_dpo/delta": 0.17101894319057465,
"fcm_dpo/margin": 20.11448860168457,
"fcm_dpo/q_t": 0.37627357244491577,
"grad_norm": 21.9239501953125,
"learning_rate": 9.03946036001449e-09,
"logits/chosen": 0.5745566487312317,
"logits/rejected": 0.5245550870895386,
"logps/chosen": -89.63140106201172,
"logps/ref_chosen": -48.30857849121094,
"logps/ref_rejected": -70.6141128540039,
"logps/rejected": -132.0514373779297,
"loss": 1.0601,
"margin_dpo/margin_mean": 20.114486694335938,
"margin_dpo/margin_std": 31.439861297607422,
"step": 611
},
{
"KL/chosen_KL_mean": -45.562416076660156,
"KL/mean": -59.08216857910156,
"KL/rejected_KL_mean": -72.6019287109375,
"KL/std": 36.2523193359375,
"epoch": 0.9251700680272109,
"fcm_dpo/beta": 0.03201327472925186,
"fcm_dpo/delta": -0.07186618447303772,
"fcm_dpo/margin": 27.039508819580078,
"fcm_dpo/q_t": 0.33631759881973267,
"grad_norm": 19.976383209228516,
"learning_rate": 8.690495320571839e-09,
"logits/chosen": 0.45067673921585083,
"logits/rejected": 0.3812732696533203,
"logps/chosen": -106.79397583007812,
"logps/ref_chosen": -61.23155975341797,
"logps/ref_rejected": -94.37979888916016,
"logps/rejected": -166.98171997070312,
"loss": 0.9933,
"margin_dpo/margin_mean": 27.039508819580078,
"margin_dpo/margin_std": 38.99497985839844,
"step": 612
},
{
"KL/chosen_KL_mean": -37.76877975463867,
"KL/mean": -54.28990936279297,
"KL/rejected_KL_mean": -70.81103515625,
"KL/std": 36.1613883972168,
"epoch": 0.926681783824641,
"fcm_dpo/beta": 0.030983448028564453,
"fcm_dpo/delta": -0.24858498573303223,
"fcm_dpo/margin": 33.042259216308594,
"fcm_dpo/q_t": 0.30614638328552246,
"grad_norm": 17.632335662841797,
"learning_rate": 8.348280226706722e-09,
"logits/chosen": 0.4682348370552063,
"logits/rejected": 0.4623478651046753,
"logps/chosen": -91.75189208984375,
"logps/ref_chosen": -53.98310852050781,
"logps/ref_rejected": -58.32208251953125,
"logps/rejected": -129.13311767578125,
"loss": 0.8478,
"margin_dpo/margin_mean": 33.042259216308594,
"margin_dpo/margin_std": 36.33470153808594,
"step": 613
},
{
"KL/chosen_KL_mean": -43.43042755126953,
"KL/mean": -56.76459503173828,
"KL/rejected_KL_mean": -70.0987548828125,
"KL/std": 31.019271850585938,
"epoch": 0.9281934996220711,
"fcm_dpo/beta": 0.0304144024848938,
"fcm_dpo/delta": -0.012328855693340302,
"fcm_dpo/margin": 26.6683349609375,
"fcm_dpo/q_t": 0.3338313102722168,
"grad_norm": 20.849937438964844,
"learning_rate": 8.012824650910937e-09,
"logits/chosen": 0.5582789182662964,
"logits/rejected": 0.5460919737815857,
"logps/chosen": -103.6734619140625,
"logps/ref_chosen": -60.24303436279297,
"logps/ref_rejected": -72.26258850097656,
"logps/rejected": -142.36134338378906,
"loss": 0.9255,
"margin_dpo/margin_mean": 26.6683349609375,
"margin_dpo/margin_std": 31.444583892822266,
"step": 614
},
{
"KL/chosen_KL_mean": -44.83760070800781,
"KL/mean": -59.19696807861328,
"KL/rejected_KL_mean": -73.55632019042969,
"KL/std": 37.00213623046875,
"epoch": 0.9297052154195011,
"fcm_dpo/beta": 0.0297965407371521,
"fcm_dpo/delta": -0.062363844364881516,
"fcm_dpo/margin": 28.718719482421875,
"fcm_dpo/q_t": 0.3398233652114868,
"grad_norm": 21.655941009521484,
"learning_rate": 7.684137976598088e-09,
"logits/chosen": 0.514799952507019,
"logits/rejected": 0.4654197692871094,
"logps/chosen": -116.93228149414062,
"logps/ref_chosen": -72.09467315673828,
"logps/ref_rejected": -104.02980041503906,
"logps/rejected": -177.58612060546875,
"loss": 0.9957,
"margin_dpo/margin_mean": 28.718719482421875,
"margin_dpo/margin_std": 40.58180236816406,
"step": 615
},
{
"KL/chosen_KL_mean": -44.345245361328125,
"KL/mean": -55.76415252685547,
"KL/rejected_KL_mean": -67.18305969238281,
"KL/std": 32.98649215698242,
"epoch": 0.9312169312169312,
"fcm_dpo/beta": 0.030331624671816826,
"fcm_dpo/delta": 0.11543683707714081,
"fcm_dpo/margin": 22.837812423706055,
"fcm_dpo/q_t": 0.370138943195343,
"grad_norm": 20.064226150512695,
"learning_rate": 7.36222939784098e-09,
"logits/chosen": 0.5760443210601807,
"logits/rejected": 0.49106380343437195,
"logps/chosen": -102.87596893310547,
"logps/ref_chosen": -58.530723571777344,
"logps/ref_rejected": -75.48025512695312,
"logps/rejected": -142.66331481933594,
"loss": 1.0384,
"margin_dpo/margin_mean": 22.837814331054688,
"margin_dpo/margin_std": 35.295989990234375,
"step": 616
},
{
"KL/chosen_KL_mean": -48.91425323486328,
"KL/mean": -60.303802490234375,
"KL/rejected_KL_mean": -71.693359375,
"KL/std": 34.04789733886719,
"epoch": 0.9327286470143613,
"fcm_dpo/beta": 0.03068365529179573,
"fcm_dpo/delta": 0.10637275129556656,
"fcm_dpo/margin": 22.77910614013672,
"fcm_dpo/q_t": 0.36347612738609314,
"grad_norm": 21.133838653564453,
"learning_rate": 7.047107919114586e-09,
"logits/chosen": 0.5448430776596069,
"logits/rejected": 0.4960317611694336,
"logps/chosen": -106.5229263305664,
"logps/ref_chosen": -57.608673095703125,
"logps/ref_rejected": -81.22109985351562,
"logps/rejected": -152.91445922851562,
"loss": 1.0212,
"margin_dpo/margin_mean": 22.77910614013672,
"margin_dpo/margin_std": 32.204254150390625,
"step": 617
},
{
"KL/chosen_KL_mean": -41.57891845703125,
"KL/mean": -54.18458557128906,
"KL/rejected_KL_mean": -66.79025268554688,
"KL/std": 35.886566162109375,
"epoch": 0.9342403628117913,
"fcm_dpo/beta": 0.03101673163473606,
"fcm_dpo/delta": 0.017431527376174927,
"fcm_dpo/margin": 25.21134376525879,
"fcm_dpo/q_t": 0.349711537361145,
"grad_norm": 21.669893264770508,
"learning_rate": 6.738782355044048e-09,
"logits/chosen": 0.5267188549041748,
"logits/rejected": 0.42298799753189087,
"logps/chosen": -98.27485656738281,
"logps/ref_chosen": -56.69594192504883,
"logps/ref_rejected": -85.92362976074219,
"logps/rejected": -152.71389770507812,
"loss": 0.9609,
"margin_dpo/margin_mean": 25.211345672607422,
"margin_dpo/margin_std": 33.00782775878906,
"step": 618
},
{
"KL/chosen_KL_mean": -39.777366638183594,
"KL/mean": -52.436004638671875,
"KL/rejected_KL_mean": -65.09465026855469,
"KL/std": 32.83123016357422,
"epoch": 0.9357520786092215,
"fcm_dpo/beta": 0.03125225752592087,
"fcm_dpo/delta": 0.009025529026985168,
"fcm_dpo/margin": 25.317279815673828,
"fcm_dpo/q_t": 0.348273903131485,
"grad_norm": 18.896320343017578,
"learning_rate": 6.437261330158206e-09,
"logits/chosen": 0.6078675985336304,
"logits/rejected": 0.5344126224517822,
"logps/chosen": -93.83578491210938,
"logps/ref_chosen": -54.05841827392578,
"logps/ref_rejected": -83.55493927001953,
"logps/rejected": -148.64959716796875,
"loss": 0.965,
"margin_dpo/margin_mean": 25.317279815673828,
"margin_dpo/margin_std": 33.63002014160156,
"step": 619
},
{
"KL/chosen_KL_mean": -39.8848991394043,
"KL/mean": -51.43940734863281,
"KL/rejected_KL_mean": -62.99391555786133,
"KL/std": 31.84799575805664,
"epoch": 0.9372637944066515,
"fcm_dpo/beta": 0.03225833922624588,
"fcm_dpo/delta": 0.05094073712825775,
"fcm_dpo/margin": 23.109012603759766,
"fcm_dpo/q_t": 0.36446672677993774,
"grad_norm": 25.099145889282227,
"learning_rate": 6.142553278648238e-09,
"logits/chosen": 0.5527920722961426,
"logits/rejected": 0.5471271276473999,
"logps/chosen": -103.2546157836914,
"logps/ref_chosen": -63.36971664428711,
"logps/ref_rejected": -65.68269348144531,
"logps/rejected": -128.67660522460938,
"loss": 1.038,
"margin_dpo/margin_mean": 23.109012603759766,
"margin_dpo/margin_std": 34.14104461669922,
"step": 620
},
{
"KL/chosen_KL_mean": -42.45032501220703,
"KL/mean": -53.225311279296875,
"KL/rejected_KL_mean": -64.00030517578125,
"KL/std": 32.297000885009766,
"epoch": 0.9387755102040817,
"fcm_dpo/beta": 0.032468922436237335,
"fcm_dpo/delta": 0.10621249675750732,
"fcm_dpo/margin": 21.549976348876953,
"fcm_dpo/q_t": 0.3690629005432129,
"grad_norm": 22.792800903320312,
"learning_rate": 5.854666444131934e-09,
"logits/chosen": 0.5810672044754028,
"logits/rejected": 0.47503602504730225,
"logps/chosen": -94.77154541015625,
"logps/ref_chosen": -52.321224212646484,
"logps/ref_rejected": -88.09001159667969,
"logps/rejected": -152.09030151367188,
"loss": 1.0761,
"margin_dpo/margin_mean": 21.549978256225586,
"margin_dpo/margin_std": 35.361324310302734,
"step": 621
},
{
"KL/chosen_KL_mean": -46.312774658203125,
"KL/mean": -57.30717468261719,
"KL/rejected_KL_mean": -68.30157470703125,
"KL/std": 35.315643310546875,
"epoch": 0.9402872260015117,
"fcm_dpo/beta": 0.0329909548163414,
"fcm_dpo/delta": 0.07938125729560852,
"fcm_dpo/margin": 21.988800048828125,
"fcm_dpo/q_t": 0.3594774603843689,
"grad_norm": 20.948509216308594,
"learning_rate": 5.573608879422875e-09,
"logits/chosen": 0.49009227752685547,
"logits/rejected": 0.4497200846672058,
"logps/chosen": -106.17823791503906,
"logps/ref_chosen": -59.86545944213867,
"logps/ref_rejected": -81.86668395996094,
"logps/rejected": -150.1682586669922,
"loss": 0.9982,
"margin_dpo/margin_mean": 21.988800048828125,
"margin_dpo/margin_std": 30.735015869140625,
"step": 622
},
{
"KL/chosen_KL_mean": -49.466217041015625,
"KL/mean": -62.440582275390625,
"KL/rejected_KL_mean": -75.41496276855469,
"KL/std": 35.49492263793945,
"epoch": 0.9417989417989417,
"fcm_dpo/beta": 0.03263657167553902,
"fcm_dpo/delta": -0.051398035138845444,
"fcm_dpo/margin": 25.94874382019043,
"fcm_dpo/q_t": 0.3343699276447296,
"grad_norm": 18.90989875793457,
"learning_rate": 5.299388446305342e-09,
"logits/chosen": 0.5144941210746765,
"logits/rejected": 0.44776612520217896,
"logps/chosen": -116.83467102050781,
"logps/ref_chosen": -67.36846160888672,
"logps/ref_rejected": -82.02733612060547,
"logps/rejected": -157.44229125976562,
"loss": 0.9179,
"margin_dpo/margin_mean": 25.948741912841797,
"margin_dpo/margin_std": 31.110553741455078,
"step": 623
},
{
"KL/chosen_KL_mean": -40.11148452758789,
"KL/mean": -53.995384216308594,
"KL/rejected_KL_mean": -67.8792724609375,
"KL/std": 34.82365417480469,
"epoch": 0.9433106575963719,
"fcm_dpo/beta": 0.032003238797187805,
"fcm_dpo/delta": -0.09814178943634033,
"fcm_dpo/margin": 27.767793655395508,
"fcm_dpo/q_t": 0.33920931816101074,
"grad_norm": 19.443069458007812,
"learning_rate": 5.03201281531429e-09,
"logits/chosen": 0.5250840187072754,
"logits/rejected": 0.42520371079444885,
"logps/chosen": -91.13803100585938,
"logps/ref_chosen": -51.02655029296875,
"logps/ref_rejected": -76.49203491210938,
"logps/rejected": -144.37130737304688,
"loss": 0.9565,
"margin_dpo/margin_mean": 27.767793655395508,
"margin_dpo/margin_std": 37.009613037109375,
"step": 624
},
{
"KL/chosen_KL_mean": -43.644832611083984,
"KL/mean": -53.9796028137207,
"KL/rejected_KL_mean": -64.31437683105469,
"KL/std": 33.18622589111328,
"epoch": 0.9448223733938019,
"fcm_dpo/beta": 0.032724544405937195,
"fcm_dpo/delta": 0.1318553388118744,
"fcm_dpo/margin": 20.669544219970703,
"fcm_dpo/q_t": 0.3774385452270508,
"grad_norm": 20.581443786621094,
"learning_rate": 4.7714894655209174e-09,
"logits/chosen": 0.5876431465148926,
"logits/rejected": 0.5019428133964539,
"logps/chosen": -97.85244750976562,
"logps/ref_chosen": -54.20761489868164,
"logps/ref_rejected": -84.93669128417969,
"logps/rejected": -149.25106811523438,
"loss": 1.1065,
"margin_dpo/margin_mean": 20.669546127319336,
"margin_dpo/margin_std": 36.78896713256836,
"step": 625
},
{
"KL/chosen_KL_mean": -40.374717712402344,
"KL/mean": -54.71905517578125,
"KL/rejected_KL_mean": -69.06339263916016,
"KL/std": 36.232276916503906,
"epoch": 0.9463340891912321,
"fcm_dpo/beta": 0.03213457390666008,
"fcm_dpo/delta": -0.13456374406814575,
"fcm_dpo/margin": 28.68867301940918,
"fcm_dpo/q_t": 0.3437504768371582,
"grad_norm": 20.765810012817383,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": 0.6345266699790955,
"logits/rejected": 0.5021851658821106,
"logps/chosen": -85.43673706054688,
"logps/ref_chosen": -45.06201934814453,
"logps/ref_rejected": -89.66368103027344,
"logps/rejected": -158.72708129882812,
"loss": 1.0454,
"margin_dpo/margin_mean": 28.688674926757812,
"margin_dpo/margin_std": 44.237403869628906,
"step": 626
},
{
"KL/chosen_KL_mean": -43.914947509765625,
"KL/mean": -60.0136833190918,
"KL/rejected_KL_mean": -76.11241149902344,
"KL/std": 34.76005554199219,
"epoch": 0.9478458049886621,
"fcm_dpo/beta": 0.03084169700741768,
"fcm_dpo/delta": -0.21548502147197723,
"fcm_dpo/margin": 32.19746398925781,
"fcm_dpo/q_t": 0.3146284818649292,
"grad_norm": 19.66023063659668,
"learning_rate": 4.271028567242818e-09,
"logits/chosen": 0.4840816259384155,
"logits/rejected": 0.3665580451488495,
"logps/chosen": -102.70600891113281,
"logps/ref_chosen": -58.791053771972656,
"logps/ref_rejected": -94.90802001953125,
"logps/rejected": -171.0204315185547,
"loss": 0.897,
"margin_dpo/margin_mean": 32.19746398925781,
"margin_dpo/margin_std": 38.98145294189453,
"step": 627
},
{
"KL/chosen_KL_mean": -37.61805725097656,
"KL/mean": -53.94782257080078,
"KL/rejected_KL_mean": -70.277587890625,
"KL/std": 35.664676666259766,
"epoch": 0.9493575207860923,
"fcm_dpo/beta": 0.03022720292210579,
"fcm_dpo/delta": -0.21070462465286255,
"fcm_dpo/margin": 32.65953063964844,
"fcm_dpo/q_t": 0.3106863796710968,
"grad_norm": 21.5578556060791,
"learning_rate": 4.0311050177251895e-09,
"logits/chosen": 0.5580893158912659,
"logits/rejected": 0.5208135843276978,
"logps/chosen": -90.421630859375,
"logps/ref_chosen": -52.80357360839844,
"logps/ref_rejected": -76.49468994140625,
"logps/rejected": -146.77227783203125,
"loss": 0.9537,
"margin_dpo/margin_mean": 32.65953063964844,
"margin_dpo/margin_std": 39.72731018066406,
"step": 628
},
{
"KL/chosen_KL_mean": -40.470176696777344,
"KL/mean": -53.423011779785156,
"KL/rejected_KL_mean": -66.37583923339844,
"KL/std": 33.411163330078125,
"epoch": 0.9508692365835223,
"fcm_dpo/beta": 0.029644351452589035,
"fcm_dpo/delta": 0.034037984907627106,
"fcm_dpo/margin": 25.905670166015625,
"fcm_dpo/q_t": 0.3487117290496826,
"grad_norm": 18.10139274597168,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": 0.5535793304443359,
"logits/rejected": 0.5457119941711426,
"logps/chosen": -111.18766784667969,
"logps/ref_chosen": -70.71749877929688,
"logps/ref_rejected": -78.96273803710938,
"logps/rejected": -145.33859252929688,
"loss": 0.9226,
"margin_dpo/margin_mean": 25.905670166015625,
"margin_dpo/margin_std": 30.098609924316406,
"step": 629
},
{
"KL/chosen_KL_mean": -42.22637939453125,
"KL/mean": -56.090721130371094,
"KL/rejected_KL_mean": -69.95506286621094,
"KL/std": 37.00312042236328,
"epoch": 0.9523809523809523,
"fcm_dpo/beta": 0.029658004641532898,
"fcm_dpo/delta": -0.025315992534160614,
"fcm_dpo/margin": 27.728683471679688,
"fcm_dpo/q_t": 0.3414066731929779,
"grad_norm": 15.803033828735352,
"learning_rate": 3.5719052736323806e-09,
"logits/chosen": 0.49068087339401245,
"logits/rejected": 0.4432094693183899,
"logps/chosen": -98.42779541015625,
"logps/ref_chosen": -56.201412200927734,
"logps/ref_rejected": -74.69807434082031,
"logps/rejected": -144.65313720703125,
"loss": 0.9297,
"margin_dpo/margin_mean": 27.728683471679688,
"margin_dpo/margin_std": 33.98920822143555,
"step": 630
},
{
"KL/chosen_KL_mean": -38.312564849853516,
"KL/mean": -54.70954895019531,
"KL/rejected_KL_mean": -71.10653686523438,
"KL/std": 36.32965087890625,
"epoch": 0.9538926681783825,
"fcm_dpo/beta": 0.02823655493557453,
"fcm_dpo/delta": -0.14881953597068787,
"fcm_dpo/margin": 32.793975830078125,
"fcm_dpo/q_t": 0.3277819752693176,
"grad_norm": 19.586389541625977,
"learning_rate": 3.352641923861144e-09,
"logits/chosen": 0.6357178688049316,
"logits/rejected": 0.5238248109817505,
"logps/chosen": -97.13316345214844,
"logps/ref_chosen": -58.82059860229492,
"logps/ref_rejected": -96.51437377929688,
"logps/rejected": -167.62091064453125,
"loss": 0.9319,
"margin_dpo/margin_mean": 32.793975830078125,
"margin_dpo/margin_std": 40.02543258666992,
"step": 631
},
{
"KL/chosen_KL_mean": -40.46467590332031,
"KL/mean": -57.193267822265625,
"KL/rejected_KL_mean": -73.92185974121094,
"KL/std": 34.62277603149414,
"epoch": 0.9554043839758125,
"fcm_dpo/beta": 0.027888979762792587,
"fcm_dpo/delta": -0.14702674746513367,
"fcm_dpo/margin": 33.45717239379883,
"fcm_dpo/q_t": 0.31367796659469604,
"grad_norm": 16.896800994873047,
"learning_rate": 3.140277830901428e-09,
"logits/chosen": 0.5784220695495605,
"logits/rejected": 0.5536450147628784,
"logps/chosen": -99.250732421875,
"logps/ref_chosen": -58.786048889160156,
"logps/ref_rejected": -67.21923828125,
"logps/rejected": -141.14109802246094,
"loss": 0.8562,
"margin_dpo/margin_mean": 33.457176208496094,
"margin_dpo/margin_std": 35.489871978759766,
"step": 632
},
{
"KL/chosen_KL_mean": -37.61022186279297,
"KL/mean": -49.89120101928711,
"KL/rejected_KL_mean": -62.17218780517578,
"KL/std": 33.52429962158203,
"epoch": 0.9569160997732427,
"fcm_dpo/beta": 0.0282943956553936,
"fcm_dpo/delta": 0.11205098032951355,
"fcm_dpo/margin": 24.561969757080078,
"fcm_dpo/q_t": 0.36849457025527954,
"grad_norm": 16.958282470703125,
"learning_rate": 2.9348189350335007e-09,
"logits/chosen": 0.5199195742607117,
"logits/rejected": 0.45689111948013306,
"logps/chosen": -89.74041748046875,
"logps/ref_chosen": -52.13019561767578,
"logps/ref_rejected": -67.23016357421875,
"logps/rejected": -129.40234375,
"loss": 1.0335,
"margin_dpo/margin_mean": 24.561969757080078,
"margin_dpo/margin_std": 37.097103118896484,
"step": 633
},
{
"KL/chosen_KL_mean": -43.55609130859375,
"KL/mean": -50.937355041503906,
"KL/rejected_KL_mean": -58.31862258911133,
"KL/std": 31.610820770263672,
"epoch": 0.9584278155706727,
"fcm_dpo/beta": 0.030082430690526962,
"fcm_dpo/delta": 0.37042272090911865,
"fcm_dpo/margin": 14.762529373168945,
"fcm_dpo/q_t": 0.420715868473053,
"grad_norm": 26.369741439819336,
"learning_rate": 2.736270983384276e-09,
"logits/chosen": 0.5791378021240234,
"logits/rejected": 0.5895206928253174,
"logps/chosen": -104.535888671875,
"logps/ref_chosen": -60.97979736328125,
"logps/ref_rejected": -58.50825119018555,
"logps/rejected": -116.82687377929688,
"loss": 1.2388,
"margin_dpo/margin_mean": 14.762530326843262,
"margin_dpo/margin_std": 35.889408111572266,
"step": 634
},
{
"KL/chosen_KL_mean": -53.46791076660156,
"KL/mean": -63.21743392944336,
"KL/rejected_KL_mean": -72.96694946289062,
"KL/std": 33.996768951416016,
"epoch": 0.9599395313681028,
"fcm_dpo/beta": 0.0314885713160038,
"fcm_dpo/delta": 0.1977309286594391,
"fcm_dpo/margin": 19.499042510986328,
"fcm_dpo/q_t": 0.39308854937553406,
"grad_norm": 22.93577766418457,
"learning_rate": 2.5446395297668287e-09,
"logits/chosen": 0.4295782148838043,
"logits/rejected": 0.365118145942688,
"logps/chosen": -119.44093322753906,
"logps/ref_chosen": -65.9730224609375,
"logps/ref_rejected": -85.61317443847656,
"logps/rejected": -158.5801239013672,
"loss": 1.2012,
"margin_dpo/margin_mean": 19.499042510986328,
"margin_dpo/margin_std": 40.97630310058594,
"step": 635
},
{
"KL/chosen_KL_mean": -40.943885803222656,
"KL/mean": -55.528175354003906,
"KL/rejected_KL_mean": -70.11245727539062,
"KL/std": 35.467247009277344,
"epoch": 0.9614512471655329,
"fcm_dpo/beta": 0.0310262069106102,
"fcm_dpo/delta": -0.11683852970600128,
"fcm_dpo/margin": 29.168590545654297,
"fcm_dpo/q_t": 0.32491472363471985,
"grad_norm": 18.515304565429688,
"learning_rate": 2.359929934524829e-09,
"logits/chosen": 0.5118346810340881,
"logits/rejected": 0.42026370763778687,
"logps/chosen": -90.08405303955078,
"logps/ref_chosen": -49.140167236328125,
"logps/ref_rejected": -81.26971435546875,
"logps/rejected": -151.38217163085938,
"loss": 0.8938,
"margin_dpo/margin_mean": 29.168590545654297,
"margin_dpo/margin_std": 34.66674041748047,
"step": 636
},
{
"KL/chosen_KL_mean": -53.92256164550781,
"KL/mean": -64.56072998046875,
"KL/rejected_KL_mean": -75.19889831542969,
"KL/std": 32.787139892578125,
"epoch": 0.9629629629629629,
"fcm_dpo/beta": 0.031797170639038086,
"fcm_dpo/delta": 0.13112817704677582,
"fcm_dpo/margin": 21.27634048461914,
"fcm_dpo/q_t": 0.3831368088722229,
"grad_norm": 23.125991821289062,
"learning_rate": 2.1821473643827137e-09,
"logits/chosen": 0.5066337585449219,
"logits/rejected": 0.4331105053424835,
"logps/chosen": -127.61914825439453,
"logps/ref_chosen": -73.69658660888672,
"logps/ref_rejected": -83.01487731933594,
"logps/rejected": -158.21377563476562,
"loss": 1.1488,
"margin_dpo/margin_mean": 21.27634048461914,
"margin_dpo/margin_std": 40.99974822998047,
"step": 637
},
{
"KL/chosen_KL_mean": -48.4141731262207,
"KL/mean": -61.80738067626953,
"KL/rejected_KL_mean": -75.20059204101562,
"KL/std": 37.1383171081543,
"epoch": 0.9644746787603931,
"fcm_dpo/beta": 0.03168731927871704,
"fcm_dpo/delta": -0.05334305390715599,
"fcm_dpo/margin": 26.78641700744629,
"fcm_dpo/q_t": 0.34307920932769775,
"grad_norm": 21.305564880371094,
"learning_rate": 2.0112967923011646e-09,
"logits/chosen": 0.5187181830406189,
"logits/rejected": 0.4680929481983185,
"logps/chosen": -111.19575500488281,
"logps/ref_chosen": -62.78158187866211,
"logps/ref_rejected": -85.40478515625,
"logps/rejected": -160.60537719726562,
"loss": 0.9674,
"margin_dpo/margin_mean": 26.786415100097656,
"margin_dpo/margin_std": 36.208106994628906,
"step": 638
},
{
"KL/chosen_KL_mean": -42.945159912109375,
"KL/mean": -56.8536262512207,
"KL/rejected_KL_mean": -70.76209259033203,
"KL/std": 36.39090347290039,
"epoch": 0.9659863945578231,
"fcm_dpo/beta": 0.03127972036600113,
"fcm_dpo/delta": -0.07683076709508896,
"fcm_dpo/margin": 27.81693458557129,
"fcm_dpo/q_t": 0.33929571509361267,
"grad_norm": 21.127824783325195,
"learning_rate": 1.847382997337943e-09,
"logits/chosen": 0.5249953269958496,
"logits/rejected": 0.41127103567123413,
"logps/chosen": -96.71174621582031,
"logps/ref_chosen": -53.76658630371094,
"logps/ref_rejected": -72.30009460449219,
"logps/rejected": -143.06219482421875,
"loss": 0.9446,
"margin_dpo/margin_mean": 27.81693458557129,
"margin_dpo/margin_std": 36.36324691772461,
"step": 639
},
{
"KL/chosen_KL_mean": -44.95122528076172,
"KL/mean": -56.254302978515625,
"KL/rejected_KL_mean": -67.55738830566406,
"KL/std": 34.88379669189453,
"epoch": 0.9674981103552532,
"fcm_dpo/beta": 0.03140745311975479,
"fcm_dpo/delta": 0.0969884991645813,
"fcm_dpo/margin": 22.606157302856445,
"fcm_dpo/q_t": 0.35654008388519287,
"grad_norm": 20.348535537719727,
"learning_rate": 1.690410564514244e-09,
"logits/chosen": 0.5839003920555115,
"logits/rejected": 0.5229287147521973,
"logps/chosen": -96.36900329589844,
"logps/ref_chosen": -51.41777801513672,
"logps/ref_rejected": -77.27879333496094,
"logps/rejected": -144.836181640625,
"loss": 1.0716,
"margin_dpo/margin_mean": 22.606155395507812,
"margin_dpo/margin_std": 36.692012786865234,
"step": 640
},
{
"KL/chosen_KL_mean": -45.83881378173828,
"KL/mean": -57.58781814575195,
"KL/rejected_KL_mean": -69.33682250976562,
"KL/std": 32.90306854248047,
"epoch": 0.9690098261526833,
"fcm_dpo/beta": 0.03183242306113243,
"fcm_dpo/delta": 0.0562346875667572,
"fcm_dpo/margin": 23.498010635375977,
"fcm_dpo/q_t": 0.35128700733184814,
"grad_norm": 21.091442108154297,
"learning_rate": 1.5403838846864692e-09,
"logits/chosen": 0.5104883909225464,
"logits/rejected": 0.4826532006263733,
"logps/chosen": -116.89347839355469,
"logps/ref_chosen": -71.0546646118164,
"logps/ref_rejected": -82.2440185546875,
"logps/rejected": -151.58084106445312,
"loss": 0.9694,
"margin_dpo/margin_mean": 23.498008728027344,
"margin_dpo/margin_std": 31.388883590698242,
"step": 641
},
{
"KL/chosen_KL_mean": -48.326541900634766,
"KL/mean": -58.25520706176758,
"KL/rejected_KL_mean": -68.18387603759766,
"KL/std": 31.67426300048828,
"epoch": 0.9705215419501134,
"fcm_dpo/beta": 0.032394275069236755,
"fcm_dpo/delta": 0.16641943156719208,
"fcm_dpo/margin": 19.857332229614258,
"fcm_dpo/q_t": 0.38337334990501404,
"grad_norm": 25.411165237426758,
"learning_rate": 1.3973071544233218e-09,
"logits/chosen": 0.4777846038341522,
"logits/rejected": 0.4858616590499878,
"logps/chosen": -117.25581359863281,
"logps/ref_chosen": -68.92927551269531,
"logps/ref_rejected": -70.85682678222656,
"logps/rejected": -139.04071044921875,
"loss": 1.1461,
"margin_dpo/margin_mean": 19.857332229614258,
"margin_dpo/margin_std": 36.84928894042969,
"step": 642
},
{
"KL/chosen_KL_mean": -46.31542205810547,
"KL/mean": -57.94270324707031,
"KL/rejected_KL_mean": -69.56997680664062,
"KL/std": 32.99165725708008,
"epoch": 0.9720332577475435,
"fcm_dpo/beta": 0.03313422203063965,
"fcm_dpo/delta": 0.03186196833848953,
"fcm_dpo/margin": 23.254549026489258,
"fcm_dpo/q_t": 0.36055275797843933,
"grad_norm": 36.48023986816406,
"learning_rate": 1.261184375888541e-09,
"logits/chosen": 0.460068941116333,
"logits/rejected": 0.3696553409099579,
"logps/chosen": -111.62445831298828,
"logps/ref_chosen": -65.30903625488281,
"logps/ref_rejected": -83.61613464355469,
"logps/rejected": -153.1861114501953,
"loss": 1.0505,
"margin_dpo/margin_mean": 23.25455093383789,
"margin_dpo/margin_std": 36.687599182128906,
"step": 643
},
{
"KL/chosen_KL_mean": -41.07984924316406,
"KL/mean": -51.94877624511719,
"KL/rejected_KL_mean": -62.81770324707031,
"KL/std": 33.44035339355469,
"epoch": 0.9735449735449735,
"fcm_dpo/beta": 0.03386765718460083,
"fcm_dpo/delta": 0.06720557808876038,
"fcm_dpo/margin": 21.737850189208984,
"fcm_dpo/q_t": 0.3682977557182312,
"grad_norm": 21.03730010986328,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": 0.5970633029937744,
"logits/rejected": 0.5632849931716919,
"logps/chosen": -92.08245086669922,
"logps/ref_chosen": -51.002601623535156,
"logps/ref_rejected": -64.46372985839844,
"logps/rejected": -127.28143310546875,
"loss": 1.1474,
"margin_dpo/margin_mean": 21.737850189208984,
"margin_dpo/margin_std": 40.25080108642578,
"step": 644
},
{
"KL/chosen_KL_mean": -45.624202728271484,
"KL/mean": -57.04932403564453,
"KL/rejected_KL_mean": -68.47444152832031,
"KL/std": 34.80279541015625,
"epoch": 0.9750566893424036,
"fcm_dpo/beta": 0.03370298445224762,
"fcm_dpo/delta": 0.03188609704375267,
"fcm_dpo/margin": 22.850234985351562,
"fcm_dpo/q_t": 0.3569079339504242,
"grad_norm": 22.33905029296875,
"learning_rate": 1.0098157099674987e-09,
"logits/chosen": 0.47687435150146484,
"logits/rejected": 0.4549427032470703,
"logps/chosen": -106.58761596679688,
"logps/ref_chosen": -60.963409423828125,
"logps/ref_rejected": -69.73353576660156,
"logps/rejected": -138.20797729492188,
"loss": 1.019,
"margin_dpo/margin_mean": 22.850234985351562,
"margin_dpo/margin_std": 34.275508880615234,
"step": 645
},
{
"KL/chosen_KL_mean": -48.04452133178711,
"KL/mean": -59.946693420410156,
"KL/rejected_KL_mean": -71.84886169433594,
"KL/std": 37.24139404296875,
"epoch": 0.9765684051398337,
"fcm_dpo/beta": 0.03389802575111389,
"fcm_dpo/delta": -0.007534122560173273,
"fcm_dpo/margin": 23.804344177246094,
"fcm_dpo/q_t": 0.3509964048862457,
"grad_norm": 22.17904281616211,
"learning_rate": 8.945768539031783e-10,
"logits/chosen": 0.5630219578742981,
"logits/rejected": 0.5091748237609863,
"logps/chosen": -110.3345947265625,
"logps/ref_chosen": -62.290069580078125,
"logps/ref_rejected": -85.54812622070312,
"logps/rejected": -157.39698791503906,
"loss": 1.0346,
"margin_dpo/margin_mean": 23.804344177246094,
"margin_dpo/margin_std": 36.64860916137695,
"step": 646
},
{
"KL/chosen_KL_mean": -47.405487060546875,
"KL/mean": -63.31657028198242,
"KL/rejected_KL_mean": -79.22764587402344,
"KL/std": 36.01445007324219,
"epoch": 0.9780801209372638,
"fcm_dpo/beta": 0.032675907015800476,
"fcm_dpo/delta": -0.2671160101890564,
"fcm_dpo/margin": 31.82217025756836,
"fcm_dpo/q_t": 0.30921974778175354,
"grad_norm": 23.366548538208008,
"learning_rate": 7.863060120144316e-10,
"logits/chosen": 0.576119601726532,
"logits/rejected": 0.4795302748680115,
"logps/chosen": -114.92135620117188,
"logps/ref_chosen": -67.515869140625,
"logps/ref_rejected": -101.50871276855469,
"logps/rejected": -180.73635864257812,
"loss": 0.8618,
"margin_dpo/margin_mean": 31.82217025756836,
"margin_dpo/margin_std": 36.936439514160156,
"step": 647
},
{
"KL/chosen_KL_mean": -47.49691390991211,
"KL/mean": -60.25159454345703,
"KL/rejected_KL_mean": -73.00627136230469,
"KL/std": 32.82928466796875,
"epoch": 0.9795918367346939,
"fcm_dpo/beta": 0.03184132277965546,
"fcm_dpo/delta": -0.01447652280330658,
"fcm_dpo/margin": 25.509355545043945,
"fcm_dpo/q_t": 0.34164726734161377,
"grad_norm": 17.782150268554688,
"learning_rate": 6.850062128694045e-10,
"logits/chosen": 0.49999916553497314,
"logits/rejected": 0.4332047700881958,
"logps/chosen": -112.09284973144531,
"logps/ref_chosen": -64.59593963623047,
"logps/ref_rejected": -83.384033203125,
"logps/rejected": -156.3903045654297,
"loss": 1.0102,
"margin_dpo/margin_mean": 25.509355545043945,
"margin_dpo/margin_std": 36.32646942138672,
"step": 648
},
{
"KL/chosen_KL_mean": -44.42068099975586,
"KL/mean": -57.07667541503906,
"KL/rejected_KL_mean": -69.732666015625,
"KL/std": 33.58146286010742,
"epoch": 0.981103552532124,
"fcm_dpo/beta": 0.031717028468847275,
"fcm_dpo/delta": -0.004996992647647858,
"fcm_dpo/margin": 25.311986923217773,
"fcm_dpo/q_t": 0.35203224420547485,
"grad_norm": 25.448650360107422,
"learning_rate": 5.906802900412788e-10,
"logits/chosen": 0.5485668182373047,
"logits/rejected": 0.48975634574890137,
"logps/chosen": -93.73033142089844,
"logps/ref_chosen": -49.30964660644531,
"logps/ref_rejected": -73.73710632324219,
"logps/rejected": -143.4697723388672,
"loss": 1.0337,
"margin_dpo/margin_mean": 25.31198501586914,
"margin_dpo/margin_std": 37.88409423828125,
"step": 649
},
{
"KL/chosen_KL_mean": -42.31673049926758,
"KL/mean": -54.86392593383789,
"KL/rejected_KL_mean": -67.41111755371094,
"KL/std": 35.4691162109375,
"epoch": 0.982615268329554,
"fcm_dpo/beta": 0.03204188868403435,
"fcm_dpo/delta": -0.004477673210203648,
"fcm_dpo/margin": 25.094390869140625,
"fcm_dpo/q_t": 0.3569669723510742,
"grad_norm": 25.507871627807617,
"learning_rate": 5.033308820289184e-10,
"logits/chosen": 0.6226690411567688,
"logits/rejected": 0.5567650198936462,
"logps/chosen": -97.37998962402344,
"logps/ref_chosen": -55.06325912475586,
"logps/ref_rejected": -77.39610290527344,
"logps/rejected": -144.80722045898438,
"loss": 1.0672,
"margin_dpo/margin_mean": 25.094392776489258,
"margin_dpo/margin_std": 40.48807144165039,
"step": 650
},
{
"KL/chosen_KL_mean": -41.40019226074219,
"KL/mean": -52.56417465209961,
"KL/rejected_KL_mean": -63.7281608581543,
"KL/std": 34.437225341796875,
"epoch": 0.9841269841269841,
"fcm_dpo/beta": 0.032618556171655655,
"fcm_dpo/delta": 0.0751466155052185,
"fcm_dpo/margin": 22.327966690063477,
"fcm_dpo/q_t": 0.3638933598995209,
"grad_norm": 21.54810905456543,
"learning_rate": 4.2296043218295606e-10,
"logits/chosen": 0.5694529414176941,
"logits/rejected": 0.4965120255947113,
"logps/chosen": -95.4653549194336,
"logps/ref_chosen": -54.065162658691406,
"logps/ref_rejected": -77.79080200195312,
"logps/rejected": -141.5189666748047,
"loss": 1.0293,
"margin_dpo/margin_mean": 22.327966690063477,
"margin_dpo/margin_std": 33.20568084716797,
"step": 651
},
{
"KL/chosen_KL_mean": -49.35215759277344,
"KL/mean": -60.68022918701172,
"KL/rejected_KL_mean": -72.00830078125,
"KL/std": 34.67872619628906,
"epoch": 0.9856386999244142,
"fcm_dpo/beta": 0.03263973072171211,
"fcm_dpo/delta": 0.06523266434669495,
"fcm_dpo/margin": 22.656145095825195,
"fcm_dpo/q_t": 0.3670682907104492,
"grad_norm": 22.150224685668945,
"learning_rate": 3.4957118863768176e-10,
"logits/chosen": 0.5378252863883972,
"logits/rejected": 0.48126500844955444,
"logps/chosen": -112.99246215820312,
"logps/ref_chosen": -63.64030456542969,
"logps/ref_rejected": -78.86882019042969,
"logps/rejected": -150.87713623046875,
"loss": 1.0563,
"margin_dpo/margin_mean": 22.656145095825195,
"margin_dpo/margin_std": 36.34861755371094,
"step": 652
},
{
"KL/chosen_KL_mean": -43.989662170410156,
"KL/mean": -57.336952209472656,
"KL/rejected_KL_mean": -70.68423461914062,
"KL/std": 34.78733825683594,
"epoch": 0.9871504157218443,
"fcm_dpo/beta": 0.03256607800722122,
"fcm_dpo/delta": -0.07587750256061554,
"fcm_dpo/margin": 26.694576263427734,
"fcm_dpo/q_t": 0.34083616733551025,
"grad_norm": 22.887805938720703,
"learning_rate": 2.831652042480093e-10,
"logits/chosen": 0.5100860595703125,
"logits/rejected": 0.4632833003997803,
"logps/chosen": -105.65803527832031,
"logps/ref_chosen": -61.668373107910156,
"logps/ref_rejected": -73.83012390136719,
"logps/rejected": -144.51437377929688,
"loss": 0.9963,
"margin_dpo/margin_mean": 26.694576263427734,
"margin_dpo/margin_std": 38.514312744140625,
"step": 653
},
{
"KL/chosen_KL_mean": -45.31245803833008,
"KL/mean": -55.97831344604492,
"KL/rejected_KL_mean": -66.64417266845703,
"KL/std": 36.80150604248047,
"epoch": 0.9886621315192744,
"fcm_dpo/beta": 0.03246813267469406,
"fcm_dpo/delta": 0.11256799101829529,
"fcm_dpo/margin": 21.331714630126953,
"fcm_dpo/q_t": 0.3787607252597809,
"grad_norm": 22.46723747253418,
"learning_rate": 2.2374433653205016e-10,
"logits/chosen": 0.5046300888061523,
"logits/rejected": 0.40765130519866943,
"logps/chosen": -102.88072204589844,
"logps/ref_chosen": -57.568267822265625,
"logps/ref_rejected": -87.74789428710938,
"logps/rejected": -154.39205932617188,
"loss": 1.0765,
"margin_dpo/margin_mean": 21.33171272277832,
"margin_dpo/margin_std": 34.9078254699707,
"step": 654
},
{
"KL/chosen_KL_mean": -37.20157241821289,
"KL/mean": -51.45652770996094,
"KL/rejected_KL_mean": -65.71148681640625,
"KL/std": 33.084171295166016,
"epoch": 0.9901738473167044,
"fcm_dpo/beta": 0.031920626759529114,
"fcm_dpo/delta": -0.13033875823020935,
"fcm_dpo/margin": 28.50991439819336,
"fcm_dpo/q_t": 0.3177955448627472,
"grad_norm": 18.4930362701416,
"learning_rate": 1.7131024761923852e-10,
"logits/chosen": 0.521104097366333,
"logits/rejected": 0.4239880442619324,
"logps/chosen": -89.34872436523438,
"logps/ref_chosen": -52.14714813232422,
"logps/ref_rejected": -80.85014343261719,
"logps/rejected": -146.56161499023438,
"loss": 0.8502,
"margin_dpo/margin_mean": 28.509910583496094,
"margin_dpo/margin_std": 28.062389373779297,
"step": 655
},
{
"KL/chosen_KL_mean": -45.38775634765625,
"KL/mean": -59.60160827636719,
"KL/rejected_KL_mean": -73.81546020507812,
"KL/std": 35.18117141723633,
"epoch": 0.9916855631141346,
"fcm_dpo/beta": 0.03162258490920067,
"fcm_dpo/delta": -0.10963227599859238,
"fcm_dpo/margin": 28.427703857421875,
"fcm_dpo/q_t": 0.33033907413482666,
"grad_norm": 17.709102630615234,
"learning_rate": 1.2586440420372934e-10,
"logits/chosen": 0.46768325567245483,
"logits/rejected": 0.4151872396469116,
"logps/chosen": -118.64448547363281,
"logps/ref_chosen": -73.25672912597656,
"logps/ref_rejected": -85.35127258300781,
"logps/rejected": -159.16673278808594,
"loss": 0.904,
"margin_dpo/margin_mean": 28.427701950073242,
"margin_dpo/margin_std": 34.47346878051758,
"step": 656
},
{
"KL/chosen_KL_mean": -42.539817810058594,
"KL/mean": -58.39175796508789,
"KL/rejected_KL_mean": -74.24369812011719,
"KL/std": 38.77171325683594,
"epoch": 0.9931972789115646,
"fcm_dpo/beta": 0.030581142753362656,
"fcm_dpo/delta": -0.18895968794822693,
"fcm_dpo/margin": 31.703880310058594,
"fcm_dpo/q_t": 0.3181675672531128,
"grad_norm": 19.23728370666504,
"learning_rate": 8.740807750345913e-11,
"logits/chosen": 0.6302456855773926,
"logits/rejected": 0.540096640586853,
"logps/chosen": -92.26321411132812,
"logps/ref_chosen": -49.72339630126953,
"logps/ref_rejected": -75.1568603515625,
"logps/rejected": -149.4005584716797,
"loss": 0.9203,
"margin_dpo/margin_mean": 31.703880310058594,
"margin_dpo/margin_std": 39.745079040527344,
"step": 657
},
{
"KL/chosen_KL_mean": -46.40203094482422,
"KL/mean": -57.525978088378906,
"KL/rejected_KL_mean": -68.64991760253906,
"KL/std": 37.033355712890625,
"epoch": 0.9947089947089947,
"fcm_dpo/beta": 0.03115946799516678,
"fcm_dpo/delta": 0.11264770478010178,
"fcm_dpo/margin": 22.24788475036621,
"fcm_dpo/q_t": 0.3732909858226776,
"grad_norm": 24.012834548950195,
"learning_rate": 5.594234322453539e-11,
"logits/chosen": 0.5549105405807495,
"logits/rejected": 0.5142882466316223,
"logps/chosen": -109.44837951660156,
"logps/ref_chosen": -63.04634094238281,
"logps/ref_rejected": -83.44963073730469,
"logps/rejected": -152.09954833984375,
"loss": 1.1425,
"margin_dpo/margin_mean": 22.24788475036621,
"margin_dpo/margin_std": 41.10093688964844,
"step": 658
},
{
"KL/chosen_KL_mean": -46.43489456176758,
"KL/mean": -56.98037338256836,
"KL/rejected_KL_mean": -67.52584838867188,
"KL/std": 34.07621765136719,
"epoch": 0.9962207105064248,
"fcm_dpo/beta": 0.03144437074661255,
"fcm_dpo/delta": 0.14566412568092346,
"fcm_dpo/margin": 21.09095573425293,
"fcm_dpo/q_t": 0.3753587007522583,
"grad_norm": 21.728607177734375,
"learning_rate": 3.146808153123293e-11,
"logits/chosen": 0.6010191440582275,
"logits/rejected": 0.5353198051452637,
"logps/chosen": -101.51509094238281,
"logps/ref_chosen": -55.0802001953125,
"logps/ref_rejected": -71.91049194335938,
"logps/rejected": -139.43634033203125,
"loss": 1.1074,
"margin_dpo/margin_mean": 21.09095573425293,
"margin_dpo/margin_std": 36.32421112060547,
"step": 659
},
{
"KL/chosen_KL_mean": -45.47954559326172,
"KL/mean": -59.227901458740234,
"KL/rejected_KL_mean": -72.97624969482422,
"KL/std": 34.6148796081543,
"epoch": 0.9977324263038548,
"fcm_dpo/beta": 0.031728580594062805,
"fcm_dpo/delta": -0.0792866051197052,
"fcm_dpo/margin": 27.49671173095703,
"fcm_dpo/q_t": 0.32858672738075256,
"grad_norm": 19.706825256347656,
"learning_rate": 1.3985977021235829e-11,
"logits/chosen": 0.6279199123382568,
"logits/rejected": 0.5537869930267334,
"logps/chosen": -100.00546264648438,
"logps/ref_chosen": -54.525917053222656,
"logps/ref_rejected": -81.23604583740234,
"logps/rejected": -154.21229553222656,
"loss": 0.8971,
"margin_dpo/margin_mean": 27.49671173095703,
"margin_dpo/margin_std": 32.02090835571289,
"step": 660
},
{
"KL/chosen_KL_mean": -51.72168731689453,
"KL/mean": -62.73213195800781,
"KL/rejected_KL_mean": -73.74258422851562,
"KL/std": 36.27117919921875,
"epoch": 0.999244142101285,
"fcm_dpo/beta": 0.032111577689647675,
"fcm_dpo/delta": 0.09923127293586731,
"fcm_dpo/margin": 22.020889282226562,
"fcm_dpo/q_t": 0.365009069442749,
"grad_norm": 22.926618576049805,
"learning_rate": 3.4965187065971735e-12,
"logits/chosen": 0.49313268065452576,
"logits/rejected": 0.40937820076942444,
"logps/chosen": -112.09432983398438,
"logps/ref_chosen": -60.37263870239258,
"logps/ref_rejected": -77.42874145507812,
"logps/rejected": -151.17132568359375,
"loss": 1.0897,
"margin_dpo/margin_mean": 22.020889282226562,
"margin_dpo/margin_std": 37.274497985839844,
"step": 661
},
{
"epoch": 0.999244142101285,
"step": 661,
"total_flos": 0.0,
"train_loss": 1.068178587349388,
"train_runtime": 1640.4212,
"train_samples_per_second": 25.808,
"train_steps_per_second": 0.403
}
],
"logging_steps": 1,
"max_steps": 661,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink