W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
0122555ec684c9712f07cfb377d405b58be4fa40
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC 0122555ec6 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8
Source: Original Platform
2026-06-01 07:43:16 +08:00

15247 lines
564 KiB
JSON
Raw Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.999244142101285,
"eval_steps": 200,
"global_step": 661,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.02867889404296875,
"KL/mean": 0.029354453086853027,
"KL/rejected_KL_mean": 0.030029296875,
"KL/std": 0.2071000635623932,
"epoch": 0.0015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0013532638549804688,
"fcm_dpo/q_t": 0.5000336766242981,
"grad_norm": 28.2196044921875,
"learning_rate": 0.0,
"logits/chosen": 0.13337239623069763,
"logits/rejected": 0.12492949515581131,
"logps/chosen": -64.5841293334961,
"logps/ref_chosen": -64.61280822753906,
"logps/ref_rejected": -64.17195129394531,
"logps/rejected": -64.14192199707031,
"loss": 1.3866,
"margin_dpo/margin_mean": -0.0013527870178222656,
"margin_dpo/margin_std": 0.2561596930027008,
"step": 1
},
{
"KL/chosen_KL_mean": -0.00289154052734375,
"KL/mean": -0.021616414189338684,
"KL/rejected_KL_mean": -0.04033660888671875,
"KL/std": 0.19624735414981842,
"epoch": 0.0030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.037450045347213745,
"fcm_dpo/q_t": 0.4990639090538025,
"grad_norm": 27.82891273498535,
"learning_rate": 7.462686567164179e-09,
"logits/chosen": 0.09414851665496826,
"logits/rejected": 0.07363267242908478,
"logps/chosen": -56.101890563964844,
"logps/ref_chosen": -56.0989990234375,
"logps/ref_rejected": -66.59971618652344,
"logps/rejected": -66.64006042480469,
"loss": 1.3828,
"margin_dpo/margin_mean": 0.03744968771934509,
"margin_dpo/margin_std": 0.27811938524246216,
"step": 2
},
{
"KL/chosen_KL_mean": 0.0621337890625,
"KL/mean": 0.06413677334785461,
"KL/rejected_KL_mean": 0.06613540649414062,
"KL/std": 0.22972196340560913,
"epoch": 0.0045351473922902496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0040013790130615234,
"fcm_dpo/q_t": 0.5000998377799988,
"grad_norm": 31.116546630859375,
"learning_rate": 1.4925373134328357e-08,
"logits/chosen": 0.09941669553518295,
"logits/rejected": 0.06143157184123993,
"logps/chosen": -65.39512634277344,
"logps/ref_chosen": -65.45726013183594,
"logps/ref_rejected": -90.82853698730469,
"logps/rejected": -90.76239776611328,
"loss": 1.3869,
"margin_dpo/margin_mean": -0.00400128960609436,
"margin_dpo/margin_std": 0.290522038936615,
"step": 3
},
{
"KL/chosen_KL_mean": 0.000263214111328125,
"KL/mean": -0.023683160543441772,
"KL/rejected_KL_mean": -0.047637939453125,
"KL/std": 0.24327343702316284,
"epoch": 0.006046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04789847135543823,
"fcm_dpo/q_t": 0.49880456924438477,
"grad_norm": 34.16933059692383,
"learning_rate": 2.2388059701492534e-08,
"logits/chosen": 0.11254644393920898,
"logits/rejected": 0.0962742418050766,
"logps/chosen": -76.85992431640625,
"logps/ref_chosen": -76.86018371582031,
"logps/ref_rejected": -79.91523742675781,
"logps/rejected": -79.96287536621094,
"loss": 1.3818,
"margin_dpo/margin_mean": 0.04789838194847107,
"margin_dpo/margin_std": 0.3381476402282715,
"step": 4
},
{
"KL/chosen_KL_mean": -0.027805328369140625,
"KL/mean": -0.015260711312294006,
"KL/rejected_KL_mean": -0.002712249755859375,
"KL/std": 0.18707606196403503,
"epoch": 0.007558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.025092214345932007,
"fcm_dpo/q_t": 0.5006270408630371,
"grad_norm": 29.659147262573242,
"learning_rate": 2.9850746268656714e-08,
"logits/chosen": 0.07833173871040344,
"logits/rejected": 0.03964802622795105,
"logps/chosen": -62.9991455078125,
"logps/ref_chosen": -62.97134017944336,
"logps/ref_rejected": -79.9192123413086,
"logps/rejected": -79.92192077636719,
"loss": 1.3889,
"margin_dpo/margin_mean": -0.02509164810180664,
"margin_dpo/margin_std": 0.23738428950309753,
"step": 5
},
{
"KL/chosen_KL_mean": -0.0323944091796875,
"KL/mean": 0.007522225379943848,
"KL/rejected_KL_mean": 0.047443389892578125,
"KL/std": 0.25012582540512085,
"epoch": 0.009070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.07983356714248657,
"fcm_dpo/q_t": 0.501994788646698,
"grad_norm": 29.858251571655273,
"learning_rate": 3.731343283582089e-08,
"logits/chosen": 0.14125987887382507,
"logits/rejected": 0.10181364417076111,
"logps/chosen": -51.339759826660156,
"logps/ref_chosen": -51.30736541748047,
"logps/ref_rejected": -82.77239227294922,
"logps/rejected": -82.7249526977539,
"loss": 1.3945,
"margin_dpo/margin_mean": -0.07983347773551941,
"margin_dpo/margin_std": 0.3165475130081177,
"step": 6
},
{
"KL/chosen_KL_mean": 0.05200386047363281,
"KL/mean": 0.038868606090545654,
"KL/rejected_KL_mean": 0.025732040405273438,
"KL/std": 0.21536785364151,
"epoch": 0.010582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02626955509185791,
"fcm_dpo/q_t": 0.4993433356285095,
"grad_norm": 27.112102508544922,
"learning_rate": 4.477611940298507e-08,
"logits/chosen": 0.019042763859033585,
"logits/rejected": -0.02486884593963623,
"logps/chosen": -51.40740966796875,
"logps/ref_chosen": -51.45941162109375,
"logps/ref_rejected": -66.3828125,
"logps/rejected": -66.3570785522461,
"loss": 1.3839,
"margin_dpo/margin_mean": 0.026269584894180298,
"margin_dpo/margin_std": 0.3134131133556366,
"step": 7
},
{
"KL/chosen_KL_mean": -0.024341583251953125,
"KL/mean": -0.011224046349525452,
"KL/rejected_KL_mean": 0.001895904541015625,
"KL/std": 0.2582741677761078,
"epoch": 0.012093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.026233285665512085,
"fcm_dpo/q_t": 0.5006557703018188,
"grad_norm": 28.65567970275879,
"learning_rate": 5.223880597014925e-08,
"logits/chosen": 0.07061031460762024,
"logits/rejected": 0.04848004877567291,
"logps/chosen": -62.221885681152344,
"logps/ref_chosen": -62.197547912597656,
"logps/ref_rejected": -74.66180419921875,
"logps/rejected": -74.659912109375,
"loss": 1.3892,
"margin_dpo/margin_mean": -0.026233792304992676,
"margin_dpo/margin_std": 0.357799768447876,
"step": 8
},
{
"KL/chosen_KL_mean": -0.012987136840820312,
"KL/mean": -0.02591407299041748,
"KL/rejected_KL_mean": -0.038845062255859375,
"KL/std": 0.2304016649723053,
"epoch": 0.013605442176870748,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.025857150554656982,
"fcm_dpo/q_t": 0.4993537664413452,
"grad_norm": 31.47567367553711,
"learning_rate": 5.970149253731343e-08,
"logits/chosen": 0.15661674737930298,
"logits/rejected": 0.09843897819519043,
"logps/chosen": -55.64270782470703,
"logps/ref_chosen": -55.629722595214844,
"logps/ref_rejected": -86.21221923828125,
"logps/rejected": -86.25106811523438,
"loss": 1.3839,
"margin_dpo/margin_mean": 0.02585729956626892,
"margin_dpo/margin_std": 0.3030700385570526,
"step": 9
},
{
"KL/chosen_KL_mean": -0.0495147705078125,
"KL/mean": -0.027920737862586975,
"KL/rejected_KL_mean": -0.006328582763671875,
"KL/std": 0.25644826889038086,
"epoch": 0.015117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.043188005685806274,
"fcm_dpo/q_t": 0.5010795593261719,
"grad_norm": 29.739234924316406,
"learning_rate": 6.71641791044776e-08,
"logits/chosen": 0.12492163479328156,
"logits/rejected": 0.09430065751075745,
"logps/chosen": -62.740116119384766,
"logps/ref_chosen": -62.69060134887695,
"logps/ref_rejected": -90.610107421875,
"logps/rejected": -90.61643981933594,
"loss": 1.391,
"margin_dpo/margin_mean": -0.043188244104385376,
"margin_dpo/margin_std": 0.3699379861354828,
"step": 10
},
{
"KL/chosen_KL_mean": 0.019266128540039062,
"KL/mean": -0.0006564557552337646,
"KL/rejected_KL_mean": -0.0205841064453125,
"KL/std": 0.20153909921646118,
"epoch": 0.016628873771730914,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03984951972961426,
"fcm_dpo/q_t": 0.49900442361831665,
"grad_norm": 29.289079666137695,
"learning_rate": 7.462686567164178e-08,
"logits/chosen": 0.1184384673833847,
"logits/rejected": 0.11146491020917892,
"logps/chosen": -65.74784851074219,
"logps/ref_chosen": -65.76712036132812,
"logps/ref_rejected": -72.4764633178711,
"logps/rejected": -72.4970474243164,
"loss": 1.3826,
"margin_dpo/margin_mean": 0.039849698543548584,
"margin_dpo/margin_std": 0.301962286233902,
"step": 11
},
{
"KL/chosen_KL_mean": -0.03316497802734375,
"KL/mean": -0.036665767431259155,
"KL/rejected_KL_mean": -0.04016876220703125,
"KL/std": 0.2160276174545288,
"epoch": 0.018140589569160998,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.007006347179412842,
"fcm_dpo/q_t": 0.4998248219490051,
"grad_norm": 28.296424865722656,
"learning_rate": 8.208955223880596e-08,
"logits/chosen": 0.00660199299454689,
"logits/rejected": -0.009146112017333508,
"logps/chosen": -60.73805618286133,
"logps/ref_chosen": -60.704891204833984,
"logps/ref_rejected": -69.41564178466797,
"logps/rejected": -69.455810546875,
"loss": 1.3858,
"margin_dpo/margin_mean": 0.007006734609603882,
"margin_dpo/margin_std": 0.2850860357284546,
"step": 12
},
{
"KL/chosen_KL_mean": 0.041828155517578125,
"KL/mean": 0.04208873212337494,
"KL/rejected_KL_mean": 0.0423583984375,
"KL/std": 0.19688871502876282,
"epoch": 0.019652305366591082,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.000529557466506958,
"fcm_dpo/q_t": 0.5000130534172058,
"grad_norm": 29.095415115356445,
"learning_rate": 8.955223880597014e-08,
"logits/chosen": 0.10334185510873795,
"logits/rejected": 0.04153688997030258,
"logps/chosen": -49.867427825927734,
"logps/ref_chosen": -49.90925598144531,
"logps/ref_rejected": -92.37818145751953,
"logps/rejected": -92.33582305908203,
"loss": 1.3866,
"margin_dpo/margin_mean": -0.0005303919315338135,
"margin_dpo/margin_std": 0.302842915058136,
"step": 13
},
{
"KL/chosen_KL_mean": 0.022100448608398438,
"KL/mean": 0.00214308500289917,
"KL/rejected_KL_mean": -0.017810821533203125,
"KL/std": 0.17682762444019318,
"epoch": 0.021164021164021163,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03991013765335083,
"fcm_dpo/q_t": 0.4990024268627167,
"grad_norm": 29.589576721191406,
"learning_rate": 9.701492537313432e-08,
"logits/chosen": 0.0741623118519783,
"logits/rejected": 0.056773897260427475,
"logps/chosen": -60.59669494628906,
"logps/ref_chosen": -60.61879348754883,
"logps/ref_rejected": -71.79306030273438,
"logps/rejected": -71.81086730957031,
"loss": 1.3825,
"margin_dpo/margin_mean": 0.039910584688186646,
"margin_dpo/margin_std": 0.25130391120910645,
"step": 14
},
{
"KL/chosen_KL_mean": -0.03235435485839844,
"KL/mean": -0.033513978123664856,
"KL/rejected_KL_mean": -0.03466796875,
"KL/std": 0.23530861735343933,
"epoch": 0.022675736961451247,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0023103058338165283,
"fcm_dpo/q_t": 0.4999410808086395,
"grad_norm": 33.306461334228516,
"learning_rate": 1.044776119402985e-07,
"logits/chosen": 0.07059284299612045,
"logits/rejected": 0.027335654944181442,
"logps/chosen": -63.50189208984375,
"logps/ref_chosen": -63.46953582763672,
"logps/ref_rejected": -88.88951110839844,
"logps/rejected": -88.92417907714844,
"loss": 1.3864,
"margin_dpo/margin_mean": 0.0023106038570404053,
"margin_dpo/margin_std": 0.35110729932785034,
"step": 15
},
{
"KL/chosen_KL_mean": 0.0051555633544921875,
"KL/mean": -0.00991896539926529,
"KL/rejected_KL_mean": -0.024997711181640625,
"KL/std": 0.20466622710227966,
"epoch": 0.02418745275888133,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03015093505382538,
"fcm_dpo/q_t": 0.49924665689468384,
"grad_norm": 26.78266716003418,
"learning_rate": 1.1194029850746268e-07,
"logits/chosen": 0.11953230202198029,
"logits/rejected": 0.08214319497346878,
"logps/chosen": -46.52714538574219,
"logps/ref_chosen": -46.53229904174805,
"logps/ref_rejected": -74.27533721923828,
"logps/rejected": -74.30033874511719,
"loss": 1.3835,
"margin_dpo/margin_mean": 0.030150994658470154,
"margin_dpo/margin_std": 0.3233643174171448,
"step": 16
},
{
"KL/chosen_KL_mean": -0.03330421447753906,
"KL/mean": -0.026638299226760864,
"KL/rejected_KL_mean": -0.0199737548828125,
"KL/std": 0.21004769206047058,
"epoch": 0.025699168556311415,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.013328969478607178,
"fcm_dpo/q_t": 0.5003334283828735,
"grad_norm": 32.7893180847168,
"learning_rate": 1.1940298507462686e-07,
"logits/chosen": 0.05357828736305237,
"logits/rejected": 0.03485024347901344,
"logps/chosen": -64.11113739013672,
"logps/ref_chosen": -64.07783508300781,
"logps/ref_rejected": -86.40876770019531,
"logps/rejected": -86.42874145507812,
"loss": 1.3878,
"margin_dpo/margin_mean": -0.013329803943634033,
"margin_dpo/margin_std": 0.27140507102012634,
"step": 17
},
{
"KL/chosen_KL_mean": -0.001186370849609375,
"KL/mean": -0.005001351237297058,
"KL/rejected_KL_mean": -0.008813858032226562,
"KL/std": 0.21888390183448792,
"epoch": 0.027210884353741496,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.007630079984664917,
"fcm_dpo/q_t": 0.4998093545436859,
"grad_norm": 27.819944381713867,
"learning_rate": 1.2686567164179106e-07,
"logits/chosen": 0.0680946335196495,
"logits/rejected": 0.024525661021471024,
"logps/chosen": -44.87552261352539,
"logps/ref_chosen": -44.87433624267578,
"logps/ref_rejected": -70.97604370117188,
"logps/rejected": -70.98486328125,
"loss": 1.3858,
"margin_dpo/margin_mean": 0.007630020380020142,
"margin_dpo/margin_std": 0.3008061647415161,
"step": 18
},
{
"KL/chosen_KL_mean": 0.01468658447265625,
"KL/mean": -0.019955851137638092,
"KL/rejected_KL_mean": -0.054599761962890625,
"KL/std": 0.20606964826583862,
"epoch": 0.02872260015117158,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06928969919681549,
"fcm_dpo/q_t": 0.4982680082321167,
"grad_norm": 30.73091697692871,
"learning_rate": 1.343283582089552e-07,
"logits/chosen": 0.0740823969244957,
"logits/rejected": 0.06053391098976135,
"logps/chosen": -68.14512634277344,
"logps/ref_chosen": -68.1598129272461,
"logps/ref_rejected": -81.17138671875,
"logps/rejected": -81.22599029541016,
"loss": 1.3795,
"margin_dpo/margin_mean": 0.06928981840610504,
"margin_dpo/margin_std": 0.22947733104228973,
"step": 19
},
{
"KL/chosen_KL_mean": 0.0164337158203125,
"KL/mean": 0.01836688071489334,
"KL/rejected_KL_mean": 0.020305633544921875,
"KL/std": 0.2009357511997223,
"epoch": 0.030234315948601664,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.003872022032737732,
"fcm_dpo/q_t": 0.5000965595245361,
"grad_norm": 29.24262046813965,
"learning_rate": 1.4179104477611938e-07,
"logits/chosen": 0.14226043224334717,
"logits/rejected": 0.1189441829919815,
"logps/chosen": -53.66212844848633,
"logps/ref_chosen": -53.67856216430664,
"logps/ref_rejected": -74.16911315917969,
"logps/rejected": -74.14881134033203,
"loss": 1.3869,
"margin_dpo/margin_mean": -0.0038716942071914673,
"margin_dpo/margin_std": 0.2613745629787445,
"step": 20
},
{
"KL/chosen_KL_mean": -0.013164520263671875,
"KL/mean": -0.004536345601081848,
"KL/rejected_KL_mean": 0.00409698486328125,
"KL/std": 0.20376834273338318,
"epoch": 0.031746031746031744,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.017256349325180054,
"fcm_dpo/q_t": 0.5004312992095947,
"grad_norm": 29.48101806640625,
"learning_rate": 1.4925373134328355e-07,
"logits/chosen": 0.1120043694972992,
"logits/rejected": 0.08625894784927368,
"logps/chosen": -64.7147216796875,
"logps/ref_chosen": -64.70155334472656,
"logps/ref_rejected": -81.02095031738281,
"logps/rejected": -81.016845703125,
"loss": 1.3882,
"margin_dpo/margin_mean": -0.01725691556930542,
"margin_dpo/margin_std": 0.2752753496170044,
"step": 21
},
{
"KL/chosen_KL_mean": 0.0014190673828125,
"KL/mean": -0.024805426597595215,
"KL/rejected_KL_mean": -0.051029205322265625,
"KL/std": 0.2457861751317978,
"epoch": 0.03325774754346183,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05245363712310791,
"fcm_dpo/q_t": 0.49868905544281006,
"grad_norm": 28.892513275146484,
"learning_rate": 1.5671641791044775e-07,
"logits/chosen": -0.0012458395212888718,
"logits/rejected": -0.02185724303126335,
"logps/chosen": -58.03457260131836,
"logps/ref_chosen": -58.03599166870117,
"logps/ref_rejected": -80.72721862792969,
"logps/rejected": -80.77824401855469,
"loss": 1.3814,
"margin_dpo/margin_mean": 0.052453696727752686,
"margin_dpo/margin_std": 0.36326271295547485,
"step": 22
},
{
"KL/chosen_KL_mean": -0.005031585693359375,
"KL/mean": -0.004315197467803955,
"KL/rejected_KL_mean": -0.003604888916015625,
"KL/std": 0.22622814774513245,
"epoch": 0.03476946334089191,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0014243721961975098,
"fcm_dpo/q_t": 0.5000349879264832,
"grad_norm": 32.9621696472168,
"learning_rate": 1.6417910447761193e-07,
"logits/chosen": 0.14867977797985077,
"logits/rejected": 0.12283647060394287,
"logps/chosen": -66.36111450195312,
"logps/ref_chosen": -66.35608673095703,
"logps/ref_rejected": -93.02769470214844,
"logps/rejected": -93.03129577636719,
"loss": 1.3867,
"margin_dpo/margin_mean": -0.0014249980449676514,
"margin_dpo/margin_std": 0.3283410668373108,
"step": 23
},
{
"KL/chosen_KL_mean": -0.023939132690429688,
"KL/mean": -0.010058224201202393,
"KL/rejected_KL_mean": 0.0038280487060546875,
"KL/std": 0.2233589142560959,
"epoch": 0.036281179138321996,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.027758657932281494,
"fcm_dpo/q_t": 0.5006934404373169,
"grad_norm": 26.37616539001465,
"learning_rate": 1.716417910447761e-07,
"logits/chosen": 0.11849376559257507,
"logits/rejected": 0.08617529273033142,
"logps/chosen": -54.48517608642578,
"logps/ref_chosen": -54.461238861083984,
"logps/ref_rejected": -68.33817291259766,
"logps/rejected": -68.3343505859375,
"loss": 1.3893,
"margin_dpo/margin_mean": -0.027759432792663574,
"margin_dpo/margin_std": 0.3043414354324341,
"step": 24
},
{
"KL/chosen_KL_mean": -0.07821273803710938,
"KL/mean": -0.042458370327949524,
"KL/rejected_KL_mean": -0.00670623779296875,
"KL/std": 0.2250480055809021,
"epoch": 0.03779289493575208,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0715077817440033,
"fcm_dpo/q_t": 0.5017871856689453,
"grad_norm": 29.881423950195312,
"learning_rate": 1.7910447761194027e-07,
"logits/chosen": 0.10595692694187164,
"logits/rejected": 0.05439333990216255,
"logps/chosen": -60.08241653442383,
"logps/ref_chosen": -60.00420379638672,
"logps/ref_rejected": -90.47376251220703,
"logps/rejected": -90.48046875,
"loss": 1.3937,
"margin_dpo/margin_mean": -0.07150781154632568,
"margin_dpo/margin_std": 0.2951575517654419,
"step": 25
},
{
"KL/chosen_KL_mean": -0.025112152099609375,
"KL/mean": -0.030633173882961273,
"KL/rejected_KL_mean": -0.036151885986328125,
"KL/std": 0.2159086912870407,
"epoch": 0.039304610733182165,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.011040762066841125,
"fcm_dpo/q_t": 0.4997239112854004,
"grad_norm": 29.78719711303711,
"learning_rate": 1.8656716417910447e-07,
"logits/chosen": 0.10197645425796509,
"logits/rejected": 0.08369824290275574,
"logps/chosen": -56.844268798828125,
"logps/ref_chosen": -56.81915283203125,
"logps/ref_rejected": -77.84333038330078,
"logps/rejected": -77.87948608398438,
"loss": 1.3855,
"margin_dpo/margin_mean": 0.011040925979614258,
"margin_dpo/margin_std": 0.3176705837249756,
"step": 26
},
{
"KL/chosen_KL_mean": -0.040157318115234375,
"KL/mean": -0.01530282199382782,
"KL/rejected_KL_mean": 0.009548187255859375,
"KL/std": 0.24493414163589478,
"epoch": 0.04081632653061224,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04971262812614441,
"fcm_dpo/q_t": 0.5012425184249878,
"grad_norm": 29.07723617553711,
"learning_rate": 1.9402985074626865e-07,
"logits/chosen": 0.0994294285774231,
"logits/rejected": 0.07478873431682587,
"logps/chosen": -62.91718292236328,
"logps/ref_chosen": -62.87702560424805,
"logps/ref_rejected": -71.34437561035156,
"logps/rejected": -71.33482360839844,
"loss": 1.3915,
"margin_dpo/margin_mean": -0.04971283674240112,
"margin_dpo/margin_std": 0.2961461544036865,
"step": 27
},
{
"KL/chosen_KL_mean": -0.025875091552734375,
"KL/mean": -0.01864849030971527,
"KL/rejected_KL_mean": -0.011423110961914062,
"KL/std": 0.2145068347454071,
"epoch": 0.042328042328042326,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.014449745416641235,
"fcm_dpo/q_t": 0.500361442565918,
"grad_norm": 27.64870834350586,
"learning_rate": 2.0149253731343282e-07,
"logits/chosen": 0.061025142669677734,
"logits/rejected": 0.05223534256219864,
"logps/chosen": -59.8592529296875,
"logps/ref_chosen": -59.8333740234375,
"logps/ref_rejected": -70.39804077148438,
"logps/rejected": -70.40946197509766,
"loss": 1.388,
"margin_dpo/margin_mean": -0.014449775218963623,
"margin_dpo/margin_std": 0.28998634219169617,
"step": 28
},
{
"KL/chosen_KL_mean": -0.011623382568359375,
"KL/mean": -0.024745166301727295,
"KL/rejected_KL_mean": -0.03786468505859375,
"KL/std": 0.22506870329380035,
"epoch": 0.04383975812547241,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02624613046646118,
"fcm_dpo/q_t": 0.49934351444244385,
"grad_norm": 32.635955810546875,
"learning_rate": 2.08955223880597e-07,
"logits/chosen": 0.1302889883518219,
"logits/rejected": 0.11286543309688568,
"logps/chosen": -74.13182067871094,
"logps/ref_chosen": -74.12020111083984,
"logps/ref_rejected": -83.33099365234375,
"logps/rejected": -83.36885070800781,
"loss": 1.3839,
"margin_dpo/margin_mean": 0.026246458292007446,
"margin_dpo/margin_std": 0.3253002464771271,
"step": 29
},
{
"KL/chosen_KL_mean": -0.0457916259765625,
"KL/mean": -0.03270387649536133,
"KL/rejected_KL_mean": -0.01961517333984375,
"KL/std": 0.2302623689174652,
"epoch": 0.045351473922902494,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.026176929473876953,
"fcm_dpo/q_t": 0.5006539225578308,
"grad_norm": 30.555559158325195,
"learning_rate": 2.1641791044776117e-07,
"logits/chosen": 0.12713855504989624,
"logits/rejected": 0.07250553369522095,
"logps/chosen": -50.797080993652344,
"logps/ref_chosen": -50.75128936767578,
"logps/ref_rejected": -89.29063415527344,
"logps/rejected": -89.31024932861328,
"loss": 1.3891,
"margin_dpo/margin_mean": -0.026176661252975464,
"margin_dpo/margin_std": 0.2921898066997528,
"step": 30
},
{
"KL/chosen_KL_mean": -0.028470993041992188,
"KL/mean": -0.032488495111465454,
"KL/rejected_KL_mean": -0.03650665283203125,
"KL/std": 0.22643673419952393,
"epoch": 0.04686318972033258,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.00803285837173462,
"fcm_dpo/q_t": 0.49979931116104126,
"grad_norm": 34.28264617919922,
"learning_rate": 2.2388059701492537e-07,
"logits/chosen": 0.09353435784578323,
"logits/rejected": 0.04833420366048813,
"logps/chosen": -65.36522674560547,
"logps/ref_chosen": -65.33675384521484,
"logps/ref_rejected": -100.76666259765625,
"logps/rejected": -100.80316925048828,
"loss": 1.3857,
"margin_dpo/margin_mean": 0.008032828569412231,
"margin_dpo/margin_std": 0.31350964307785034,
"step": 31
},
{
"KL/chosen_KL_mean": 0.023860931396484375,
"KL/mean": -0.0067330896854400635,
"KL/rejected_KL_mean": -0.037326812744140625,
"KL/std": 0.2645628750324249,
"epoch": 0.04837490551776266,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.061192214488983154,
"fcm_dpo/q_t": 0.49847108125686646,
"grad_norm": 29.951520919799805,
"learning_rate": 2.3134328358208954e-07,
"logits/chosen": 0.06752718985080719,
"logits/rejected": 0.059862636029720306,
"logps/chosen": -67.15946960449219,
"logps/ref_chosen": -67.18333435058594,
"logps/ref_rejected": -82.80763244628906,
"logps/rejected": -82.84496307373047,
"loss": 1.3805,
"margin_dpo/margin_mean": 0.06119275093078613,
"margin_dpo/margin_std": 0.3593711256980896,
"step": 32
},
{
"KL/chosen_KL_mean": -0.0693359375,
"KL/mean": -0.0568530336022377,
"KL/rejected_KL_mean": -0.04436492919921875,
"KL/std": 0.23845943808555603,
"epoch": 0.049886621315192746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0249737948179245,
"fcm_dpo/q_t": 0.5006238222122192,
"grad_norm": 31.129165649414062,
"learning_rate": 2.388059701492537e-07,
"logits/chosen": 0.026731964200735092,
"logits/rejected": 0.0009631272405385971,
"logps/chosen": -64.10881805419922,
"logps/ref_chosen": -64.03948211669922,
"logps/ref_rejected": -75.68357849121094,
"logps/rejected": -75.72795104980469,
"loss": 1.3891,
"margin_dpo/margin_mean": -0.024973228573799133,
"margin_dpo/margin_std": 0.34394294023513794,
"step": 33
},
{
"KL/chosen_KL_mean": -0.00998687744140625,
"KL/mean": -0.04686702787876129,
"KL/rejected_KL_mean": -0.08374404907226562,
"KL/std": 0.2107549011707306,
"epoch": 0.05139833711262283,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07376006245613098,
"fcm_dpo/q_t": 0.4981565773487091,
"grad_norm": 28.369434356689453,
"learning_rate": 2.4626865671641786e-07,
"logits/chosen": 0.11266010254621506,
"logits/rejected": 0.08217991888523102,
"logps/chosen": -53.67428207397461,
"logps/ref_chosen": -53.6642951965332,
"logps/ref_rejected": -65.77989959716797,
"logps/rejected": -65.8636474609375,
"loss": 1.3792,
"margin_dpo/margin_mean": 0.07376009225845337,
"margin_dpo/margin_std": 0.28919702768325806,
"step": 34
},
{
"KL/chosen_KL_mean": -0.05603218078613281,
"KL/mean": -0.08405140042304993,
"KL/rejected_KL_mean": -0.11207199096679688,
"KL/std": 0.21035067737102509,
"epoch": 0.05291005291005291,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.056039512157440186,
"fcm_dpo/q_t": 0.4985993802547455,
"grad_norm": 27.643247604370117,
"learning_rate": 2.537313432835821e-07,
"logits/chosen": 0.04645001143217087,
"logits/rejected": 0.024315200746059418,
"logps/chosen": -61.07289505004883,
"logps/ref_chosen": -61.01686096191406,
"logps/ref_rejected": -72.78598022460938,
"logps/rejected": -72.89805603027344,
"loss": 1.3809,
"margin_dpo/margin_mean": 0.05603942275047302,
"margin_dpo/margin_std": 0.2715159058570862,
"step": 35
},
{
"KL/chosen_KL_mean": -0.06983184814453125,
"KL/mean": -0.08555865287780762,
"KL/rejected_KL_mean": -0.10128402709960938,
"KL/std": 0.23037990927696228,
"epoch": 0.05442176870748299,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03145325183868408,
"fcm_dpo/q_t": 0.49921417236328125,
"grad_norm": 28.695158004760742,
"learning_rate": 2.611940298507462e-07,
"logits/chosen": 0.09576475620269775,
"logits/rejected": 0.043359238654375076,
"logps/chosen": -50.60719299316406,
"logps/ref_chosen": -50.53736114501953,
"logps/ref_rejected": -78.11678314208984,
"logps/rejected": -78.21806335449219,
"loss": 1.3834,
"margin_dpo/margin_mean": 0.03145378828048706,
"margin_dpo/margin_std": 0.34016674757003784,
"step": 36
},
{
"KL/chosen_KL_mean": -0.032245635986328125,
"KL/mean": -0.10205593705177307,
"KL/rejected_KL_mean": -0.17187118530273438,
"KL/std": 0.2938510775566101,
"epoch": 0.055933484504913075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.13962048292160034,
"fcm_dpo/q_t": 0.4965134263038635,
"grad_norm": 37.48423767089844,
"learning_rate": 2.686567164179104e-07,
"logits/chosen": 0.10579690337181091,
"logits/rejected": 0.025799859315156937,
"logps/chosen": -59.5861930847168,
"logps/ref_chosen": -59.55394744873047,
"logps/ref_rejected": -108.27702331542969,
"logps/rejected": -108.44889831542969,
"loss": 1.3729,
"margin_dpo/margin_mean": 0.13962051272392273,
"margin_dpo/margin_std": 0.4410157799720764,
"step": 37
},
{
"KL/chosen_KL_mean": -0.0857086181640625,
"KL/mean": -0.11043363809585571,
"KL/rejected_KL_mean": -0.13515663146972656,
"KL/std": 0.25614839792251587,
"epoch": 0.05744520030234316,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04944509267807007,
"fcm_dpo/q_t": 0.4987648129463196,
"grad_norm": 29.67615509033203,
"learning_rate": 2.761194029850746e-07,
"logits/chosen": 0.08986853808164597,
"logits/rejected": 0.0756976380944252,
"logps/chosen": -65.87406921386719,
"logps/ref_chosen": -65.78836059570312,
"logps/ref_rejected": -76.1619873046875,
"logps/rejected": -76.2971420288086,
"loss": 1.3817,
"margin_dpo/margin_mean": 0.04944491386413574,
"margin_dpo/margin_std": 0.3537583351135254,
"step": 38
},
{
"KL/chosen_KL_mean": -0.10344505310058594,
"KL/mean": -0.12965010106563568,
"KL/rejected_KL_mean": -0.155853271484375,
"KL/std": 0.25784462690353394,
"epoch": 0.05895691609977324,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05241057276725769,
"fcm_dpo/q_t": 0.4986896514892578,
"grad_norm": 29.232746124267578,
"learning_rate": 2.8358208955223876e-07,
"logits/chosen": 0.1692015826702118,
"logits/rejected": 0.14214356243610382,
"logps/chosen": -57.28025436401367,
"logps/ref_chosen": -57.17681121826172,
"logps/ref_rejected": -79.486328125,
"logps/rejected": -79.64218139648438,
"loss": 1.3814,
"margin_dpo/margin_mean": 0.052410632371902466,
"margin_dpo/margin_std": 0.3558116555213928,
"step": 39
},
{
"KL/chosen_KL_mean": -0.11075210571289062,
"KL/mean": -0.1190851703286171,
"KL/rejected_KL_mean": -0.12742233276367188,
"KL/std": 0.24136286973953247,
"epoch": 0.06046863189720333,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.016666367650032043,
"fcm_dpo/q_t": 0.49958330392837524,
"grad_norm": 31.398401260375977,
"learning_rate": 2.9104477611940296e-07,
"logits/chosen": 0.1105370819568634,
"logits/rejected": 0.06097438558936119,
"logps/chosen": -61.44491958618164,
"logps/ref_chosen": -61.33416748046875,
"logps/ref_rejected": -79.10697174072266,
"logps/rejected": -79.23439025878906,
"loss": 1.3849,
"margin_dpo/margin_mean": 0.016666486859321594,
"margin_dpo/margin_std": 0.327433705329895,
"step": 40
},
{
"KL/chosen_KL_mean": -0.11747360229492188,
"KL/mean": -0.16613994538784027,
"KL/rejected_KL_mean": -0.21480178833007812,
"KL/std": 0.2806848883628845,
"epoch": 0.06198034769463341,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09732410311698914,
"fcm_dpo/q_t": 0.49756765365600586,
"grad_norm": 30.464834213256836,
"learning_rate": 2.985074626865671e-07,
"logits/chosen": 0.06427644938230515,
"logits/rejected": 0.04391469433903694,
"logps/chosen": -67.66419982910156,
"logps/ref_chosen": -67.5467300415039,
"logps/ref_rejected": -83.87788391113281,
"logps/rejected": -84.09268951416016,
"loss": 1.377,
"margin_dpo/margin_mean": 0.09732472896575928,
"margin_dpo/margin_std": 0.3751354217529297,
"step": 41
},
{
"KL/chosen_KL_mean": -0.10732078552246094,
"KL/mean": -0.1636602282524109,
"KL/rejected_KL_mean": -0.22000503540039062,
"KL/std": 0.3067892789840698,
"epoch": 0.06349206349206349,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11268658936023712,
"fcm_dpo/q_t": 0.4971844553947449,
"grad_norm": 29.092308044433594,
"learning_rate": 3.059701492537313e-07,
"logits/chosen": 0.05194007605314255,
"logits/rejected": 0.0304647795855999,
"logps/chosen": -61.37217712402344,
"logps/ref_chosen": -61.26485824584961,
"logps/ref_rejected": -76.3629150390625,
"logps/rejected": -76.58291625976562,
"loss": 1.3755,
"margin_dpo/margin_mean": 0.1126861423254013,
"margin_dpo/margin_std": 0.4176199436187744,
"step": 42
},
{
"KL/chosen_KL_mean": -0.09345245361328125,
"KL/mean": -0.111243337392807,
"KL/rejected_KL_mean": -0.12903594970703125,
"KL/std": 0.28760355710983276,
"epoch": 0.06500377928949358,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.035582125186920166,
"fcm_dpo/q_t": 0.49911123514175415,
"grad_norm": 34.529136657714844,
"learning_rate": 3.134328358208955e-07,
"logits/chosen": 0.08271262049674988,
"logits/rejected": 0.07184702157974243,
"logps/chosen": -71.90248107910156,
"logps/ref_chosen": -71.80902862548828,
"logps/ref_rejected": -81.12464141845703,
"logps/rejected": -81.25367736816406,
"loss": 1.3832,
"margin_dpo/margin_mean": 0.035581886768341064,
"margin_dpo/margin_std": 0.3950834274291992,
"step": 43
},
{
"KL/chosen_KL_mean": -0.16973114013671875,
"KL/mean": -0.1733190417289734,
"KL/rejected_KL_mean": -0.176910400390625,
"KL/std": 0.3061409592628479,
"epoch": 0.06651549508692366,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.007180750370025635,
"fcm_dpo/q_t": 0.4998205602169037,
"grad_norm": 32.519447326660156,
"learning_rate": 3.2089552238805965e-07,
"logits/chosen": 0.04154222086071968,
"logits/rejected": 0.011357433162629604,
"logps/chosen": -66.72016143798828,
"logps/ref_chosen": -66.55043029785156,
"logps/ref_rejected": -85.06198120117188,
"logps/rejected": -85.2388916015625,
"loss": 1.3861,
"margin_dpo/margin_mean": 0.0071804821491241455,
"margin_dpo/margin_std": 0.45991164445877075,
"step": 44
},
{
"KL/chosen_KL_mean": -0.13983726501464844,
"KL/mean": -0.17917409539222717,
"KL/rejected_KL_mean": -0.21851348876953125,
"KL/std": 0.2951315641403198,
"epoch": 0.06802721088435375,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0786718875169754,
"fcm_dpo/q_t": 0.49803340435028076,
"grad_norm": 31.8658504486084,
"learning_rate": 3.2835820895522385e-07,
"logits/chosen": 0.13182783126831055,
"logits/rejected": 0.07822854816913605,
"logps/chosen": -62.383689880371094,
"logps/ref_chosen": -62.24385452270508,
"logps/ref_rejected": -92.96665954589844,
"logps/rejected": -93.1851806640625,
"loss": 1.3789,
"margin_dpo/margin_mean": 0.07867246866226196,
"margin_dpo/margin_std": 0.4071173369884491,
"step": 45
},
{
"KL/chosen_KL_mean": -0.11602401733398438,
"KL/mean": -0.19016796350479126,
"KL/rejected_KL_mean": -0.2643165588378906,
"KL/std": 0.31932127475738525,
"epoch": 0.06953892668178382,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.14828699827194214,
"fcm_dpo/q_t": 0.4962952733039856,
"grad_norm": 31.104398727416992,
"learning_rate": 3.3582089552238805e-07,
"logits/chosen": 0.13636288046836853,
"logits/rejected": 0.08986735343933105,
"logps/chosen": -61.61492919921875,
"logps/ref_chosen": -61.498905181884766,
"logps/ref_rejected": -78.91172790527344,
"logps/rejected": -79.17604064941406,
"loss": 1.372,
"margin_dpo/margin_mean": 0.14828673005104065,
"margin_dpo/margin_std": 0.4339235723018646,
"step": 46
},
{
"KL/chosen_KL_mean": -0.14241790771484375,
"KL/mean": -0.20898690819740295,
"KL/rejected_KL_mean": -0.2755584716796875,
"KL/std": 0.2778022289276123,
"epoch": 0.0710506424792139,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.13313710689544678,
"fcm_dpo/q_t": 0.49667197465896606,
"grad_norm": 28.65785789489746,
"learning_rate": 3.432835820895522e-07,
"logits/chosen": 0.05303880572319031,
"logits/rejected": 0.009075134992599487,
"logps/chosen": -51.72076416015625,
"logps/ref_chosen": -51.578346252441406,
"logps/ref_rejected": -68.2215576171875,
"logps/rejected": -68.49711608886719,
"loss": 1.3734,
"margin_dpo/margin_mean": 0.13313719630241394,
"margin_dpo/margin_std": 0.38025137782096863,
"step": 47
},
{
"KL/chosen_KL_mean": -0.2553062438964844,
"KL/mean": -0.2546156048774719,
"KL/rejected_KL_mean": -0.2539253234863281,
"KL/std": 0.34144163131713867,
"epoch": 0.07256235827664399,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0013832151889801025,
"fcm_dpo/q_t": 0.5000360608100891,
"grad_norm": 26.89451789855957,
"learning_rate": 3.507462686567164e-07,
"logits/chosen": 0.16787520051002502,
"logits/rejected": 0.1375715285539627,
"logps/chosen": -52.04895782470703,
"logps/ref_chosen": -51.79365158081055,
"logps/ref_rejected": -64.22503662109375,
"logps/rejected": -64.47896575927734,
"loss": 1.3871,
"margin_dpo/margin_mean": -0.0013829171657562256,
"margin_dpo/margin_std": 0.4911916255950928,
"step": 48
},
{
"KL/chosen_KL_mean": -0.20473098754882812,
"KL/mean": -0.23833119869232178,
"KL/rejected_KL_mean": -0.2719306945800781,
"KL/std": 0.333074688911438,
"epoch": 0.07407407407407407,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06719930469989777,
"fcm_dpo/q_t": 0.4983193874359131,
"grad_norm": 27.35091781616211,
"learning_rate": 3.5820895522388055e-07,
"logits/chosen": 0.03351443260908127,
"logits/rejected": 0.011932183057069778,
"logps/chosen": -58.339332580566406,
"logps/ref_chosen": -58.13460159301758,
"logps/ref_rejected": -64.63206481933594,
"logps/rejected": -64.90399932861328,
"loss": 1.3801,
"margin_dpo/margin_mean": 0.06719934940338135,
"margin_dpo/margin_std": 0.4344923794269562,
"step": 49
},
{
"KL/chosen_KL_mean": -0.25493812561035156,
"KL/mean": -0.3313453197479248,
"KL/rejected_KL_mean": -0.40775299072265625,
"KL/std": 0.3598587214946747,
"epoch": 0.07558578987150416,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15281718969345093,
"fcm_dpo/q_t": 0.4961821436882019,
"grad_norm": 27.900310516357422,
"learning_rate": 3.6567164179104475e-07,
"logits/chosen": 0.11298177391290665,
"logits/rejected": 0.08316892385482788,
"logps/chosen": -53.11137771606445,
"logps/ref_chosen": -52.85643768310547,
"logps/ref_rejected": -72.17460632324219,
"logps/rejected": -72.58235168457031,
"loss": 1.3715,
"margin_dpo/margin_mean": 0.15281730890274048,
"margin_dpo/margin_std": 0.40361297130584717,
"step": 50
},
{
"KL/chosen_KL_mean": -0.3104381561279297,
"KL/mean": -0.41523122787475586,
"KL/rejected_KL_mean": -0.5200233459472656,
"KL/std": 0.4115421772003174,
"epoch": 0.07709750566893424,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20958328247070312,
"fcm_dpo/q_t": 0.49476635456085205,
"grad_norm": 30.999216079711914,
"learning_rate": 3.7313432835820895e-07,
"logits/chosen": 0.07924918085336685,
"logits/rejected": 0.05170644819736481,
"logps/chosen": -63.966880798339844,
"logps/ref_chosen": -63.65644073486328,
"logps/ref_rejected": -86.13229370117188,
"logps/rejected": -86.6523208618164,
"loss": 1.3661,
"margin_dpo/margin_mean": 0.209583580493927,
"margin_dpo/margin_std": 0.49226850271224976,
"step": 51
},
{
"KL/chosen_KL_mean": -0.32818603515625,
"KL/mean": -0.4305916428565979,
"KL/rejected_KL_mean": -0.5329971313476562,
"KL/std": 0.40797537565231323,
"epoch": 0.07860922146636433,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2048064023256302,
"fcm_dpo/q_t": 0.4948863089084625,
"grad_norm": 32.73704528808594,
"learning_rate": 3.805970149253731e-07,
"logits/chosen": 0.07293161749839783,
"logits/rejected": 0.024682261049747467,
"logps/chosen": -68.16839599609375,
"logps/ref_chosen": -67.8402099609375,
"logps/ref_rejected": -96.97090911865234,
"logps/rejected": -97.50390625,
"loss": 1.3666,
"margin_dpo/margin_mean": 0.2048058956861496,
"margin_dpo/margin_std": 0.5083526372909546,
"step": 52
},
{
"KL/chosen_KL_mean": -0.34824371337890625,
"KL/mean": -0.43194323778152466,
"KL/rejected_KL_mean": -0.5156440734863281,
"KL/std": 0.3921298384666443,
"epoch": 0.0801209372637944,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1674003303050995,
"fcm_dpo/q_t": 0.4958154559135437,
"grad_norm": 27.480405807495117,
"learning_rate": 3.880597014925373e-07,
"logits/chosen": 0.07155004888772964,
"logits/rejected": 0.06118874251842499,
"logps/chosen": -57.226383209228516,
"logps/ref_chosen": -56.87813949584961,
"logps/ref_rejected": -60.75569152832031,
"logps/rejected": -61.271339416503906,
"loss": 1.3702,
"margin_dpo/margin_mean": 0.1674005091190338,
"margin_dpo/margin_std": 0.4693652391433716,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3873310089111328,
"KL/mean": -0.46201127767562866,
"KL/rejected_KL_mean": -0.5366916656494141,
"KL/std": 0.39615002274513245,
"epoch": 0.08163265306122448,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.14936020970344543,
"fcm_dpo/q_t": 0.4962688088417053,
"grad_norm": 26.520944595336914,
"learning_rate": 3.9552238805970144e-07,
"logits/chosen": 0.03022252582013607,
"logits/rejected": 0.015316802076995373,
"logps/chosen": -47.65425109863281,
"logps/ref_chosen": -47.26692199707031,
"logps/ref_rejected": -62.19426727294922,
"logps/rejected": -62.73095703125,
"loss": 1.3721,
"margin_dpo/margin_mean": 0.14936044812202454,
"margin_dpo/margin_std": 0.5132778286933899,
"step": 54
},
{
"KL/chosen_KL_mean": -0.3409919738769531,
"KL/mean": -0.4967942535877228,
"KL/rejected_KL_mean": -0.6526031494140625,
"KL/std": 0.48860490322113037,
"epoch": 0.08314436885865457,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3116055130958557,
"fcm_dpo/q_t": 0.49222302436828613,
"grad_norm": 31.283021926879883,
"learning_rate": 4.0298507462686564e-07,
"logits/chosen": 0.04874131828546524,
"logits/rejected": -0.029031695798039436,
"logps/chosen": -50.66718292236328,
"logps/ref_chosen": -50.32619094848633,
"logps/ref_rejected": -92.44389343261719,
"logps/rejected": -93.09649658203125,
"loss": 1.3565,
"margin_dpo/margin_mean": 0.31160497665405273,
"margin_dpo/margin_std": 0.6381343603134155,
"step": 55
},
{
"KL/chosen_KL_mean": -0.34792327880859375,
"KL/mean": -0.44482094049453735,
"KL/rejected_KL_mean": -0.5417137145996094,
"KL/std": 0.4808979034423828,
"epoch": 0.08465608465608465,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19379210472106934,
"fcm_dpo/q_t": 0.49515196681022644,
"grad_norm": 27.554279327392578,
"learning_rate": 4.1044776119402984e-07,
"logits/chosen": 0.13023659586906433,
"logits/rejected": 0.10778436064720154,
"logps/chosen": -57.11489486694336,
"logps/ref_chosen": -56.766971588134766,
"logps/ref_rejected": -66.30504608154297,
"logps/rejected": -66.84675598144531,
"loss": 1.3682,
"margin_dpo/margin_mean": 0.19379198551177979,
"margin_dpo/margin_std": 0.671954333782196,
"step": 56
},
{
"KL/chosen_KL_mean": -0.5123443603515625,
"KL/mean": -0.6745944023132324,
"KL/rejected_KL_mean": -0.8368415832519531,
"KL/std": 0.6111703515052795,
"epoch": 0.08616780045351474,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3244953751564026,
"fcm_dpo/q_t": 0.49190065264701843,
"grad_norm": 30.53541374206543,
"learning_rate": 4.17910447761194e-07,
"logits/chosen": 0.12122000753879547,
"logits/rejected": 0.05556311830878258,
"logps/chosen": -58.28009033203125,
"logps/ref_chosen": -57.76774597167969,
"logps/ref_rejected": -82.75698852539062,
"logps/rejected": -83.59382629394531,
"loss": 1.3553,
"margin_dpo/margin_mean": 0.3244949281215668,
"margin_dpo/margin_std": 0.6604194641113281,
"step": 57
},
{
"KL/chosen_KL_mean": -0.5639533996582031,
"KL/mean": -0.6952039003372192,
"KL/rejected_KL_mean": -0.8264541625976562,
"KL/std": 0.6729289889335632,
"epoch": 0.08767951625094482,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.26249879598617554,
"fcm_dpo/q_t": 0.4935028851032257,
"grad_norm": 30.190555572509766,
"learning_rate": 4.253731343283582e-07,
"logits/chosen": 0.05941343307495117,
"logits/rejected": 0.04385446757078171,
"logps/chosen": -73.32804107666016,
"logps/ref_chosen": -72.76408386230469,
"logps/ref_rejected": -84.49275207519531,
"logps/rejected": -85.31920623779297,
"loss": 1.363,
"margin_dpo/margin_mean": 0.2624986469745636,
"margin_dpo/margin_std": 0.9827470183372498,
"step": 58
},
{
"KL/chosen_KL_mean": -0.5376720428466797,
"KL/mean": -0.6911669969558716,
"KL/rejected_KL_mean": -0.8446674346923828,
"KL/std": 0.6432867050170898,
"epoch": 0.08919123204837491,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3069899380207062,
"fcm_dpo/q_t": 0.4923567771911621,
"grad_norm": 26.56601905822754,
"learning_rate": 4.3283582089552234e-07,
"logits/chosen": 0.12795722484588623,
"logits/rejected": 0.06226852163672447,
"logps/chosen": -50.35844802856445,
"logps/ref_chosen": -49.820777893066406,
"logps/ref_rejected": -77.14368438720703,
"logps/rejected": -77.98834991455078,
"loss": 1.3577,
"margin_dpo/margin_mean": 0.3069910705089569,
"margin_dpo/margin_std": 0.8052564859390259,
"step": 59
},
{
"KL/chosen_KL_mean": -0.6147499084472656,
"KL/mean": -0.6438243389129639,
"KL/rejected_KL_mean": -0.6729011535644531,
"KL/std": 0.6342208981513977,
"epoch": 0.09070294784580499,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05814960598945618,
"fcm_dpo/q_t": 0.498546302318573,
"grad_norm": 29.609012603759766,
"learning_rate": 4.4029850746268654e-07,
"logits/chosen": 0.13567467033863068,
"logits/rejected": 0.13422152400016785,
"logps/chosen": -63.83952331542969,
"logps/ref_chosen": -63.22477340698242,
"logps/ref_rejected": -61.360477447509766,
"logps/rejected": -62.03337860107422,
"loss": 1.3822,
"margin_dpo/margin_mean": 0.05814942717552185,
"margin_dpo/margin_std": 0.809841513633728,
"step": 60
},
{
"KL/chosen_KL_mean": -0.7202339172363281,
"KL/mean": -0.7759703397750854,
"KL/rejected_KL_mean": -0.8317070007324219,
"KL/std": 0.7063132524490356,
"epoch": 0.09221466364323508,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11147007346153259,
"fcm_dpo/q_t": 0.49721381068229675,
"grad_norm": 27.97510528564453,
"learning_rate": 4.4776119402985074e-07,
"logits/chosen": 0.14454835653305054,
"logits/rejected": 0.11166666448116302,
"logps/chosen": -49.737030029296875,
"logps/ref_chosen": -49.01679992675781,
"logps/ref_rejected": -74.90817260742188,
"logps/rejected": -75.73987579345703,
"loss": 1.3776,
"margin_dpo/margin_mean": 0.11146968603134155,
"margin_dpo/margin_std": 0.9815603494644165,
"step": 61
},
{
"KL/chosen_KL_mean": -0.7713375091552734,
"KL/mean": -0.8586764931678772,
"KL/rejected_KL_mean": -0.9460220336914062,
"KL/std": 0.6308267712593079,
"epoch": 0.09372637944066516,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1746814250946045,
"fcm_dpo/q_t": 0.4956405758857727,
"grad_norm": 28.683074951171875,
"learning_rate": 4.552238805970149e-07,
"logits/chosen": 0.09312943369150162,
"logits/rejected": 0.0548982173204422,
"logps/chosen": -63.52320861816406,
"logps/ref_chosen": -62.751869201660156,
"logps/ref_rejected": -78.93360900878906,
"logps/rejected": -79.87963104248047,
"loss": 1.3708,
"margin_dpo/margin_mean": 0.1746809184551239,
"margin_dpo/margin_std": 0.8726956248283386,
"step": 62
},
{
"KL/chosen_KL_mean": -0.5630893707275391,
"KL/mean": -0.8304413557052612,
"KL/rejected_KL_mean": -1.0978012084960938,
"KL/std": 0.6817377209663391,
"epoch": 0.09523809523809523,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5347055196762085,
"fcm_dpo/q_t": 0.48666688799858093,
"grad_norm": 31.4731502532959,
"learning_rate": 4.626865671641791e-07,
"logits/chosen": 0.1990186870098114,
"logits/rejected": 0.1737050712108612,
"logps/chosen": -61.078338623046875,
"logps/ref_chosen": -60.51525115966797,
"logps/ref_rejected": -85.11021423339844,
"logps/rejected": -86.20801544189453,
"loss": 1.3355,
"margin_dpo/margin_mean": 0.534705638885498,
"margin_dpo/margin_std": 0.8916192054748535,
"step": 63
},
{
"KL/chosen_KL_mean": -0.8345146179199219,
"KL/mean": -0.8636895418167114,
"KL/rejected_KL_mean": -0.8928680419921875,
"KL/std": 0.7093801498413086,
"epoch": 0.09674981103552532,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.058351099491119385,
"fcm_dpo/q_t": 0.4985370635986328,
"grad_norm": 26.187711715698242,
"learning_rate": 4.701492537313433e-07,
"logits/chosen": 0.07552579045295715,
"logits/rejected": 0.05123044550418854,
"logps/chosen": -52.041358947753906,
"logps/ref_chosen": -51.20684814453125,
"logps/ref_rejected": -66.93081665039062,
"logps/rejected": -67.82368469238281,
"loss": 1.3824,
"margin_dpo/margin_mean": 0.058351099491119385,
"margin_dpo/margin_std": 0.8890206813812256,
"step": 64
},
{
"KL/chosen_KL_mean": -0.8652439117431641,
"KL/mean": -1.0923452377319336,
"KL/rejected_KL_mean": -1.3194541931152344,
"KL/std": 0.8883547782897949,
"epoch": 0.0982615268329554,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4542124569416046,
"fcm_dpo/q_t": 0.48869743943214417,
"grad_norm": 30.891233444213867,
"learning_rate": 4.776119402985074e-07,
"logits/chosen": 0.1984306275844574,
"logits/rejected": 0.16880089044570923,
"logps/chosen": -68.15393829345703,
"logps/ref_chosen": -67.2886962890625,
"logps/ref_rejected": -74.44281005859375,
"logps/rejected": -75.76226043701172,
"loss": 1.3448,
"margin_dpo/margin_mean": 0.4542120397090912,
"margin_dpo/margin_std": 1.1676384210586548,
"step": 65
},
{
"KL/chosen_KL_mean": -0.9264984130859375,
"KL/mean": -1.099461555480957,
"KL/rejected_KL_mean": -1.2724266052246094,
"KL/std": 0.8448681831359863,
"epoch": 0.09977324263038549,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.34592562913894653,
"fcm_dpo/q_t": 0.4913817346096039,
"grad_norm": 29.38481330871582,
"learning_rate": 4.850746268656717e-07,
"logits/chosen": 0.11555850505828857,
"logits/rejected": 0.09126327931880951,
"logps/chosen": -71.66990661621094,
"logps/ref_chosen": -70.743408203125,
"logps/ref_rejected": -77.26499938964844,
"logps/rejected": -78.53742980957031,
"loss": 1.3552,
"margin_dpo/margin_mean": 0.34592553973197937,
"margin_dpo/margin_std": 1.115427017211914,
"step": 66
},
{
"KL/chosen_KL_mean": -0.7729225158691406,
"KL/mean": -0.9254127144813538,
"KL/rejected_KL_mean": -1.0778999328613281,
"KL/std": 0.7637161612510681,
"epoch": 0.10128495842781557,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.30497562885284424,
"fcm_dpo/q_t": 0.49239760637283325,
"grad_norm": 29.26283073425293,
"learning_rate": 4.925373134328357e-07,
"logits/chosen": 0.10749173164367676,
"logits/rejected": 0.05099410563707352,
"logps/chosen": -61.375526428222656,
"logps/ref_chosen": -60.60260009765625,
"logps/ref_rejected": -75.22235870361328,
"logps/rejected": -76.30026245117188,
"loss": 1.3582,
"margin_dpo/margin_mean": 0.3049759268760681,
"margin_dpo/margin_std": 0.9200187921524048,
"step": 67
},
{
"KL/chosen_KL_mean": -1.2506217956542969,
"KL/mean": -1.421552062034607,
"KL/rejected_KL_mean": -1.5924835205078125,
"KL/std": 0.9510532021522522,
"epoch": 0.10279667422524566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.34186407923698425,
"fcm_dpo/q_t": 0.49147123098373413,
"grad_norm": 31.39417266845703,
"learning_rate": 5e-07,
"logits/chosen": 0.06202474609017372,
"logits/rejected": 0.03223402798175812,
"logps/chosen": -78.77899169921875,
"logps/ref_chosen": -77.52836608886719,
"logps/ref_rejected": -93.17778015136719,
"logps/rejected": -94.770263671875,
"loss": 1.3563,
"margin_dpo/margin_mean": 0.34186458587646484,
"margin_dpo/margin_std": 1.2264301776885986,
"step": 68
},
{
"KL/chosen_KL_mean": -1.184225082397461,
"KL/mean": -1.4764790534973145,
"KL/rejected_KL_mean": -1.7687301635742188,
"KL/std": 1.024730920791626,
"epoch": 0.10430839002267574,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5845098495483398,
"fcm_dpo/q_t": 0.4854779541492462,
"grad_norm": 30.998098373413086,
"learning_rate": 4.999965034812934e-07,
"logits/chosen": 0.08792998641729355,
"logits/rejected": 0.045593686401844025,
"logps/chosen": -67.12727355957031,
"logps/ref_chosen": -65.94305419921875,
"logps/ref_rejected": -89.7735595703125,
"logps/rejected": -91.54228973388672,
"loss": 1.3326,
"margin_dpo/margin_mean": 0.5845092535018921,
"margin_dpo/margin_std": 1.2174975872039795,
"step": 69
},
{
"KL/chosen_KL_mean": -1.2389240264892578,
"KL/mean": -1.3959846496582031,
"KL/rejected_KL_mean": -1.5530376434326172,
"KL/std": 0.9079196453094482,
"epoch": 0.10582010582010581,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.31411120295524597,
"fcm_dpo/q_t": 0.49217674136161804,
"grad_norm": 28.801374435424805,
"learning_rate": 4.999860140229787e-07,
"logits/chosen": 0.16327008605003357,
"logits/rejected": 0.1396973729133606,
"logps/chosen": -63.19683837890625,
"logps/ref_chosen": -61.95791244506836,
"logps/ref_rejected": -75.80945587158203,
"logps/rejected": -77.36249542236328,
"loss": 1.3588,
"margin_dpo/margin_mean": 0.3141113817691803,
"margin_dpo/margin_std": 1.214660406112671,
"step": 70
},
{
"KL/chosen_KL_mean": -1.4482479095458984,
"KL/mean": -1.5614473819732666,
"KL/rejected_KL_mean": -1.6746501922607422,
"KL/std": 1.0249958038330078,
"epoch": 0.1073318216175359,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.22640283405780792,
"fcm_dpo/q_t": 0.49432241916656494,
"grad_norm": 28.454469680786133,
"learning_rate": 4.999685319184688e-07,
"logits/chosen": 0.11360330879688263,
"logits/rejected": 0.09736478328704834,
"logps/chosen": -64.79582214355469,
"logps/ref_chosen": -63.34757995605469,
"logps/ref_rejected": -67.49658203125,
"logps/rejected": -69.17123413085938,
"loss": 1.3693,
"margin_dpo/margin_mean": 0.22640304267406464,
"margin_dpo/margin_std": 1.474951982498169,
"step": 71
},
{
"KL/chosen_KL_mean": -1.1897659301757812,
"KL/mean": -1.553931713104248,
"KL/rejected_KL_mean": -1.9180984497070312,
"KL/std": 1.0742740631103516,
"epoch": 0.10884353741496598,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7283300161361694,
"fcm_dpo/q_t": 0.48187729716300964,
"grad_norm": 30.26369857788086,
"learning_rate": 4.999440576567755e-07,
"logits/chosen": 0.1204986423254013,
"logits/rejected": 0.05717724561691284,
"logps/chosen": -57.04906463623047,
"logps/ref_chosen": -55.85929870605469,
"logps/ref_rejected": -68.45423889160156,
"logps/rejected": -70.37234497070312,
"loss": 1.3191,
"margin_dpo/margin_mean": 0.7283297181129456,
"margin_dpo/margin_std": 1.2839291095733643,
"step": 72
},
{
"KL/chosen_KL_mean": -1.8362503051757812,
"KL/mean": -1.8976032733917236,
"KL/rejected_KL_mean": -1.9589519500732422,
"KL/std": 1.2357733249664307,
"epoch": 0.11035525321239607,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12270593643188477,
"fcm_dpo/q_t": 0.4970957636833191,
"grad_norm": 31.9867000579834,
"learning_rate": 4.999125919224965e-07,
"logits/chosen": 0.11336830258369446,
"logits/rejected": 0.09915591776371002,
"logps/chosen": -70.97505187988281,
"logps/ref_chosen": -69.13880920410156,
"logps/ref_rejected": -79.04586791992188,
"logps/rejected": -81.00482177734375,
"loss": 1.3809,
"margin_dpo/margin_mean": 0.12270653247833252,
"margin_dpo/margin_std": 1.5762428045272827,
"step": 73
},
{
"KL/chosen_KL_mean": -1.2705440521240234,
"KL/mean": -1.6108319759368896,
"KL/rejected_KL_mean": -1.9511184692382812,
"KL/std": 1.0245332717895508,
"epoch": 0.11186696900982615,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6805762648582458,
"fcm_dpo/q_t": 0.48308998346328735,
"grad_norm": 27.936552047729492,
"learning_rate": 4.998741355957963e-07,
"logits/chosen": 0.1540667563676834,
"logits/rejected": 0.09951502084732056,
"logps/chosen": -51.19428253173828,
"logps/ref_chosen": -49.923736572265625,
"logps/ref_rejected": -81.73213958740234,
"logps/rejected": -83.68325805664062,
"loss": 1.3243,
"margin_dpo/margin_mean": 0.6805763244628906,
"margin_dpo/margin_std": 1.3530315160751343,
"step": 74
},
{
"KL/chosen_KL_mean": -1.3861274719238281,
"KL/mean": -1.7660695314407349,
"KL/rejected_KL_mean": -2.146007537841797,
"KL/std": 1.1102235317230225,
"epoch": 0.11337868480725624,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7598824501037598,
"fcm_dpo/q_t": 0.4811212420463562,
"grad_norm": 26.1428165435791,
"learning_rate": 4.998286897523808e-07,
"logits/chosen": 0.12207367271184921,
"logits/rejected": 0.0893176943063736,
"logps/chosen": -47.45487976074219,
"logps/ref_chosen": -46.06875228881836,
"logps/ref_rejected": -66.1181411743164,
"logps/rejected": -68.26415252685547,
"loss": 1.3163,
"margin_dpo/margin_mean": 0.7598823308944702,
"margin_dpo/margin_std": 1.3477892875671387,
"step": 75
},
{
"KL/chosen_KL_mean": -1.5627460479736328,
"KL/mean": -1.734074354171753,
"KL/rejected_KL_mean": -1.9054031372070312,
"KL/std": 1.2022194862365723,
"epoch": 0.11489040060468632,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3426612615585327,
"fcm_dpo/q_t": 0.49147289991378784,
"grad_norm": 29.220705032348633,
"learning_rate": 4.997762556634679e-07,
"logits/chosen": 0.12868475914001465,
"logits/rejected": 0.08463678508996964,
"logps/chosen": -55.62549591064453,
"logps/ref_chosen": -54.06275177001953,
"logps/ref_rejected": -74.87464141845703,
"logps/rejected": -76.78004455566406,
"loss": 1.3586,
"margin_dpo/margin_mean": 0.3426612913608551,
"margin_dpo/margin_std": 1.5809069871902466,
"step": 76
},
{
"KL/chosen_KL_mean": -1.6425457000732422,
"KL/mean": -2.0247294902801514,
"KL/rejected_KL_mean": -2.406909942626953,
"KL/std": 1.2334535121917725,
"epoch": 0.1164021164021164,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7643673419952393,
"fcm_dpo/q_t": 0.4810134768486023,
"grad_norm": 29.239410400390625,
"learning_rate": 4.99716834795752e-07,
"logits/chosen": 0.17115390300750732,
"logits/rejected": 0.12893466651439667,
"logps/chosen": -54.71863555908203,
"logps/ref_chosen": -53.07609176635742,
"logps/ref_rejected": -74.45601654052734,
"logps/rejected": -76.86293029785156,
"loss": 1.3169,
"margin_dpo/margin_mean": 0.7643671631813049,
"margin_dpo/margin_std": 1.4656544923782349,
"step": 77
},
{
"KL/chosen_KL_mean": -1.8553199768066406,
"KL/mean": -2.048530340194702,
"KL/rejected_KL_mean": -2.2417449951171875,
"KL/std": 1.2667438983917236,
"epoch": 0.11791383219954649,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.38642293214797974,
"fcm_dpo/q_t": 0.4903751313686371,
"grad_norm": 29.433731079101562,
"learning_rate": 4.996504288113623e-07,
"logits/chosen": 0.11265414953231812,
"logits/rejected": 0.09231537580490112,
"logps/chosen": -69.58074188232422,
"logps/ref_chosen": -67.72541809082031,
"logps/ref_rejected": -79.03926849365234,
"logps/rejected": -81.28101348876953,
"loss": 1.3531,
"margin_dpo/margin_mean": 0.3864227533340454,
"margin_dpo/margin_std": 1.4329341650009155,
"step": 78
},
{
"KL/chosen_KL_mean": -1.7077217102050781,
"KL/mean": -2.3071465492248535,
"KL/rejected_KL_mean": -2.9065704345703125,
"KL/std": 1.5384122133255005,
"epoch": 0.11942554799697656,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1988508701324463,
"fcm_dpo/q_t": 0.47053754329681396,
"grad_norm": 30.798582077026367,
"learning_rate": 4.995770395678171e-07,
"logits/chosen": 0.18950387835502625,
"logits/rejected": 0.12797753512859344,
"logps/chosen": -53.86836624145508,
"logps/ref_chosen": -52.16064453125,
"logps/ref_rejected": -83.31062316894531,
"logps/rejected": -86.21719360351562,
"loss": 1.2795,
"margin_dpo/margin_mean": 1.198850393295288,
"margin_dpo/margin_std": 1.959941029548645,
"step": 79
},
{
"KL/chosen_KL_mean": -2.024747848510742,
"KL/mean": -2.3166556358337402,
"KL/rejected_KL_mean": -2.6085662841796875,
"KL/std": 1.369431495666504,
"epoch": 0.12093726379440665,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5838208198547363,
"fcm_dpo/q_t": 0.48558056354522705,
"grad_norm": 28.381168365478516,
"learning_rate": 4.994966691179711e-07,
"logits/chosen": 0.14689680933952332,
"logits/rejected": 0.08690305054187775,
"logps/chosen": -63.43531036376953,
"logps/ref_chosen": -61.410560607910156,
"logps/ref_rejected": -78.66004943847656,
"logps/rejected": -81.26861572265625,
"loss": 1.3383,
"margin_dpo/margin_mean": 0.5838209390640259,
"margin_dpo/margin_std": 1.8922240734100342,
"step": 80
},
{
"KL/chosen_KL_mean": -2.1597976684570312,
"KL/mean": -2.5934157371520996,
"KL/rejected_KL_mean": -3.0270309448242188,
"KL/std": 1.434091329574585,
"epoch": 0.12244897959183673,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8672326803207397,
"fcm_dpo/q_t": 0.47862696647644043,
"grad_norm": 29.007713317871094,
"learning_rate": 4.994093197099587e-07,
"logits/chosen": 0.13177847862243652,
"logits/rejected": 0.09794065356254578,
"logps/chosen": -65.96417236328125,
"logps/ref_chosen": -63.80437088012695,
"logps/ref_rejected": -79.3484115600586,
"logps/rejected": -82.37544250488281,
"loss": 1.3097,
"margin_dpo/margin_mean": 0.8672324419021606,
"margin_dpo/margin_std": 1.7289981842041016,
"step": 81
},
{
"KL/chosen_KL_mean": -1.7621707916259766,
"KL/mean": -2.3001327514648438,
"KL/rejected_KL_mean": -2.8380966186523438,
"KL/std": 1.359740972518921,
"epoch": 0.12396069538926682,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.075925588607788,
"fcm_dpo/q_t": 0.4732835590839386,
"grad_norm": 26.572818756103516,
"learning_rate": 4.993149937871306e-07,
"logits/chosen": 0.12116063386201859,
"logits/rejected": 0.057523056864738464,
"logps/chosen": -50.58006286621094,
"logps/ref_chosen": -48.817893981933594,
"logps/ref_rejected": -70.31497955322266,
"logps/rejected": -73.153076171875,
"loss": 1.2876,
"margin_dpo/margin_mean": 1.0759259462356567,
"margin_dpo/margin_std": 1.5349102020263672,
"step": 82
},
{
"KL/chosen_KL_mean": -2.0419349670410156,
"KL/mean": -2.62204647064209,
"KL/rejected_KL_mean": -3.202159881591797,
"KL/std": 1.4831633567810059,
"epoch": 0.1254724111866969,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1602277755737305,
"fcm_dpo/q_t": 0.47140026092529297,
"grad_norm": 29.156784057617188,
"learning_rate": 4.992136939879856e-07,
"logits/chosen": 0.18060043454170227,
"logits/rejected": 0.13148349523544312,
"logps/chosen": -59.192710876464844,
"logps/ref_chosen": -57.15077209472656,
"logps/ref_rejected": -75.1710205078125,
"logps/rejected": -78.37318420410156,
"loss": 1.2821,
"margin_dpo/margin_mean": 1.1602275371551514,
"margin_dpo/margin_std": 1.8395249843597412,
"step": 83
},
{
"KL/chosen_KL_mean": -2.5159435272216797,
"KL/mean": -2.952871322631836,
"KL/rejected_KL_mean": -3.389801025390625,
"KL/std": 1.8310933113098145,
"epoch": 0.12698412698412698,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8738585710525513,
"fcm_dpo/q_t": 0.4783720076084137,
"grad_norm": 30.70692253112793,
"learning_rate": 4.991054231460969e-07,
"logits/chosen": 0.16691604256629944,
"logits/rejected": 0.1250467747449875,
"logps/chosen": -67.29324340820312,
"logps/ref_chosen": -64.77729797363281,
"logps/ref_rejected": -84.71949768066406,
"logps/rejected": -88.10929870605469,
"loss": 1.3101,
"margin_dpo/margin_mean": 0.8738582134246826,
"margin_dpo/margin_std": 1.9217841625213623,
"step": 84
},
{
"KL/chosen_KL_mean": -2.2303695678710938,
"KL/mean": -2.8911349773406982,
"KL/rejected_KL_mean": -3.5518951416015625,
"KL/std": 1.786381721496582,
"epoch": 0.12849584278155707,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3215255737304688,
"fcm_dpo/q_t": 0.46737951040267944,
"grad_norm": 26.60832405090332,
"learning_rate": 4.989901842900325e-07,
"logits/chosen": 0.14956089854240417,
"logits/rejected": 0.10656014084815979,
"logps/chosen": -52.48206329345703,
"logps/ref_chosen": -50.25169372558594,
"logps/ref_rejected": -66.55439758300781,
"logps/rejected": -70.10629272460938,
"loss": 1.2688,
"margin_dpo/margin_mean": 1.3215255737304688,
"margin_dpo/margin_std": 1.9796284437179565,
"step": 85
},
{
"KL/chosen_KL_mean": -2.857900619506836,
"KL/mean": -3.3198752403259277,
"KL/rejected_KL_mean": -3.781848907470703,
"KL/std": 1.9224777221679688,
"epoch": 0.13000755857898716,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9239518642425537,
"fcm_dpo/q_t": 0.47717535495758057,
"grad_norm": 27.272216796875,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": 0.16544556617736816,
"logits/rejected": 0.147491455078125,
"logps/chosen": -63.58708190917969,
"logps/ref_chosen": -60.72917938232422,
"logps/ref_rejected": -72.30961608886719,
"logps/rejected": -76.09146118164062,
"loss": 1.3065,
"margin_dpo/margin_mean": 0.9239521026611328,
"margin_dpo/margin_std": 2.0516607761383057,
"step": 86
},
{
"KL/chosen_KL_mean": -3.028636932373047,
"KL/mean": -3.547091007232666,
"KL/rejected_KL_mean": -4.065544128417969,
"KL/std": 2.0648305416107178,
"epoch": 0.13151927437641722,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0369104146957397,
"fcm_dpo/q_t": 0.4744616746902466,
"grad_norm": 30.57183837890625,
"learning_rate": 4.987388156241114e-07,
"logits/chosen": 0.14023475348949432,
"logits/rejected": 0.08374907076358795,
"logps/chosen": -68.78660583496094,
"logps/ref_chosen": -65.75796508789062,
"logps/ref_rejected": -84.81159973144531,
"logps/rejected": -88.87713623046875,
"loss": 1.3007,
"margin_dpo/margin_mean": 1.0369102954864502,
"margin_dpo/margin_std": 2.476595640182495,
"step": 87
},
{
"KL/chosen_KL_mean": -3.0699501037597656,
"KL/mean": -3.4808826446533203,
"KL/rejected_KL_mean": -3.8918113708496094,
"KL/std": 2.3378701210021973,
"epoch": 0.1330309901738473,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8218619227409363,
"fcm_dpo/q_t": 0.4799662232398987,
"grad_norm": 30.38210105895996,
"learning_rate": 4.986026928455767e-07,
"logits/chosen": 0.23296837508678436,
"logits/rejected": 0.2060338854789734,
"logps/chosen": -65.89398193359375,
"logps/ref_chosen": -62.82402801513672,
"logps/ref_rejected": -74.9607162475586,
"logps/rejected": -78.85252380371094,
"loss": 1.3288,
"margin_dpo/margin_mean": 0.821861982345581,
"margin_dpo/margin_std": 2.9647645950317383,
"step": 88
},
{
"KL/chosen_KL_mean": -2.5609130859375,
"KL/mean": -3.2582736015319824,
"KL/rejected_KL_mean": -3.9556312561035156,
"KL/std": 2.156515598297119,
"epoch": 0.1345427059712774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3947181701660156,
"fcm_dpo/q_t": 0.4660217761993408,
"grad_norm": 29.258838653564453,
"learning_rate": 4.984596161153135e-07,
"logits/chosen": 0.237514466047287,
"logits/rejected": 0.15626181662082672,
"logps/chosen": -43.752349853515625,
"logps/ref_chosen": -41.191436767578125,
"logps/ref_rejected": -85.44769287109375,
"logps/rejected": -89.40332794189453,
"loss": 1.2685,
"margin_dpo/margin_mean": 1.3947186470031738,
"margin_dpo/margin_std": 2.536036968231201,
"step": 89
},
{
"KL/chosen_KL_mean": -3.1376190185546875,
"KL/mean": -3.7834763526916504,
"KL/rejected_KL_mean": -4.4293365478515625,
"KL/std": 2.46217942237854,
"epoch": 0.1360544217687075,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2917141914367676,
"fcm_dpo/q_t": 0.4683051109313965,
"grad_norm": 29.12344741821289,
"learning_rate": 4.983095894354857e-07,
"logits/chosen": 0.168723464012146,
"logits/rejected": 0.11255622655153275,
"logps/chosen": -59.721527099609375,
"logps/ref_chosen": -56.58390808105469,
"logps/ref_rejected": -86.86978149414062,
"logps/rejected": -91.29911804199219,
"loss": 1.2769,
"margin_dpo/margin_mean": 1.291713833808899,
"margin_dpo/margin_std": 2.4582650661468506,
"step": 90
},
{
"KL/chosen_KL_mean": -3.149148941040039,
"KL/mean": -3.722820281982422,
"KL/rejected_KL_mean": -4.296489715576172,
"KL/std": 2.303582191467285,
"epoch": 0.13756613756613756,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1473394632339478,
"fcm_dpo/q_t": 0.47223398089408875,
"grad_norm": 25.720582962036133,
"learning_rate": 4.98152617002662e-07,
"logits/chosen": 0.1862848550081253,
"logits/rejected": 0.14085903763771057,
"logps/chosen": -55.531494140625,
"logps/ref_chosen": -52.38234329223633,
"logps/ref_rejected": -72.17642211914062,
"logps/rejected": -76.47291564941406,
"loss": 1.2941,
"margin_dpo/margin_mean": 1.147339105606079,
"margin_dpo/margin_std": 2.7885167598724365,
"step": 91
},
{
"KL/chosen_KL_mean": -3.230426788330078,
"KL/mean": -4.01956844329834,
"KL/rejected_KL_mean": -4.808712005615234,
"KL/std": 2.6578006744384766,
"epoch": 0.13907785336356765,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.578284740447998,
"fcm_dpo/q_t": 0.4619179964065552,
"grad_norm": 26.956829071044922,
"learning_rate": 4.979887032076988e-07,
"logits/chosen": 0.22094112634658813,
"logits/rejected": 0.1804642379283905,
"logps/chosen": -56.23912811279297,
"logps/ref_chosen": -53.00870132446289,
"logps/ref_rejected": -79.77812957763672,
"logps/rejected": -84.58683776855469,
"loss": 1.2588,
"margin_dpo/margin_mean": 1.578284502029419,
"margin_dpo/margin_std": 3.120558738708496,
"step": 92
},
{
"KL/chosen_KL_mean": -3.615804672241211,
"KL/mean": -4.103872299194336,
"KL/rejected_KL_mean": -4.591936111450195,
"KL/std": 2.629575729370117,
"epoch": 0.14058956916099774,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9761362075805664,
"fcm_dpo/q_t": 0.4767376184463501,
"grad_norm": 24.554616928100586,
"learning_rate": 4.978178526356172e-07,
"logits/chosen": 0.20379352569580078,
"logits/rejected": 0.17453384399414062,
"logps/chosen": -48.522857666015625,
"logps/ref_chosen": -44.90705108642578,
"logps/ref_rejected": -58.7879524230957,
"logps/rejected": -63.37989044189453,
"loss": 1.3171,
"margin_dpo/margin_mean": 0.9761360883712769,
"margin_dpo/margin_std": 3.2695860862731934,
"step": 93
},
{
"KL/chosen_KL_mean": -3.2867393493652344,
"KL/mean": -4.221029758453369,
"KL/rejected_KL_mean": -5.1553192138671875,
"KL/std": 3.2257328033447266,
"epoch": 0.1421012849584278,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8685789108276367,
"fcm_dpo/q_t": 0.45501282811164856,
"grad_norm": 27.455610275268555,
"learning_rate": 4.976400700654751e-07,
"logits/chosen": 0.2643049359321594,
"logits/rejected": 0.22353461384773254,
"logps/chosen": -63.224510192871094,
"logps/ref_chosen": -59.93777084350586,
"logps/ref_rejected": -79.3138427734375,
"logps/rejected": -84.46916198730469,
"loss": 1.2467,
"margin_dpo/margin_mean": 1.868578553199768,
"margin_dpo/margin_std": 3.965810537338257,
"step": 94
},
{
"KL/chosen_KL_mean": -4.280330657958984,
"KL/mean": -5.080137252807617,
"KL/rejected_KL_mean": -5.87994384765625,
"KL/std": 2.722566604614258,
"epoch": 0.1436130007558579,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5996131896972656,
"fcm_dpo/q_t": 0.4613925516605377,
"grad_norm": 29.14002799987793,
"learning_rate": 4.974553604702332e-07,
"logits/chosen": 0.15258441865444183,
"logits/rejected": 0.0883299708366394,
"logps/chosen": -64.44882202148438,
"logps/ref_chosen": -60.168487548828125,
"logps/ref_rejected": -90.73665618896484,
"logps/rejected": -96.6166000366211,
"loss": 1.2618,
"margin_dpo/margin_mean": 1.5996129512786865,
"margin_dpo/margin_std": 3.4419288635253906,
"step": 95
},
{
"KL/chosen_KL_mean": -4.186677932739258,
"KL/mean": -5.113858222961426,
"KL/rejected_KL_mean": -6.041038513183594,
"KL/std": 3.425692081451416,
"epoch": 0.14512471655328799,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8543610572814941,
"fcm_dpo/q_t": 0.4570516049861908,
"grad_norm": 27.39577293395996,
"learning_rate": 4.972637290166157e-07,
"logits/chosen": 0.18779154121875763,
"logits/rejected": 0.14556936919689178,
"logps/chosen": -64.85545349121094,
"logps/ref_chosen": -60.66877746582031,
"logps/ref_rejected": -88.30673217773438,
"logps/rejected": -94.34777069091797,
"loss": 1.2594,
"margin_dpo/margin_mean": 1.854360818862915,
"margin_dpo/margin_std": 4.501690864562988,
"step": 96
},
{
"KL/chosen_KL_mean": -4.962148666381836,
"KL/mean": -5.490826606750488,
"KL/rejected_KL_mean": -6.019508361816406,
"KL/std": 3.6154727935791016,
"epoch": 0.14663643235071808,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0573649406433105,
"fcm_dpo/q_t": 0.47493094205856323,
"grad_norm": 36.246150970458984,
"learning_rate": 4.970651810649666e-07,
"logits/chosen": 0.12958383560180664,
"logits/rejected": 0.08639142662286758,
"logps/chosen": -70.00627136230469,
"logps/ref_chosen": -65.04412078857422,
"logps/ref_rejected": -78.42092895507812,
"logps/rejected": -84.44043731689453,
"loss": 1.3422,
"margin_dpo/margin_mean": 1.0573647022247314,
"margin_dpo/margin_std": 4.932512283325195,
"step": 97
},
{
"KL/chosen_KL_mean": -4.633974075317383,
"KL/mean": -4.971208095550537,
"KL/rejected_KL_mean": -5.308444976806641,
"KL/std": 3.280994415283203,
"epoch": 0.14814814814814814,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6744681596755981,
"fcm_dpo/q_t": 0.4831414818763733,
"grad_norm": 30.31377410888672,
"learning_rate": 4.968597221690985e-07,
"logits/chosen": 0.23023685812950134,
"logits/rejected": 0.2029840350151062,
"logps/chosen": -60.13720703125,
"logps/ref_chosen": -55.503231048583984,
"logps/ref_rejected": -72.81553649902344,
"logps/rejected": -78.12398529052734,
"loss": 1.3469,
"margin_dpo/margin_mean": 0.6744686365127563,
"margin_dpo/margin_std": 3.2570295333862305,
"step": 98
},
{
"KL/chosen_KL_mean": -4.691219329833984,
"KL/mean": -5.205721855163574,
"KL/rejected_KL_mean": -5.7202301025390625,
"KL/std": 3.546834945678711,
"epoch": 0.14965986394557823,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0290096998214722,
"fcm_dpo/q_t": 0.4757050573825836,
"grad_norm": 32.59518814086914,
"learning_rate": 4.966473580761389e-07,
"logits/chosen": 0.2541065216064453,
"logits/rejected": 0.21417078375816345,
"logps/chosen": -63.2668571472168,
"logps/ref_chosen": -58.57563781738281,
"logps/ref_rejected": -78.693603515625,
"logps/rejected": -84.41383361816406,
"loss": 1.3378,
"margin_dpo/margin_mean": 1.0290095806121826,
"margin_dpo/margin_std": 4.595114707946777,
"step": 99
},
{
"KL/chosen_KL_mean": -5.293548583984375,
"KL/mean": -6.042011260986328,
"KL/rejected_KL_mean": -6.790477752685547,
"KL/std": 4.170048236846924,
"epoch": 0.15117157974300832,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.496924638748169,
"fcm_dpo/q_t": 0.46660345792770386,
"grad_norm": 32.89773941040039,
"learning_rate": 4.964280947263676e-07,
"logits/chosen": 0.20524325966835022,
"logits/rejected": 0.19783976674079895,
"logps/chosen": -84.87698364257812,
"logps/ref_chosen": -79.58343505859375,
"logps/ref_rejected": -92.152587890625,
"logps/rejected": -98.94306182861328,
"loss": 1.3287,
"margin_dpo/margin_mean": 1.4969251155853271,
"margin_dpo/margin_std": 5.857483386993408,
"step": 100
},
{
"KL/chosen_KL_mean": -4.28514289855957,
"KL/mean": -5.326085090637207,
"KL/rejected_KL_mean": -6.367027282714844,
"KL/std": 3.696342945098877,
"epoch": 0.15268329554043839,
"fcm_dpo/beta": 0.22752070426940918,
"fcm_dpo/delta": 0.07919151335954666,
"fcm_dpo/margin": 2.0818839073181152,
"fcm_dpo/q_t": 0.4497947096824646,
"grad_norm": 81.98783111572266,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": 0.21574577689170837,
"logits/rejected": 0.17738720774650574,
"logps/chosen": -56.61792755126953,
"logps/ref_chosen": -52.332786560058594,
"logps/ref_rejected": -69.55589294433594,
"logps/rejected": -75.92292785644531,
"loss": 1.2433,
"margin_dpo/margin_mean": 2.0818839073181152,
"margin_dpo/margin_std": 4.205405235290527,
"step": 101
},
{
"KL/chosen_KL_mean": -5.278135299682617,
"KL/mean": -5.710550308227539,
"KL/rejected_KL_mean": -6.142971038818359,
"KL/std": 3.6688332557678223,
"epoch": 0.15419501133786848,
"fcm_dpo/beta": 0.35483452677726746,
"fcm_dpo/delta": -7.2879483923316e-05,
"fcm_dpo/margin": 0.8648367524147034,
"fcm_dpo/q_t": 0.47156286239624023,
"grad_norm": 162.13876342773438,
"learning_rate": 4.959688949822748e-07,
"logits/chosen": 0.13888651132583618,
"logits/rejected": 0.10068652778863907,
"logps/chosen": -70.02162170410156,
"logps/ref_chosen": -64.74348449707031,
"logps/ref_rejected": -69.06132507324219,
"logps/rejected": -75.20429992675781,
"loss": 1.7198,
"margin_dpo/margin_mean": 0.8648366928100586,
"margin_dpo/margin_std": 5.406614303588867,
"step": 102
},
{
"KL/chosen_KL_mean": -4.759641647338867,
"KL/mean": -5.5327959060668945,
"KL/rejected_KL_mean": -6.305950164794922,
"KL/std": 3.811723232269287,
"epoch": 0.15570672713529857,
"fcm_dpo/beta": 0.2257329523563385,
"fcm_dpo/delta": -0.23946459591388702,
"fcm_dpo/margin": 1.5463120937347412,
"fcm_dpo/q_t": 0.3969665467739105,
"grad_norm": 121.77664947509766,
"learning_rate": 4.957289714327572e-07,
"logits/chosen": 0.2512431740760803,
"logits/rejected": 0.2177349030971527,
"logps/chosen": -68.59628295898438,
"logps/ref_chosen": -63.83664321899414,
"logps/ref_rejected": -79.32362365722656,
"logps/rejected": -85.62957763671875,
"loss": 1.4546,
"margin_dpo/margin_mean": 1.546311855316162,
"margin_dpo/margin_std": 3.6003336906433105,
"step": 103
},
{
"KL/chosen_KL_mean": -4.559057235717773,
"KL/mean": -5.69294548034668,
"KL/rejected_KL_mean": -6.826839447021484,
"KL/std": 4.804152011871338,
"epoch": 0.15721844293272866,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.2677783966064453,
"fcm_dpo/q_t": 0.49564671516418457,
"grad_norm": 3.038849353790283,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": 0.23869748413562775,
"logits/rejected": 0.15296530723571777,
"logps/chosen": -65.55826568603516,
"logps/ref_chosen": -60.99920654296875,
"logps/ref_rejected": -98.84645080566406,
"logps/rejected": -105.67329406738281,
"loss": 1.3695,
"margin_dpo/margin_mean": 2.267778158187866,
"margin_dpo/margin_std": 6.126588821411133,
"step": 104
},
{
"KL/chosen_KL_mean": -5.151271820068359,
"KL/mean": -6.049022197723389,
"KL/rejected_KL_mean": -6.94677734375,
"KL/std": 3.8394556045532227,
"epoch": 0.15873015873015872,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.7955060005187988,
"fcm_dpo/q_t": 0.4965510368347168,
"grad_norm": 2.853419303894043,
"learning_rate": 4.952285105344791e-07,
"logits/chosen": 0.17378970980644226,
"logits/rejected": 0.12029208987951279,
"logps/chosen": -76.1015396118164,
"logps/ref_chosen": -70.95027160644531,
"logps/ref_rejected": -87.88340759277344,
"logps/rejected": -94.83018493652344,
"loss": 1.3729,
"margin_dpo/margin_mean": 1.7955060005187988,
"margin_dpo/margin_std": 4.681982040405273,
"step": 105
},
{
"KL/chosen_KL_mean": -4.813375473022461,
"KL/mean": -5.589751243591309,
"KL/rejected_KL_mean": -6.366130828857422,
"KL/std": 3.7389588356018066,
"epoch": 0.1602418745275888,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5527547597885132,
"fcm_dpo/q_t": 0.4970164895057678,
"grad_norm": 2.309171438217163,
"learning_rate": 4.949679871846857e-07,
"logits/chosen": 0.2168569266796112,
"logits/rejected": 0.20323088765144348,
"logps/chosen": -67.272705078125,
"logps/ref_chosen": -62.45933151245117,
"logps/ref_rejected": -67.00595092773438,
"logps/rejected": -73.37207794189453,
"loss": 1.3748,
"margin_dpo/margin_mean": 1.5527546405792236,
"margin_dpo/margin_std": 5.025470733642578,
"step": 106
},
{
"KL/chosen_KL_mean": -6.303882598876953,
"KL/mean": -6.7031097412109375,
"KL/rejected_KL_mean": -7.102336883544922,
"KL/std": 4.748565196990967,
"epoch": 0.1617535903250189,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7984510064125061,
"fcm_dpo/q_t": 0.49846479296684265,
"grad_norm": 2.6205790042877197,
"learning_rate": 4.947006115536947e-07,
"logits/chosen": 0.19034229218959808,
"logits/rejected": 0.16588297486305237,
"logps/chosen": -82.141845703125,
"logps/ref_chosen": -75.83796691894531,
"logps/ref_rejected": -87.74038696289062,
"logps/rejected": -94.84272003173828,
"loss": 1.3807,
"margin_dpo/margin_mean": 0.7984509468078613,
"margin_dpo/margin_std": 5.902289390563965,
"step": 107
},
{
"KL/chosen_KL_mean": -5.396709442138672,
"KL/mean": -6.335811138153076,
"KL/rejected_KL_mean": -7.274906158447266,
"KL/std": 4.074840545654297,
"epoch": 0.16326530612244897,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8781964778900146,
"fcm_dpo/q_t": 0.49639254808425903,
"grad_norm": 2.582226037979126,
"learning_rate": 4.944263911205772e-07,
"logits/chosen": 0.19318030774593353,
"logits/rejected": 0.16133251786231995,
"logps/chosen": -73.78994750976562,
"logps/ref_chosen": -68.39323425292969,
"logps/ref_rejected": -83.24267578125,
"logps/rejected": -90.517578125,
"loss": 1.3723,
"margin_dpo/margin_mean": 1.8781968355178833,
"margin_dpo/margin_std": 5.379508972167969,
"step": 108
},
{
"KL/chosen_KL_mean": -4.847797393798828,
"KL/mean": -6.345134735107422,
"KL/rejected_KL_mean": -7.842475891113281,
"KL/std": 5.034552574157715,
"epoch": 0.16477702191987906,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.994675636291504,
"fcm_dpo/q_t": 0.49424993991851807,
"grad_norm": 2.9282801151275635,
"learning_rate": 4.941453335558681e-07,
"logits/chosen": 0.20009146630764008,
"logits/rejected": 0.14455196261405945,
"logps/chosen": -60.37527847290039,
"logps/ref_chosen": -55.52748107910156,
"logps/ref_rejected": -83.55218505859375,
"logps/rejected": -91.39466857910156,
"loss": 1.3638,
"margin_dpo/margin_mean": 2.994675636291504,
"margin_dpo/margin_std": 5.442989349365234,
"step": 109
},
{
"KL/chosen_KL_mean": -6.40728759765625,
"KL/mean": -6.496090888977051,
"KL/rejected_KL_mean": -6.584896087646484,
"KL/std": 4.201725006103516,
"epoch": 0.16628873771730915,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.17760717868804932,
"fcm_dpo/q_t": 0.49965718388557434,
"grad_norm": 2.743898630142212,
"learning_rate": 4.938574467213517e-07,
"logits/chosen": 0.16291868686676025,
"logits/rejected": 0.17184513807296753,
"logps/chosen": -87.56603240966797,
"logps/ref_chosen": -81.15874481201172,
"logps/ref_rejected": -72.56021118164062,
"logps/rejected": -79.14511108398438,
"loss": 1.3854,
"margin_dpo/margin_mean": 0.1776072382926941,
"margin_dpo/margin_std": 5.59630012512207,
"step": 110
},
{
"KL/chosen_KL_mean": -5.86181640625,
"KL/mean": -6.897708892822266,
"KL/rejected_KL_mean": -7.933605194091797,
"KL/std": 4.601965427398682,
"epoch": 0.16780045351473924,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.0717906951904297,
"fcm_dpo/q_t": 0.4960215389728546,
"grad_norm": 2.4579412937164307,
"learning_rate": 4.935627386698418e-07,
"logits/chosen": 0.2789857089519501,
"logits/rejected": 0.24208299815654755,
"logps/chosen": -58.220802307128906,
"logps/ref_chosen": -52.358985900878906,
"logps/ref_rejected": -77.06150817871094,
"logps/rejected": -84.99510955810547,
"loss": 1.3709,
"margin_dpo/margin_mean": 2.0717906951904297,
"margin_dpo/margin_std": 5.603476047515869,
"step": 111
},
{
"KL/chosen_KL_mean": -5.752780914306641,
"KL/mean": -7.0450944900512695,
"KL/rejected_KL_mean": -8.337406158447266,
"KL/std": 4.755733966827393,
"epoch": 0.1693121693121693,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.5846307277679443,
"fcm_dpo/q_t": 0.49503466486930847,
"grad_norm": 3.0572729110717773,
"learning_rate": 4.932612176449559e-07,
"logits/chosen": 0.1905369758605957,
"logits/rejected": 0.12637881934642792,
"logps/chosen": -68.77284240722656,
"logps/ref_chosen": -63.02006530761719,
"logps/ref_rejected": -111.36941528320312,
"logps/rejected": -119.70681762695312,
"loss": 1.3669,
"margin_dpo/margin_mean": 2.584629535675049,
"margin_dpo/margin_std": 5.197779655456543,
"step": 112
},
{
"KL/chosen_KL_mean": -5.822654724121094,
"KL/mean": -6.4955949783325195,
"KL/rejected_KL_mean": -7.1685333251953125,
"KL/std": 3.995758533477783,
"epoch": 0.1708238851095994,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3458778858184814,
"fcm_dpo/q_t": 0.49741414189338684,
"grad_norm": 2.6555185317993164,
"learning_rate": 4.929528920808854e-07,
"logits/chosen": 0.2019399255514145,
"logits/rejected": 0.16312995553016663,
"logps/chosen": -61.63031768798828,
"logps/ref_chosen": -55.80766296386719,
"logps/ref_rejected": -69.84014129638672,
"logps/rejected": -77.00867462158203,
"loss": 1.3764,
"margin_dpo/margin_mean": 1.3458775281906128,
"margin_dpo/margin_std": 5.301340103149414,
"step": 113
},
{
"KL/chosen_KL_mean": -5.752845764160156,
"KL/mean": -6.942575454711914,
"KL/rejected_KL_mean": -8.132308959960938,
"KL/std": 4.493397235870361,
"epoch": 0.17233560090702948,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.3794596195220947,
"fcm_dpo/q_t": 0.495430052280426,
"grad_norm": 2.5707952976226807,
"learning_rate": 4.92637770602159e-07,
"logits/chosen": 0.26456063985824585,
"logits/rejected": 0.20437636971473694,
"logps/chosen": -72.08561706542969,
"logps/ref_chosen": -66.33277130126953,
"logps/ref_rejected": -71.61489868164062,
"logps/rejected": -79.74720764160156,
"loss": 1.3685,
"margin_dpo/margin_mean": 2.3794591426849365,
"margin_dpo/margin_std": 5.449493885040283,
"step": 114
},
{
"KL/chosen_KL_mean": -6.265972137451172,
"KL/mean": -7.2780961990356445,
"KL/rejected_KL_mean": -8.290224075317383,
"KL/std": 4.36235237121582,
"epoch": 0.17384731670445955,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.0242466926574707,
"fcm_dpo/q_t": 0.49611204862594604,
"grad_norm": 2.4445128440856934,
"learning_rate": 4.923158620234019e-07,
"logits/chosen": 0.2416526824235916,
"logits/rejected": 0.17938242852687836,
"logps/chosen": -62.015010833740234,
"logps/ref_chosen": -55.74903869628906,
"logps/ref_rejected": -79.59849548339844,
"logps/rejected": -87.88871765136719,
"loss": 1.3712,
"margin_dpo/margin_mean": 2.0242466926574707,
"margin_dpo/margin_std": 5.242649078369141,
"step": 115
},
{
"KL/chosen_KL_mean": -6.046346664428711,
"KL/mean": -7.712983131408691,
"KL/rejected_KL_mean": -9.379619598388672,
"KL/std": 4.501278877258301,
"epoch": 0.17535903250188964,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3332784175872803,
"fcm_dpo/q_t": 0.49359679222106934,
"grad_norm": 2.852839708328247,
"learning_rate": 4.91987175349089e-07,
"logits/chosen": 0.2487809658050537,
"logits/rejected": 0.1809816211462021,
"logps/chosen": -55.41151428222656,
"logps/ref_chosen": -49.36516571044922,
"logps/ref_rejected": -72.84671020507812,
"logps/rejected": -82.22632598876953,
"loss": 1.3612,
"margin_dpo/margin_mean": 3.3332786560058594,
"margin_dpo/margin_std": 5.188892364501953,
"step": 116
},
{
"KL/chosen_KL_mean": -6.292646408081055,
"KL/mean": -7.328824043273926,
"KL/rejected_KL_mean": -8.365001678466797,
"KL/std": 4.449042320251465,
"epoch": 0.17687074829931973,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.0723490715026855,
"fcm_dpo/q_t": 0.496019184589386,
"grad_norm": 2.2913970947265625,
"learning_rate": 4.916517197732933e-07,
"logits/chosen": 0.26744014024734497,
"logits/rejected": 0.23043423891067505,
"logps/chosen": -64.0035400390625,
"logps/ref_chosen": -57.710899353027344,
"logps/ref_rejected": -69.77253723144531,
"logps/rejected": -78.13754272460938,
"loss": 1.371,
"margin_dpo/margin_mean": 2.0723495483398438,
"margin_dpo/margin_std": 5.764100551605225,
"step": 117
},
{
"KL/chosen_KL_mean": -6.316926956176758,
"KL/mean": -7.789030075073242,
"KL/rejected_KL_mean": -9.261127471923828,
"KL/std": 5.045524597167969,
"epoch": 0.17838246409674982,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.9442031383514404,
"fcm_dpo/q_t": 0.4943440854549408,
"grad_norm": 2.639897346496582,
"learning_rate": 4.913095046794281e-07,
"logits/chosen": 0.33026766777038574,
"logits/rejected": 0.28865814208984375,
"logps/chosen": -58.79682540893555,
"logps/ref_chosen": -52.479896545410156,
"logps/ref_rejected": -81.359130859375,
"logps/rejected": -90.62025451660156,
"loss": 1.3642,
"margin_dpo/margin_mean": 2.9442026615142822,
"margin_dpo/margin_std": 5.081968307495117,
"step": 118
},
{
"KL/chosen_KL_mean": -8.139110565185547,
"KL/mean": -9.161100387573242,
"KL/rejected_KL_mean": -10.183094024658203,
"KL/std": 6.225261688232422,
"epoch": 0.17989417989417988,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.0439817905426025,
"fcm_dpo/q_t": 0.4960786998271942,
"grad_norm": 2.5201234817504883,
"learning_rate": 4.909605396399855e-07,
"logits/chosen": 0.2730048894882202,
"logits/rejected": 0.2327132672071457,
"logps/chosen": -69.49678039550781,
"logps/ref_chosen": -61.35767364501953,
"logps/ref_rejected": -75.71510314941406,
"logps/rejected": -85.898193359375,
"loss": 1.3716,
"margin_dpo/margin_mean": 2.0439815521240234,
"margin_dpo/margin_std": 7.831289768218994,
"step": 119
},
{
"KL/chosen_KL_mean": -6.7714385986328125,
"KL/mean": -8.515937805175781,
"KL/rejected_KL_mean": -10.260440826416016,
"KL/std": 5.0966644287109375,
"epoch": 0.18140589569160998,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.4890010356903076,
"fcm_dpo/q_t": 0.49329859018325806,
"grad_norm": 2.945573091506958,
"learning_rate": 4.906048344162676e-07,
"logits/chosen": 0.23838064074516296,
"logits/rejected": 0.17867352068424225,
"logps/chosen": -66.67900848388672,
"logps/ref_chosen": -59.907569885253906,
"logps/ref_rejected": -79.6910629272461,
"logps/rejected": -89.95150756835938,
"loss": 1.3601,
"margin_dpo/margin_mean": 3.4890012741088867,
"margin_dpo/margin_std": 5.204916000366211,
"step": 120
},
{
"KL/chosen_KL_mean": -7.601043701171875,
"KL/mean": -9.675811767578125,
"KL/rejected_KL_mean": -11.75058364868164,
"KL/std": 6.724109649658203,
"epoch": 0.18291761148904007,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.149538516998291,
"fcm_dpo/q_t": 0.49204063415527344,
"grad_norm": 3.3024706840515137,
"learning_rate": 4.902423989581143e-07,
"logits/chosen": 0.35299158096313477,
"logits/rejected": 0.26683202385902405,
"logps/chosen": -63.267086029052734,
"logps/ref_chosen": -55.66604232788086,
"logps/ref_rejected": -101.56233978271484,
"logps/rejected": -113.31292724609375,
"loss": 1.3556,
"margin_dpo/margin_mean": 4.149538516998291,
"margin_dpo/margin_std": 7.892666816711426,
"step": 121
},
{
"KL/chosen_KL_mean": -8.613723754882812,
"KL/mean": -10.285037994384766,
"KL/rejected_KL_mean": -11.956352233886719,
"KL/std": 7.303105354309082,
"epoch": 0.18442932728647016,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3426246643066406,
"fcm_dpo/q_t": 0.49359560012817383,
"grad_norm": 2.7032148838043213,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": 0.26826468110084534,
"logits/rejected": 0.2332470566034317,
"logps/chosen": -71.9480972290039,
"logps/ref_chosen": -63.334373474121094,
"logps/ref_rejected": -73.67523193359375,
"logps/rejected": -85.631591796875,
"loss": 1.362,
"margin_dpo/margin_mean": 3.3426246643066406,
"margin_dpo/margin_std": 9.229589462280273,
"step": 122
},
{
"KL/chosen_KL_mean": -8.551752090454102,
"KL/mean": -10.23425579071045,
"KL/rejected_KL_mean": -11.916763305664062,
"KL/std": 6.5701446533203125,
"epoch": 0.18594104308390022,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3650121688842773,
"fcm_dpo/q_t": 0.4935457706451416,
"grad_norm": 2.6148831844329834,
"learning_rate": 4.894973780788722e-07,
"logits/chosen": 0.3093302845954895,
"logits/rejected": 0.2672804594039917,
"logps/chosen": -65.45049285888672,
"logps/ref_chosen": -56.89874267578125,
"logps/ref_rejected": -78.97028350830078,
"logps/rejected": -90.88704681396484,
"loss": 1.3614,
"margin_dpo/margin_mean": 3.3650124073028564,
"margin_dpo/margin_std": 7.255990505218506,
"step": 123
},
{
"KL/chosen_KL_mean": -9.072261810302734,
"KL/mean": -11.368237495422363,
"KL/rejected_KL_mean": -13.664215087890625,
"KL/std": 7.325522422790527,
"epoch": 0.1874527588813303,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.591949939727783,
"fcm_dpo/q_t": 0.491183876991272,
"grad_norm": 2.9130699634552,
"learning_rate": 4.89114813497619e-07,
"logits/chosen": 0.3153066039085388,
"logits/rejected": 0.25329259037971497,
"logps/chosen": -66.18834686279297,
"logps/ref_chosen": -57.116085052490234,
"logps/ref_rejected": -87.93074035644531,
"logps/rejected": -101.59495544433594,
"loss": 1.3524,
"margin_dpo/margin_mean": 4.591949462890625,
"margin_dpo/margin_std": 8.780609130859375,
"step": 124
},
{
"KL/chosen_KL_mean": -10.442159652709961,
"KL/mean": -12.644015312194824,
"KL/rejected_KL_mean": -14.845863342285156,
"KL/std": 8.506006240844727,
"epoch": 0.1889644746787604,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.4036970138549805,
"fcm_dpo/q_t": 0.4915594458580017,
"grad_norm": 3.0990495681762695,
"learning_rate": 4.887255603610184e-07,
"logits/chosen": 0.34969115257263184,
"logits/rejected": 0.28866562247276306,
"logps/chosen": -76.1483383178711,
"logps/ref_chosen": -65.7061767578125,
"logps/ref_rejected": -91.72711944580078,
"logps/rejected": -106.57298278808594,
"loss": 1.3542,
"margin_dpo/margin_mean": 4.4036970138549805,
"margin_dpo/margin_std": 9.770685195922852,
"step": 125
},
{
"KL/chosen_KL_mean": -10.139768600463867,
"KL/mean": -10.864148139953613,
"KL/rejected_KL_mean": -11.588525772094727,
"KL/std": 6.056389808654785,
"epoch": 0.19047619047619047,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4487552642822266,
"fcm_dpo/q_t": 0.4972168207168579,
"grad_norm": 2.5714292526245117,
"learning_rate": 4.883296295573176e-07,
"logits/chosen": 0.16045671701431274,
"logits/rejected": 0.15517394244670868,
"logps/chosen": -78.31585693359375,
"logps/ref_chosen": -68.17608642578125,
"logps/ref_rejected": -65.1175537109375,
"logps/rejected": -76.7060775756836,
"loss": 1.3758,
"margin_dpo/margin_mean": 1.4487552642822266,
"margin_dpo/margin_std": 6.551729679107666,
"step": 126
},
{
"KL/chosen_KL_mean": -10.118179321289062,
"KL/mean": -12.15770149230957,
"KL/rejected_KL_mean": -14.197227478027344,
"KL/std": 5.957196235656738,
"epoch": 0.19198790627362056,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.07904577255249,
"fcm_dpo/q_t": 0.49216848611831665,
"grad_norm": 2.511080265045166,
"learning_rate": 4.87927032161552e-07,
"logits/chosen": 0.27716439962387085,
"logits/rejected": 0.2475191354751587,
"logps/chosen": -71.9984130859375,
"logps/ref_chosen": -61.88023376464844,
"logps/ref_rejected": -68.46012878417969,
"logps/rejected": -82.6573486328125,
"loss": 1.356,
"margin_dpo/margin_mean": 4.07904577255249,
"margin_dpo/margin_std": 7.116820812225342,
"step": 127
},
{
"KL/chosen_KL_mean": -11.079221725463867,
"KL/mean": -14.211316108703613,
"KL/rejected_KL_mean": -17.343414306640625,
"KL/std": 9.948829650878906,
"epoch": 0.19349962207105065,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.264186859130859,
"fcm_dpo/q_t": 0.4880082905292511,
"grad_norm": 3.778376817703247,
"learning_rate": 4.875177794352363e-07,
"logits/chosen": 0.3209880590438843,
"logits/rejected": 0.2622468173503876,
"logps/chosen": -77.7882080078125,
"logps/ref_chosen": -66.708984375,
"logps/ref_rejected": -94.97969055175781,
"logps/rejected": -112.32310485839844,
"loss": 1.3411,
"margin_dpo/margin_mean": 6.264187812805176,
"margin_dpo/margin_std": 12.51683235168457,
"step": 128
},
{
"KL/chosen_KL_mean": -13.11799430847168,
"KL/mean": -14.13327407836914,
"KL/rejected_KL_mean": -15.148551940917969,
"KL/std": 9.112518310546875,
"epoch": 0.19501133786848074,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.0305609703063965,
"fcm_dpo/q_t": 0.49610602855682373,
"grad_norm": 2.6070330142974854,
"learning_rate": 4.871018828260491e-07,
"logits/chosen": 0.30411797761917114,
"logits/rejected": 0.2988065481185913,
"logps/chosen": -78.45681762695312,
"logps/ref_chosen": -65.33882904052734,
"logps/ref_rejected": -68.06109619140625,
"logps/rejected": -83.20964050292969,
"loss": 1.3727,
"margin_dpo/margin_mean": 2.0305609703063965,
"margin_dpo/margin_std": 11.41251277923584,
"step": 129
},
{
"KL/chosen_KL_mean": -12.702459335327148,
"KL/mean": -15.399238586425781,
"KL/rejected_KL_mean": -18.09601593017578,
"KL/std": 9.932580947875977,
"epoch": 0.1965230536659108,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.393557548522949,
"fcm_dpo/q_t": 0.4896560311317444,
"grad_norm": 3.132603883743286,
"learning_rate": 4.866793539675126e-07,
"logits/chosen": 0.26491034030914307,
"logits/rejected": 0.21366286277770996,
"logps/chosen": -71.36320495605469,
"logps/ref_chosen": -58.660743713378906,
"logps/ref_rejected": -79.24510192871094,
"logps/rejected": -97.34111785888672,
"loss": 1.3468,
"margin_dpo/margin_mean": 5.393556594848633,
"margin_dpo/margin_std": 10.183626174926758,
"step": 130
},
{
"KL/chosen_KL_mean": -11.906011581420898,
"KL/mean": -15.45162582397461,
"KL/rejected_KL_mean": -18.997241973876953,
"KL/std": 10.712788581848145,
"epoch": 0.1980347694633409,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.091231346130371,
"fcm_dpo/q_t": 0.4864232838153839,
"grad_norm": 3.4537715911865234,
"learning_rate": 4.86250204678667e-07,
"logits/chosen": 0.28533366322517395,
"logits/rejected": 0.2199201136827469,
"logps/chosen": -64.42054748535156,
"logps/ref_chosen": -52.51453399658203,
"logps/ref_rejected": -85.18299865722656,
"logps/rejected": -104.18023681640625,
"loss": 1.3349,
"margin_dpo/margin_mean": 7.091231346130371,
"margin_dpo/margin_std": 12.101551055908203,
"step": 131
},
{
"KL/chosen_KL_mean": -13.843952178955078,
"KL/mean": -16.509641647338867,
"KL/rejected_KL_mean": -19.175325393676758,
"KL/std": 11.187821388244629,
"epoch": 0.19954648526077098,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.331372261047363,
"fcm_dpo/q_t": 0.4897724986076355,
"grad_norm": 2.8171606063842773,
"learning_rate": 4.858144469637408e-07,
"logits/chosen": 0.3724517822265625,
"logits/rejected": 0.341764897108078,
"logps/chosen": -79.52908325195312,
"logps/ref_chosen": -65.68513488769531,
"logps/ref_rejected": -69.54120635986328,
"logps/rejected": -88.71653747558594,
"loss": 1.3473,
"margin_dpo/margin_mean": 5.331372261047363,
"margin_dpo/margin_std": 10.194868087768555,
"step": 132
},
{
"KL/chosen_KL_mean": -15.300018310546875,
"KL/mean": -16.778060913085938,
"KL/rejected_KL_mean": -18.25609588623047,
"KL/std": 9.998517990112305,
"epoch": 0.20105820105820105,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.956071376800537,
"fcm_dpo/q_t": 0.4943294823169708,
"grad_norm": 2.656231164932251,
"learning_rate": 4.853720930118138e-07,
"logits/chosen": 0.2910245954990387,
"logits/rejected": 0.28055471181869507,
"logps/chosen": -78.89813232421875,
"logps/ref_chosen": -63.598114013671875,
"logps/ref_rejected": -73.72798156738281,
"logps/rejected": -91.98407745361328,
"loss": 1.3658,
"margin_dpo/margin_mean": 2.956071376800537,
"margin_dpo/margin_std": 11.712414741516113,
"step": 133
},
{
"KL/chosen_KL_mean": -14.085258483886719,
"KL/mean": -17.977962493896484,
"KL/rejected_KL_mean": -21.87066650390625,
"KL/std": 12.072915077209473,
"epoch": 0.20256991685563114,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.785408973693848,
"fcm_dpo/q_t": 0.4851023554801941,
"grad_norm": 3.1521494388580322,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": 0.41877007484436035,
"logits/rejected": 0.36156511306762695,
"logps/chosen": -67.87982940673828,
"logps/ref_chosen": -53.79457092285156,
"logps/ref_rejected": -74.16741943359375,
"logps/rejected": -96.0380859375,
"loss": 1.3303,
"margin_dpo/margin_mean": 7.785409927368164,
"margin_dpo/margin_std": 14.063751220703125,
"step": 134
},
{
"KL/chosen_KL_mean": -14.210046768188477,
"KL/mean": -16.44684410095215,
"KL/rejected_KL_mean": -18.683643341064453,
"KL/std": 10.793325424194336,
"epoch": 0.20408163265306123,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.473599910736084,
"fcm_dpo/q_t": 0.4914175271987915,
"grad_norm": 2.542938232421875,
"learning_rate": 4.844676460754862e-07,
"logits/chosen": 0.3599017262458801,
"logits/rejected": 0.32789885997772217,
"logps/chosen": -63.651123046875,
"logps/ref_chosen": -49.441078186035156,
"logps/ref_rejected": -65.96878051757812,
"logps/rejected": -84.65242004394531,
"loss": 1.354,
"margin_dpo/margin_mean": 4.473599433898926,
"margin_dpo/margin_std": 11.142325401306152,
"step": 135
},
{
"KL/chosen_KL_mean": -18.304481506347656,
"KL/mean": -21.979602813720703,
"KL/rejected_KL_mean": -25.654720306396484,
"KL/std": 14.195917129516602,
"epoch": 0.20559334845049132,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.350245475769043,
"fcm_dpo/q_t": 0.4859556555747986,
"grad_norm": 3.6188459396362305,
"learning_rate": 4.840055783904106e-07,
"logits/chosen": 0.35441240668296814,
"logits/rejected": 0.27984628081321716,
"logps/chosen": -85.0637435913086,
"logps/ref_chosen": -66.75926208496094,
"logps/ref_rejected": -94.61787414550781,
"logps/rejected": -120.27259826660156,
"loss": 1.3349,
"margin_dpo/margin_mean": 7.350246429443359,
"margin_dpo/margin_std": 16.540786743164062,
"step": 136
},
{
"KL/chosen_KL_mean": -16.800689697265625,
"KL/mean": -20.018104553222656,
"KL/rejected_KL_mean": -23.235515594482422,
"KL/std": 12.882331848144531,
"epoch": 0.20710506424792138,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.434828281402588,
"fcm_dpo/q_t": 0.48770397901535034,
"grad_norm": 2.851135492324829,
"learning_rate": 4.835369650662767e-07,
"logits/chosen": 0.39496833086013794,
"logits/rejected": 0.3645436465740204,
"logps/chosen": -73.58448791503906,
"logps/ref_chosen": -56.78379821777344,
"logps/ref_rejected": -69.89952087402344,
"logps/rejected": -93.13504028320312,
"loss": 1.3409,
"margin_dpo/margin_mean": 6.434828758239746,
"margin_dpo/margin_std": 14.978425979614258,
"step": 137
},
{
"KL/chosen_KL_mean": -20.657804489135742,
"KL/mean": -23.541412353515625,
"KL/rejected_KL_mean": -26.425025939941406,
"KL/std": 14.655818939208984,
"epoch": 0.20861678004535147,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.767213821411133,
"fcm_dpo/q_t": 0.48898112773895264,
"grad_norm": 2.8203554153442383,
"learning_rate": 4.830618192112065e-07,
"logits/chosen": 0.3970043361186981,
"logits/rejected": 0.36176303029060364,
"logps/chosen": -79.42381286621094,
"logps/ref_chosen": -58.766014099121094,
"logps/ref_rejected": -68.12371826171875,
"logps/rejected": -94.54874420166016,
"loss": 1.347,
"margin_dpo/margin_mean": 5.767213821411133,
"margin_dpo/margin_std": 17.624059677124023,
"step": 138
},
{
"KL/chosen_KL_mean": -22.82883071899414,
"KL/mean": -25.602848052978516,
"KL/rejected_KL_mean": -28.376876831054688,
"KL/std": 12.521333694458008,
"epoch": 0.21012849584278157,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.548043727874756,
"fcm_dpo/q_t": 0.48936471343040466,
"grad_norm": 3.1577208042144775,
"learning_rate": 4.825801541160509e-07,
"logits/chosen": 0.3622744083404541,
"logits/rejected": 0.3314181864261627,
"logps/chosen": -94.0544204711914,
"logps/ref_chosen": -71.2255859375,
"logps/ref_rejected": -82.1834716796875,
"logps/rejected": -110.56034851074219,
"loss": 1.3473,
"margin_dpo/margin_mean": 5.548043727874756,
"margin_dpo/margin_std": 14.130794525146484,
"step": 139
},
{
"KL/chosen_KL_mean": -21.173185348510742,
"KL/mean": -25.737133026123047,
"KL/rejected_KL_mean": -30.30107879638672,
"KL/std": 16.705089569091797,
"epoch": 0.21164021164021163,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.127891540527344,
"fcm_dpo/q_t": 0.4825606942176819,
"grad_norm": 3.3629658222198486,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": 0.35938477516174316,
"logits/rejected": 0.3098704516887665,
"logps/chosen": -84.45085144042969,
"logps/ref_chosen": -63.27766418457031,
"logps/ref_rejected": -83.30647277832031,
"logps/rejected": -113.60755157470703,
"loss": 1.324,
"margin_dpo/margin_mean": 9.127891540527344,
"margin_dpo/margin_std": 21.27337646484375,
"step": 140
},
{
"KL/chosen_KL_mean": -23.774330139160156,
"KL/mean": -28.25469970703125,
"KL/rejected_KL_mean": -32.73507308959961,
"KL/std": 16.478158950805664,
"epoch": 0.21315192743764172,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.960737228393555,
"fcm_dpo/q_t": 0.48290276527404785,
"grad_norm": 3.510462999343872,
"learning_rate": 4.815973202802966e-07,
"logits/chosen": 0.43517443537712097,
"logits/rejected": 0.38708174228668213,
"logps/chosen": -85.54109954833984,
"logps/ref_chosen": -61.76676940917969,
"logps/ref_rejected": -88.60601806640625,
"logps/rejected": -121.34109497070312,
"loss": 1.3247,
"margin_dpo/margin_mean": 8.960738182067871,
"margin_dpo/margin_std": 19.960668563842773,
"step": 141
},
{
"KL/chosen_KL_mean": -23.207942962646484,
"KL/mean": -26.78273582458496,
"KL/rejected_KL_mean": -30.357532501220703,
"KL/std": 14.630638122558594,
"epoch": 0.2146636432350718,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.149584770202637,
"fcm_dpo/q_t": 0.4863271713256836,
"grad_norm": 3.2684221267700195,
"learning_rate": 4.810961790316729e-07,
"logits/chosen": 0.425723671913147,
"logits/rejected": 0.40205299854278564,
"logps/chosen": -88.48271942138672,
"logps/ref_chosen": -65.2747802734375,
"logps/ref_rejected": -81.1378173828125,
"logps/rejected": -111.49534606933594,
"loss": 1.3353,
"margin_dpo/margin_mean": 7.149584770202637,
"margin_dpo/margin_std": 14.595829010009766,
"step": 142
},
{
"KL/chosen_KL_mean": -25.313261032104492,
"KL/mean": -27.440980911254883,
"KL/rejected_KL_mean": -29.56869888305664,
"KL/std": 18.696537017822266,
"epoch": 0.2161753590325019,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.255438327789307,
"fcm_dpo/q_t": 0.49176695942878723,
"grad_norm": 3.4273107051849365,
"learning_rate": 4.805885735261454e-07,
"logits/chosen": 0.4723702371120453,
"logits/rejected": 0.4537476897239685,
"logps/chosen": -87.93109130859375,
"logps/ref_chosen": -62.617828369140625,
"logps/ref_rejected": -70.39239501953125,
"logps/rejected": -99.96109008789062,
"loss": 1.3612,
"margin_dpo/margin_mean": 4.255438327789307,
"margin_dpo/margin_std": 22.319290161132812,
"step": 143
},
{
"KL/chosen_KL_mean": -26.418071746826172,
"KL/mean": -30.407001495361328,
"KL/rejected_KL_mean": -34.39592361450195,
"KL/std": 20.060935974121094,
"epoch": 0.21768707482993196,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.977849006652832,
"fcm_dpo/q_t": 0.4849676489830017,
"grad_norm": 3.241164207458496,
"learning_rate": 4.800745179625307e-07,
"logits/chosen": 0.4205203056335449,
"logits/rejected": 0.38700443506240845,
"logps/chosen": -87.22076416015625,
"logps/ref_chosen": -60.80268859863281,
"logps/ref_rejected": -79.07284545898438,
"logps/rejected": -113.46876525878906,
"loss": 1.3353,
"margin_dpo/margin_mean": 7.977849006652832,
"margin_dpo/margin_std": 25.233558654785156,
"step": 144
},
{
"KL/chosen_KL_mean": -27.356273651123047,
"KL/mean": -31.281274795532227,
"KL/rejected_KL_mean": -35.206275939941406,
"KL/std": 23.064613342285156,
"epoch": 0.21919879062736206,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.850009918212891,
"fcm_dpo/q_t": 0.48525863885879517,
"grad_norm": 3.983025074005127,
"learning_rate": 4.795540267200686e-07,
"logits/chosen": 0.4053303003311157,
"logits/rejected": 0.4174395799636841,
"logps/chosen": -101.96773529052734,
"logps/ref_chosen": -74.61146545410156,
"logps/ref_rejected": -83.24461364746094,
"logps/rejected": -118.45088958740234,
"loss": 1.3407,
"margin_dpo/margin_mean": 7.850009441375732,
"margin_dpo/margin_std": 30.250947952270508,
"step": 145
},
{
"KL/chosen_KL_mean": -26.02267837524414,
"KL/mean": -29.3727970123291,
"KL/rejected_KL_mean": -32.72291946411133,
"KL/std": 19.601165771484375,
"epoch": 0.22071050642479215,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.700242519378662,
"fcm_dpo/q_t": 0.4872305989265442,
"grad_norm": 3.2747786045074463,
"learning_rate": 4.790271143580173e-07,
"logits/chosen": 0.4032752215862274,
"logits/rejected": 0.39034542441368103,
"logps/chosen": -83.86366271972656,
"logps/ref_chosen": -57.84098434448242,
"logps/ref_rejected": -67.47422790527344,
"logps/rejected": -100.1971435546875,
"loss": 1.3418,
"margin_dpo/margin_mean": 6.70024299621582,
"margin_dpo/margin_std": 20.67666244506836,
"step": 146
},
{
"KL/chosen_KL_mean": -29.016862869262695,
"KL/mean": -32.25214385986328,
"KL/rejected_KL_mean": -35.48743438720703,
"KL/std": 21.1727237701416,
"epoch": 0.2222222222222222,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.470568656921387,
"fcm_dpo/q_t": 0.4874805808067322,
"grad_norm": 3.599782943725586,
"learning_rate": 4.784937956152489e-07,
"logits/chosen": 0.414762020111084,
"logits/rejected": 0.3716977834701538,
"logps/chosen": -95.83033752441406,
"logps/ref_chosen": -66.81346893310547,
"logps/ref_rejected": -81.1796875,
"logps/rejected": -116.66712951660156,
"loss": 1.3455,
"margin_dpo/margin_mean": 6.470569610595703,
"margin_dpo/margin_std": 23.536331176757812,
"step": 147
},
{
"KL/chosen_KL_mean": -24.12586212158203,
"KL/mean": -30.170164108276367,
"KL/rejected_KL_mean": -36.21445846557617,
"KL/std": 20.31801414489746,
"epoch": 0.2237339380196523,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.088604927062988,
"fcm_dpo/q_t": 0.4769914448261261,
"grad_norm": 3.680696487426758,
"learning_rate": 4.779540854098347e-07,
"logits/chosen": 0.5683670043945312,
"logits/rejected": 0.49711206555366516,
"logps/chosen": -72.81361389160156,
"logps/ref_chosen": -48.6877555847168,
"logps/ref_rejected": -67.50503540039062,
"logps/rejected": -103.71949768066406,
"loss": 1.3031,
"margin_dpo/margin_mean": 12.088605880737305,
"margin_dpo/margin_std": 22.556541442871094,
"step": 148
},
{
"KL/chosen_KL_mean": -29.621259689331055,
"KL/mean": -35.046180725097656,
"KL/rejected_KL_mean": -40.471099853515625,
"KL/std": 22.03968620300293,
"epoch": 0.2252456538170824,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.849838256835938,
"fcm_dpo/q_t": 0.47930651903152466,
"grad_norm": 3.6473169326782227,
"learning_rate": 4.774079988386296e-07,
"logits/chosen": 0.46499621868133545,
"logits/rejected": 0.4208984971046448,
"logps/chosen": -84.7650375366211,
"logps/ref_chosen": -55.143775939941406,
"logps/ref_rejected": -64.79888916015625,
"logps/rejected": -105.26998901367188,
"loss": 1.3114,
"margin_dpo/margin_mean": 10.849838256835938,
"margin_dpo/margin_std": 21.40958595275879,
"step": 149
},
{
"KL/chosen_KL_mean": -29.565271377563477,
"KL/mean": -36.95281219482422,
"KL/rejected_KL_mean": -44.34034729003906,
"KL/std": 23.726795196533203,
"epoch": 0.22675736961451248,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 14.775071144104004,
"fcm_dpo/q_t": 0.47204089164733887,
"grad_norm": 3.51343035697937,
"learning_rate": 4.768555511768486e-07,
"logits/chosen": 0.4633652865886688,
"logits/rejected": 0.4186779260635376,
"logps/chosen": -97.03601837158203,
"logps/ref_chosen": -67.47074890136719,
"logps/ref_rejected": -89.21170806884766,
"logps/rejected": -133.55206298828125,
"loss": 1.2891,
"margin_dpo/margin_mean": 14.775071144104004,
"margin_dpo/margin_std": 29.063199996948242,
"step": 150
},
{
"KL/chosen_KL_mean": -21.63004493713379,
"KL/mean": -31.206939697265625,
"KL/rejected_KL_mean": -40.78384017944336,
"KL/std": 26.35299301147461,
"epoch": 0.22826908541194255,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 19.153793334960938,
"fcm_dpo/q_t": 0.46416527032852173,
"grad_norm": 4.2144775390625,
"learning_rate": 4.762967578776406e-07,
"logits/chosen": 0.4840549826622009,
"logits/rejected": 0.4279939532279968,
"logps/chosen": -74.08958435058594,
"logps/ref_chosen": -52.45954132080078,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -119.84684753417969,
"loss": 1.2569,
"margin_dpo/margin_mean": 19.153793334960938,
"margin_dpo/margin_std": 28.657949447631836,
"step": 151
},
{
"KL/chosen_KL_mean": -28.358936309814453,
"KL/mean": -34.87946701049805,
"KL/rejected_KL_mean": -41.39999771118164,
"KL/std": 23.711170196533203,
"epoch": 0.22978080120937264,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 13.041062355041504,
"fcm_dpo/q_t": 0.47520697116851807,
"grad_norm": 3.737586498260498,
"learning_rate": 4.757316345716553e-07,
"logits/chosen": 0.5636520981788635,
"logits/rejected": 0.5086890459060669,
"logps/chosen": -84.91276550292969,
"logps/ref_chosen": -56.5538330078125,
"logps/ref_rejected": -76.55074310302734,
"logps/rejected": -117.95074462890625,
"loss": 1.3005,
"margin_dpo/margin_mean": 13.041061401367188,
"margin_dpo/margin_std": 28.46988868713379,
"step": 152
},
{
"KL/chosen_KL_mean": -30.26366424560547,
"KL/mean": -36.74250030517578,
"KL/rejected_KL_mean": -43.22133255004883,
"KL/std": 24.817947387695312,
"epoch": 0.23129251700680273,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.957663536071777,
"fcm_dpo/q_t": 0.4754427969455719,
"grad_norm": 3.9588053226470947,
"learning_rate": 4.751601970666064e-07,
"logits/chosen": 0.48824167251586914,
"logits/rejected": 0.4558285176753998,
"logps/chosen": -98.27055358886719,
"logps/ref_chosen": -68.00689697265625,
"logps/ref_rejected": -74.83482360839844,
"logps/rejected": -118.05615234375,
"loss": 1.2993,
"margin_dpo/margin_mean": 12.957662582397461,
"margin_dpo/margin_std": 25.99228286743164,
"step": 153
},
{
"KL/chosen_KL_mean": -29.722576141357422,
"KL/mean": -34.363807678222656,
"KL/rejected_KL_mean": -39.00503921508789,
"KL/std": 29.986373901367188,
"epoch": 0.2328042328042328,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.282459259033203,
"fcm_dpo/q_t": 0.4823741316795349,
"grad_norm": 3.928870677947998,
"learning_rate": 4.745824613468292e-07,
"logits/chosen": 0.5707046389579773,
"logits/rejected": 0.5677356719970703,
"logps/chosen": -88.94511413574219,
"logps/ref_chosen": -59.222537994384766,
"logps/ref_rejected": -64.19131469726562,
"logps/rejected": -103.19635772705078,
"loss": 1.3314,
"margin_dpo/margin_mean": 9.282459259033203,
"margin_dpo/margin_std": 31.800521850585938,
"step": 154
},
{
"KL/chosen_KL_mean": -33.82355499267578,
"KL/mean": -41.13679122924805,
"KL/rejected_KL_mean": -48.45003128051758,
"KL/std": 29.099594116210938,
"epoch": 0.23431594860166288,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 14.62647819519043,
"fcm_dpo/q_t": 0.4723411202430725,
"grad_norm": 4.301799297332764,
"learning_rate": 4.7399844357283393e-07,
"logits/chosen": 0.580405592918396,
"logits/rejected": 0.5614101886749268,
"logps/chosen": -102.27825164794922,
"logps/ref_chosen": -68.45469665527344,
"logps/ref_rejected": -77.91763305664062,
"logps/rejected": -126.36766815185547,
"loss": 1.296,
"margin_dpo/margin_mean": 14.626476287841797,
"margin_dpo/margin_std": 35.91053009033203,
"step": 155
},
{
"KL/chosen_KL_mean": -34.29534149169922,
"KL/mean": -44.44483184814453,
"KL/rejected_KL_mean": -54.594322204589844,
"KL/std": 32.78551483154297,
"epoch": 0.23582766439909297,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 20.29897689819336,
"fcm_dpo/q_t": 0.4618450999259949,
"grad_norm": 4.438568115234375,
"learning_rate": 4.7340816008085305e-07,
"logits/chosen": 0.5352226495742798,
"logits/rejected": 0.49391427636146545,
"logps/chosen": -101.56494140625,
"logps/ref_chosen": -67.26959991455078,
"logps/ref_rejected": -86.95914459228516,
"logps/rejected": -141.553466796875,
"loss": 1.2513,
"margin_dpo/margin_mean": 20.29897689819336,
"margin_dpo/margin_std": 31.937244415283203,
"step": 156
},
{
"KL/chosen_KL_mean": -28.4747371673584,
"KL/mean": -34.87397003173828,
"KL/rejected_KL_mean": -41.2732048034668,
"KL/std": 27.206077575683594,
"epoch": 0.23733938019652306,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.798463821411133,
"fcm_dpo/q_t": 0.4756951630115509,
"grad_norm": 4.1154913902282715,
"learning_rate": 4.728116273823847e-07,
"logits/chosen": 0.5524776577949524,
"logits/rejected": 0.5371452569961548,
"logps/chosen": -83.24761199951172,
"logps/ref_chosen": -54.77287292480469,
"logps/ref_rejected": -63.87866973876953,
"logps/rejected": -105.15187072753906,
"loss": 1.3008,
"margin_dpo/margin_mean": 12.798463821411133,
"margin_dpo/margin_std": 25.809242248535156,
"step": 157
},
{
"KL/chosen_KL_mean": -34.29819107055664,
"KL/mean": -42.93422317504883,
"KL/rejected_KL_mean": -51.570255279541016,
"KL/std": 35.774105072021484,
"epoch": 0.23885109599395313,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 17.272056579589844,
"fcm_dpo/q_t": 0.46861717104911804,
"grad_norm": 4.501484394073486,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": 0.5918360948562622,
"logits/rejected": 0.5534389019012451,
"logps/chosen": -99.22090911865234,
"logps/ref_chosen": -64.92271423339844,
"logps/ref_rejected": -82.23789978027344,
"logps/rejected": -133.8081512451172,
"loss": 1.286,
"margin_dpo/margin_mean": 17.272056579589844,
"margin_dpo/margin_std": 43.68254089355469,
"step": 158
},
{
"KL/chosen_KL_mean": -33.22077178955078,
"KL/mean": -41.89005661010742,
"KL/rejected_KL_mean": -50.559349060058594,
"KL/std": 29.38928985595703,
"epoch": 0.24036281179138322,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 17.338571548461914,
"fcm_dpo/q_t": 0.46729815006256104,
"grad_norm": 4.624864101409912,
"learning_rate": 4.715998812855304e-07,
"logits/chosen": 0.5776375532150269,
"logits/rejected": 0.5369961857795715,
"logps/chosen": -90.26776123046875,
"logps/ref_chosen": -57.046993255615234,
"logps/ref_rejected": -73.32441711425781,
"logps/rejected": -123.8837661743164,
"loss": 1.2746,
"margin_dpo/margin_mean": 17.338571548461914,
"margin_dpo/margin_std": 34.329891204833984,
"step": 159
},
{
"KL/chosen_KL_mean": -34.37767791748047,
"KL/mean": -43.98714065551758,
"KL/rejected_KL_mean": -53.59660339355469,
"KL/std": 33.58818054199219,
"epoch": 0.2418745275888133,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 19.21892547607422,
"fcm_dpo/q_t": 0.46424031257629395,
"grad_norm": 5.148886203765869,
"learning_rate": 4.7098470178228755e-07,
"logits/chosen": 0.5072331428527832,
"logits/rejected": 0.4576966464519501,
"logps/chosen": -84.1845932006836,
"logps/ref_chosen": -49.806915283203125,
"logps/ref_rejected": -68.3370132446289,
"logps/rejected": -121.9336166381836,
"loss": 1.265,
"margin_dpo/margin_mean": 19.21892547607422,
"margin_dpo/margin_std": 37.81275177001953,
"step": 160
},
{
"KL/chosen_KL_mean": -35.280982971191406,
"KL/mean": -45.04498291015625,
"KL/rejected_KL_mean": -54.808990478515625,
"KL/std": 29.070030212402344,
"epoch": 0.24338624338624337,
"fcm_dpo/beta": 0.007687800098210573,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 19.527999877929688,
"fcm_dpo/q_t": 0.46312597393989563,
"grad_norm": 5.976346015930176,
"learning_rate": 4.703633408618955e-07,
"logits/chosen": 0.5991806983947754,
"logits/rejected": 0.5647536516189575,
"logps/chosen": -87.7814712524414,
"logps/ref_chosen": -52.50048828125,
"logps/ref_rejected": -66.04540252685547,
"logps/rejected": -120.8543930053711,
"loss": 1.2569,
"margin_dpo/margin_mean": 19.52800178527832,
"margin_dpo/margin_std": 31.474956512451172,
"step": 161
},
{
"KL/chosen_KL_mean": -45.48345947265625,
"KL/mean": -58.5546989440918,
"KL/rejected_KL_mean": -71.62594604492188,
"KL/std": 45.28847885131836,
"epoch": 0.24489795918367346,
"fcm_dpo/beta": 0.019494613632559776,
"fcm_dpo/delta": 0.08775182068347931,
"fcm_dpo/margin": 26.142498016357422,
"fcm_dpo/q_t": 0.45138120651245117,
"grad_norm": 21.5263614654541,
"learning_rate": 4.697358159051549e-07,
"logits/chosen": 0.6398231983184814,
"logits/rejected": 0.5949274301528931,
"logps/chosen": -114.95265197753906,
"logps/ref_chosen": -69.46919250488281,
"logps/ref_rejected": -92.00952911376953,
"logps/rejected": -163.63546752929688,
"loss": 1.1634,
"margin_dpo/margin_mean": 26.142494201660156,
"margin_dpo/margin_std": 50.92291259765625,
"step": 162
},
{
"KL/chosen_KL_mean": -32.73987579345703,
"KL/mean": -46.96985626220703,
"KL/rejected_KL_mean": -61.1998405456543,
"KL/std": 35.424747467041016,
"epoch": 0.24640967498110355,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": -0.12022696435451508,
"fcm_dpo/margin": 28.459957122802734,
"fcm_dpo/q_t": 0.4269000291824341,
"grad_norm": 3.908928155899048,
"learning_rate": 4.691021444652876e-07,
"logits/chosen": 0.6149941682815552,
"logits/rejected": 0.5667312145233154,
"logps/chosen": -83.35371398925781,
"logps/ref_chosen": -50.613834381103516,
"logps/ref_rejected": -74.62033081054688,
"logps/rejected": -135.82015991210938,
"loss": 1.2688,
"margin_dpo/margin_mean": 28.459957122802734,
"margin_dpo/margin_std": 38.75524139404297,
"step": 163
},
{
"KL/chosen_KL_mean": -39.49297332763672,
"KL/mean": -53.75981140136719,
"KL/rejected_KL_mean": -68.02665710449219,
"KL/std": 41.714744567871094,
"epoch": 0.24792139077853365,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 28.533679962158203,
"fcm_dpo/q_t": 0.46781498193740845,
"grad_norm": 4.191322326660156,
"learning_rate": 4.6846234426744624e-07,
"logits/chosen": 0.5729007720947266,
"logits/rejected": 0.5150625705718994,
"logps/chosen": -94.34107971191406,
"logps/ref_chosen": -54.848114013671875,
"logps/ref_rejected": -79.0630111694336,
"logps/rejected": -147.08966064453125,
"loss": 1.2704,
"margin_dpo/margin_mean": 28.53368377685547,
"margin_dpo/margin_std": 42.69341278076172,
"step": 164
},
{
"KL/chosen_KL_mean": -45.441650390625,
"KL/mean": -58.43135070800781,
"KL/rejected_KL_mean": -71.42105102539062,
"KL/std": 41.49935531616211,
"epoch": 0.2494331065759637,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 25.979393005371094,
"fcm_dpo/q_t": 0.4706171154975891,
"grad_norm": 4.249202251434326,
"learning_rate": 4.678164332082175e-07,
"logits/chosen": 0.6801586151123047,
"logits/rejected": 0.6297128200531006,
"logps/chosen": -96.5308609008789,
"logps/ref_chosen": -51.089210510253906,
"logps/ref_rejected": -71.23370361328125,
"logps/rejected": -142.65475463867188,
"loss": 1.2815,
"margin_dpo/margin_mean": 25.979393005371094,
"margin_dpo/margin_std": 44.35696029663086,
"step": 165
},
{
"KL/chosen_KL_mean": -49.10633850097656,
"KL/mean": -61.71703338623047,
"KL/rejected_KL_mean": -74.3277359008789,
"KL/std": 45.09733581542969,
"epoch": 0.2509448223733938,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 25.221389770507812,
"fcm_dpo/q_t": 0.47154539823532104,
"grad_norm": 4.738693714141846,
"learning_rate": 4.6716442935512214e-07,
"logits/chosen": 0.6274275779724121,
"logits/rejected": 0.5359787940979004,
"logps/chosen": -112.29714965820312,
"logps/ref_chosen": -63.19081115722656,
"logps/ref_rejected": -93.8402099609375,
"logps/rejected": -168.16793823242188,
"loss": 1.2842,
"margin_dpo/margin_mean": 25.221389770507812,
"margin_dpo/margin_std": 43.412750244140625,
"step": 166
},
{
"KL/chosen_KL_mean": -42.655426025390625,
"KL/mean": -56.4607048034668,
"KL/rejected_KL_mean": -70.26599884033203,
"KL/std": 39.476051330566406,
"epoch": 0.25245653817082386,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 27.610572814941406,
"fcm_dpo/q_t": 0.4687613248825073,
"grad_norm": 5.55941915512085,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": 0.6002979278564453,
"logits/rejected": 0.5730512142181396,
"logps/chosen": -101.57969665527344,
"logps/ref_chosen": -58.92427062988281,
"logps/ref_rejected": -72.97377014160156,
"logps/rejected": -143.23977661132812,
"loss": 1.2729,
"margin_dpo/margin_mean": 27.610572814941406,
"margin_dpo/margin_std": 41.076995849609375,
"step": 167
},
{
"KL/chosen_KL_mean": -52.67481231689453,
"KL/mean": -66.61177062988281,
"KL/rejected_KL_mean": -80.54873657226562,
"KL/std": 47.3160400390625,
"epoch": 0.25396825396825395,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 27.873924255371094,
"fcm_dpo/q_t": 0.4689233899116516,
"grad_norm": 4.620728492736816,
"learning_rate": 4.6584221638904767e-07,
"logits/chosen": 0.6193612813949585,
"logits/rejected": 0.5847088098526001,
"logps/chosen": -118.3261947631836,
"logps/ref_chosen": -65.65138244628906,
"logps/ref_rejected": -79.71418762207031,
"logps/rejected": -160.26292419433594,
"loss": 1.2779,
"margin_dpo/margin_mean": 27.873924255371094,
"margin_dpo/margin_std": 53.34053039550781,
"step": 168
},
{
"KL/chosen_KL_mean": -43.973472595214844,
"KL/mean": -56.64049530029297,
"KL/rejected_KL_mean": -69.30752563476562,
"KL/std": 43.73821258544922,
"epoch": 0.25547996976568405,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 25.33405113220215,
"fcm_dpo/q_t": 0.47141051292419434,
"grad_norm": 4.314192295074463,
"learning_rate": 4.651720442612075e-07,
"logits/chosen": 0.7225455641746521,
"logits/rejected": 0.6896216869354248,
"logps/chosen": -105.39933776855469,
"logps/ref_chosen": -61.425865173339844,
"logps/ref_rejected": -76.09590148925781,
"logps/rejected": -145.40342712402344,
"loss": 1.2896,
"margin_dpo/margin_mean": 25.33405113220215,
"margin_dpo/margin_std": 54.63517379760742,
"step": 169
},
{
"KL/chosen_KL_mean": -39.26036834716797,
"KL/mean": -53.16771697998047,
"KL/rejected_KL_mean": -67.07505798339844,
"KL/std": 41.21588134765625,
"epoch": 0.25699168556311414,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 27.814695358276367,
"fcm_dpo/q_t": 0.46871745586395264,
"grad_norm": 5.0231099128723145,
"learning_rate": 4.6449585330874425e-07,
"logits/chosen": 0.689607560634613,
"logits/rejected": 0.6932332515716553,
"logps/chosen": -95.91355895996094,
"logps/ref_chosen": -56.65319061279297,
"logps/ref_rejected": -63.45965576171875,
"logps/rejected": -130.5347137451172,
"loss": 1.2806,
"margin_dpo/margin_mean": 27.814697265625,
"margin_dpo/margin_std": 54.60728454589844,
"step": 170
},
{
"KL/chosen_KL_mean": -52.41896438598633,
"KL/mean": -69.64653778076172,
"KL/rejected_KL_mean": -86.87411499023438,
"KL/std": 57.269317626953125,
"epoch": 0.2585034013605442,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 34.45514678955078,
"fcm_dpo/q_t": 0.46149614453315735,
"grad_norm": 4.3705902099609375,
"learning_rate": 4.6381366244617224e-07,
"logits/chosen": 0.767911434173584,
"logits/rejected": 0.7232061624526978,
"logps/chosen": -116.15372467041016,
"logps/ref_chosen": -63.73476028442383,
"logps/ref_rejected": -78.50328063964844,
"logps/rejected": -165.37738037109375,
"loss": 1.2571,
"margin_dpo/margin_mean": 34.45514678955078,
"margin_dpo/margin_std": 64.5282974243164,
"step": 171
},
{
"KL/chosen_KL_mean": -55.21881866455078,
"KL/mean": -71.57858276367188,
"KL/rejected_KL_mean": -87.93833923339844,
"KL/std": 53.955963134765625,
"epoch": 0.2600151171579743,
"fcm_dpo/beta": 0.004572372883558273,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 32.719520568847656,
"fcm_dpo/q_t": 0.4635423421859741,
"grad_norm": 6.561391353607178,
"learning_rate": 4.631254907558365e-07,
"logits/chosen": 0.7488380670547485,
"logits/rejected": 0.686285674571991,
"logps/chosen": -107.42057800292969,
"logps/ref_chosen": -52.201759338378906,
"logps/ref_rejected": -82.85285949707031,
"logps/rejected": -170.79119873046875,
"loss": 1.259,
"margin_dpo/margin_mean": 32.719520568847656,
"margin_dpo/margin_std": 55.974552154541016,
"step": 172
},
{
"KL/chosen_KL_mean": -52.73722839355469,
"KL/mean": -72.90831756591797,
"KL/rejected_KL_mean": -93.07940673828125,
"KL/std": 59.073570251464844,
"epoch": 0.2615268329554044,
"fcm_dpo/beta": 0.014145536348223686,
"fcm_dpo/delta": 0.06264997273683548,
"fcm_dpo/margin": 40.342193603515625,
"fcm_dpo/q_t": 0.4352329969406128,
"grad_norm": 13.129067420959473,
"learning_rate": 4.624313574873786e-07,
"logits/chosen": 0.7639884352684021,
"logits/rejected": 0.6851507425308228,
"logps/chosen": -108.17195129394531,
"logps/ref_chosen": -55.434722900390625,
"logps/ref_rejected": -77.81967163085938,
"logps/rejected": -170.89907836914062,
"loss": 1.1037,
"margin_dpo/margin_mean": 40.342193603515625,
"margin_dpo/margin_std": 72.60908508300781,
"step": 173
},
{
"KL/chosen_KL_mean": -61.133522033691406,
"KL/mean": -83.47229766845703,
"KL/rejected_KL_mean": -105.81106567382812,
"KL/std": 60.41809844970703,
"epoch": 0.26303854875283444,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": -0.1022532507777214,
"fcm_dpo/margin": 44.67753601074219,
"fcm_dpo/q_t": 0.41912031173706055,
"grad_norm": 3.89009428024292,
"learning_rate": 4.61731282057198e-07,
"logits/chosen": 0.7750641107559204,
"logits/rejected": 0.7090386748313904,
"logps/chosen": -118.30548095703125,
"logps/ref_chosen": -57.17195129394531,
"logps/ref_rejected": -85.47578430175781,
"logps/rejected": -191.28684997558594,
"loss": 1.2858,
"margin_dpo/margin_mean": 44.67753601074219,
"margin_dpo/margin_std": 57.747344970703125,
"step": 174
},
{
"KL/chosen_KL_mean": -57.48314666748047,
"KL/mean": -79.11009216308594,
"KL/rejected_KL_mean": -100.73703002929688,
"KL/std": 61.092506408691406,
"epoch": 0.26455026455026454,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 43.253883361816406,
"fcm_dpo/q_t": 0.47405892610549927,
"grad_norm": 3.5935990810394287,
"learning_rate": 4.6102528404790965e-07,
"logits/chosen": 0.799199104309082,
"logits/rejected": 0.7752283811569214,
"logps/chosen": -125.14877319335938,
"logps/ref_chosen": -67.6656265258789,
"logps/ref_rejected": -84.36766815185547,
"logps/rejected": -185.10470581054688,
"loss": 1.2931,
"margin_dpo/margin_mean": 43.253883361816406,
"margin_dpo/margin_std": 77.93309020996094,
"step": 175
},
{
"KL/chosen_KL_mean": -75.63528442382812,
"KL/mean": -89.86036682128906,
"KL/rejected_KL_mean": -104.08544158935547,
"KL/std": 67.660888671875,
"epoch": 0.2660619803476946,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 28.450157165527344,
"fcm_dpo/q_t": 0.4830489456653595,
"grad_norm": 4.446000576019287,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": 0.7807189226150513,
"logits/rejected": 0.762683093547821,
"logps/chosen": -153.49404907226562,
"logps/ref_chosen": -77.8587646484375,
"logps/ref_rejected": -81.08732604980469,
"logps/rejected": -185.17276000976562,
"loss": 1.3297,
"margin_dpo/margin_mean": 28.450157165527344,
"margin_dpo/margin_std": 85.36820220947266,
"step": 176
},
{
"KL/chosen_KL_mean": -56.47824478149414,
"KL/mean": -86.89440155029297,
"KL/rejected_KL_mean": -117.310546875,
"KL/std": 66.56551361083984,
"epoch": 0.2675736961451247,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 60.83230209350586,
"fcm_dpo/q_t": 0.4634092152118683,
"grad_norm": 4.204408645629883,
"learning_rate": 4.5959559945025183e-07,
"logits/chosen": 0.8780996799468994,
"logits/rejected": 0.7931272983551025,
"logps/chosen": -111.69863891601562,
"logps/ref_chosen": -55.22039794921875,
"logps/ref_rejected": -92.54973602294922,
"logps/rejected": -209.86029052734375,
"loss": 1.2499,
"margin_dpo/margin_mean": 60.832298278808594,
"margin_dpo/margin_std": 62.22540283203125,
"step": 177
},
{
"KL/chosen_KL_mean": -71.03286743164062,
"KL/mean": -90.96412658691406,
"KL/rejected_KL_mean": -110.89537048339844,
"KL/std": 68.37006378173828,
"epoch": 0.2690854119425548,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 39.86250305175781,
"fcm_dpo/q_t": 0.47601377964019775,
"grad_norm": 5.275820732116699,
"learning_rate": 4.588719528532341e-07,
"logits/chosen": 0.7893873453140259,
"logits/rejected": 0.7460411787033081,
"logps/chosen": -131.84335327148438,
"logps/ref_chosen": -60.81049346923828,
"logps/ref_rejected": -81.12973022460938,
"logps/rejected": -192.0251007080078,
"loss": 1.2979,
"margin_dpo/margin_mean": 39.86250686645508,
"margin_dpo/margin_std": 62.40123748779297,
"step": 178
},
{
"KL/chosen_KL_mean": -76.19467163085938,
"KL/mean": -97.18215942382812,
"KL/rejected_KL_mean": -118.16964721679688,
"KL/std": 71.81806945800781,
"epoch": 0.2705971277399849,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 41.9749755859375,
"fcm_dpo/q_t": 0.474868506193161,
"grad_norm": 3.9109606742858887,
"learning_rate": 4.581424636586928e-07,
"logits/chosen": 0.8630691170692444,
"logits/rejected": 0.8520238995552063,
"logps/chosen": -141.86639404296875,
"logps/ref_chosen": -65.67171478271484,
"logps/ref_rejected": -75.32586669921875,
"logps/rejected": -193.49551391601562,
"loss": 1.2974,
"margin_dpo/margin_mean": 41.9749755859375,
"margin_dpo/margin_std": 83.55865478515625,
"step": 179
},
{
"KL/chosen_KL_mean": -63.65786361694336,
"KL/mean": -79.41153717041016,
"KL/rejected_KL_mean": -95.16521453857422,
"KL/std": 63.367218017578125,
"epoch": 0.272108843537415,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 31.507352828979492,
"fcm_dpo/q_t": 0.4810509979724884,
"grad_norm": 3.097667932510376,
"learning_rate": 4.5740715227200897e-07,
"logits/chosen": 0.7191129922866821,
"logits/rejected": 0.7070032358169556,
"logps/chosen": -120.34066772460938,
"logps/ref_chosen": -56.68280792236328,
"logps/ref_rejected": -64.94414520263672,
"logps/rejected": -160.10935974121094,
"loss": 1.3188,
"margin_dpo/margin_mean": 31.507354736328125,
"margin_dpo/margin_std": 71.05530548095703,
"step": 180
},
{
"KL/chosen_KL_mean": -61.297054290771484,
"KL/mean": -88.07896423339844,
"KL/rejected_KL_mean": -114.86087036132812,
"KL/std": 77.27511596679688,
"epoch": 0.273620559334845,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 53.56382751464844,
"fcm_dpo/q_t": 0.4678885340690613,
"grad_norm": 4.882580280303955,
"learning_rate": 4.566660392614228e-07,
"logits/chosen": 0.89243084192276,
"logits/rejected": 0.8556175827980042,
"logps/chosen": -122.07310485839844,
"logps/ref_chosen": -60.77604675292969,
"logps/ref_rejected": -83.98361206054688,
"logps/rejected": -198.844482421875,
"loss": 1.2695,
"margin_dpo/margin_mean": 53.56382369995117,
"margin_dpo/margin_std": 77.41310119628906,
"step": 181
},
{
"KL/chosen_KL_mean": -78.02749633789062,
"KL/mean": -106.99266815185547,
"KL/rejected_KL_mean": -135.95785522460938,
"KL/std": 78.79426574707031,
"epoch": 0.2751322751322751,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 57.93036651611328,
"fcm_dpo/q_t": 0.4655247628688812,
"grad_norm": 5.966950416564941,
"learning_rate": 4.5591914535745817e-07,
"logits/chosen": 0.8725330829620361,
"logits/rejected": 0.7979344129562378,
"logps/chosen": -138.28128051757812,
"logps/ref_chosen": -60.2537841796875,
"logps/ref_rejected": -89.7706298828125,
"logps/rejected": -225.72848510742188,
"loss": 1.2631,
"margin_dpo/margin_mean": 57.93036651611328,
"margin_dpo/margin_std": 90.61637115478516,
"step": 182
},
{
"KL/chosen_KL_mean": -96.92032623291016,
"KL/mean": -109.78304290771484,
"KL/rejected_KL_mean": -122.645751953125,
"KL/std": 79.40963745117188,
"epoch": 0.2766439909297052,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 25.725440979003906,
"fcm_dpo/q_t": 0.48464155197143555,
"grad_norm": 4.779506683349609,
"learning_rate": 4.551664914523433e-07,
"logits/chosen": 0.8302876949310303,
"logits/rejected": 0.8160231113433838,
"logps/chosen": -158.68174743652344,
"logps/ref_chosen": -61.76142120361328,
"logps/ref_rejected": -72.54627990722656,
"logps/rejected": -195.19204711914062,
"loss": 1.3378,
"margin_dpo/margin_mean": 25.725440979003906,
"margin_dpo/margin_std": 93.7728271484375,
"step": 183
},
{
"KL/chosen_KL_mean": -62.56498718261719,
"KL/mean": -87.00849151611328,
"KL/rejected_KL_mean": -111.45199584960938,
"KL/std": 66.37240600585938,
"epoch": 0.2781557067271353,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 48.887001037597656,
"fcm_dpo/q_t": 0.4707995057106018,
"grad_norm": 6.9039998054504395,
"learning_rate": 4.544080985994258e-07,
"logits/chosen": 0.9358631372451782,
"logits/rejected": 0.8840564489364624,
"logps/chosen": -109.40570831298828,
"logps/ref_chosen": -46.840721130371094,
"logps/ref_rejected": -69.3609390258789,
"logps/rejected": -180.81292724609375,
"loss": 1.2804,
"margin_dpo/margin_mean": 48.887001037597656,
"margin_dpo/margin_std": 79.2603759765625,
"step": 184
},
{
"KL/chosen_KL_mean": -80.97999572753906,
"KL/mean": -106.63510131835938,
"KL/rejected_KL_mean": -132.29022216796875,
"KL/std": 85.46723937988281,
"epoch": 0.2796674225245654,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 51.310218811035156,
"fcm_dpo/q_t": 0.46929919719696045,
"grad_norm": 6.995913028717041,
"learning_rate": 4.5364398801258394e-07,
"logits/chosen": 0.9254462122917175,
"logits/rejected": 0.8940056562423706,
"logps/chosen": -133.3011474609375,
"logps/ref_chosen": -52.32114028930664,
"logps/ref_rejected": -68.3885726928711,
"logps/rejected": -200.6787872314453,
"loss": 1.2813,
"margin_dpo/margin_mean": 51.31022262573242,
"margin_dpo/margin_std": 100.49153137207031,
"step": 185
},
{
"KL/chosen_KL_mean": -84.80559539794922,
"KL/mean": -113.83717346191406,
"KL/rejected_KL_mean": -142.86874389648438,
"KL/std": 107.6434555053711,
"epoch": 0.2811791383219955,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 58.063148498535156,
"fcm_dpo/q_t": 0.46672147512435913,
"grad_norm": 6.379158973693848,
"learning_rate": 4.5287418106563354e-07,
"logits/chosen": 0.8682913184165955,
"logits/rejected": 0.8399237394332886,
"logps/chosen": -152.22572326660156,
"logps/ref_chosen": -67.42012786865234,
"logps/ref_rejected": -82.50968933105469,
"logps/rejected": -225.37843322753906,
"loss": 1.2795,
"margin_dpo/margin_mean": 58.063148498535156,
"margin_dpo/margin_std": 143.8511962890625,
"step": 186
},
{
"KL/chosen_KL_mean": -106.72709655761719,
"KL/mean": -132.92059326171875,
"KL/rejected_KL_mean": -159.11410522460938,
"KL/std": 99.10030364990234,
"epoch": 0.28269085411942557,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 52.38701248168945,
"fcm_dpo/q_t": 0.46877333521842957,
"grad_norm": 5.65391206741333,
"learning_rate": 4.520986992917297e-07,
"logits/chosen": 0.8852934241294861,
"logits/rejected": 0.8451642990112305,
"logps/chosen": -182.25259399414062,
"logps/ref_chosen": -75.52549743652344,
"logps/ref_rejected": -94.76289367675781,
"logps/rejected": -253.8769989013672,
"loss": 1.2815,
"margin_dpo/margin_mean": 52.38701629638672,
"margin_dpo/margin_std": 109.4050521850586,
"step": 187
},
{
"KL/chosen_KL_mean": -94.13689422607422,
"KL/mean": -118.1536865234375,
"KL/rejected_KL_mean": -142.17047119140625,
"KL/std": 93.5236587524414,
"epoch": 0.2842025699168556,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 48.0335807800293,
"fcm_dpo/q_t": 0.4715660512447357,
"grad_norm": 5.645846366882324,
"learning_rate": 4.5131756438276466e-07,
"logits/chosen": 0.9208471775054932,
"logits/rejected": 0.8986366987228394,
"logps/chosen": -165.66021728515625,
"logps/ref_chosen": -71.52333068847656,
"logps/ref_rejected": -78.29949951171875,
"logps/rejected": -220.469970703125,
"loss": 1.2893,
"margin_dpo/margin_mean": 48.03358459472656,
"margin_dpo/margin_std": 105.69105529785156,
"step": 188
},
{
"KL/chosen_KL_mean": -113.9881591796875,
"KL/mean": -126.44110107421875,
"KL/rejected_KL_mean": -138.89404296875,
"KL/std": 106.01944732666016,
"epoch": 0.2857142857142857,
"fcm_dpo/beta": 0.0024263300001621246,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 24.9058895111084,
"fcm_dpo/q_t": 0.48495352268218994,
"grad_norm": 7.330818176269531,
"learning_rate": 4.5053079818876096e-07,
"logits/chosen": 0.984262228012085,
"logits/rejected": 1.0013400316238403,
"logps/chosen": -186.1644287109375,
"logps/ref_chosen": -72.17626953125,
"logps/ref_rejected": -75.26313781738281,
"logps/rejected": -214.15716552734375,
"loss": 1.3461,
"margin_dpo/margin_mean": 24.90589141845703,
"margin_dpo/margin_std": 109.00585174560547,
"step": 189
},
{
"KL/chosen_KL_mean": -87.60731506347656,
"KL/mean": -137.9809112548828,
"KL/rejected_KL_mean": -188.35450744628906,
"KL/std": 113.34170532226562,
"epoch": 0.2872260015117158,
"fcm_dpo/beta": 0.00413139583542943,
"fcm_dpo/delta": -0.01652386784553528,
"fcm_dpo/margin": 100.7471923828125,
"fcm_dpo/q_t": 0.4048711657524109,
"grad_norm": 11.910893440246582,
"learning_rate": 4.4973842271726024e-07,
"logits/chosen": 1.0319868326187134,
"logits/rejected": 0.919386625289917,
"logps/chosen": -142.23158264160156,
"logps/ref_chosen": -54.624271392822266,
"logps/ref_rejected": -101.47068786621094,
"logps/rejected": -289.8251953125,
"loss": 1.0797,
"margin_dpo/margin_mean": 100.74720001220703,
"margin_dpo/margin_std": 116.15534973144531,
"step": 190
},
{
"KL/chosen_KL_mean": -128.3540802001953,
"KL/mean": -150.7650146484375,
"KL/rejected_KL_mean": -173.17596435546875,
"KL/std": 121.36798095703125,
"epoch": 0.2887377173091459,
"fcm_dpo/beta": 0.001862645847722888,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 44.8218994140625,
"fcm_dpo/q_t": 0.4793294072151184,
"grad_norm": 7.475344181060791,
"learning_rate": 4.48940460132708e-07,
"logits/chosen": 0.9936637878417969,
"logits/rejected": 0.9684906005859375,
"logps/chosen": -201.28659057617188,
"logps/ref_chosen": -72.93251037597656,
"logps/ref_rejected": -89.95103454589844,
"logps/rejected": -263.12701416015625,
"loss": 1.3288,
"margin_dpo/margin_mean": 44.821895599365234,
"margin_dpo/margin_std": 163.15121459960938,
"step": 191
},
{
"KL/chosen_KL_mean": -115.16537475585938,
"KL/mean": -129.2557830810547,
"KL/rejected_KL_mean": -143.34620666503906,
"KL/std": 98.83819580078125,
"epoch": 0.29024943310657597,
"fcm_dpo/beta": 0.001862645847722888,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 28.18083953857422,
"fcm_dpo/q_t": 0.48696666955947876,
"grad_norm": 3.313401222229004,
"learning_rate": 4.481369327558329e-07,
"logits/chosen": 1.02927827835083,
"logits/rejected": 1.007131576538086,
"logps/chosen": -169.16648864746094,
"logps/ref_chosen": -54.001121520996094,
"logps/ref_rejected": -63.531551361083984,
"logps/rejected": -206.8777618408203,
"loss": 1.3415,
"margin_dpo/margin_mean": 28.180843353271484,
"margin_dpo/margin_std": 89.10528564453125,
"step": 192
},
{
"KL/chosen_KL_mean": -104.46321105957031,
"KL/mean": -128.8253173828125,
"KL/rejected_KL_mean": -153.18743896484375,
"KL/std": 88.49467468261719,
"epoch": 0.29176114890400606,
"fcm_dpo/beta": 0.001862645847722888,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 48.724239349365234,
"fcm_dpo/q_t": 0.47747933864593506,
"grad_norm": 3.7360472679138184,
"learning_rate": 4.47327863063023e-07,
"logits/chosen": 1.0135523080825806,
"logits/rejected": 1.0168956518173218,
"logps/chosen": -161.21249389648438,
"logps/ref_chosen": -56.74927520751953,
"logps/ref_rejected": -58.80629348754883,
"logps/rejected": -211.99374389648438,
"loss": 1.3057,
"margin_dpo/margin_mean": 48.7242431640625,
"margin_dpo/margin_std": 94.30784606933594,
"step": 193
},
{
"KL/chosen_KL_mean": -115.06211853027344,
"KL/mean": -141.86691284179688,
"KL/rejected_KL_mean": -168.67172241210938,
"KL/std": 117.1404800415039,
"epoch": 0.29327286470143615,
"fcm_dpo/beta": 0.001862645847722888,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 53.60961151123047,
"fcm_dpo/q_t": 0.47572097182273865,
"grad_norm": 6.185281276702881,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": 1.052534818649292,
"logits/rejected": 1.035210371017456,
"logps/chosen": -171.7115478515625,
"logps/ref_chosen": -56.64944076538086,
"logps/ref_rejected": -69.98954772949219,
"logps/rejected": -238.66127014160156,
"loss": 1.3049,
"margin_dpo/margin_mean": 53.60961151123047,
"margin_dpo/margin_std": 132.62509155273438,
"step": 194
},
{
"KL/chosen_KL_mean": -144.16168212890625,
"KL/mean": -165.70416259765625,
"KL/rejected_KL_mean": -187.2466278076172,
"KL/std": 127.5001220703125,
"epoch": 0.2947845804988662,
"fcm_dpo/beta": 0.001862645847722888,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 43.0849494934082,
"fcm_dpo/q_t": 0.4802978038787842,
"grad_norm": 5.549439430236816,
"learning_rate": 4.4569318740967043e-07,
"logits/chosen": 0.9337140321731567,
"logits/rejected": 0.9414522051811218,
"logps/chosen": -214.5714569091797,
"logps/ref_chosen": -70.40977478027344,
"logps/ref_rejected": -74.39448547363281,
"logps/rejected": -261.64111328125,
"loss": 1.326,
"margin_dpo/margin_mean": 43.08494567871094,
"margin_dpo/margin_std": 144.31869506835938,
"step": 195
},
{
"KL/chosen_KL_mean": -121.95760345458984,
"KL/mean": -155.3830108642578,
"KL/rejected_KL_mean": -188.80841064453125,
"KL/std": 146.87850952148438,
"epoch": 0.2962962962962963,
"fcm_dpo/beta": 0.001862645847722888,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 66.8508071899414,
"fcm_dpo/q_t": 0.4697552025318146,
"grad_norm": 5.599655628204346,
"learning_rate": 4.448676271745197e-07,
"logits/chosen": 1.0035507678985596,
"logits/rejected": 0.9772269129753113,
"logps/chosen": -181.1851806640625,
"logps/ref_chosen": -59.227577209472656,
"logps/ref_rejected": -83.54757690429688,
"logps/rejected": -272.3559875488281,
"loss": 1.2815,
"margin_dpo/margin_mean": 66.8508071899414,
"margin_dpo/margin_std": 129.74188232421875,
"step": 196
},
{
"KL/chosen_KL_mean": -116.84405517578125,
"KL/mean": -151.54763793945312,
"KL/rejected_KL_mean": -186.251220703125,
"KL/std": 130.40406799316406,
"epoch": 0.29780801209372637,
"fcm_dpo/beta": 0.001862645847722888,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 69.40716552734375,
"fcm_dpo/q_t": 0.4688175320625305,
"grad_norm": 4.895405292510986,
"learning_rate": 4.440366160729392e-07,
"logits/chosen": 1.0863635540008545,
"logits/rejected": 1.0468790531158447,
"logps/chosen": -168.37318420410156,
"logps/ref_chosen": -51.52912902832031,
"logps/ref_rejected": -73.70631408691406,
"logps/rejected": -259.95751953125,
"loss": 1.2843,
"margin_dpo/margin_mean": 69.40716552734375,
"margin_dpo/margin_std": 164.8914794921875,
"step": 197
},
{
"KL/chosen_KL_mean": -125.76316833496094,
"KL/mean": -156.66259765625,
"KL/rejected_KL_mean": -187.56204223632812,
"KL/std": 126.10319519042969,
"epoch": 0.29931972789115646,
"fcm_dpo/beta": 0.001862645847722888,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 61.798851013183594,
"fcm_dpo/q_t": 0.4717727303504944,
"grad_norm": 4.650595664978027,
"learning_rate": 4.432001773500957e-07,
"logits/chosen": 1.1201140880584717,
"logits/rejected": 1.0980479717254639,
"logps/chosen": -185.54583740234375,
"logps/ref_chosen": -59.78268051147461,
"logps/ref_rejected": -72.24533081054688,
"logps/rejected": -259.807373046875,
"loss": 1.2896,
"margin_dpo/margin_mean": 61.79884719848633,
"margin_dpo/margin_std": 131.0926513671875,
"step": 198
},
{
"KL/chosen_KL_mean": -128.4625244140625,
"KL/mean": -160.48162841796875,
"KL/rejected_KL_mean": -192.500732421875,
"KL/std": 118.23385620117188,
"epoch": 0.30083144368858655,
"fcm_dpo/beta": 0.001862645847722888,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 64.0382308959961,
"fcm_dpo/q_t": 0.47144871950149536,
"grad_norm": 5.723056316375732,
"learning_rate": 4.4235833440297856e-07,
"logits/chosen": 1.049532175064087,
"logits/rejected": 0.9984226226806641,
"logps/chosen": -184.84930419921875,
"logps/ref_chosen": -56.38677215576172,
"logps/ref_rejected": -74.56779479980469,
"logps/rejected": -267.06854248046875,
"loss": 1.2908,
"margin_dpo/margin_mean": 64.0382308959961,
"margin_dpo/margin_std": 147.04953002929688,
"step": 199
},
{
"KL/chosen_KL_mean": -128.42446899414062,
"KL/mean": -175.69747924804688,
"KL/rejected_KL_mean": -222.97048950195312,
"KL/std": 161.18472290039062,
"epoch": 0.30234315948601664,
"fcm_dpo/beta": 0.006587632466107607,
"fcm_dpo/delta": 0.0778110921382904,
"fcm_dpo/margin": 94.5460205078125,
"fcm_dpo/q_t": 0.4322563111782074,
"grad_norm": 19.645002365112305,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": 1.102787971496582,
"logits/rejected": 1.0492348670959473,
"logps/chosen": -186.248779296875,
"logps/ref_chosen": -57.82432556152344,
"logps/ref_rejected": -89.28246307373047,
"logps/rejected": -312.2529296875,
"loss": 1.1653,
"margin_dpo/margin_mean": 94.5460205078125,
"margin_dpo/margin_std": 190.74359130859375,
"step": 200
},
{
"KL/chosen_KL_mean": -153.81317138671875,
"KL/mean": -185.98416137695312,
"KL/rejected_KL_mean": -218.15512084960938,
"KL/std": 170.94137573242188,
"epoch": 0.30385487528344673,
"fcm_dpo/beta": 0.0015425452729687095,
"fcm_dpo/delta": -0.08959634602069855,
"fcm_dpo/margin": 64.3419418334961,
"fcm_dpo/q_t": 0.4359002113342285,
"grad_norm": 4.09077787399292,
"learning_rate": 4.4065853017905953e-07,
"logits/chosen": 1.068925380706787,
"logits/rejected": 1.0387152433395386,
"logps/chosen": -212.8129425048828,
"logps/ref_chosen": -58.999759674072266,
"logps/ref_rejected": -84.67575073242188,
"logps/rejected": -302.83087158203125,
"loss": 1.3112,
"margin_dpo/margin_mean": 64.34194946289062,
"margin_dpo/margin_std": 189.60845947265625,
"step": 201
},
{
"KL/chosen_KL_mean": -113.20198822021484,
"KL/mean": -160.37185668945312,
"KL/rejected_KL_mean": -207.541748046875,
"KL/std": 120.42765808105469,
"epoch": 0.30536659108087677,
"fcm_dpo/beta": 0.0015425452729687095,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 94.33975982666016,
"fcm_dpo/q_t": 0.46428173780441284,
"grad_norm": 5.2453508377075195,
"learning_rate": 4.3980061644943575e-07,
"logits/chosen": 1.0624161958694458,
"logits/rejected": 1.0226997137069702,
"logps/chosen": -160.86264038085938,
"logps/ref_chosen": -47.660648345947266,
"logps/ref_rejected": -73.63249969482422,
"logps/rejected": -281.17425537109375,
"loss": 1.2583,
"margin_dpo/margin_mean": 94.33975982666016,
"margin_dpo/margin_std": 137.9876708984375,
"step": 202
},
{
"KL/chosen_KL_mean": -150.9142303466797,
"KL/mean": -200.44468688964844,
"KL/rejected_KL_mean": -249.97515869140625,
"KL/std": 143.50985717773438,
"epoch": 0.30687830687830686,
"fcm_dpo/beta": 0.0015425452729687095,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.0609130859375,
"fcm_dpo/q_t": 0.4625791609287262,
"grad_norm": 6.239743232727051,
"learning_rate": 4.3893739358856455e-07,
"logits/chosen": 1.1303417682647705,
"logits/rejected": 1.0701119899749756,
"logps/chosen": -213.23977661132812,
"logps/ref_chosen": -62.32553482055664,
"logps/ref_rejected": -99.37226104736328,
"logps/rejected": -349.347412109375,
"loss": 1.2536,
"margin_dpo/margin_mean": 99.0609130859375,
"margin_dpo/margin_std": 154.81906127929688,
"step": 203
},
{
"KL/chosen_KL_mean": -129.41946411132812,
"KL/mean": -163.46804809570312,
"KL/rejected_KL_mean": -197.5166015625,
"KL/std": 131.462158203125,
"epoch": 0.30839002267573695,
"fcm_dpo/beta": 0.0015425452729687095,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 68.09713745117188,
"fcm_dpo/q_t": 0.4740549921989441,
"grad_norm": 4.741433143615723,
"learning_rate": 4.380688857426449e-07,
"logits/chosen": 1.1037497520446777,
"logits/rejected": 1.0652885437011719,
"logps/chosen": -180.0487823486328,
"logps/ref_chosen": -50.62931823730469,
"logps/ref_rejected": -66.60475158691406,
"logps/rejected": -264.121337890625,
"loss": 1.3004,
"margin_dpo/margin_mean": 68.09713745117188,
"margin_dpo/margin_std": 162.0704803466797,
"step": 204
},
{
"KL/chosen_KL_mean": -178.677978515625,
"KL/mean": -224.63375854492188,
"KL/rejected_KL_mean": -270.58953857421875,
"KL/std": 211.23817443847656,
"epoch": 0.30990173847316704,
"fcm_dpo/beta": 0.0015425452729687095,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 91.91157531738281,
"fcm_dpo/q_t": 0.4660814702510834,
"grad_norm": 4.968754291534424,
"learning_rate": 4.3719511720570814e-07,
"logits/chosen": 1.1142351627349854,
"logits/rejected": 1.0800881385803223,
"logps/chosen": -249.03414916992188,
"logps/ref_chosen": -70.3561782836914,
"logps/ref_rejected": -93.39848327636719,
"logps/rejected": -363.988037109375,
"loss": 1.2784,
"margin_dpo/margin_mean": 91.91158294677734,
"margin_dpo/margin_std": 221.230224609375,
"step": 205
},
{
"KL/chosen_KL_mean": -190.87460327148438,
"KL/mean": -210.524169921875,
"KL/rejected_KL_mean": -230.17376708984375,
"KL/std": 179.7605743408203,
"epoch": 0.31141345427059713,
"fcm_dpo/beta": 0.0015425452729687095,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 39.29918670654297,
"fcm_dpo/q_t": 0.48608922958374023,
"grad_norm": 6.895484447479248,
"learning_rate": 4.363161124189387e-07,
"logits/chosen": 1.153550624847412,
"logits/rejected": 1.1396384239196777,
"logps/chosen": -258.52008056640625,
"logps/ref_chosen": -67.64547729492188,
"logps/ref_rejected": -79.89584350585938,
"logps/rejected": -310.0696105957031,
"loss": 1.3557,
"margin_dpo/margin_mean": 39.2991828918457,
"margin_dpo/margin_std": 221.7179412841797,
"step": 206
},
{
"KL/chosen_KL_mean": -192.2279052734375,
"KL/mean": -239.26425170898438,
"KL/rejected_KL_mean": -286.30059814453125,
"KL/std": 181.48193359375,
"epoch": 0.3129251700680272,
"fcm_dpo/beta": 0.0015425452729687095,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 94.07268524169922,
"fcm_dpo/q_t": 0.465423583984375,
"grad_norm": 4.847929954528809,
"learning_rate": 4.3543189596998986e-07,
"logits/chosen": 1.124064326286316,
"logits/rejected": 1.0906260013580322,
"logps/chosen": -259.8921203613281,
"logps/ref_chosen": -67.66419219970703,
"logps/ref_rejected": -85.10249328613281,
"logps/rejected": -371.403076171875,
"loss": 1.2714,
"margin_dpo/margin_mean": 94.07269287109375,
"margin_dpo/margin_std": 202.92820739746094,
"step": 207
},
{
"KL/chosen_KL_mean": -176.33616638183594,
"KL/mean": -196.31283569335938,
"KL/rejected_KL_mean": -216.28952026367188,
"KL/std": 157.74716186523438,
"epoch": 0.3144368858654573,
"fcm_dpo/beta": 0.0015425452729687095,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 39.953346252441406,
"fcm_dpo/q_t": 0.48543232679367065,
"grad_norm": 5.455617427825928,
"learning_rate": 4.3454249259229664e-07,
"logits/chosen": 1.1541500091552734,
"logits/rejected": 1.136106014251709,
"logps/chosen": -234.06787109375,
"logps/ref_chosen": -57.731712341308594,
"logps/ref_rejected": -74.19276428222656,
"logps/rejected": -290.4822998046875,
"loss": 1.3437,
"margin_dpo/margin_mean": 39.95335388183594,
"margin_dpo/margin_std": 167.1756134033203,
"step": 208
},
{
"KL/chosen_KL_mean": -220.13638305664062,
"KL/mean": -263.32781982421875,
"KL/rejected_KL_mean": -306.5192565917969,
"KL/std": 255.47915649414062,
"epoch": 0.31594860166288735,
"fcm_dpo/beta": 0.0015425452729687095,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 86.38284301757812,
"fcm_dpo/q_t": 0.46814075112342834,
"grad_norm": 6.359311580657959,
"learning_rate": 4.336479271643833e-07,
"logits/chosen": 1.1173896789550781,
"logits/rejected": 1.0917067527770996,
"logps/chosen": -288.68646240234375,
"logps/ref_chosen": -68.55007934570312,
"logps/ref_rejected": -87.90541076660156,
"logps/rejected": -394.4246826171875,
"loss": 1.3044,
"margin_dpo/margin_mean": 86.38285064697266,
"margin_dpo/margin_std": 283.98956298828125,
"step": 209
},
{
"KL/chosen_KL_mean": -179.193603515625,
"KL/mean": -246.36642456054688,
"KL/rejected_KL_mean": -313.53924560546875,
"KL/std": 254.68429565429688,
"epoch": 0.31746031746031744,
"fcm_dpo/beta": 0.0019031364936381578,
"fcm_dpo/delta": 0.023973697796463966,
"fcm_dpo/margin": 134.34561157226562,
"fcm_dpo/q_t": 0.45283159613609314,
"grad_norm": 9.294327735900879,
"learning_rate": 4.327482247091679e-07,
"logits/chosen": 1.2122018337249756,
"logits/rejected": 1.1607775688171387,
"logps/chosen": -236.46188354492188,
"logps/ref_chosen": -57.268272399902344,
"logps/ref_rejected": -85.72807312011719,
"logps/rejected": -399.2673034667969,
"loss": 1.198,
"margin_dpo/margin_mean": 134.34561157226562,
"margin_dpo/margin_std": 250.78106689453125,
"step": 210
},
{
"KL/chosen_KL_mean": -167.25848388671875,
"KL/mean": -236.9961395263672,
"KL/rejected_KL_mean": -306.7337951660156,
"KL/std": 225.39352416992188,
"epoch": 0.31897203325774753,
"fcm_dpo/beta": 0.004614786710590124,
"fcm_dpo/delta": -0.5421129465103149,
"fcm_dpo/margin": 139.4752960205078,
"fcm_dpo/q_t": 0.3901931047439575,
"grad_norm": 23.795278549194336,
"learning_rate": 4.3184341039326217e-07,
"logits/chosen": 1.2343952655792236,
"logits/rejected": 1.1764979362487793,
"logps/chosen": -220.89920043945312,
"logps/ref_chosen": -53.640708923339844,
"logps/ref_rejected": -93.0387954711914,
"logps/rejected": -399.7725830078125,
"loss": 1.3759,
"margin_dpo/margin_mean": 139.4752960205078,
"margin_dpo/margin_std": 275.88897705078125,
"step": 211
},
{
"KL/chosen_KL_mean": -185.3031463623047,
"KL/mean": -240.04290771484375,
"KL/rejected_KL_mean": -294.78265380859375,
"KL/std": 222.0595703125,
"epoch": 0.3204837490551776,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 109.47950744628906,
"fcm_dpo/q_t": 0.49998939037323,
"grad_norm": 0.001861194847151637,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": 1.2033497095108032,
"logits/rejected": 1.167579174041748,
"logps/chosen": -242.66989135742188,
"logps/ref_chosen": -57.36674499511719,
"logps/ref_rejected": -79.89643096923828,
"logps/rejected": -374.6790771484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 109.47950744628906,
"margin_dpo/margin_std": 288.2134704589844,
"step": 212
},
{
"KL/chosen_KL_mean": -211.92974853515625,
"KL/mean": -252.25604248046875,
"KL/rejected_KL_mean": -292.5823974609375,
"KL/std": 271.2462463378906,
"epoch": 0.3219954648526077,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 80.65261840820312,
"fcm_dpo/q_t": 0.4999921917915344,
"grad_norm": 0.0020023249089717865,
"learning_rate": 4.3001854756006724e-07,
"logits/chosen": 1.1829383373260498,
"logits/rejected": 1.178153157234192,
"logps/chosen": -277.15087890625,
"logps/ref_chosen": -65.22111511230469,
"logps/ref_rejected": -80.1810302734375,
"logps/rejected": -372.7633972167969,
"loss": 1.3863,
"margin_dpo/margin_mean": 80.65261840820312,
"margin_dpo/margin_std": 359.29901123046875,
"step": 213
},
{
"KL/chosen_KL_mean": -217.9564208984375,
"KL/mean": -248.85366821289062,
"KL/rejected_KL_mean": -279.75091552734375,
"KL/std": 257.87176513671875,
"epoch": 0.3235071806500378,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 61.79451370239258,
"fcm_dpo/q_t": 0.49999403953552246,
"grad_norm": 0.0014851855812594295,
"learning_rate": 4.290985500881143e-07,
"logits/chosen": 1.1230930089950562,
"logits/rejected": 1.1280714273452759,
"logps/chosen": -279.2487487792969,
"logps/ref_chosen": -61.292327880859375,
"logps/ref_rejected": -67.69841003417969,
"logps/rejected": -347.4493408203125,
"loss": 1.3863,
"margin_dpo/margin_mean": 61.794517517089844,
"margin_dpo/margin_std": 338.4102783203125,
"step": 214
},
{
"KL/chosen_KL_mean": -213.97157287597656,
"KL/mean": -296.07861328125,
"KL/rejected_KL_mean": -378.18560791015625,
"KL/std": 297.4847106933594,
"epoch": 0.3250188964474679,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 164.21408081054688,
"fcm_dpo/q_t": 0.49998411536216736,
"grad_norm": 0.003093123435974121,
"learning_rate": 4.281735428447157e-07,
"logits/chosen": 1.1404362916946411,
"logits/rejected": 1.096791386604309,
"logps/chosen": -277.8406982421875,
"logps/ref_chosen": -63.869136810302734,
"logps/ref_rejected": -98.7657241821289,
"logps/rejected": -476.95135498046875,
"loss": 1.3862,
"margin_dpo/margin_mean": 164.21408081054688,
"margin_dpo/margin_std": 372.910400390625,
"step": 215
},
{
"KL/chosen_KL_mean": -207.86676025390625,
"KL/mean": -284.94354248046875,
"KL/rejected_KL_mean": -362.020263671875,
"KL/std": 262.0420837402344,
"epoch": 0.32653061224489793,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 154.15354919433594,
"fcm_dpo/q_t": 0.49998509883880615,
"grad_norm": 0.003247037995606661,
"learning_rate": 4.2724355170431247e-07,
"logits/chosen": 1.1896278858184814,
"logits/rejected": 1.1537840366363525,
"logps/chosen": -275.69171142578125,
"logps/ref_chosen": -67.824951171875,
"logps/ref_rejected": -96.40231323242188,
"logps/rejected": -458.422607421875,
"loss": 1.3862,
"margin_dpo/margin_mean": 154.15353393554688,
"margin_dpo/margin_std": 316.47894287109375,
"step": 216
},
{
"KL/chosen_KL_mean": -208.4380645751953,
"KL/mean": -280.11431884765625,
"KL/rejected_KL_mean": -351.79052734375,
"KL/std": 250.4637908935547,
"epoch": 0.328042328042328,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 143.35247802734375,
"fcm_dpo/q_t": 0.49998611211776733,
"grad_norm": 0.002652729395776987,
"learning_rate": 4.26308602680756e-07,
"logits/chosen": 1.1905633211135864,
"logits/rejected": 1.1464626789093018,
"logps/chosen": -268.94305419921875,
"logps/ref_chosen": -60.5049934387207,
"logps/ref_rejected": -84.26618194580078,
"logps/rejected": -436.0567321777344,
"loss": 1.3862,
"margin_dpo/margin_mean": 143.35247802734375,
"margin_dpo/margin_std": 322.68109130859375,
"step": 217
},
{
"KL/chosen_KL_mean": -261.1753845214844,
"KL/mean": -280.609130859375,
"KL/rejected_KL_mean": -300.0428466796875,
"KL/std": 274.44744873046875,
"epoch": 0.3295540438397581,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 38.86746597290039,
"fcm_dpo/q_t": 0.49999624490737915,
"grad_norm": 0.0014461593236774206,
"learning_rate": 4.253687219265803e-07,
"logits/chosen": 1.2026422023773193,
"logits/rejected": 1.20313560962677,
"logps/chosen": -331.76971435546875,
"logps/ref_chosen": -70.59431457519531,
"logps/ref_rejected": -73.89038848876953,
"logps/rejected": -373.9332275390625,
"loss": 1.3863,
"margin_dpo/margin_mean": 38.867462158203125,
"margin_dpo/margin_std": 299.7601318359375,
"step": 218
},
{
"KL/chosen_KL_mean": -212.3046112060547,
"KL/mean": -256.71380615234375,
"KL/rejected_KL_mean": -301.1230163574219,
"KL/std": 195.33700561523438,
"epoch": 0.3310657596371882,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 88.81842041015625,
"fcm_dpo/q_t": 0.49999141693115234,
"grad_norm": 0.0015157065354287624,
"learning_rate": 4.2442393573227043e-07,
"logits/chosen": 1.2033343315124512,
"logits/rejected": 1.186316967010498,
"logps/chosen": -272.7955322265625,
"logps/ref_chosen": -60.490943908691406,
"logps/ref_rejected": -75.85001373291016,
"logps/rejected": -376.9730224609375,
"loss": 1.3863,
"margin_dpo/margin_mean": 88.81842041015625,
"margin_dpo/margin_std": 246.12535095214844,
"step": 219
},
{
"KL/chosen_KL_mean": -152.8551025390625,
"KL/mean": -207.81396484375,
"KL/rejected_KL_mean": -262.7728271484375,
"KL/std": 192.8766326904297,
"epoch": 0.3325774754346183,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 109.91773986816406,
"fcm_dpo/q_t": 0.4999893307685852,
"grad_norm": 0.00206238916143775,
"learning_rate": 4.234742705255272e-07,
"logits/chosen": 1.2526534795761108,
"logits/rejected": 1.2216923236846924,
"logps/chosen": -197.86849975585938,
"logps/ref_chosen": -45.013397216796875,
"logps/ref_rejected": -70.49369812011719,
"logps/rejected": -333.26654052734375,
"loss": 1.3863,
"margin_dpo/margin_mean": 109.91773986816406,
"margin_dpo/margin_std": 220.33956909179688,
"step": 220
},
{
"KL/chosen_KL_mean": -189.33047485351562,
"KL/mean": -266.7629089355469,
"KL/rejected_KL_mean": -344.19537353515625,
"KL/std": 269.7894287109375,
"epoch": 0.3340891912320484,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 154.86489868164062,
"fcm_dpo/q_t": 0.4999850392341614,
"grad_norm": 0.002508159726858139,
"learning_rate": 4.22519752870528e-07,
"logits/chosen": 1.1693766117095947,
"logits/rejected": 1.1354807615280151,
"logps/chosen": -248.4263153076172,
"logps/ref_chosen": -59.09584045410156,
"logps/ref_rejected": -88.64388275146484,
"logps/rejected": -432.8392333984375,
"loss": 1.3862,
"margin_dpo/margin_mean": 154.86488342285156,
"margin_dpo/margin_std": 335.72869873046875,
"step": 221
},
{
"KL/chosen_KL_mean": -219.88815307617188,
"KL/mean": -301.087890625,
"KL/rejected_KL_mean": -382.28759765625,
"KL/std": 298.183349609375,
"epoch": 0.3356009070294785,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 162.39944458007812,
"fcm_dpo/q_t": 0.4999842941761017,
"grad_norm": 0.003092326922342181,
"learning_rate": 4.2156040946718343e-07,
"logits/chosen": 1.2328643798828125,
"logits/rejected": 1.1719518899917603,
"logps/chosen": -275.8858337402344,
"logps/ref_chosen": -55.9976921081543,
"logps/ref_rejected": -111.94727325439453,
"logps/rejected": -494.2348937988281,
"loss": 1.3862,
"margin_dpo/margin_mean": 162.39944458007812,
"margin_dpo/margin_std": 360.7342529296875,
"step": 222
},
{
"KL/chosen_KL_mean": -218.1814422607422,
"KL/mean": -282.19677734375,
"KL/rejected_KL_mean": -346.21209716796875,
"KL/std": 276.27166748046875,
"epoch": 0.3371126228269085,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 128.03065490722656,
"fcm_dpo/q_t": 0.4999876022338867,
"grad_norm": 0.0024099198635667562,
"learning_rate": 4.2059626715039065e-07,
"logits/chosen": 1.2278566360473633,
"logits/rejected": 1.2019100189208984,
"logps/chosen": -278.0728759765625,
"logps/ref_chosen": -59.891422271728516,
"logps/ref_rejected": -86.28954315185547,
"logps/rejected": -432.50164794921875,
"loss": 1.3862,
"margin_dpo/margin_mean": 128.03065490722656,
"margin_dpo/margin_std": 309.4208679199219,
"step": 223
},
{
"KL/chosen_KL_mean": -233.63507080078125,
"KL/mean": -257.69244384765625,
"KL/rejected_KL_mean": -281.74981689453125,
"KL/std": 263.045166015625,
"epoch": 0.3386243386243386,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 48.114749908447266,
"fcm_dpo/q_t": 0.4999953508377075,
"grad_norm": 0.0018048178171738982,
"learning_rate": 4.1962735288928304e-07,
"logits/chosen": 1.2750532627105713,
"logits/rejected": 1.2633410692214966,
"logps/chosen": -297.6797180175781,
"logps/ref_chosen": -64.04463195800781,
"logps/ref_rejected": -75.05450439453125,
"logps/rejected": -356.8043212890625,
"loss": 1.3863,
"margin_dpo/margin_mean": 48.11475372314453,
"margin_dpo/margin_std": 270.31732177734375,
"step": 224
},
{
"KL/chosen_KL_mean": -257.35723876953125,
"KL/mean": -354.2248840332031,
"KL/rejected_KL_mean": -451.092529296875,
"KL/std": 388.58221435546875,
"epoch": 0.3401360544217687,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 193.7353057861328,
"fcm_dpo/q_t": 0.49998122453689575,
"grad_norm": 0.003719078376889229,
"learning_rate": 4.186536937864752e-07,
"logits/chosen": 1.2433876991271973,
"logits/rejected": 1.2075812816619873,
"logps/chosen": -323.45306396484375,
"logps/ref_chosen": -66.0958251953125,
"logps/ref_rejected": -97.68675231933594,
"logps/rejected": -548.779296875,
"loss": 1.3862,
"margin_dpo/margin_mean": 193.73529052734375,
"margin_dpo/margin_std": 417.85693359375,
"step": 225
},
{
"KL/chosen_KL_mean": -184.22708129882812,
"KL/mean": -231.06289672851562,
"KL/rejected_KL_mean": -277.8987121582031,
"KL/std": 214.7318115234375,
"epoch": 0.3416477702191988,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 93.67162322998047,
"fcm_dpo/q_t": 0.49999094009399414,
"grad_norm": 0.0018737937789410353,
"learning_rate": 4.176753170773052e-07,
"logits/chosen": 1.2286086082458496,
"logits/rejected": 1.2162175178527832,
"logps/chosen": -235.64395141601562,
"logps/ref_chosen": -51.4168701171875,
"logps/ref_rejected": -66.30068969726562,
"logps/rejected": -344.19940185546875,
"loss": 1.3863,
"margin_dpo/margin_mean": 93.67161560058594,
"margin_dpo/margin_std": 288.7680358886719,
"step": 226
},
{
"KL/chosen_KL_mean": -203.26730346679688,
"KL/mean": -247.32821655273438,
"KL/rejected_KL_mean": -291.38916015625,
"KL/std": 236.57278442382812,
"epoch": 0.3431594860166289,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 88.12184143066406,
"fcm_dpo/q_t": 0.4999914765357971,
"grad_norm": 0.0017340828198939562,
"learning_rate": 4.166922501290729e-07,
"logits/chosen": 1.2588739395141602,
"logits/rejected": 1.2475314140319824,
"logps/chosen": -261.257080078125,
"logps/ref_chosen": -57.989776611328125,
"logps/ref_rejected": -75.05464172363281,
"logps/rejected": -366.44378662109375,
"loss": 1.3863,
"margin_dpo/margin_mean": 88.12184143066406,
"margin_dpo/margin_std": 271.02618408203125,
"step": 227
},
{
"KL/chosen_KL_mean": -190.28192138671875,
"KL/mean": -252.29940795898438,
"KL/rejected_KL_mean": -314.31695556640625,
"KL/std": 251.63345336914062,
"epoch": 0.34467120181405897,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 124.0350112915039,
"fcm_dpo/q_t": 0.49998798966407776,
"grad_norm": 0.0025014770217239857,
"learning_rate": 4.1570452044027405e-07,
"logits/chosen": 1.2646236419677734,
"logits/rejected": 1.2317761182785034,
"logps/chosen": -245.84127807617188,
"logps/ref_chosen": -55.55936813354492,
"logps/ref_rejected": -77.02364349365234,
"logps/rejected": -391.340576171875,
"loss": 1.3862,
"margin_dpo/margin_mean": 124.0350112915039,
"margin_dpo/margin_std": 312.650390625,
"step": 228
},
{
"KL/chosen_KL_mean": -173.30197143554688,
"KL/mean": -235.5060272216797,
"KL/rejected_KL_mean": -297.7100830078125,
"KL/std": 271.6685485839844,
"epoch": 0.34618291761148906,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 124.40812683105469,
"fcm_dpo/q_t": 0.49998795986175537,
"grad_norm": 0.0023492288310080767,
"learning_rate": 4.147121556398312e-07,
"logits/chosen": 1.2890279293060303,
"logits/rejected": 1.2542376518249512,
"logps/chosen": -224.09661865234375,
"logps/ref_chosen": -50.79466247558594,
"logps/ref_rejected": -78.4474105834961,
"logps/rejected": -376.157470703125,
"loss": 1.3862,
"margin_dpo/margin_mean": 124.40812683105469,
"margin_dpo/margin_std": 323.8258972167969,
"step": 229
},
{
"KL/chosen_KL_mean": -214.48695373535156,
"KL/mean": -247.63502502441406,
"KL/rejected_KL_mean": -280.7830810546875,
"KL/std": 275.4522705078125,
"epoch": 0.3476946334089191,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 66.2961196899414,
"fcm_dpo/q_t": 0.49999356269836426,
"grad_norm": 0.0015789554454386234,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": 1.2542227506637573,
"logits/rejected": 1.2686011791229248,
"logps/chosen": -271.2161865234375,
"logps/ref_chosen": -56.729225158691406,
"logps/ref_rejected": -62.99180603027344,
"logps/rejected": -343.77490234375,
"loss": 1.3863,
"margin_dpo/margin_mean": 66.2961196899414,
"margin_dpo/margin_std": 330.39410400390625,
"step": 230
},
{
"KL/chosen_KL_mean": -251.24246215820312,
"KL/mean": -291.47467041015625,
"KL/rejected_KL_mean": -331.7068786621094,
"KL/std": 262.79315185546875,
"epoch": 0.3492063492063492,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 80.46441650390625,
"fcm_dpo/q_t": 0.4999921917915344,
"grad_norm": 0.0015881173312664032,
"learning_rate": 4.1271363186719835e-07,
"logits/chosen": 1.2051804065704346,
"logits/rejected": 1.2067768573760986,
"logps/chosen": -323.83953857421875,
"logps/ref_chosen": -72.59709930419922,
"logps/ref_rejected": -86.2322998046875,
"logps/rejected": -417.9391784667969,
"loss": 1.3863,
"margin_dpo/margin_mean": 80.46441650390625,
"margin_dpo/margin_std": 341.13519287109375,
"step": 231
},
{
"KL/chosen_KL_mean": -247.03854370117188,
"KL/mean": -286.765869140625,
"KL/rejected_KL_mean": -326.4931945800781,
"KL/std": 314.61639404296875,
"epoch": 0.3507180650037793,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 79.45465087890625,
"fcm_dpo/q_t": 0.499992311000824,
"grad_norm": 0.0016828159568831325,
"learning_rate": 4.1170752879801436e-07,
"logits/chosen": 1.212486982345581,
"logits/rejected": 1.1974080801010132,
"logps/chosen": -315.1571044921875,
"logps/ref_chosen": -68.1185302734375,
"logps/ref_rejected": -83.79415893554688,
"logps/rejected": -410.287353515625,
"loss": 1.3863,
"margin_dpo/margin_mean": 79.45465087890625,
"margin_dpo/margin_std": 385.42352294921875,
"step": 232
},
{
"KL/chosen_KL_mean": -209.6459503173828,
"KL/mean": -239.39019775390625,
"KL/rejected_KL_mean": -269.1343994140625,
"KL/std": 206.01983642578125,
"epoch": 0.35222978080120937,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 59.48847198486328,
"fcm_dpo/q_t": 0.4999942183494568,
"grad_norm": 0.0019632186740636826,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": 1.2710832357406616,
"logits/rejected": 1.2419123649597168,
"logps/chosen": -264.7160949707031,
"logps/ref_chosen": -55.070152282714844,
"logps/ref_rejected": -66.61845397949219,
"logps/rejected": -335.75286865234375,
"loss": 1.3863,
"margin_dpo/margin_mean": 59.48847198486328,
"margin_dpo/margin_std": 254.86038208007812,
"step": 233
},
{
"KL/chosen_KL_mean": -177.97811889648438,
"KL/mean": -201.326416015625,
"KL/rejected_KL_mean": -224.67471313476562,
"KL/std": 178.39566040039062,
"epoch": 0.35374149659863946,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 46.69660568237305,
"fcm_dpo/q_t": 0.49999547004699707,
"grad_norm": 0.0013678089017048478,
"learning_rate": 4.09681781007452e-07,
"logits/chosen": 1.230893611907959,
"logits/rejected": 1.2400652170181274,
"logps/chosen": -233.9040069580078,
"logps/ref_chosen": -55.92589569091797,
"logps/ref_rejected": -51.11608123779297,
"logps/rejected": -275.7908020019531,
"loss": 1.3863,
"margin_dpo/margin_mean": 46.69660568237305,
"margin_dpo/margin_std": 229.29837036132812,
"step": 234
},
{
"KL/chosen_KL_mean": -215.87887573242188,
"KL/mean": -269.4683532714844,
"KL/rejected_KL_mean": -323.057861328125,
"KL/std": 284.41656494140625,
"epoch": 0.35525321239606955,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 107.17898559570312,
"fcm_dpo/q_t": 0.4999896287918091,
"grad_norm": 0.0015956625575199723,
"learning_rate": 4.08662192950594e-07,
"logits/chosen": 1.2453619241714478,
"logits/rejected": 1.2564499378204346,
"logps/chosen": -280.4186096191406,
"logps/ref_chosen": -64.53972625732422,
"logps/ref_rejected": -77.69151306152344,
"logps/rejected": -400.7493896484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 107.17898559570312,
"margin_dpo/margin_std": 307.4293212890625,
"step": 235
},
{
"KL/chosen_KL_mean": -268.6509704589844,
"KL/mean": -299.5308837890625,
"KL/rejected_KL_mean": -330.4107971191406,
"KL/std": 262.4644775390625,
"epoch": 0.35676492819349964,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 61.75979995727539,
"fcm_dpo/q_t": 0.4999940097332001,
"grad_norm": 0.0013398700393736362,
"learning_rate": 4.076381667711306e-07,
"logits/chosen": 1.2493544816970825,
"logits/rejected": 1.2446727752685547,
"logps/chosen": -339.80572509765625,
"logps/ref_chosen": -71.15473937988281,
"logps/ref_rejected": -84.88541412353516,
"logps/rejected": -415.29620361328125,
"loss": 1.3863,
"margin_dpo/margin_mean": 61.759796142578125,
"margin_dpo/margin_std": 344.2894287109375,
"step": 236
},
{
"KL/chosen_KL_mean": -239.0124969482422,
"KL/mean": -271.1570739746094,
"KL/rejected_KL_mean": -303.30169677734375,
"KL/std": 226.06419372558594,
"epoch": 0.35827664399092973,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 64.28919219970703,
"fcm_dpo/q_t": 0.49999377131462097,
"grad_norm": 0.0016555647598579526,
"learning_rate": 4.066097311132753e-07,
"logits/chosen": 1.203998327255249,
"logits/rejected": 1.211648941040039,
"logps/chosen": -315.1545104980469,
"logps/ref_chosen": -76.14201354980469,
"logps/ref_rejected": -80.88479614257812,
"logps/rejected": -384.1864929199219,
"loss": 1.3863,
"margin_dpo/margin_mean": 64.2891845703125,
"margin_dpo/margin_std": 271.71954345703125,
"step": 237
},
{
"KL/chosen_KL_mean": -209.5856475830078,
"KL/mean": -240.08590698242188,
"KL/rejected_KL_mean": -270.586181640625,
"KL/std": 247.57437133789062,
"epoch": 0.35978835978835977,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 61.00053405761719,
"fcm_dpo/q_t": 0.49999409914016724,
"grad_norm": 0.0015964685007929802,
"learning_rate": 4.0557691474458414e-07,
"logits/chosen": 1.198110580444336,
"logits/rejected": 1.1851861476898193,
"logps/chosen": -278.4704895019531,
"logps/ref_chosen": -68.88484954833984,
"logps/ref_rejected": -75.8946304321289,
"logps/rejected": -346.48077392578125,
"loss": 1.3863,
"margin_dpo/margin_mean": 61.000526428222656,
"margin_dpo/margin_std": 281.09130859375,
"step": 238
},
{
"KL/chosen_KL_mean": -206.03028869628906,
"KL/mean": -302.91278076171875,
"KL/rejected_KL_mean": -399.7953186035156,
"KL/std": 299.86492919921875,
"epoch": 0.36130007558578986,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 193.76502990722656,
"fcm_dpo/q_t": 0.49998122453689575,
"grad_norm": 0.0037673686165362597,
"learning_rate": 4.045397465551513e-07,
"logits/chosen": 1.2611656188964844,
"logits/rejected": 1.1864782571792603,
"logps/chosen": -262.8021240234375,
"logps/ref_chosen": -56.771827697753906,
"logps/ref_rejected": -116.23050689697266,
"logps/rejected": -516.0258178710938,
"loss": 1.3862,
"margin_dpo/margin_mean": 193.76504516601562,
"margin_dpo/margin_std": 343.26318359375,
"step": 239
},
{
"KL/chosen_KL_mean": -220.20172119140625,
"KL/mean": -302.418212890625,
"KL/rejected_KL_mean": -384.63470458984375,
"KL/std": 321.66656494140625,
"epoch": 0.36281179138321995,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 164.43296813964844,
"fcm_dpo/q_t": 0.49998408555984497,
"grad_norm": 0.00302553316578269,
"learning_rate": 4.0349825555680045e-07,
"logits/chosen": 1.2720229625701904,
"logits/rejected": 1.2404829263687134,
"logps/chosen": -273.55584716796875,
"logps/ref_chosen": -53.35411071777344,
"logps/ref_rejected": -80.12019348144531,
"logps/rejected": -464.7548828125,
"loss": 1.3862,
"margin_dpo/margin_mean": 164.43295288085938,
"margin_dpo/margin_std": 391.212890625,
"step": 240
},
{
"KL/chosen_KL_mean": -226.899658203125,
"KL/mean": -280.0133056640625,
"KL/rejected_KL_mean": -333.1269226074219,
"KL/std": 283.84722900390625,
"epoch": 0.36432350718065004,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 106.22727966308594,
"fcm_dpo/q_t": 0.49998971819877625,
"grad_norm": 0.0025036621373146772,
"learning_rate": 4.0245247088227377e-07,
"logits/chosen": 1.2458418607711792,
"logits/rejected": 1.2394804954528809,
"logps/chosen": -298.7950744628906,
"logps/ref_chosen": -71.89541625976562,
"logps/ref_rejected": -83.03492736816406,
"logps/rejected": -416.161865234375,
"loss": 1.3863,
"margin_dpo/margin_mean": 106.22727966308594,
"margin_dpo/margin_std": 347.1982421875,
"step": 241
},
{
"KL/chosen_KL_mean": -213.5513916015625,
"KL/mean": -256.9989318847656,
"KL/rejected_KL_mean": -300.44647216796875,
"KL/std": 220.97384643554688,
"epoch": 0.36583522297808013,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 86.89505004882812,
"fcm_dpo/q_t": 0.49999159574508667,
"grad_norm": 0.0013653092319145799,
"learning_rate": 4.0140242178441665e-07,
"logits/chosen": 1.2530593872070312,
"logits/rejected": 1.252440094947815,
"logps/chosen": -271.47882080078125,
"logps/ref_chosen": -57.927433013916016,
"logps/ref_rejected": -67.838623046875,
"logps/rejected": -368.28509521484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 86.89505767822266,
"margin_dpo/margin_std": 233.89271545410156,
"step": 242
},
{
"KL/chosen_KL_mean": -261.82867431640625,
"KL/mean": -280.63238525390625,
"KL/rejected_KL_mean": -299.43609619140625,
"KL/std": 240.62506103515625,
"epoch": 0.3673469387755102,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 37.60741424560547,
"fcm_dpo/q_t": 0.4999963641166687,
"grad_norm": 0.0014760139165446162,
"learning_rate": 4.003481376353596e-07,
"logits/chosen": 1.2306547164916992,
"logits/rejected": 1.242539405822754,
"logps/chosen": -336.1053466796875,
"logps/ref_chosen": -74.27667236328125,
"logps/ref_rejected": -73.24340057373047,
"logps/rejected": -372.67950439453125,
"loss": 1.3863,
"margin_dpo/margin_mean": 37.60741424560547,
"margin_dpo/margin_std": 283.18731689453125,
"step": 243
},
{
"KL/chosen_KL_mean": -188.20855712890625,
"KL/mean": -253.76040649414062,
"KL/rejected_KL_mean": -319.312255859375,
"KL/std": 258.85791015625,
"epoch": 0.3688586545729403,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 131.1037139892578,
"fcm_dpo/q_t": 0.49998730421066284,
"grad_norm": 0.0026807633694261312,
"learning_rate": 3.9928964792569654e-07,
"logits/chosen": 1.2482179403305054,
"logits/rejected": 1.2198402881622314,
"logps/chosen": -241.57244873046875,
"logps/ref_chosen": -53.36390686035156,
"logps/ref_rejected": -71.10276794433594,
"logps/rejected": -390.4150390625,
"loss": 1.3862,
"margin_dpo/margin_mean": 131.1037139892578,
"margin_dpo/margin_std": 323.0280456542969,
"step": 244
},
{
"KL/chosen_KL_mean": -225.77105712890625,
"KL/mean": -294.74444580078125,
"KL/rejected_KL_mean": -363.7178649902344,
"KL/std": 293.7945556640625,
"epoch": 0.37037037037037035,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 137.94680786132812,
"fcm_dpo/q_t": 0.4999866485595703,
"grad_norm": 0.0021245733369141817,
"learning_rate": 3.982269822636601e-07,
"logits/chosen": 1.2210156917572021,
"logits/rejected": 1.2317429780960083,
"logps/chosen": -296.9661560058594,
"logps/ref_chosen": -71.19510650634766,
"logps/ref_rejected": -80.76235961914062,
"logps/rejected": -444.480224609375,
"loss": 1.3862,
"margin_dpo/margin_mean": 137.94680786132812,
"margin_dpo/margin_std": 270.6390075683594,
"step": 245
},
{
"KL/chosen_KL_mean": -266.89068603515625,
"KL/mean": -320.71197509765625,
"KL/rejected_KL_mean": -374.5332336425781,
"KL/std": 293.4222412109375,
"epoch": 0.37188208616780044,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 107.6425552368164,
"fcm_dpo/q_t": 0.4999895691871643,
"grad_norm": 0.0021127404179424047,
"learning_rate": 3.971601703742932e-07,
"logits/chosen": 1.265270471572876,
"logits/rejected": 1.237593650817871,
"logps/chosen": -338.5117492675781,
"logps/ref_chosen": -71.62104797363281,
"logps/ref_rejected": -94.03392028808594,
"logps/rejected": -468.567138671875,
"loss": 1.3863,
"margin_dpo/margin_mean": 107.64256286621094,
"margin_dpo/margin_std": 309.1661376953125,
"step": 246
},
{
"KL/chosen_KL_mean": -302.8369140625,
"KL/mean": -326.8985595703125,
"KL/rejected_KL_mean": -350.9602355957031,
"KL/std": 334.1150817871094,
"epoch": 0.37339380196523053,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 48.12330627441406,
"fcm_dpo/q_t": 0.4999953508377075,
"grad_norm": 0.002026443835347891,
"learning_rate": 3.960892420986177e-07,
"logits/chosen": 1.2411327362060547,
"logits/rejected": 1.2402052879333496,
"logps/chosen": -382.85943603515625,
"logps/ref_chosen": -80.02254486083984,
"logps/ref_rejected": -89.22705841064453,
"logps/rejected": -440.1872863769531,
"loss": 1.3863,
"margin_dpo/margin_mean": 48.12330627441406,
"margin_dpo/margin_std": 404.8681640625,
"step": 247
},
{
"KL/chosen_KL_mean": -273.5013427734375,
"KL/mean": -274.2215881347656,
"KL/rejected_KL_mean": -274.94183349609375,
"KL/std": 300.5850830078125,
"epoch": 0.3749055177626606,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.44049072265625,
"fcm_dpo/q_t": 0.49999985098838806,
"grad_norm": 0.0020725768990814686,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": 1.2465314865112305,
"logits/rejected": 1.2722697257995605,
"logps/chosen": -338.8793029785156,
"logps/ref_chosen": -65.37796020507812,
"logps/ref_rejected": -61.365787506103516,
"logps/rejected": -336.3076171875,
"loss": 1.3863,
"margin_dpo/margin_mean": 1.44049072265625,
"margin_dpo/margin_std": 416.7322998046875,
"step": 248
},
{
"KL/chosen_KL_mean": -266.708984375,
"KL/mean": -261.5863037109375,
"KL/rejected_KL_mean": -256.463623046875,
"KL/std": 257.3307800292969,
"epoch": 0.3764172335600907,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -10.245379447937012,
"fcm_dpo/q_t": 0.5000010132789612,
"grad_norm": 0.0021689594723284245,
"learning_rate": 3.9393515632731094e-07,
"logits/chosen": 1.2218635082244873,
"logits/rejected": 1.247019648551941,
"logps/chosen": -341.3104553222656,
"logps/ref_chosen": -74.60145568847656,
"logps/ref_rejected": -63.79338455200195,
"logps/rejected": -320.25701904296875,
"loss": 1.3863,
"margin_dpo/margin_mean": -10.245381355285645,
"margin_dpo/margin_std": 330.6340026855469,
"step": 249
},
{
"KL/chosen_KL_mean": -243.35177612304688,
"KL/mean": -302.5293884277344,
"KL/rejected_KL_mean": -361.70703125,
"KL/std": 364.4193115234375,
"epoch": 0.3779289493575208,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 118.3552474975586,
"fcm_dpo/q_t": 0.49998852610588074,
"grad_norm": 0.0022773679811507463,
"learning_rate": 3.9285205908608934e-07,
"logits/chosen": 1.2806836366653442,
"logits/rejected": 1.28066885471344,
"logps/chosen": -305.28997802734375,
"logps/ref_chosen": -61.938209533691406,
"logps/ref_rejected": -72.21602630615234,
"logps/rejected": -433.9230651855469,
"loss": 1.3862,
"margin_dpo/margin_mean": 118.3552474975586,
"margin_dpo/margin_std": 424.7938537597656,
"step": 250
},
{
"KL/chosen_KL_mean": -238.873779296875,
"KL/mean": -284.80145263671875,
"KL/rejected_KL_mean": -330.7291259765625,
"KL/std": 258.41424560546875,
"epoch": 0.3794406651549509,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 91.85538482666016,
"fcm_dpo/q_t": 0.49999111890792847,
"grad_norm": 0.002100482815876603,
"learning_rate": 3.9176496596569265e-07,
"logits/chosen": 1.240132451057434,
"logits/rejected": 1.2237201929092407,
"logps/chosen": -305.730712890625,
"logps/ref_chosen": -66.85694885253906,
"logps/ref_rejected": -84.83396911621094,
"logps/rejected": -415.5631103515625,
"loss": 1.3863,
"margin_dpo/margin_mean": 91.85537719726562,
"margin_dpo/margin_std": 292.9080810546875,
"step": 251
},
{
"KL/chosen_KL_mean": -204.33717346191406,
"KL/mean": -248.474853515625,
"KL/rejected_KL_mean": -292.612548828125,
"KL/std": 251.9514923095703,
"epoch": 0.38095238095238093,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 88.27537536621094,
"fcm_dpo/q_t": 0.4999914765357971,
"grad_norm": 0.0017355838790535927,
"learning_rate": 3.9067390737445254e-07,
"logits/chosen": 1.2478840351104736,
"logits/rejected": 1.2181034088134766,
"logps/chosen": -260.56109619140625,
"logps/ref_chosen": -56.22393035888672,
"logps/ref_rejected": -77.1136245727539,
"logps/rejected": -369.7261657714844,
"loss": 1.3863,
"margin_dpo/margin_mean": 88.27537536621094,
"margin_dpo/margin_std": 345.6888122558594,
"step": 252
},
{
"KL/chosen_KL_mean": -171.81988525390625,
"KL/mean": -219.31829833984375,
"KL/rejected_KL_mean": -266.81671142578125,
"KL/std": 209.09054565429688,
"epoch": 0.382464096749811,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 94.99684143066406,
"fcm_dpo/q_t": 0.4999908208847046,
"grad_norm": 0.0018323827534914017,
"learning_rate": 3.8957891383162304e-07,
"logits/chosen": 1.2669950723648071,
"logits/rejected": 1.26656174659729,
"logps/chosen": -224.02987670898438,
"logps/ref_chosen": -52.21001434326172,
"logps/ref_rejected": -58.75764846801758,
"logps/rejected": -325.5743408203125,
"loss": 1.3863,
"margin_dpo/margin_mean": 94.99683380126953,
"margin_dpo/margin_std": 259.6316223144531,
"step": 253
},
{
"KL/chosen_KL_mean": -270.77874755859375,
"KL/mean": -316.91168212890625,
"KL/rejected_KL_mean": -363.0445556640625,
"KL/std": 343.09942626953125,
"epoch": 0.3839758125472411,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 92.26583862304688,
"fcm_dpo/q_t": 0.4999910593032837,
"grad_norm": 0.0022480462212115526,
"learning_rate": 3.884800159665276e-07,
"logits/chosen": 1.238560438156128,
"logits/rejected": 1.2194675207138062,
"logps/chosen": -336.4150695800781,
"logps/ref_chosen": -65.63632202148438,
"logps/ref_rejected": -82.34425354003906,
"logps/rejected": -445.3888244628906,
"loss": 1.3863,
"margin_dpo/margin_mean": 92.26583862304688,
"margin_dpo/margin_std": 337.8890380859375,
"step": 254
},
{
"KL/chosen_KL_mean": -255.44696044921875,
"KL/mean": -305.6675720214844,
"KL/rejected_KL_mean": -355.88818359375,
"KL/std": 318.76397705078125,
"epoch": 0.3854875283446712,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 100.44125366210938,
"fcm_dpo/q_t": 0.4999902844429016,
"grad_norm": 0.0018314458429813385,
"learning_rate": 3.873772445177015e-07,
"logits/chosen": 1.2495830059051514,
"logits/rejected": 1.2430639266967773,
"logps/chosen": -323.3580627441406,
"logps/ref_chosen": -67.91108703613281,
"logps/ref_rejected": -83.89114379882812,
"logps/rejected": -439.77935791015625,
"loss": 1.3863,
"margin_dpo/margin_mean": 100.44124603271484,
"margin_dpo/margin_std": 398.81390380859375,
"step": 255
},
{
"KL/chosen_KL_mean": -259.4943542480469,
"KL/mean": -310.67401123046875,
"KL/rejected_KL_mean": -361.85369873046875,
"KL/std": 321.3037414550781,
"epoch": 0.3869992441421013,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.35935974121094,
"fcm_dpo/q_t": 0.4999900758266449,
"grad_norm": 0.0024800151586532593,
"learning_rate": 3.862706303320329e-07,
"logits/chosen": 1.2634713649749756,
"logits/rejected": 1.2241191864013672,
"logps/chosen": -322.99432373046875,
"logps/ref_chosen": -63.49998474121094,
"logps/ref_rejected": -90.77104187011719,
"logps/rejected": -452.624755859375,
"loss": 1.3863,
"margin_dpo/margin_mean": 102.35935974121094,
"margin_dpo/margin_std": 402.4932556152344,
"step": 256
},
{
"KL/chosen_KL_mean": -262.64410400390625,
"KL/mean": -331.921875,
"KL/rejected_KL_mean": -401.1995849609375,
"KL/std": 297.68072509765625,
"epoch": 0.3885109599395314,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 138.5554656982422,
"fcm_dpo/q_t": 0.49998658895492554,
"grad_norm": 0.0026133295614272356,
"learning_rate": 3.851602043638994e-07,
"logits/chosen": 1.2339143753051758,
"logits/rejected": 1.192386507987976,
"logps/chosen": -333.2447509765625,
"logps/ref_chosen": -70.60064697265625,
"logps/ref_rejected": -108.58313751220703,
"logps/rejected": -509.78271484375,
"loss": 1.3862,
"margin_dpo/margin_mean": 138.5554656982422,
"margin_dpo/margin_std": 377.41680908203125,
"step": 257
},
{
"KL/chosen_KL_mean": -226.23171997070312,
"KL/mean": -291.40777587890625,
"KL/rejected_KL_mean": -356.5838623046875,
"KL/std": 314.12420654296875,
"epoch": 0.3900226757369615,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.35211181640625,
"fcm_dpo/q_t": 0.49998739361763,
"grad_norm": 0.002391485497355461,
"learning_rate": 3.840459976743023e-07,
"logits/chosen": 1.226086139678955,
"logits/rejected": 1.204946517944336,
"logps/chosen": -285.48590087890625,
"logps/ref_chosen": -59.25416564941406,
"logps/ref_rejected": -85.58709716796875,
"logps/rejected": -442.1709289550781,
"loss": 1.3862,
"margin_dpo/margin_mean": 130.35211181640625,
"margin_dpo/margin_std": 371.4399108886719,
"step": 258
},
{
"KL/chosen_KL_mean": -232.07115173339844,
"KL/mean": -321.46673583984375,
"KL/rejected_KL_mean": -410.86236572265625,
"KL/std": 316.74798583984375,
"epoch": 0.3915343915343915,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 178.79119873046875,
"fcm_dpo/q_t": 0.49998271465301514,
"grad_norm": 0.003150097792968154,
"learning_rate": 3.8292804142999796e-07,
"logits/chosen": 1.2068517208099365,
"logits/rejected": 1.1704658269882202,
"logps/chosen": -297.50604248046875,
"logps/ref_chosen": -65.43487548828125,
"logps/ref_rejected": -95.41731262207031,
"logps/rejected": -506.2796630859375,
"loss": 1.3862,
"margin_dpo/margin_mean": 178.79119873046875,
"margin_dpo/margin_std": 385.9300537109375,
"step": 259
},
{
"KL/chosen_KL_mean": -184.13815307617188,
"KL/mean": -257.88519287109375,
"KL/rejected_KL_mean": -331.63226318359375,
"KL/std": 256.04803466796875,
"epoch": 0.3930461073318216,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 147.49411010742188,
"fcm_dpo/q_t": 0.4999857246875763,
"grad_norm": 0.002410466317087412,
"learning_rate": 3.818063669026256e-07,
"logits/chosen": 1.246681809425354,
"logits/rejected": 1.2082734107971191,
"logps/chosen": -233.2277374267578,
"logps/ref_chosen": -49.08958435058594,
"logps/ref_rejected": -79.01708221435547,
"logps/rejected": -410.64935302734375,
"loss": 1.3862,
"margin_dpo/margin_mean": 147.49411010742188,
"margin_dpo/margin_std": 310.56329345703125,
"step": 260
},
{
"KL/chosen_KL_mean": -234.46389770507812,
"KL/mean": -250.5965576171875,
"KL/rejected_KL_mean": -266.7292175292969,
"KL/std": 255.46902465820312,
"epoch": 0.3945578231292517,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 32.26532745361328,
"fcm_dpo/q_t": 0.4999969005584717,
"grad_norm": 0.001933022402226925,
"learning_rate": 3.806810054678331e-07,
"logits/chosen": 1.184006929397583,
"logits/rejected": 1.216488242149353,
"logps/chosen": -305.3363037109375,
"logps/ref_chosen": -70.87239074707031,
"logps/ref_rejected": -65.01522064208984,
"logps/rejected": -331.74444580078125,
"loss": 1.3863,
"margin_dpo/margin_mean": 32.265323638916016,
"margin_dpo/margin_std": 333.6747741699219,
"step": 261
},
{
"KL/chosen_KL_mean": -257.26776123046875,
"KL/mean": -322.60302734375,
"KL/rejected_KL_mean": -387.93829345703125,
"KL/std": 322.8201904296875,
"epoch": 0.3960695389266818,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.67054748535156,
"fcm_dpo/q_t": 0.4999873638153076,
"grad_norm": 0.0025448398664593697,
"learning_rate": 3.7955198860439887e-07,
"logits/chosen": 1.293022871017456,
"logits/rejected": 1.268181324005127,
"logps/chosen": -325.13836669921875,
"logps/ref_chosen": -67.8706283569336,
"logps/ref_rejected": -88.7205810546875,
"logps/rejected": -476.65887451171875,
"loss": 1.3862,
"margin_dpo/margin_mean": 130.67056274414062,
"margin_dpo/margin_std": 383.709716796875,
"step": 262
},
{
"KL/chosen_KL_mean": -204.814208984375,
"KL/mean": -286.87152099609375,
"KL/rejected_KL_mean": -368.9288024902344,
"KL/std": 260.5648193359375,
"epoch": 0.3975812547241119,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 164.11460876464844,
"fcm_dpo/q_t": 0.49998411536216736,
"grad_norm": 0.0031319339759647846,
"learning_rate": 3.784193478933516e-07,
"logits/chosen": 1.2445664405822754,
"logits/rejected": 1.1973962783813477,
"logps/chosen": -260.0087890625,
"logps/ref_chosen": -55.194583892822266,
"logps/ref_rejected": -80.54048156738281,
"logps/rejected": -449.46929931640625,
"loss": 1.3862,
"margin_dpo/margin_mean": 164.11460876464844,
"margin_dpo/margin_std": 305.0133056640625,
"step": 263
},
{
"KL/chosen_KL_mean": -304.02825927734375,
"KL/mean": -360.13616943359375,
"KL/rejected_KL_mean": -416.244140625,
"KL/std": 354.64105224609375,
"epoch": 0.39909297052154197,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 112.21583557128906,
"fcm_dpo/q_t": 0.4999891221523285,
"grad_norm": 0.0019485560478642583,
"learning_rate": 3.7728311501708674e-07,
"logits/chosen": 1.213137149810791,
"logits/rejected": 1.212262511253357,
"logps/chosen": -387.198974609375,
"logps/ref_chosen": -83.17068481445312,
"logps/ref_rejected": -88.33625793457031,
"logps/rejected": -504.58038330078125,
"loss": 1.3863,
"margin_dpo/margin_mean": 112.21582794189453,
"margin_dpo/margin_std": 419.0724792480469,
"step": 264
},
{
"KL/chosen_KL_mean": -178.7972412109375,
"KL/mean": -248.697021484375,
"KL/rejected_KL_mean": -318.5968017578125,
"KL/std": 211.929931640625,
"epoch": 0.40060468631897206,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 139.79954528808594,
"fcm_dpo/q_t": 0.499986469745636,
"grad_norm": 0.0025794145185500383,
"learning_rate": 3.7614332175848027e-07,
"logits/chosen": 1.2378535270690918,
"logits/rejected": 1.2246911525726318,
"logps/chosen": -230.4600830078125,
"logps/ref_chosen": -51.66284942626953,
"logps/ref_rejected": -67.1720962524414,
"logps/rejected": -385.7688903808594,
"loss": 1.3862,
"margin_dpo/margin_mean": 139.79954528808594,
"margin_dpo/margin_std": 223.987548828125,
"step": 265
},
{
"KL/chosen_KL_mean": -226.0594024658203,
"KL/mean": -293.44244384765625,
"KL/rejected_KL_mean": -360.8254699707031,
"KL/std": 271.87078857421875,
"epoch": 0.4021164021164021,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 134.7660675048828,
"fcm_dpo/q_t": 0.4999869465827942,
"grad_norm": 0.0027317116037011147,
"learning_rate": 3.75e-07,
"logits/chosen": 1.2706271409988403,
"logits/rejected": 1.2448146343231201,
"logps/chosen": -283.5098876953125,
"logps/ref_chosen": -57.45049285888672,
"logps/ref_rejected": -77.60826110839844,
"logps/rejected": -438.4337158203125,
"loss": 1.3862,
"margin_dpo/margin_mean": 134.76608276367188,
"margin_dpo/margin_std": 255.4769744873047,
"step": 266
},
{
"KL/chosen_KL_mean": -187.67311096191406,
"KL/mean": -220.25643920898438,
"KL/rejected_KL_mean": -252.83978271484375,
"KL/std": 226.619140625,
"epoch": 0.4036281179138322,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 65.16666412353516,
"fcm_dpo/q_t": 0.4999936819076538,
"grad_norm": 0.0015339229721575975,
"learning_rate": 3.738531817228131e-07,
"logits/chosen": 1.2775146961212158,
"logits/rejected": 1.2753092050552368,
"logps/chosen": -242.70846557617188,
"logps/ref_chosen": -55.03535079956055,
"logps/ref_rejected": -66.0953369140625,
"logps/rejected": -318.93511962890625,
"loss": 1.3863,
"margin_dpo/margin_mean": 65.16666412353516,
"margin_dpo/margin_std": 247.80099487304688,
"step": 267
},
{
"KL/chosen_KL_mean": -225.61331176757812,
"KL/mean": -269.3935546875,
"KL/rejected_KL_mean": -313.173828125,
"KL/std": 249.95361328125,
"epoch": 0.4051398337112623,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 87.56051635742188,
"fcm_dpo/q_t": 0.4999915361404419,
"grad_norm": 0.0014259773306548595,
"learning_rate": 3.7270289900589204e-07,
"logits/chosen": 1.2123204469680786,
"logits/rejected": 1.2190499305725098,
"logps/chosen": -290.68505859375,
"logps/ref_chosen": -65.07174682617188,
"logps/ref_rejected": -71.42485809326172,
"logps/rejected": -384.59869384765625,
"loss": 1.3863,
"margin_dpo/margin_mean": 87.56051635742188,
"margin_dpo/margin_std": 257.95928955078125,
"step": 268
},
{
"KL/chosen_KL_mean": -245.9951171875,
"KL/mean": -291.13848876953125,
"KL/rejected_KL_mean": -336.2818603515625,
"KL/std": 273.3773498535156,
"epoch": 0.40665154950869237,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 90.28675079345703,
"fcm_dpo/q_t": 0.499991238117218,
"grad_norm": 0.002284642308950424,
"learning_rate": 3.7154918402511714e-07,
"logits/chosen": 1.2732834815979004,
"logits/rejected": 1.2535003423690796,
"logps/chosen": -313.13134765625,
"logps/ref_chosen": -67.1362075805664,
"logps/ref_rejected": -82.55778503417969,
"logps/rejected": -418.83966064453125,
"loss": 1.3863,
"margin_dpo/margin_mean": 90.28675842285156,
"margin_dpo/margin_std": 311.7049560546875,
"step": 269
},
{
"KL/chosen_KL_mean": -231.08285522460938,
"KL/mean": -305.00152587890625,
"KL/rejected_KL_mean": -378.920166015625,
"KL/std": 334.8675537109375,
"epoch": 0.40816326530612246,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 147.83731079101562,
"fcm_dpo/q_t": 0.4999856948852539,
"grad_norm": 0.0029855347238481045,
"learning_rate": 3.7039206905237656e-07,
"logits/chosen": 1.265000343322754,
"logits/rejected": 1.234115481376648,
"logps/chosen": -297.77154541015625,
"logps/ref_chosen": -66.6886978149414,
"logps/ref_rejected": -85.16129302978516,
"logps/rejected": -464.0814208984375,
"loss": 1.3862,
"margin_dpo/margin_mean": 147.83731079101562,
"margin_dpo/margin_std": 414.43560791015625,
"step": 270
},
{
"KL/chosen_KL_mean": -261.5472106933594,
"KL/mean": -319.3465576171875,
"KL/rejected_KL_mean": -377.14593505859375,
"KL/std": 378.90277099609375,
"epoch": 0.40967498110355255,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 115.59872436523438,
"fcm_dpo/q_t": 0.4999887943267822,
"grad_norm": 0.002353003015741706,
"learning_rate": 3.692315864546635e-07,
"logits/chosen": 1.3018203973770142,
"logits/rejected": 1.2758502960205078,
"logps/chosen": -333.95477294921875,
"logps/ref_chosen": -72.40754699707031,
"logps/ref_rejected": -92.06311798095703,
"logps/rejected": -469.20904541015625,
"loss": 1.3862,
"margin_dpo/margin_mean": 115.5987319946289,
"margin_dpo/margin_std": 363.4158935546875,
"step": 271
},
{
"KL/chosen_KL_mean": -240.21646118164062,
"KL/mean": -269.6424255371094,
"KL/rejected_KL_mean": -299.06842041015625,
"KL/std": 277.5693359375,
"epoch": 0.41118669690098264,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 58.8519287109375,
"fcm_dpo/q_t": 0.49999427795410156,
"grad_norm": 0.001313323387876153,
"learning_rate": 3.6806776869317067e-07,
"logits/chosen": 1.288849949836731,
"logits/rejected": 1.30269455909729,
"logps/chosen": -306.81787109375,
"logps/ref_chosen": -66.60140228271484,
"logps/ref_rejected": -67.74340057373047,
"logps/rejected": -366.8117980957031,
"loss": 1.3863,
"margin_dpo/margin_mean": 58.8519287109375,
"margin_dpo/margin_std": 330.32196044921875,
"step": 272
},
{
"KL/chosen_KL_mean": -218.98214721679688,
"KL/mean": -287.33990478515625,
"KL/rejected_KL_mean": -355.6976318359375,
"KL/std": 279.4718017578125,
"epoch": 0.4126984126984127,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 136.71548461914062,
"fcm_dpo/q_t": 0.49998676776885986,
"grad_norm": 0.0026562055572867393,
"learning_rate": 3.669006483223828e-07,
"logits/chosen": 1.2323360443115234,
"logits/rejected": 1.204160451889038,
"logps/chosen": -276.3370361328125,
"logps/ref_chosen": -57.35487747192383,
"logps/ref_rejected": -84.17168426513672,
"logps/rejected": -439.86932373046875,
"loss": 1.3862,
"margin_dpo/margin_mean": 136.71546936035156,
"margin_dpo/margin_std": 341.8489074707031,
"step": 273
},
{
"KL/chosen_KL_mean": -227.13885498046875,
"KL/mean": -261.77813720703125,
"KL/rejected_KL_mean": -296.41741943359375,
"KL/std": 222.0265350341797,
"epoch": 0.41421012849584277,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 69.27857971191406,
"fcm_dpo/q_t": 0.4999932646751404,
"grad_norm": 0.001474517397582531,
"learning_rate": 3.657302579891656e-07,
"logits/chosen": 1.2174702882766724,
"logits/rejected": 1.2115811109542847,
"logps/chosen": -286.78033447265625,
"logps/ref_chosen": -59.64149475097656,
"logps/ref_rejected": -68.29348754882812,
"logps/rejected": -364.71087646484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 69.27857208251953,
"margin_dpo/margin_std": 245.24844360351562,
"step": 274
},
{
"KL/chosen_KL_mean": -202.14822387695312,
"KL/mean": -266.6926574707031,
"KL/rejected_KL_mean": -331.2370910644531,
"KL/std": 231.38563537597656,
"epoch": 0.41572184429327286,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 129.0888671875,
"fcm_dpo/q_t": 0.49998751282691956,
"grad_norm": 0.0028466566000133753,
"learning_rate": 3.645566304318526e-07,
"logits/chosen": 1.286808967590332,
"logits/rejected": 1.2461142539978027,
"logps/chosen": -255.4148712158203,
"logps/ref_chosen": -53.26664352416992,
"logps/ref_rejected": -73.84062194824219,
"logps/rejected": -405.07769775390625,
"loss": 1.3862,
"margin_dpo/margin_mean": 129.0888671875,
"margin_dpo/margin_std": 244.29052734375,
"step": 275
},
{
"KL/chosen_KL_mean": -184.27493286132812,
"KL/mean": -225.28565979003906,
"KL/rejected_KL_mean": -266.2964172363281,
"KL/std": 173.41802978515625,
"epoch": 0.41723356009070295,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 82.021484375,
"fcm_dpo/q_t": 0.4999920725822449,
"grad_norm": 0.0015512220561504364,
"learning_rate": 3.633797984793294e-07,
"logits/chosen": 1.2420344352722168,
"logits/rejected": 1.2362403869628906,
"logps/chosen": -237.29571533203125,
"logps/ref_chosen": -53.02079772949219,
"logps/ref_rejected": -61.56678771972656,
"logps/rejected": -327.86322021484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 82.021484375,
"margin_dpo/margin_std": 235.60247802734375,
"step": 276
},
{
"KL/chosen_KL_mean": -272.57080078125,
"KL/mean": -275.81597900390625,
"KL/rejected_KL_mean": -279.0611572265625,
"KL/std": 280.1846618652344,
"epoch": 0.41874527588813304,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.490352630615234,
"fcm_dpo/q_t": 0.49999934434890747,
"grad_norm": 0.0013179422821849585,
"learning_rate": 3.6219979505011555e-07,
"logits/chosen": 1.2742948532104492,
"logits/rejected": 1.291219711303711,
"logps/chosen": -344.0037841796875,
"logps/ref_chosen": -71.43299102783203,
"logps/ref_rejected": -67.65852355957031,
"logps/rejected": -346.71966552734375,
"loss": 1.3863,
"margin_dpo/margin_mean": 6.490348815917969,
"margin_dpo/margin_std": 291.0250244140625,
"step": 277
},
{
"KL/chosen_KL_mean": -271.00958251953125,
"KL/mean": -318.64300537109375,
"KL/rejected_KL_mean": -366.27642822265625,
"KL/std": 313.33929443359375,
"epoch": 0.42025699168556313,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 95.26683807373047,
"fcm_dpo/q_t": 0.4999907612800598,
"grad_norm": 0.0023758604656904936,
"learning_rate": 3.6101665315144353e-07,
"logits/chosen": 1.2502532005310059,
"logits/rejected": 1.2217738628387451,
"logps/chosen": -338.120361328125,
"logps/ref_chosen": -67.11076354980469,
"logps/ref_rejected": -88.74851989746094,
"logps/rejected": -455.02496337890625,
"loss": 1.3863,
"margin_dpo/margin_mean": 95.26683044433594,
"margin_dpo/margin_std": 256.470947265625,
"step": 278
},
{
"KL/chosen_KL_mean": -200.28704833984375,
"KL/mean": -251.19207763671875,
"KL/rejected_KL_mean": -302.09710693359375,
"KL/std": 222.035400390625,
"epoch": 0.4217687074829932,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 101.81007385253906,
"fcm_dpo/q_t": 0.49999016523361206,
"grad_norm": 0.002032017568126321,
"learning_rate": 3.5983040587833563e-07,
"logits/chosen": 1.242897868156433,
"logits/rejected": 1.225640058517456,
"logps/chosen": -254.78451538085938,
"logps/ref_chosen": -54.49748611450195,
"logps/ref_rejected": -70.42373657226562,
"logps/rejected": -372.5208435058594,
"loss": 1.3863,
"margin_dpo/margin_mean": 101.8100814819336,
"margin_dpo/margin_std": 249.4945068359375,
"step": 279
},
{
"KL/chosen_KL_mean": -205.01568603515625,
"KL/mean": -279.7425842285156,
"KL/rejected_KL_mean": -354.469482421875,
"KL/std": 266.3010559082031,
"epoch": 0.42328042328042326,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 149.45379638671875,
"fcm_dpo/q_t": 0.4999855160713196,
"grad_norm": 0.002008727751672268,
"learning_rate": 3.586410864126781e-07,
"logits/chosen": 1.2439789772033691,
"logits/rejected": 1.2364301681518555,
"logps/chosen": -265.4485168457031,
"logps/ref_chosen": -60.43281173706055,
"logps/ref_rejected": -78.39051818847656,
"logps/rejected": -432.8599853515625,
"loss": 1.3862,
"margin_dpo/margin_mean": 149.45379638671875,
"margin_dpo/margin_std": 288.43994140625,
"step": 280
},
{
"KL/chosen_KL_mean": -224.62643432617188,
"KL/mean": -239.99485778808594,
"KL/rejected_KL_mean": -255.36328125,
"KL/std": 230.7904052734375,
"epoch": 0.42479213907785335,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 30.736845016479492,
"fcm_dpo/q_t": 0.49999701976776123,
"grad_norm": 0.0013363017933443189,
"learning_rate": 3.574487280222929e-07,
"logits/chosen": 1.2372968196868896,
"logits/rejected": 1.2535717487335205,
"logps/chosen": -284.90850830078125,
"logps/ref_chosen": -60.2820930480957,
"logps/ref_rejected": -62.04009246826172,
"logps/rejected": -317.40338134765625,
"loss": 1.3863,
"margin_dpo/margin_mean": 30.736841201782227,
"margin_dpo/margin_std": 231.83021545410156,
"step": 281
},
{
"KL/chosen_KL_mean": -230.42965698242188,
"KL/mean": -267.6971130371094,
"KL/rejected_KL_mean": -304.96453857421875,
"KL/std": 253.47097778320312,
"epoch": 0.42630385487528344,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 74.53489685058594,
"fcm_dpo/q_t": 0.4999927878379822,
"grad_norm": 0.0015713232569396496,
"learning_rate": 3.562533640600075e-07,
"logits/chosen": 1.2360237836837769,
"logits/rejected": 1.222773790359497,
"logps/chosen": -291.0535888671875,
"logps/ref_chosen": -60.623924255371094,
"logps/ref_rejected": -68.67400360107422,
"logps/rejected": -373.6385498046875,
"loss": 1.3863,
"margin_dpo/margin_mean": 74.5348892211914,
"margin_dpo/margin_std": 305.5966796875,
"step": 282
},
{
"KL/chosen_KL_mean": -207.01063537597656,
"KL/mean": -313.7869873046875,
"KL/rejected_KL_mean": -420.56329345703125,
"KL/std": 381.99566650390625,
"epoch": 0.42781557067271353,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 213.55267333984375,
"fcm_dpo/q_t": 0.4999793469905853,
"grad_norm": 0.004121567588299513,
"learning_rate": 3.550550279627215e-07,
"logits/chosen": 1.2334487438201904,
"logits/rejected": 1.195540189743042,
"logps/chosen": -274.65838623046875,
"logps/ref_chosen": -67.64775085449219,
"logps/ref_rejected": -99.96835327148438,
"logps/rejected": -520.5316772460938,
"loss": 1.3862,
"margin_dpo/margin_mean": 213.5526580810547,
"margin_dpo/margin_std": 493.8293151855469,
"step": 283
},
{
"KL/chosen_KL_mean": -213.08843994140625,
"KL/mean": -276.4241027832031,
"KL/rejected_KL_mean": -339.759765625,
"KL/std": 258.0605773925781,
"epoch": 0.4293272864701436,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 126.67131805419922,
"fcm_dpo/q_t": 0.49998775124549866,
"grad_norm": 0.0023277702275663614,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": 1.2803890705108643,
"logits/rejected": 1.249000906944275,
"logps/chosen": -270.0558776855469,
"logps/ref_chosen": -56.96742630004883,
"logps/ref_rejected": -86.36236572265625,
"logps/rejected": -426.12213134765625,
"loss": 1.3862,
"margin_dpo/margin_mean": 126.67131805419922,
"margin_dpo/margin_std": 305.05230712890625,
"step": 284
},
{
"KL/chosen_KL_mean": -243.08445739746094,
"KL/mean": -282.5973815917969,
"KL/rejected_KL_mean": -322.11029052734375,
"KL/std": 253.41421508789062,
"epoch": 0.4308390022675737,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 79.02581024169922,
"fcm_dpo/q_t": 0.49999234080314636,
"grad_norm": 0.0040868110954761505,
"learning_rate": 3.5264957352549375e-07,
"logits/chosen": 1.2414788007736206,
"logits/rejected": 1.229378342628479,
"logps/chosen": -314.7405700683594,
"logps/ref_chosen": -71.65611267089844,
"logps/ref_rejected": -81.63829803466797,
"logps/rejected": -403.74859619140625,
"loss": 1.3863,
"margin_dpo/margin_mean": 79.02581787109375,
"margin_dpo/margin_std": 294.8158264160156,
"step": 285
},
{
"KL/chosen_KL_mean": -242.94229125976562,
"KL/mean": -322.85455322265625,
"KL/rejected_KL_mean": -402.766845703125,
"KL/std": 337.2320251464844,
"epoch": 0.4323507180650038,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 159.82455444335938,
"fcm_dpo/q_t": 0.4999845325946808,
"grad_norm": 0.0029409886337816715,
"learning_rate": 3.514425224712835e-07,
"logits/chosen": 1.2100770473480225,
"logits/rejected": 1.176485300064087,
"logps/chosen": -304.0218200683594,
"logps/ref_chosen": -61.07952117919922,
"logps/ref_rejected": -91.28128051757812,
"logps/rejected": -494.0481262207031,
"loss": 1.3862,
"margin_dpo/margin_mean": 159.82455444335938,
"margin_dpo/margin_std": 381.1865539550781,
"step": 286
},
{
"KL/chosen_KL_mean": -163.37152099609375,
"KL/mean": -214.27389526367188,
"KL/rejected_KL_mean": -265.1762390136719,
"KL/std": 199.1695556640625,
"epoch": 0.43386243386243384,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 101.80474090576172,
"fcm_dpo/q_t": 0.49999016523361206,
"grad_norm": 0.0017581077991053462,
"learning_rate": 3.502326338516534e-07,
"logits/chosen": 1.243018388748169,
"logits/rejected": 1.237166166305542,
"logps/chosen": -209.40731811523438,
"logps/ref_chosen": -46.035789489746094,
"logps/ref_rejected": -59.95293426513672,
"logps/rejected": -325.1291809082031,
"loss": 1.3863,
"margin_dpo/margin_mean": 101.80474090576172,
"margin_dpo/margin_std": 249.35662841796875,
"step": 287
},
{
"KL/chosen_KL_mean": -247.86630249023438,
"KL/mean": -297.19573974609375,
"KL/rejected_KL_mean": -346.5251159667969,
"KL/std": 270.19976806640625,
"epoch": 0.43537414965986393,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 98.65884399414062,
"fcm_dpo/q_t": 0.49999046325683594,
"grad_norm": 0.002023276872932911,
"learning_rate": 3.490199415097892e-07,
"logits/chosen": 1.1788934469223022,
"logits/rejected": 1.1556057929992676,
"logps/chosen": -313.25714111328125,
"logps/ref_chosen": -65.3908462524414,
"logps/ref_rejected": -88.53607940673828,
"logps/rejected": -435.06121826171875,
"loss": 1.3863,
"margin_dpo/margin_mean": 98.6588363647461,
"margin_dpo/margin_std": 299.93707275390625,
"step": 288
},
{
"KL/chosen_KL_mean": -218.3133087158203,
"KL/mean": -268.06744384765625,
"KL/rejected_KL_mean": -317.82159423828125,
"KL/std": 313.28009033203125,
"epoch": 0.436885865457294,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.50828552246094,
"fcm_dpo/q_t": 0.4999903440475464,
"grad_norm": 0.00193495349958539,
"learning_rate": 3.4780447936730247e-07,
"logits/chosen": 1.2431552410125732,
"logits/rejected": 1.243152141571045,
"logps/chosen": -272.90692138671875,
"logps/ref_chosen": -54.5936279296875,
"logps/ref_rejected": -67.20855712890625,
"logps/rejected": -385.0301513671875,
"loss": 1.3863,
"margin_dpo/margin_mean": 99.50830078125,
"margin_dpo/margin_std": 335.50543212890625,
"step": 289
},
{
"KL/chosen_KL_mean": -221.37855529785156,
"KL/mean": -276.5778503417969,
"KL/rejected_KL_mean": -331.77716064453125,
"KL/std": 264.361083984375,
"epoch": 0.4383975812547241,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 110.39859771728516,
"fcm_dpo/q_t": 0.4999893307685852,
"grad_norm": 0.001686166855506599,
"learning_rate": 3.465862814232821e-07,
"logits/chosen": 1.2381784915924072,
"logits/rejected": 1.1985375881195068,
"logps/chosen": -282.76312255859375,
"logps/ref_chosen": -61.38457489013672,
"logps/ref_rejected": -91.92778015136719,
"logps/rejected": -423.7049255371094,
"loss": 1.3863,
"margin_dpo/margin_mean": 110.39860534667969,
"margin_dpo/margin_std": 287.36102294921875,
"step": 290
},
{
"KL/chosen_KL_mean": -188.7603759765625,
"KL/mean": -280.0110168457031,
"KL/rejected_KL_mean": -371.2616882324219,
"KL/std": 305.65924072265625,
"epoch": 0.4399092970521542,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 182.5012969970703,
"fcm_dpo/q_t": 0.4999823570251465,
"grad_norm": 0.0030123190954327583,
"learning_rate": 3.4536538175334343e-07,
"logits/chosen": 1.3322393894195557,
"logits/rejected": 1.3048069477081299,
"logps/chosen": -239.6234130859375,
"logps/ref_chosen": -50.863037109375,
"logps/ref_rejected": -82.20868682861328,
"logps/rejected": -453.4703674316406,
"loss": 1.3862,
"margin_dpo/margin_mean": 182.5012969970703,
"margin_dpo/margin_std": 367.24835205078125,
"step": 291
},
{
"KL/chosen_KL_mean": -225.71701049804688,
"KL/mean": -268.5120849609375,
"KL/rejected_KL_mean": -311.30718994140625,
"KL/std": 296.009033203125,
"epoch": 0.4414210128495843,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 85.59013366699219,
"fcm_dpo/q_t": 0.4999917149543762,
"grad_norm": 0.0019401878817006946,
"learning_rate": 3.4414181450867465e-07,
"logits/chosen": 1.244645118713379,
"logits/rejected": 1.2302597761154175,
"logps/chosen": -290.06591796875,
"logps/ref_chosen": -64.34888458251953,
"logps/ref_rejected": -72.86434173583984,
"logps/rejected": -384.1715087890625,
"loss": 1.3863,
"margin_dpo/margin_mean": 85.59012603759766,
"margin_dpo/margin_std": 375.63006591796875,
"step": 292
},
{
"KL/chosen_KL_mean": -197.04302978515625,
"KL/mean": -276.2276306152344,
"KL/rejected_KL_mean": -355.4122314453125,
"KL/std": 286.4289855957031,
"epoch": 0.4429327286470144,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 158.36923217773438,
"fcm_dpo/q_t": 0.4999846816062927,
"grad_norm": 0.002683456754311919,
"learning_rate": 3.4291561391508185e-07,
"logits/chosen": 1.3139989376068115,
"logits/rejected": 1.2797266244888306,
"logps/chosen": -251.91250610351562,
"logps/ref_chosen": -54.869468688964844,
"logps/ref_rejected": -81.858642578125,
"logps/rejected": -437.2708740234375,
"loss": 1.3862,
"margin_dpo/margin_mean": 158.36923217773438,
"margin_dpo/margin_std": 348.2574157714844,
"step": 293
},
{
"KL/chosen_KL_mean": -204.25213623046875,
"KL/mean": -246.90603637695312,
"KL/rejected_KL_mean": -289.5599365234375,
"KL/std": 281.533447265625,
"epoch": 0.4444444444444444,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 85.30781555175781,
"fcm_dpo/q_t": 0.4999917447566986,
"grad_norm": 0.0014272573171183467,
"learning_rate": 3.4168681427203153e-07,
"logits/chosen": 1.244971752166748,
"logits/rejected": 1.2297112941741943,
"logps/chosen": -260.92303466796875,
"logps/ref_chosen": -56.670902252197266,
"logps/ref_rejected": -70.32819366455078,
"logps/rejected": -359.88812255859375,
"loss": 1.3863,
"margin_dpo/margin_mean": 85.30781555175781,
"margin_dpo/margin_std": 254.677978515625,
"step": 294
},
{
"KL/chosen_KL_mean": -193.20266723632812,
"KL/mean": -270.8068542480469,
"KL/rejected_KL_mean": -348.4110107421875,
"KL/std": 270.56781005859375,
"epoch": 0.4459561602418745,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 155.20838928222656,
"fcm_dpo/q_t": 0.4999849796295166,
"grad_norm": 0.0031207390129566193,
"learning_rate": 3.4045544995169125e-07,
"logits/chosen": 1.2604093551635742,
"logits/rejected": 1.2129838466644287,
"logps/chosen": -243.60354614257812,
"logps/ref_chosen": -50.40088653564453,
"logps/ref_rejected": -83.43521881103516,
"logps/rejected": -431.84625244140625,
"loss": 1.3862,
"margin_dpo/margin_mean": 155.20838928222656,
"margin_dpo/margin_std": 278.3273010253906,
"step": 295
},
{
"KL/chosen_KL_mean": -232.06796264648438,
"KL/mean": -298.0061340332031,
"KL/rejected_KL_mean": -363.9443359375,
"KL/std": 285.2636413574219,
"epoch": 0.4474678760393046,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 131.87635803222656,
"fcm_dpo/q_t": 0.49998724460601807,
"grad_norm": 0.0020598177798092365,
"learning_rate": 3.392215553979679e-07,
"logits/chosen": 1.1887896060943604,
"logits/rejected": 1.1723339557647705,
"logps/chosen": -301.21832275390625,
"logps/ref_chosen": -69.15034484863281,
"logps/ref_rejected": -89.60166931152344,
"logps/rejected": -453.5459899902344,
"loss": 1.3862,
"margin_dpo/margin_mean": 131.87635803222656,
"margin_dpo/margin_std": 339.91888427734375,
"step": 296
},
{
"KL/chosen_KL_mean": -209.5154266357422,
"KL/mean": -267.8277587890625,
"KL/rejected_KL_mean": -326.14007568359375,
"KL/std": 264.28759765625,
"epoch": 0.4489795918367347,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 116.6246566772461,
"fcm_dpo/q_t": 0.49998870491981506,
"grad_norm": 0.0021352798212319613,
"learning_rate": 3.3798516512554485e-07,
"logits/chosen": 1.2609082460403442,
"logits/rejected": 1.251166820526123,
"logps/chosen": -267.53173828125,
"logps/ref_chosen": -58.01630401611328,
"logps/ref_rejected": -69.95780944824219,
"logps/rejected": -396.0978698730469,
"loss": 1.3862,
"margin_dpo/margin_mean": 116.6246566772461,
"margin_dpo/margin_std": 218.6939239501953,
"step": 297
},
{
"KL/chosen_KL_mean": -233.57615661621094,
"KL/mean": -274.2930908203125,
"KL/rejected_KL_mean": -315.010009765625,
"KL/std": 299.4732360839844,
"epoch": 0.4504913076341648,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 81.43384552001953,
"fcm_dpo/q_t": 0.49999210238456726,
"grad_norm": 0.0023514467757195234,
"learning_rate": 3.367463137189156e-07,
"logits/chosen": 1.2886168956756592,
"logits/rejected": 1.2632899284362793,
"logps/chosen": -289.7454833984375,
"logps/ref_chosen": -56.1693115234375,
"logps/ref_rejected": -68.55052185058594,
"logps/rejected": -383.5605163574219,
"loss": 1.3863,
"margin_dpo/margin_mean": 81.43385314941406,
"margin_dpo/margin_std": 265.78594970703125,
"step": 298
},
{
"KL/chosen_KL_mean": -219.94754028320312,
"KL/mean": -259.0394287109375,
"KL/rejected_KL_mean": -298.13128662109375,
"KL/std": 257.43145751953125,
"epoch": 0.4520030234315949,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 78.1837387084961,
"fcm_dpo/q_t": 0.4999924302101135,
"grad_norm": 0.0015664704842492938,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": 1.1981974840164185,
"logits/rejected": 1.1902220249176025,
"logps/chosen": -282.2653503417969,
"logps/ref_chosen": -62.31780242919922,
"logps/ref_rejected": -72.60028839111328,
"logps/rejected": -370.7315673828125,
"loss": 1.3863,
"margin_dpo/margin_mean": 78.18374633789062,
"margin_dpo/margin_std": 312.6253967285156,
"step": 299
},
{
"KL/chosen_KL_mean": -220.5366973876953,
"KL/mean": -281.3580322265625,
"KL/rejected_KL_mean": -342.1794128417969,
"KL/std": 277.378173828125,
"epoch": 0.45351473922902497,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 121.64270782470703,
"fcm_dpo/q_t": 0.4999881982803345,
"grad_norm": 0.0024519728031009436,
"learning_rate": 3.3426136618426043e-07,
"logits/chosen": 1.2705652713775635,
"logits/rejected": 1.2518296241760254,
"logps/chosen": -280.91827392578125,
"logps/ref_chosen": -60.38157653808594,
"logps/ref_rejected": -75.45442199707031,
"logps/rejected": -417.63385009765625,
"loss": 1.3862,
"margin_dpo/margin_mean": 121.6427001953125,
"margin_dpo/margin_std": 316.0571594238281,
"step": 300
},
{
"KL/chosen_KL_mean": -221.11685180664062,
"KL/mean": -255.0898895263672,
"KL/rejected_KL_mean": -289.06292724609375,
"KL/std": 244.7019805908203,
"epoch": 0.455026455026455,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 67.9460678100586,
"fcm_dpo/q_t": 0.4999934434890747,
"grad_norm": 0.0015637945616617799,
"learning_rate": 3.3301533956555885e-07,
"logits/chosen": 1.2348217964172363,
"logits/rejected": 1.2190877199172974,
"logps/chosen": -273.9677429199219,
"logps/ref_chosen": -52.85089111328125,
"logps/ref_rejected": -69.97584533691406,
"logps/rejected": -359.03875732421875,
"loss": 1.3863,
"margin_dpo/margin_mean": 67.94606018066406,
"margin_dpo/margin_std": 319.27197265625,
"step": 301
},
{
"KL/chosen_KL_mean": -285.2535400390625,
"KL/mean": -327.0502624511719,
"KL/rejected_KL_mean": -368.8470153808594,
"KL/std": 318.28948974609375,
"epoch": 0.4565381708238851,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 83.593505859375,
"fcm_dpo/q_t": 0.49999189376831055,
"grad_norm": 0.002295486629009247,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": 1.2259061336517334,
"logits/rejected": 1.203969955444336,
"logps/chosen": -352.22003173828125,
"logps/ref_chosen": -66.96650695800781,
"logps/ref_rejected": -88.09510803222656,
"logps/rejected": -456.942138671875,
"loss": 1.3863,
"margin_dpo/margin_mean": 83.593505859375,
"margin_dpo/margin_std": 367.9168395996094,
"step": 302
},
{
"KL/chosen_KL_mean": -222.12728881835938,
"KL/mean": -289.8243103027344,
"KL/rejected_KL_mean": -357.5213317871094,
"KL/std": 335.4458923339844,
"epoch": 0.4580498866213152,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 135.39398193359375,
"fcm_dpo/q_t": 0.4999868869781494,
"grad_norm": 0.0023559981491416693,
"learning_rate": 3.3051635489464793e-07,
"logits/chosen": 1.1870148181915283,
"logits/rejected": 1.1635546684265137,
"logps/chosen": -284.24884033203125,
"logps/ref_chosen": -62.12152862548828,
"logps/ref_rejected": -90.31204223632812,
"logps/rejected": -447.8333740234375,
"loss": 1.3862,
"margin_dpo/margin_mean": 135.3939971923828,
"margin_dpo/margin_std": 403.8096923828125,
"step": 303
},
{
"KL/chosen_KL_mean": -235.18692016601562,
"KL/mean": -295.52618408203125,
"KL/rejected_KL_mean": -355.86541748046875,
"KL/std": 304.51690673828125,
"epoch": 0.4595616024187453,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 120.678466796875,
"fcm_dpo/q_t": 0.499988317489624,
"grad_norm": 0.0019933602306991816,
"learning_rate": 3.292634667444117e-07,
"logits/chosen": 1.2784829139709473,
"logits/rejected": 1.2663378715515137,
"logps/chosen": -295.88201904296875,
"logps/ref_chosen": -60.695091247558594,
"logps/ref_rejected": -78.2525405883789,
"logps/rejected": -434.1179504394531,
"loss": 1.3862,
"margin_dpo/margin_mean": 120.67847442626953,
"margin_dpo/margin_std": 319.9189453125,
"step": 304
},
{
"KL/chosen_KL_mean": -272.2154541015625,
"KL/mean": -285.18829345703125,
"KL/rejected_KL_mean": -298.16107177734375,
"KL/std": 275.7799072265625,
"epoch": 0.46107331821617537,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 25.945667266845703,
"fcm_dpo/q_t": 0.49999749660491943,
"grad_norm": 0.0017613671952858567,
"learning_rate": 3.280083614246217e-07,
"logits/chosen": 1.2120630741119385,
"logits/rejected": 1.2468100786209106,
"logps/chosen": -344.9145812988281,
"logps/ref_chosen": -72.69914245605469,
"logps/ref_rejected": -65.65670776367188,
"logps/rejected": -363.81781005859375,
"loss": 1.3863,
"margin_dpo/margin_mean": 25.945667266845703,
"margin_dpo/margin_std": 326.3166198730469,
"step": 305
},
{
"KL/chosen_KL_mean": -198.40487670898438,
"KL/mean": -271.77508544921875,
"KL/rejected_KL_mean": -345.14532470703125,
"KL/std": 292.58148193359375,
"epoch": 0.46258503401360546,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 146.74044799804688,
"fcm_dpo/q_t": 0.49998578429222107,
"grad_norm": 0.002900703577324748,
"learning_rate": 3.267510740432719e-07,
"logits/chosen": 1.2891820669174194,
"logits/rejected": 1.249337077140808,
"logps/chosen": -252.37539672851562,
"logps/ref_chosen": -53.97052764892578,
"logps/ref_rejected": -71.02423095703125,
"logps/rejected": -416.1695556640625,
"loss": 1.3862,
"margin_dpo/margin_mean": 146.74044799804688,
"margin_dpo/margin_std": 371.2115478515625,
"step": 306
},
{
"KL/chosen_KL_mean": -208.95254516601562,
"KL/mean": -243.92665100097656,
"KL/rejected_KL_mean": -278.9007568359375,
"KL/std": 252.12908935546875,
"epoch": 0.46409674981103555,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 69.94822692871094,
"fcm_dpo/q_t": 0.4999932646751404,
"grad_norm": 0.0016844801139086485,
"learning_rate": 3.2549163976939285e-07,
"logits/chosen": 1.2810804843902588,
"logits/rejected": 1.2607386112213135,
"logps/chosen": -266.36566162109375,
"logps/ref_chosen": -57.413108825683594,
"logps/ref_rejected": -68.68010711669922,
"logps/rejected": -347.58087158203125,
"loss": 1.3863,
"margin_dpo/margin_mean": 69.94822692871094,
"margin_dpo/margin_std": 359.750732421875,
"step": 307
},
{
"KL/chosen_KL_mean": -280.36407470703125,
"KL/mean": -300.9876708984375,
"KL/rejected_KL_mean": -321.61126708984375,
"KL/std": 320.69512939453125,
"epoch": 0.4656084656084656,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 41.247154235839844,
"fcm_dpo/q_t": 0.49999600648880005,
"grad_norm": 0.00144182366784662,
"learning_rate": 3.2423009383206874e-07,
"logits/chosen": 1.2935481071472168,
"logits/rejected": 1.287482738494873,
"logps/chosen": -346.962890625,
"logps/ref_chosen": -66.59879302978516,
"logps/ref_rejected": -74.337158203125,
"logps/rejected": -395.9483947753906,
"loss": 1.3863,
"margin_dpo/margin_mean": 41.247154235839844,
"margin_dpo/margin_std": 429.9683837890625,
"step": 308
},
{
"KL/chosen_KL_mean": -248.10739135742188,
"KL/mean": -301.85650634765625,
"KL/rejected_KL_mean": -355.60565185546875,
"KL/std": 268.0228271484375,
"epoch": 0.4671201814058957,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 107.498291015625,
"fcm_dpo/q_t": 0.4999895989894867,
"grad_norm": 0.002324699657037854,
"learning_rate": 3.229664715194511e-07,
"logits/chosen": 1.25961434841156,
"logits/rejected": 1.245445728302002,
"logps/chosen": -313.50213623046875,
"logps/ref_chosen": -65.39474487304688,
"logps/ref_rejected": -75.70930480957031,
"logps/rejected": -431.31494140625,
"loss": 1.3863,
"margin_dpo/margin_mean": 107.49828338623047,
"margin_dpo/margin_std": 310.15728759765625,
"step": 309
},
{
"KL/chosen_KL_mean": -295.44293212890625,
"KL/mean": -333.5084228515625,
"KL/rejected_KL_mean": -371.5738830566406,
"KL/std": 341.388671875,
"epoch": 0.46863189720332576,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 76.13092041015625,
"fcm_dpo/q_t": 0.49999260902404785,
"grad_norm": 0.0016792939277365804,
"learning_rate": 3.2170080817777257e-07,
"logits/chosen": 1.264609456062317,
"logits/rejected": 1.2663490772247314,
"logps/chosen": -370.1112060546875,
"logps/ref_chosen": -74.66827392578125,
"logps/ref_rejected": -80.5689697265625,
"logps/rejected": -452.142822265625,
"loss": 1.3863,
"margin_dpo/margin_mean": 76.13092803955078,
"margin_dpo/margin_std": 361.7975769042969,
"step": 310
},
{
"KL/chosen_KL_mean": -204.85330200195312,
"KL/mean": -294.4068603515625,
"KL/rejected_KL_mean": -383.9604187011719,
"KL/std": 327.627685546875,
"epoch": 0.47014361300075586,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 179.10711669921875,
"fcm_dpo/q_t": 0.49998265504837036,
"grad_norm": 0.0032888497225940228,
"learning_rate": 3.204331392103574e-07,
"logits/chosen": 1.2235231399536133,
"logits/rejected": 1.1665198802947998,
"logps/chosen": -264.5913391113281,
"logps/ref_chosen": -59.738033294677734,
"logps/ref_rejected": -93.60757446289062,
"logps/rejected": -477.5679931640625,
"loss": 1.3862,
"margin_dpo/margin_mean": 179.10711669921875,
"margin_dpo/margin_std": 429.142822265625,
"step": 311
},
{
"KL/chosen_KL_mean": -197.45437622070312,
"KL/mean": -267.0279541015625,
"KL/rejected_KL_mean": -336.6015319824219,
"KL/std": 230.93252563476562,
"epoch": 0.47165532879818595,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 139.14715576171875,
"fcm_dpo/q_t": 0.49998652935028076,
"grad_norm": 0.0022931136190891266,
"learning_rate": 3.1916350007663176e-07,
"logits/chosen": 1.2868523597717285,
"logits/rejected": 1.2605751752853394,
"logps/chosen": -251.27081298828125,
"logps/ref_chosen": -53.816436767578125,
"logps/ref_rejected": -68.6575698852539,
"logps/rejected": -405.25909423828125,
"loss": 1.3862,
"margin_dpo/margin_mean": 139.1471710205078,
"margin_dpo/margin_std": 282.67626953125,
"step": 312
},
{
"KL/chosen_KL_mean": -232.93560791015625,
"KL/mean": -266.92388916015625,
"KL/rejected_KL_mean": -300.91217041015625,
"KL/std": 304.7332763671875,
"epoch": 0.47316704459561604,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 67.97655487060547,
"fcm_dpo/q_t": 0.4999934136867523,
"grad_norm": 0.0016308584017679095,
"learning_rate": 3.178919262911314e-07,
"logits/chosen": 1.3045995235443115,
"logits/rejected": 1.302783489227295,
"logps/chosen": -292.8929748535156,
"logps/ref_chosen": -59.957359313964844,
"logps/ref_rejected": -69.31729888916016,
"logps/rejected": -370.2294921875,
"loss": 1.3863,
"margin_dpo/margin_mean": 67.9765625,
"margin_dpo/margin_std": 373.98345947265625,
"step": 313
},
{
"KL/chosen_KL_mean": -240.1637420654297,
"KL/mean": -261.7298583984375,
"KL/rejected_KL_mean": -283.29595947265625,
"KL/std": 226.59564208984375,
"epoch": 0.47467876039304613,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 43.13225555419922,
"fcm_dpo/q_t": 0.4999958276748657,
"grad_norm": 0.0014463362749665976,
"learning_rate": 3.166184534225087e-07,
"logits/chosen": 1.2623672485351562,
"logits/rejected": 1.2793068885803223,
"logps/chosen": -310.431884765625,
"logps/ref_chosen": -70.26815795898438,
"logps/ref_rejected": -69.23971557617188,
"logps/rejected": -352.53570556640625,
"loss": 1.3863,
"margin_dpo/margin_mean": 43.13225555419922,
"margin_dpo/margin_std": 263.3436279296875,
"step": 314
},
{
"KL/chosen_KL_mean": -246.52500915527344,
"KL/mean": -288.8182373046875,
"KL/rejected_KL_mean": -331.1114196777344,
"KL/std": 295.01458740234375,
"epoch": 0.47619047619047616,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 84.58641052246094,
"fcm_dpo/q_t": 0.49999183416366577,
"grad_norm": 0.0019202815601602197,
"learning_rate": 3.1534311709253723e-07,
"logits/chosen": 1.2083520889282227,
"logits/rejected": 1.2015604972839355,
"logps/chosen": -314.3197021484375,
"logps/ref_chosen": -67.79469299316406,
"logps/ref_rejected": -74.55148315429688,
"logps/rejected": -405.66290283203125,
"loss": 1.3863,
"margin_dpo/margin_mean": 84.58641052246094,
"margin_dpo/margin_std": 364.86529541015625,
"step": 315
},
{
"KL/chosen_KL_mean": -200.84915161132812,
"KL/mean": -298.4619445800781,
"KL/rejected_KL_mean": -396.07470703125,
"KL/std": 297.4620666503906,
"epoch": 0.47770219198790626,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 195.2255859375,
"fcm_dpo/q_t": 0.4999811053276062,
"grad_norm": 0.0035327484365552664,
"learning_rate": 3.1406595297511564e-07,
"logits/chosen": 1.211460828781128,
"logits/rejected": 1.1569902896881104,
"logps/chosen": -256.13763427734375,
"logps/ref_chosen": -55.288482666015625,
"logps/ref_rejected": -96.15723419189453,
"logps/rejected": -492.2319641113281,
"loss": 1.3862,
"margin_dpo/margin_mean": 195.22557067871094,
"margin_dpo/margin_std": 394.2180480957031,
"step": 316
},
{
"KL/chosen_KL_mean": -203.72149658203125,
"KL/mean": -248.2875518798828,
"KL/rejected_KL_mean": -292.8536376953125,
"KL/std": 228.49501037597656,
"epoch": 0.47921390778533635,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.13215637207031,
"fcm_dpo/q_t": 0.49999138712882996,
"grad_norm": 0.0016370030352845788,
"learning_rate": 3.1278699679526975e-07,
"logits/chosen": 1.2447960376739502,
"logits/rejected": 1.2230143547058105,
"logps/chosen": -258.3028564453125,
"logps/ref_chosen": -54.58137512207031,
"logps/ref_rejected": -72.77232360839844,
"logps/rejected": -365.6259460449219,
"loss": 1.3863,
"margin_dpo/margin_mean": 89.13215637207031,
"margin_dpo/margin_std": 290.63079833984375,
"step": 317
},
{
"KL/chosen_KL_mean": -209.48098754882812,
"KL/mean": -272.79022216796875,
"KL/rejected_KL_mean": -336.09942626953125,
"KL/std": 281.35205078125,
"epoch": 0.48072562358276644,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 126.61843872070312,
"fcm_dpo/q_t": 0.49998778104782104,
"grad_norm": 0.0026248276699334383,
"learning_rate": 3.1150628432815336e-07,
"logits/chosen": 1.296776294708252,
"logits/rejected": 1.268967866897583,
"logps/chosen": -262.3692321777344,
"logps/ref_chosen": -52.88822937011719,
"logps/ref_rejected": -80.63988494873047,
"logps/rejected": -416.73931884765625,
"loss": 1.3862,
"margin_dpo/margin_mean": 126.6184310913086,
"margin_dpo/margin_std": 386.39801025390625,
"step": 318
},
{
"KL/chosen_KL_mean": -246.42294311523438,
"KL/mean": -298.90435791015625,
"KL/rejected_KL_mean": -351.38580322265625,
"KL/std": 296.37994384765625,
"epoch": 0.48223733938019653,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.962890625,
"fcm_dpo/q_t": 0.4999898672103882,
"grad_norm": 0.0018176967278122902,
"learning_rate": 3.1022385139804707e-07,
"logits/chosen": 1.2094240188598633,
"logits/rejected": 1.211363434791565,
"logps/chosen": -310.7862548828125,
"logps/ref_chosen": -64.36333465576172,
"logps/ref_rejected": -79.47296142578125,
"logps/rejected": -430.8587646484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 104.96288299560547,
"margin_dpo/margin_std": 335.21063232421875,
"step": 319
},
{
"KL/chosen_KL_mean": -198.46250915527344,
"KL/mean": -244.89617919921875,
"KL/rejected_KL_mean": -291.32989501953125,
"KL/std": 290.3658142089844,
"epoch": 0.4837490551776266,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 92.86738586425781,
"fcm_dpo/q_t": 0.4999910295009613,
"grad_norm": 0.0021061012521386147,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": 1.236513614654541,
"logits/rejected": 1.227114200592041,
"logps/chosen": -248.02125549316406,
"logps/ref_chosen": -49.558746337890625,
"logps/ref_rejected": -71.23444366455078,
"logps/rejected": -362.5643310546875,
"loss": 1.3863,
"margin_dpo/margin_mean": 92.86739349365234,
"margin_dpo/margin_std": 356.69232177734375,
"step": 320
},
{
"KL/chosen_KL_mean": -178.01779174804688,
"KL/mean": -218.20223999023438,
"KL/rejected_KL_mean": -258.3866882324219,
"KL/std": 188.52374267578125,
"epoch": 0.4852607709750567,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 80.36886596679688,
"fcm_dpo/q_t": 0.4999922215938568,
"grad_norm": 0.0019223628332838416,
"learning_rate": 3.0765396768561004e-07,
"logits/chosen": 1.2715182304382324,
"logits/rejected": 1.2804362773895264,
"logps/chosen": -230.1030731201172,
"logps/ref_chosen": -52.08526611328125,
"logps/ref_rejected": -55.58674621582031,
"logps/rejected": -313.9734191894531,
"loss": 1.3863,
"margin_dpo/margin_mean": 80.36886596679688,
"margin_dpo/margin_std": 233.33819580078125,
"step": 321
},
{
"KL/chosen_KL_mean": -205.60140991210938,
"KL/mean": -281.4060974121094,
"KL/rejected_KL_mean": -357.21075439453125,
"KL/std": 279.41717529296875,
"epoch": 0.48677248677248675,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 151.609375,
"fcm_dpo/q_t": 0.49998533725738525,
"grad_norm": 0.0024833802599459887,
"learning_rate": 3.063665887884511e-07,
"logits/chosen": 1.2712690830230713,
"logits/rejected": 1.2434958219528198,
"logps/chosen": -253.00552368164062,
"logps/ref_chosen": -47.404109954833984,
"logps/ref_rejected": -73.4260025024414,
"logps/rejected": -430.63677978515625,
"loss": 1.3862,
"margin_dpo/margin_mean": 151.609375,
"margin_dpo/margin_std": 249.66696166992188,
"step": 322
},
{
"KL/chosen_KL_mean": -240.9448699951172,
"KL/mean": -307.22540283203125,
"KL/rejected_KL_mean": -373.50592041015625,
"KL/std": 300.21624755859375,
"epoch": 0.48828420256991684,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 132.5610809326172,
"fcm_dpo/q_t": 0.4999871551990509,
"grad_norm": 0.0025218951050192118,
"learning_rate": 3.0507763319663517e-07,
"logits/chosen": 1.174757480621338,
"logits/rejected": 1.1535224914550781,
"logps/chosen": -310.951171875,
"logps/ref_chosen": -70.00630187988281,
"logps/ref_rejected": -86.96690368652344,
"logps/rejected": -460.47283935546875,
"loss": 1.3862,
"margin_dpo/margin_mean": 132.5610809326172,
"margin_dpo/margin_std": 364.93914794921875,
"step": 323
},
{
"KL/chosen_KL_mean": -213.75881958007812,
"KL/mean": -267.80682373046875,
"KL/rejected_KL_mean": -321.8548583984375,
"KL/std": 297.6637268066406,
"epoch": 0.4897959183673469,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 108.09601593017578,
"fcm_dpo/q_t": 0.4999895393848419,
"grad_norm": 0.00212281895801425,
"learning_rate": 3.0378713696502097e-07,
"logits/chosen": 1.2725311517715454,
"logits/rejected": 1.250030755996704,
"logps/chosen": -269.64764404296875,
"logps/ref_chosen": -55.88882064819336,
"logps/ref_rejected": -75.23088073730469,
"logps/rejected": -397.0857238769531,
"loss": 1.3863,
"margin_dpo/margin_mean": 108.09602355957031,
"margin_dpo/margin_std": 297.931640625,
"step": 324
},
{
"KL/chosen_KL_mean": -228.80210876464844,
"KL/mean": -300.92083740234375,
"KL/rejected_KL_mean": -373.03955078125,
"KL/std": 327.6212158203125,
"epoch": 0.491307634164777,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 144.23745727539062,
"fcm_dpo/q_t": 0.49998605251312256,
"grad_norm": 0.002541890600696206,
"learning_rate": 3.0249513619156206e-07,
"logits/chosen": 1.2248497009277344,
"logits/rejected": 1.2180355787277222,
"logps/chosen": -292.9491271972656,
"logps/ref_chosen": -64.14701843261719,
"logps/ref_rejected": -79.91143798828125,
"logps/rejected": -452.95098876953125,
"loss": 1.3862,
"margin_dpo/margin_mean": 144.23745727539062,
"margin_dpo/margin_std": 414.3074645996094,
"step": 325
},
{
"KL/chosen_KL_mean": -288.5466003417969,
"KL/mean": -298.1192626953125,
"KL/rejected_KL_mean": -307.69195556640625,
"KL/std": 281.703125,
"epoch": 0.4928193499622071,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 19.14533233642578,
"fcm_dpo/q_t": 0.49999815225601196,
"grad_norm": 0.0015972210094332695,
"learning_rate": 3.012016670162977e-07,
"logits/chosen": 1.2351655960083008,
"logits/rejected": 1.2426598072052002,
"logps/chosen": -364.07794189453125,
"logps/ref_chosen": -75.53131103515625,
"logps/ref_rejected": -76.5898666381836,
"logps/rejected": -384.28179931640625,
"loss": 1.3863,
"margin_dpo/margin_mean": 19.14533233642578,
"margin_dpo/margin_std": 339.36480712890625,
"step": 326
},
{
"KL/chosen_KL_mean": -287.528564453125,
"KL/mean": -324.689453125,
"KL/rejected_KL_mean": -361.85040283203125,
"KL/std": 321.2103271484375,
"epoch": 0.4943310657596372,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 74.32182312011719,
"fcm_dpo/q_t": 0.49999281764030457,
"grad_norm": 0.0020750907715409994,
"learning_rate": 2.99906765620341e-07,
"logits/chosen": 1.1873157024383545,
"logits/rejected": 1.1866263151168823,
"logps/chosen": -356.86572265625,
"logps/ref_chosen": -69.33717346191406,
"logps/ref_rejected": -73.37751770019531,
"logps/rejected": -435.2279052734375,
"loss": 1.3863,
"margin_dpo/margin_mean": 74.32182312011719,
"margin_dpo/margin_std": 364.2970275878906,
"step": 327
},
{
"KL/chosen_KL_mean": -234.11676025390625,
"KL/mean": -285.86553955078125,
"KL/rejected_KL_mean": -337.6143798828125,
"KL/std": 277.8924865722656,
"epoch": 0.4958427815570673,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 103.49760437011719,
"fcm_dpo/q_t": 0.49998998641967773,
"grad_norm": 0.0018194402800872922,
"learning_rate": 2.9861046822486766e-07,
"logits/chosen": 1.1889562606811523,
"logits/rejected": 1.171844244003296,
"logps/chosen": -295.822998046875,
"logps/ref_chosen": -61.70623016357422,
"logps/ref_rejected": -83.73808288574219,
"logps/rejected": -421.3524475097656,
"loss": 1.3863,
"margin_dpo/margin_mean": 103.49760437011719,
"margin_dpo/margin_std": 310.90997314453125,
"step": 328
},
{
"KL/chosen_KL_mean": -241.38299560546875,
"KL/mean": -305.01055908203125,
"KL/rejected_KL_mean": -368.63812255859375,
"KL/std": 312.40496826171875,
"epoch": 0.4973544973544973,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 127.255126953125,
"fcm_dpo/q_t": 0.4999876618385315,
"grad_norm": 0.002292838878929615,
"learning_rate": 2.9731281109010253e-07,
"logits/chosen": 1.2591251134872437,
"logits/rejected": 1.2390198707580566,
"logps/chosen": -305.88140869140625,
"logps/ref_chosen": -64.4984130859375,
"logps/ref_rejected": -83.6591796875,
"logps/rejected": -452.29730224609375,
"loss": 1.3862,
"margin_dpo/margin_mean": 127.255126953125,
"margin_dpo/margin_std": 396.40557861328125,
"step": 329
},
{
"KL/chosen_KL_mean": -189.281494140625,
"KL/mean": -244.40090942382812,
"KL/rejected_KL_mean": -299.52032470703125,
"KL/std": 252.27133178710938,
"epoch": 0.4988662131519274,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 110.23881530761719,
"fcm_dpo/q_t": 0.4999893307685852,
"grad_norm": 0.0021174049470573664,
"learning_rate": 2.9601383051430505e-07,
"logits/chosen": 1.2592849731445312,
"logits/rejected": 1.2317944765090942,
"logps/chosen": -244.08615112304688,
"logps/ref_chosen": -54.80464172363281,
"logps/ref_rejected": -75.3194351196289,
"logps/rejected": -374.83978271484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 110.23881530761719,
"margin_dpo/margin_std": 354.689697265625,
"step": 330
},
{
"KL/chosen_KL_mean": -210.95858764648438,
"KL/mean": -264.14520263671875,
"KL/rejected_KL_mean": -317.331787109375,
"KL/std": 248.91436767578125,
"epoch": 0.5003779289493575,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 106.37322235107422,
"fcm_dpo/q_t": 0.49998968839645386,
"grad_norm": 0.0013487640535458922,
"learning_rate": 2.947135628327544e-07,
"logits/chosen": 1.2729997634887695,
"logits/rejected": 1.273748755455017,
"logps/chosen": -270.201171875,
"logps/ref_chosen": -59.242584228515625,
"logps/ref_rejected": -69.87483215332031,
"logps/rejected": -387.2066345214844,
"loss": 1.3863,
"margin_dpo/margin_mean": 106.37321472167969,
"margin_dpo/margin_std": 286.5001220703125,
"step": 331
},
{
"KL/chosen_KL_mean": -226.46791076660156,
"KL/mean": -281.8634033203125,
"KL/rejected_KL_mean": -337.2588806152344,
"KL/std": 250.1396484375,
"epoch": 0.5018896447467877,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 110.7909927368164,
"fcm_dpo/q_t": 0.4999893009662628,
"grad_norm": 0.0019972885493189096,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": 1.2130635976791382,
"logits/rejected": 1.2021818161010742,
"logps/chosen": -293.5776672363281,
"logps/ref_chosen": -67.10975646972656,
"logps/ref_rejected": -77.11839294433594,
"logps/rejected": -414.37725830078125,
"loss": 1.3863,
"margin_dpo/margin_mean": 110.7909927368164,
"margin_dpo/margin_std": 291.0926208496094,
"step": 332
},
{
"KL/chosen_KL_mean": -233.87643432617188,
"KL/mean": -303.2012939453125,
"KL/rejected_KL_mean": -372.526123046875,
"KL/std": 304.5091247558594,
"epoch": 0.5034013605442177,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 138.64968872070312,
"fcm_dpo/q_t": 0.49998658895492554,
"grad_norm": 0.0025588450953364372,
"learning_rate": 2.921093116725076e-07,
"logits/chosen": 1.2254364490509033,
"logits/rejected": 1.1889737844467163,
"logps/chosen": -292.257568359375,
"logps/ref_chosen": -58.381134033203125,
"logps/ref_rejected": -85.02839660644531,
"logps/rejected": -457.55450439453125,
"loss": 1.3862,
"margin_dpo/margin_mean": 138.64968872070312,
"margin_dpo/margin_std": 301.95068359375,
"step": 333
},
{
"KL/chosen_KL_mean": -223.56558227539062,
"KL/mean": -282.3580322265625,
"KL/rejected_KL_mean": -341.15045166015625,
"KL/std": 278.36016845703125,
"epoch": 0.5049130763416477,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 117.58486938476562,
"fcm_dpo/q_t": 0.4999886155128479,
"grad_norm": 0.0021759173832833767,
"learning_rate": 2.9080540104031484e-07,
"logits/chosen": 1.2243032455444336,
"logits/rejected": 1.1974499225616455,
"logps/chosen": -290.45758056640625,
"logps/ref_chosen": -66.89199829101562,
"logps/ref_rejected": -91.83695220947266,
"logps/rejected": -432.9873962402344,
"loss": 1.3862,
"margin_dpo/margin_mean": 117.58486938476562,
"margin_dpo/margin_std": 335.0116882324219,
"step": 334
},
{
"KL/chosen_KL_mean": -209.07431030273438,
"KL/mean": -258.6519775390625,
"KL/rejected_KL_mean": -308.2296142578125,
"KL/std": 264.89276123046875,
"epoch": 0.5064247921390779,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.15528869628906,
"fcm_dpo/q_t": 0.49999040365219116,
"grad_norm": 0.0017754482105374336,
"learning_rate": 2.895003489933375e-07,
"logits/chosen": 1.212792992591858,
"logits/rejected": 1.2027299404144287,
"logps/chosen": -270.5887756347656,
"logps/ref_chosen": -61.51445770263672,
"logps/ref_rejected": -75.68916320800781,
"logps/rejected": -383.9187927246094,
"loss": 1.3863,
"margin_dpo/margin_mean": 99.15528869628906,
"margin_dpo/margin_std": 291.6207275390625,
"step": 335
},
{
"KL/chosen_KL_mean": -274.1320495605469,
"KL/mean": -322.80322265625,
"KL/rejected_KL_mean": -371.474365234375,
"KL/std": 263.66119384765625,
"epoch": 0.5079365079365079,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 97.34229278564453,
"fcm_dpo/q_t": 0.4999905824661255,
"grad_norm": 0.002034288365393877,
"learning_rate": 2.8819419203668675e-07,
"logits/chosen": 1.2427122592926025,
"logits/rejected": 1.2363805770874023,
"logps/chosen": -342.98211669921875,
"logps/ref_chosen": -68.85006713867188,
"logps/ref_rejected": -92.99603271484375,
"logps/rejected": -464.4703674316406,
"loss": 1.3863,
"margin_dpo/margin_mean": 97.34230041503906,
"margin_dpo/margin_std": 293.6593017578125,
"step": 336
},
{
"KL/chosen_KL_mean": -289.3245849609375,
"KL/mean": -334.5281066894531,
"KL/rejected_KL_mean": -379.7316589355469,
"KL/std": 313.8486328125,
"epoch": 0.509448223733938,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 90.40706634521484,
"fcm_dpo/q_t": 0.4999912679195404,
"grad_norm": 0.0020725426729768515,
"learning_rate": 2.8688696670638053e-07,
"logits/chosen": 1.1902989149093628,
"logits/rejected": 1.1826756000518799,
"logps/chosen": -362.51239013671875,
"logps/ref_chosen": -73.18783569335938,
"logps/ref_rejected": -86.89118957519531,
"logps/rejected": -466.62286376953125,
"loss": 1.3863,
"margin_dpo/margin_mean": 90.40706634521484,
"margin_dpo/margin_std": 352.4085693359375,
"step": 337
},
{
"KL/chosen_KL_mean": -244.63876342773438,
"KL/mean": -279.78643798828125,
"KL/rejected_KL_mean": -314.93414306640625,
"KL/std": 228.41464233398438,
"epoch": 0.5109599395313681,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 70.29537963867188,
"fcm_dpo/q_t": 0.4999932050704956,
"grad_norm": 0.0021237812470644712,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": 1.2304810285568237,
"logits/rejected": 1.2233942747116089,
"logps/chosen": -308.578369140625,
"logps/ref_chosen": -63.939613342285156,
"logps/ref_rejected": -75.34243774414062,
"logps/rejected": -390.27655029296875,
"loss": 1.3863,
"margin_dpo/margin_mean": 70.29537963867188,
"margin_dpo/margin_std": 193.20977783203125,
"step": 338
},
{
"KL/chosen_KL_mean": -184.398681640625,
"KL/mean": -237.96548461914062,
"KL/rejected_KL_mean": -291.5323181152344,
"KL/std": 243.76315307617188,
"epoch": 0.5124716553287982,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 107.13363647460938,
"fcm_dpo/q_t": 0.4999896287918091,
"grad_norm": 0.002144779311493039,
"learning_rate": 2.842694572172736e-07,
"logits/chosen": 1.333993673324585,
"logits/rejected": 1.2993600368499756,
"logps/chosen": -229.94781494140625,
"logps/ref_chosen": -45.54913330078125,
"logps/ref_rejected": -67.0482177734375,
"logps/rejected": -358.5805358886719,
"loss": 1.3863,
"margin_dpo/margin_mean": 107.13363647460938,
"margin_dpo/margin_std": 302.3402099609375,
"step": 339
},
{
"KL/chosen_KL_mean": -211.99609375,
"KL/mean": -246.68463134765625,
"KL/rejected_KL_mean": -281.3731994628906,
"KL/std": 209.9356689453125,
"epoch": 0.5139833711262283,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 69.37712860107422,
"fcm_dpo/q_t": 0.49999329447746277,
"grad_norm": 0.0015614746371284127,
"learning_rate": 2.8295924627584004e-07,
"logits/chosen": 1.2425800561904907,
"logits/rejected": 1.241082787513733,
"logps/chosen": -266.0017395019531,
"logps/ref_chosen": -54.00564956665039,
"logps/ref_rejected": -61.314430236816406,
"logps/rejected": -342.6876220703125,
"loss": 1.3863,
"margin_dpo/margin_mean": 69.37713623046875,
"margin_dpo/margin_std": 277.55560302734375,
"step": 340
},
{
"KL/chosen_KL_mean": -240.11708068847656,
"KL/mean": -302.804931640625,
"KL/rejected_KL_mean": -365.4927062988281,
"KL/std": 279.3319091796875,
"epoch": 0.5154950869236583,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 125.37564086914062,
"fcm_dpo/q_t": 0.4999878406524658,
"grad_norm": 0.002194821136072278,
"learning_rate": 2.816481133934373e-07,
"logits/chosen": 1.2894377708435059,
"logits/rejected": 1.27662992477417,
"logps/chosen": -303.5121765136719,
"logps/ref_chosen": -63.39509582519531,
"logps/ref_rejected": -76.20973205566406,
"logps/rejected": -441.70245361328125,
"loss": 1.3862,
"margin_dpo/margin_mean": 125.37564086914062,
"margin_dpo/margin_std": 277.63555908203125,
"step": 341
},
{
"KL/chosen_KL_mean": -215.72718811035156,
"KL/mean": -266.249755859375,
"KL/rejected_KL_mean": -316.7723388671875,
"KL/std": 273.38848876953125,
"epoch": 0.5170068027210885,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 101.045166015625,
"fcm_dpo/q_t": 0.49999022483825684,
"grad_norm": 0.0017220752779394388,
"learning_rate": 2.8033609524527046e-07,
"logits/chosen": 1.278684139251709,
"logits/rejected": 1.2664589881896973,
"logps/chosen": -268.7749938964844,
"logps/ref_chosen": -53.047813415527344,
"logps/ref_rejected": -68.2854232788086,
"logps/rejected": -385.0577697753906,
"loss": 1.3863,
"margin_dpo/margin_mean": 101.045166015625,
"margin_dpo/margin_std": 286.5677795410156,
"step": 342
},
{
"KL/chosen_KL_mean": -263.2760009765625,
"KL/mean": -300.4712829589844,
"KL/rejected_KL_mean": -337.6665344238281,
"KL/std": 306.28350830078125,
"epoch": 0.5185185185185185,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 74.39053344726562,
"fcm_dpo/q_t": 0.4999927878379822,
"grad_norm": 0.0017567041795700788,
"learning_rate": 2.7902322853130753e-07,
"logits/chosen": 1.2100257873535156,
"logits/rejected": 1.210863471031189,
"logps/chosen": -333.85455322265625,
"logps/ref_chosen": -70.57852935791016,
"logps/ref_rejected": -84.73873901367188,
"logps/rejected": -422.4052734375,
"loss": 1.3863,
"margin_dpo/margin_mean": 74.3905258178711,
"margin_dpo/margin_std": 372.44439697265625,
"step": 343
},
{
"KL/chosen_KL_mean": -235.8098907470703,
"KL/mean": -311.6300048828125,
"KL/rejected_KL_mean": -387.4500732421875,
"KL/std": 312.3414001464844,
"epoch": 0.5200302343159486,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 151.64019775390625,
"fcm_dpo/q_t": 0.49998533725738525,
"grad_norm": 0.002630658680573106,
"learning_rate": 2.7770954997525274e-07,
"logits/chosen": 1.2740308046340942,
"logits/rejected": 1.2405366897583008,
"logps/chosen": -291.62091064453125,
"logps/ref_chosen": -55.811004638671875,
"logps/ref_rejected": -84.77637481689453,
"logps/rejected": -472.2264404296875,
"loss": 1.3862,
"margin_dpo/margin_mean": 151.6402130126953,
"margin_dpo/margin_std": 380.27703857421875,
"step": 344
},
{
"KL/chosen_KL_mean": -196.2252655029297,
"KL/mean": -252.94921875,
"KL/rejected_KL_mean": -309.6731872558594,
"KL/std": 231.7432861328125,
"epoch": 0.5215419501133787,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 113.44795227050781,
"fcm_dpo/q_t": 0.49998903274536133,
"grad_norm": 0.0021876078099012375,
"learning_rate": 2.7639509632351927e-07,
"logits/chosen": 1.2337894439697266,
"logits/rejected": 1.21903395652771,
"logps/chosen": -254.0113525390625,
"logps/ref_chosen": -57.78609848022461,
"logps/ref_rejected": -78.91847229003906,
"logps/rejected": -388.5916748046875,
"loss": 1.3863,
"margin_dpo/margin_mean": 113.44795227050781,
"margin_dpo/margin_std": 229.27005004882812,
"step": 345
},
{
"KL/chosen_KL_mean": -210.19422912597656,
"KL/mean": -292.7418212890625,
"KL/rejected_KL_mean": -375.2893981933594,
"KL/std": 307.3424072265625,
"epoch": 0.5230536659108088,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 165.09515380859375,
"fcm_dpo/q_t": 0.4999840259552002,
"grad_norm": 0.002935102442279458,
"learning_rate": 2.7507990434420123e-07,
"logits/chosen": 1.247345209121704,
"logits/rejected": 1.2206707000732422,
"logps/chosen": -266.4793701171875,
"logps/ref_chosen": -56.285125732421875,
"logps/ref_rejected": -91.15303039550781,
"logps/rejected": -466.44244384765625,
"loss": 1.3862,
"margin_dpo/margin_mean": 165.09515380859375,
"margin_dpo/margin_std": 327.62249755859375,
"step": 346
},
{
"KL/chosen_KL_mean": -215.6938018798828,
"KL/mean": -268.2672424316406,
"KL/rejected_KL_mean": -320.8406677246094,
"KL/std": 256.3416442871094,
"epoch": 0.5245653817082389,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 105.14689636230469,
"fcm_dpo/q_t": 0.4999898076057434,
"grad_norm": 0.001989042619243264,
"learning_rate": 2.737640108260456e-07,
"logits/chosen": 1.3327654600143433,
"logits/rejected": 1.3226079940795898,
"logps/chosen": -269.193359375,
"logps/ref_chosen": -53.499542236328125,
"logps/ref_rejected": -72.52565002441406,
"logps/rejected": -393.3663330078125,
"loss": 1.3863,
"margin_dpo/margin_mean": 105.14688873291016,
"margin_dpo/margin_std": 277.4522705078125,
"step": 347
},
{
"KL/chosen_KL_mean": -194.62255859375,
"KL/mean": -241.3387908935547,
"KL/rejected_KL_mean": -288.0550537109375,
"KL/std": 223.19094848632812,
"epoch": 0.5260770975056689,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 93.4324722290039,
"fcm_dpo/q_t": 0.49999094009399414,
"grad_norm": 0.0018931980011984706,
"learning_rate": 2.724474525774229e-07,
"logits/chosen": 1.2694389820098877,
"logits/rejected": 1.2611613273620605,
"logps/chosen": -245.409423828125,
"logps/ref_chosen": -50.78684997558594,
"logps/ref_rejected": -68.63732147216797,
"logps/rejected": -356.6923522949219,
"loss": 1.3863,
"margin_dpo/margin_mean": 93.43247985839844,
"margin_dpo/margin_std": 223.0704345703125,
"step": 348
},
{
"KL/chosen_KL_mean": -211.4373779296875,
"KL/mean": -280.389404296875,
"KL/rejected_KL_mean": -349.3414306640625,
"KL/std": 292.76397705078125,
"epoch": 0.527588813303099,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 137.904052734375,
"fcm_dpo/q_t": 0.4999866485595703,
"grad_norm": 0.0022273960057646036,
"learning_rate": 2.711302664252973e-07,
"logits/chosen": 1.2621893882751465,
"logits/rejected": 1.2139443159103394,
"logps/chosen": -264.76239013671875,
"logps/ref_chosen": -53.325008392333984,
"logps/ref_rejected": -83.21236419677734,
"logps/rejected": -432.55377197265625,
"loss": 1.3862,
"margin_dpo/margin_mean": 137.904052734375,
"margin_dpo/margin_std": 315.9725036621094,
"step": 349
},
{
"KL/chosen_KL_mean": -224.52761840820312,
"KL/mean": -307.943359375,
"KL/rejected_KL_mean": -391.3591003417969,
"KL/std": 310.1175537109375,
"epoch": 0.5291005291005291,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 166.83148193359375,
"fcm_dpo/q_t": 0.49998384714126587,
"grad_norm": 0.002880116691812873,
"learning_rate": 2.698124892141971e-07,
"logits/chosen": 1.258007526397705,
"logits/rejected": 1.247691035270691,
"logps/chosen": -286.15338134765625,
"logps/ref_chosen": -61.625770568847656,
"logps/ref_rejected": -87.63627624511719,
"logps/rejected": -478.995361328125,
"loss": 1.3862,
"margin_dpo/margin_mean": 166.8314971923828,
"margin_dpo/margin_std": 310.433837890625,
"step": 350
},
{
"KL/chosen_KL_mean": -202.2751007080078,
"KL/mean": -260.5731506347656,
"KL/rejected_KL_mean": -318.87115478515625,
"KL/std": 201.6959991455078,
"epoch": 0.5306122448979592,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 116.59608459472656,
"fcm_dpo/q_t": 0.49998873472213745,
"grad_norm": 0.0021962763275951147,
"learning_rate": 2.6849415780518357e-07,
"logits/chosen": 1.2232160568237305,
"logits/rejected": 1.193381905555725,
"logps/chosen": -258.53143310546875,
"logps/ref_chosen": -56.2563362121582,
"logps/ref_rejected": -79.11589813232422,
"logps/rejected": -397.987060546875,
"loss": 1.3862,
"margin_dpo/margin_mean": 116.59608459472656,
"margin_dpo/margin_std": 279.30438232421875,
"step": 351
},
{
"KL/chosen_KL_mean": -229.2626953125,
"KL/mean": -293.04437255859375,
"KL/rejected_KL_mean": -356.8260498046875,
"KL/std": 303.4278564453125,
"epoch": 0.5321239606953893,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 127.56336975097656,
"fcm_dpo/q_t": 0.4999876618385315,
"grad_norm": 0.002237893408164382,
"learning_rate": 2.6717530907482024e-07,
"logits/chosen": 1.2531712055206299,
"logits/rejected": 1.2404941320419312,
"logps/chosen": -292.31463623046875,
"logps/ref_chosen": -63.05195236206055,
"logps/ref_rejected": -85.52035522460938,
"logps/rejected": -442.346435546875,
"loss": 1.3862,
"margin_dpo/margin_mean": 127.5633773803711,
"margin_dpo/margin_std": 380.3623962402344,
"step": 352
},
{
"KL/chosen_KL_mean": -227.08648681640625,
"KL/mean": -265.6427307128906,
"KL/rejected_KL_mean": -304.198974609375,
"KL/std": 260.7290954589844,
"epoch": 0.5336356764928194,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 77.11247253417969,
"fcm_dpo/q_t": 0.4999925494194031,
"grad_norm": 0.0013403355842456222,
"learning_rate": 2.658559799141411e-07,
"logits/chosen": 1.2461869716644287,
"logits/rejected": 1.2634531259536743,
"logps/chosen": -296.0956726074219,
"logps/ref_chosen": -69.00918579101562,
"logps/ref_rejected": -72.65840148925781,
"logps/rejected": -376.85736083984375,
"loss": 1.3863,
"margin_dpo/margin_mean": 77.11247253417969,
"margin_dpo/margin_std": 259.31402587890625,
"step": 353
},
{
"KL/chosen_KL_mean": -149.09295654296875,
"KL/mean": -222.05581665039062,
"KL/rejected_KL_mean": -295.0186767578125,
"KL/std": 179.0128936767578,
"epoch": 0.5351473922902494,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 145.92572021484375,
"fcm_dpo/q_t": 0.49998587369918823,
"grad_norm": 0.002911218209192157,
"learning_rate": 2.6453620722761895e-07,
"logits/chosen": 1.290527105331421,
"logits/rejected": 1.233759880065918,
"logps/chosen": -188.8813018798828,
"logps/ref_chosen": -39.78833770751953,
"logps/ref_rejected": -69.56885528564453,
"logps/rejected": -364.5875244140625,
"loss": 1.3862,
"margin_dpo/margin_mean": 145.92572021484375,
"margin_dpo/margin_std": 210.240966796875,
"step": 354
},
{
"KL/chosen_KL_mean": -183.08291625976562,
"KL/mean": -279.84710693359375,
"KL/rejected_KL_mean": -376.611328125,
"KL/std": 313.0096740722656,
"epoch": 0.5366591080876795,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 193.52838134765625,
"fcm_dpo/q_t": 0.49998125433921814,
"grad_norm": 0.0035407356917858124,
"learning_rate": 2.632160279321328e-07,
"logits/chosen": 1.2896933555603027,
"logits/rejected": 1.2377452850341797,
"logps/chosen": -229.3383026123047,
"logps/ref_chosen": -46.25537872314453,
"logps/ref_rejected": -78.20236206054688,
"logps/rejected": -454.8136901855469,
"loss": 1.3862,
"margin_dpo/margin_mean": 193.5283966064453,
"margin_dpo/margin_std": 347.1357421875,
"step": 355
},
{
"KL/chosen_KL_mean": -173.40316772460938,
"KL/mean": -241.93011474609375,
"KL/rejected_KL_mean": -310.45709228515625,
"KL/std": 237.36767578125,
"epoch": 0.5381708238851096,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 137.0539093017578,
"fcm_dpo/q_t": 0.4999867379665375,
"grad_norm": 0.0023308955132961273,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": 1.2501168251037598,
"logits/rejected": 1.2139482498168945,
"logps/chosen": -221.309326171875,
"logps/ref_chosen": -47.906158447265625,
"logps/ref_rejected": -74.29397583007812,
"logps/rejected": -384.7510681152344,
"loss": 1.3862,
"margin_dpo/margin_mean": 137.0539093017578,
"margin_dpo/margin_std": 294.431640625,
"step": 356
},
{
"KL/chosen_KL_mean": -230.25607299804688,
"KL/mean": -282.31524658203125,
"KL/rejected_KL_mean": -334.3744201660156,
"KL/std": 269.92694091796875,
"epoch": 0.5396825396825397,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.11831665039062,
"fcm_dpo/q_t": 0.49998992681503296,
"grad_norm": 0.0023283795453608036,
"learning_rate": 2.6057459723762076e-07,
"logits/chosen": 1.2416390180587769,
"logits/rejected": 1.2526752948760986,
"logps/chosen": -292.8910827636719,
"logps/ref_chosen": -62.63500213623047,
"logps/ref_rejected": -65.11399841308594,
"logps/rejected": -399.4884033203125,
"loss": 1.3863,
"margin_dpo/margin_mean": 104.11831665039062,
"margin_dpo/margin_std": 328.5772705078125,
"step": 357
},
{
"KL/chosen_KL_mean": -257.3409423828125,
"KL/mean": -282.57147216796875,
"KL/rejected_KL_mean": -307.80206298828125,
"KL/std": 286.5031433105469,
"epoch": 0.5411942554799698,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 50.46110153198242,
"fcm_dpo/q_t": 0.4999951124191284,
"grad_norm": 0.001599806360900402,
"learning_rate": 2.5925341972508954e-07,
"logits/chosen": 1.2319351434707642,
"logits/rejected": 1.2462717294692993,
"logps/chosen": -324.550537109375,
"logps/ref_chosen": -67.20960998535156,
"logps/ref_rejected": -69.34715270996094,
"logps/rejected": -377.1492004394531,
"loss": 1.3863,
"margin_dpo/margin_mean": 50.46110153198242,
"margin_dpo/margin_std": 362.09979248046875,
"step": 358
},
{
"KL/chosen_KL_mean": -271.7464599609375,
"KL/mean": -301.0337219238281,
"KL/rejected_KL_mean": -330.32098388671875,
"KL/std": 316.7093811035156,
"epoch": 0.5427059712773998,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 58.574562072753906,
"fcm_dpo/q_t": 0.49999430775642395,
"grad_norm": 0.001978958724066615,
"learning_rate": 2.579319833745169e-07,
"logits/chosen": 1.267923355102539,
"logits/rejected": 1.248403787612915,
"logps/chosen": -334.272216796875,
"logps/ref_chosen": -62.52578353881836,
"logps/ref_rejected": -76.63114929199219,
"logps/rejected": -406.9521484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 58.574562072753906,
"margin_dpo/margin_std": 401.56103515625,
"step": 359
},
{
"KL/chosen_KL_mean": -245.56210327148438,
"KL/mean": -321.9141540527344,
"KL/rejected_KL_mean": -398.26617431640625,
"KL/std": 338.3564147949219,
"epoch": 0.54421768707483,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 152.7041015625,
"fcm_dpo/q_t": 0.4999852180480957,
"grad_norm": 0.0029005235992372036,
"learning_rate": 2.5661032514931834e-07,
"logits/chosen": 1.2112646102905273,
"logits/rejected": 1.183484673500061,
"logps/chosen": -309.0498352050781,
"logps/ref_chosen": -63.48772048950195,
"logps/ref_rejected": -90.6891098022461,
"logps/rejected": -488.9552917480469,
"loss": 1.3862,
"margin_dpo/margin_mean": 152.7041015625,
"margin_dpo/margin_std": 392.829345703125,
"step": 360
},
{
"KL/chosen_KL_mean": -215.9862060546875,
"KL/mean": -271.5434875488281,
"KL/rejected_KL_mean": -327.1007385253906,
"KL/std": 243.81903076171875,
"epoch": 0.54572940287226,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 111.11454772949219,
"fcm_dpo/q_t": 0.49998927116394043,
"grad_norm": 0.0022065271623432636,
"learning_rate": 2.552884820191154e-07,
"logits/chosen": 1.2864508628845215,
"logits/rejected": 1.2714757919311523,
"logps/chosen": -273.9033508300781,
"logps/ref_chosen": -57.917144775390625,
"logps/ref_rejected": -72.39089965820312,
"logps/rejected": -399.49163818359375,
"loss": 1.3863,
"margin_dpo/margin_mean": 111.11454772949219,
"margin_dpo/margin_std": 294.0084228515625,
"step": 361
},
{
"KL/chosen_KL_mean": -237.84043884277344,
"KL/mean": -358.2422790527344,
"KL/rejected_KL_mean": -478.6441345214844,
"KL/std": 465.93792724609375,
"epoch": 0.54724111866969,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 240.80369567871094,
"fcm_dpo/q_t": 0.4999766945838928,
"grad_norm": 0.004349971190094948,
"learning_rate": 2.53966490958702e-07,
"logits/chosen": 1.2654731273651123,
"logits/rejected": 1.2318336963653564,
"logps/chosen": -301.2839050292969,
"logps/ref_chosen": -63.4434700012207,
"logps/ref_rejected": -103.45516967773438,
"logps/rejected": -582.0993041992188,
"loss": 1.3862,
"margin_dpo/margin_mean": 240.8037109375,
"margin_dpo/margin_std": 607.7028198242188,
"step": 362
},
{
"KL/chosen_KL_mean": -184.51370239257812,
"KL/mean": -285.02001953125,
"KL/rejected_KL_mean": -385.5263366699219,
"KL/std": 271.80780029296875,
"epoch": 0.5487528344671202,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 201.0126495361328,
"fcm_dpo/q_t": 0.4999805688858032,
"grad_norm": 0.003892081556841731,
"learning_rate": 2.526443889470099e-07,
"logits/chosen": 1.2803268432617188,
"logits/rejected": 1.2195180654525757,
"logps/chosen": -233.16552734375,
"logps/ref_chosen": -48.65182876586914,
"logps/ref_rejected": -88.65904235839844,
"logps/rejected": -474.1853942871094,
"loss": 1.3862,
"margin_dpo/margin_mean": 201.0126495361328,
"margin_dpo/margin_std": 300.37652587890625,
"step": 363
},
{
"KL/chosen_KL_mean": -215.3506622314453,
"KL/mean": -287.34576416015625,
"KL/rejected_KL_mean": -359.3408508300781,
"KL/std": 300.54931640625,
"epoch": 0.5502645502645502,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 143.99021911621094,
"fcm_dpo/q_t": 0.49998608231544495,
"grad_norm": 0.0026705956552177668,
"learning_rate": 2.513222129660744e-07,
"logits/chosen": 1.2083289623260498,
"logits/rejected": 1.180536150932312,
"logps/chosen": -273.22174072265625,
"logps/ref_chosen": -57.87107467651367,
"logps/ref_rejected": -80.95503234863281,
"logps/rejected": -440.2958984375,
"loss": 1.3862,
"margin_dpo/margin_mean": 143.99021911621094,
"margin_dpo/margin_std": 418.8135986328125,
"step": 364
},
{
"KL/chosen_KL_mean": -223.299072265625,
"KL/mean": -262.5470275878906,
"KL/rejected_KL_mean": -301.79498291015625,
"KL/std": 329.4648742675781,
"epoch": 0.5517762660619804,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 78.49591064453125,
"fcm_dpo/q_t": 0.49999240040779114,
"grad_norm": 0.001699400832876563,
"learning_rate": 2.5e-07,
"logits/chosen": 1.2776089906692505,
"logits/rejected": 1.2857856750488281,
"logps/chosen": -288.2412414550781,
"logps/ref_chosen": -64.94217681884766,
"logps/ref_rejected": -74.8599853515625,
"logps/rejected": -376.65496826171875,
"loss": 1.3863,
"margin_dpo/margin_mean": 78.49592590332031,
"margin_dpo/margin_std": 294.7704772949219,
"step": 365
},
{
"KL/chosen_KL_mean": -215.96405029296875,
"KL/mean": -249.5860595703125,
"KL/rejected_KL_mean": -283.2080383300781,
"KL/std": 302.95806884765625,
"epoch": 0.5532879818594104,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 67.2439956665039,
"fcm_dpo/q_t": 0.4999935030937195,
"grad_norm": 0.0016670352779328823,
"learning_rate": 2.486777870339255e-07,
"logits/chosen": 1.2348253726959229,
"logits/rejected": 1.2384909391403198,
"logps/chosen": -271.1300354003906,
"logps/ref_chosen": -55.16598129272461,
"logps/ref_rejected": -65.26121520996094,
"logps/rejected": -348.46923828125,
"loss": 1.3863,
"margin_dpo/margin_mean": 67.2439956665039,
"margin_dpo/margin_std": 308.7232666015625,
"step": 366
},
{
"KL/chosen_KL_mean": -227.30758666992188,
"KL/mean": -292.5162048339844,
"KL/rejected_KL_mean": -357.724853515625,
"KL/std": 294.52685546875,
"epoch": 0.5547996976568406,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.4172821044922,
"fcm_dpo/q_t": 0.4999873638153076,
"grad_norm": 0.0024976215790957212,
"learning_rate": 2.4735561105299014e-07,
"logits/chosen": 1.2854868173599243,
"logits/rejected": 1.2474390268325806,
"logps/chosen": -283.31805419921875,
"logps/ref_chosen": -56.01046371459961,
"logps/ref_rejected": -77.31010437011719,
"logps/rejected": -435.03497314453125,
"loss": 1.3862,
"margin_dpo/margin_mean": 130.4172821044922,
"margin_dpo/margin_std": 318.589111328125,
"step": 367
},
{
"KL/chosen_KL_mean": -271.27618408203125,
"KL/mean": -293.41290283203125,
"KL/rejected_KL_mean": -315.54962158203125,
"KL/std": 280.5533447265625,
"epoch": 0.5563114134542706,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 44.273414611816406,
"fcm_dpo/q_t": 0.4999956488609314,
"grad_norm": 0.0014242986217141151,
"learning_rate": 2.46033509041298e-07,
"logits/chosen": 1.193532109260559,
"logits/rejected": 1.199931025505066,
"logps/chosen": -346.10546875,
"logps/ref_chosen": -74.82927703857422,
"logps/ref_rejected": -76.11680603027344,
"logps/rejected": -391.66644287109375,
"loss": 1.3863,
"margin_dpo/margin_mean": 44.2734260559082,
"margin_dpo/margin_std": 339.95147705078125,
"step": 368
},
{
"KL/chosen_KL_mean": -228.98980712890625,
"KL/mean": -280.967529296875,
"KL/rejected_KL_mean": -332.9452209472656,
"KL/std": 287.7408752441406,
"epoch": 0.5578231292517006,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 103.9554214477539,
"fcm_dpo/q_t": 0.49998992681503296,
"grad_norm": 0.002357279881834984,
"learning_rate": 2.447115179808846e-07,
"logits/chosen": 1.2877874374389648,
"logits/rejected": 1.2739431858062744,
"logps/chosen": -287.3160400390625,
"logps/ref_chosen": -58.32621765136719,
"logps/ref_rejected": -80.92183685302734,
"logps/rejected": -413.8670654296875,
"loss": 1.3863,
"margin_dpo/margin_mean": 103.9554214477539,
"margin_dpo/margin_std": 325.900634765625,
"step": 369
},
{
"KL/chosen_KL_mean": -201.3767852783203,
"KL/mean": -259.1314392089844,
"KL/rejected_KL_mean": -316.8861083984375,
"KL/std": 232.7076873779297,
"epoch": 0.5593348450491308,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 115.50931549072266,
"fcm_dpo/q_t": 0.4999887943267822,
"grad_norm": 0.002263600006699562,
"learning_rate": 2.4338967485068164e-07,
"logits/chosen": 1.2925589084625244,
"logits/rejected": 1.2621781826019287,
"logps/chosen": -254.260498046875,
"logps/ref_chosen": -52.88372039794922,
"logps/ref_rejected": -79.43692016601562,
"logps/rejected": -396.322998046875,
"loss": 1.3862,
"margin_dpo/margin_mean": 115.50932312011719,
"margin_dpo/margin_std": 267.6164855957031,
"step": 370
},
{
"KL/chosen_KL_mean": -184.48826599121094,
"KL/mean": -229.01632690429688,
"KL/rejected_KL_mean": -273.54437255859375,
"KL/std": 192.02371215820312,
"epoch": 0.5608465608465608,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.05611419677734,
"fcm_dpo/q_t": 0.49999138712882996,
"grad_norm": 0.0017809628043323755,
"learning_rate": 2.420680166254831e-07,
"logits/chosen": 1.3020191192626953,
"logits/rejected": 1.2898714542388916,
"logps/chosen": -233.7124786376953,
"logps/ref_chosen": -49.224212646484375,
"logps/ref_rejected": -63.348472595214844,
"logps/rejected": -336.8928527832031,
"loss": 1.3863,
"margin_dpo/margin_mean": 89.05611419677734,
"margin_dpo/margin_std": 218.2142333984375,
"step": 371
},
{
"KL/chosen_KL_mean": -195.20889282226562,
"KL/mean": -253.14254760742188,
"KL/rejected_KL_mean": -311.076171875,
"KL/std": 244.80682373046875,
"epoch": 0.562358276643991,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 115.86727142333984,
"fcm_dpo/q_t": 0.49998876452445984,
"grad_norm": 0.002640601247549057,
"learning_rate": 2.4074658027491044e-07,
"logits/chosen": 1.274523138999939,
"logits/rejected": 1.2315776348114014,
"logps/chosen": -247.47845458984375,
"logps/ref_chosen": -52.269554138183594,
"logps/ref_rejected": -72.99522399902344,
"logps/rejected": -384.0714111328125,
"loss": 1.3862,
"margin_dpo/margin_mean": 115.86727905273438,
"margin_dpo/margin_std": 293.59796142578125,
"step": 372
},
{
"KL/chosen_KL_mean": -264.95257568359375,
"KL/mean": -320.5806884765625,
"KL/rejected_KL_mean": -376.20880126953125,
"KL/std": 349.17852783203125,
"epoch": 0.563869992441421,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 111.25619506835938,
"fcm_dpo/q_t": 0.49998921155929565,
"grad_norm": 0.0023425230756402016,
"learning_rate": 2.394254027623792e-07,
"logits/chosen": 1.284815788269043,
"logits/rejected": 1.2553908824920654,
"logps/chosen": -326.0655822753906,
"logps/ref_chosen": -61.112998962402344,
"logps/ref_rejected": -76.24851989746094,
"logps/rejected": -452.4573059082031,
"loss": 1.3863,
"margin_dpo/margin_mean": 111.25619506835938,
"margin_dpo/margin_std": 393.34063720703125,
"step": 373
},
{
"KL/chosen_KL_mean": -264.5904541015625,
"KL/mean": -314.9049072265625,
"KL/rejected_KL_mean": -365.2193298339844,
"KL/std": 318.2697448730469,
"epoch": 0.5653817082388511,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 100.6288833618164,
"fcm_dpo/q_t": 0.4999902546405792,
"grad_norm": 0.0017044921405613422,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": 1.231360912322998,
"logits/rejected": 1.247937798500061,
"logps/chosen": -337.2596435546875,
"logps/ref_chosen": -72.66920471191406,
"logps/ref_rejected": -76.83158874511719,
"logps/rejected": -442.0509033203125,
"loss": 1.3863,
"margin_dpo/margin_mean": 100.6288833618164,
"margin_dpo/margin_std": 454.28070068359375,
"step": 374
},
{
"KL/chosen_KL_mean": -232.35386657714844,
"KL/mean": -285.4949951171875,
"KL/rejected_KL_mean": -338.63616943359375,
"KL/std": 343.39801025390625,
"epoch": 0.5668934240362812,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 106.28231811523438,
"fcm_dpo/q_t": 0.49998971819877625,
"grad_norm": 0.002389972796663642,
"learning_rate": 2.3678397206786715e-07,
"logits/chosen": 1.2415052652359009,
"logits/rejected": 1.22267484664917,
"logps/chosen": -290.03717041015625,
"logps/ref_chosen": -57.68330383300781,
"logps/ref_rejected": -79.34097290039062,
"logps/rejected": -417.9771423339844,
"loss": 1.3863,
"margin_dpo/margin_mean": 106.28231811523438,
"margin_dpo/margin_std": 401.72052001953125,
"step": 375
},
{
"KL/chosen_KL_mean": -196.7454071044922,
"KL/mean": -273.0303039550781,
"KL/rejected_KL_mean": -349.315185546875,
"KL/std": 331.4114990234375,
"epoch": 0.5684051398337112,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 152.56979370117188,
"fcm_dpo/q_t": 0.4999852180480957,
"grad_norm": 0.0025670428294688463,
"learning_rate": 2.3546379277238103e-07,
"logits/chosen": 1.2817575931549072,
"logits/rejected": 1.2636425495147705,
"logps/chosen": -248.41946411132812,
"logps/ref_chosen": -51.674072265625,
"logps/ref_rejected": -75.69713592529297,
"logps/rejected": -425.0123291015625,
"loss": 1.3862,
"margin_dpo/margin_mean": 152.56979370117188,
"margin_dpo/margin_std": 369.35845947265625,
"step": 376
},
{
"KL/chosen_KL_mean": -192.5317840576172,
"KL/mean": -222.54251098632812,
"KL/rejected_KL_mean": -252.55325317382812,
"KL/std": 172.49461364746094,
"epoch": 0.5699168556311414,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 60.02146911621094,
"fcm_dpo/q_t": 0.4999941885471344,
"grad_norm": 0.0016488871769979596,
"learning_rate": 2.3414402008585886e-07,
"logits/chosen": 1.273218035697937,
"logits/rejected": 1.2637633085250854,
"logps/chosen": -238.7103271484375,
"logps/ref_chosen": -46.17853546142578,
"logps/ref_rejected": -57.756500244140625,
"logps/rejected": -310.30975341796875,
"loss": 1.3863,
"margin_dpo/margin_mean": 60.02146911621094,
"margin_dpo/margin_std": 183.21182250976562,
"step": 377
},
{
"KL/chosen_KL_mean": -234.7102813720703,
"KL/mean": -283.94964599609375,
"KL/rejected_KL_mean": -333.1890563964844,
"KL/std": 307.97674560546875,
"epoch": 0.5714285714285714,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 98.47879028320312,
"fcm_dpo/q_t": 0.49999046325683594,
"grad_norm": 0.0019810660742223263,
"learning_rate": 2.3282469092517977e-07,
"logits/chosen": 1.2981789112091064,
"logits/rejected": 1.2841894626617432,
"logps/chosen": -293.92913818359375,
"logps/ref_chosen": -59.21887969970703,
"logps/ref_rejected": -71.24818420410156,
"logps/rejected": -404.437255859375,
"loss": 1.3863,
"margin_dpo/margin_mean": 98.47879028320312,
"margin_dpo/margin_std": 306.84649658203125,
"step": 378
},
{
"KL/chosen_KL_mean": -273.62860107421875,
"KL/mean": -351.3236083984375,
"KL/rejected_KL_mean": -429.01861572265625,
"KL/std": 358.9712829589844,
"epoch": 0.5729402872260015,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 155.38998413085938,
"fcm_dpo/q_t": 0.4999849498271942,
"grad_norm": 0.003134517464786768,
"learning_rate": 2.3150584219481643e-07,
"logits/chosen": 1.227163553237915,
"logits/rejected": 1.1967265605926514,
"logps/chosen": -349.9451904296875,
"logps/ref_chosen": -76.31658935546875,
"logps/ref_rejected": -104.26200103759766,
"logps/rejected": -533.2806396484375,
"loss": 1.3862,
"margin_dpo/margin_mean": 155.38998413085938,
"margin_dpo/margin_std": 411.5760192871094,
"step": 379
},
{
"KL/chosen_KL_mean": -208.5601806640625,
"KL/mean": -263.50933837890625,
"KL/rejected_KL_mean": -318.45849609375,
"KL/std": 250.38766479492188,
"epoch": 0.5744520030234316,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 109.89828491210938,
"fcm_dpo/q_t": 0.49998939037323,
"grad_norm": 0.0019253168720752,
"learning_rate": 2.3018751078580283e-07,
"logits/chosen": 1.2732884883880615,
"logits/rejected": 1.262009620666504,
"logps/chosen": -269.8433532714844,
"logps/ref_chosen": -61.283164978027344,
"logps/ref_rejected": -72.38892364501953,
"logps/rejected": -390.847412109375,
"loss": 1.3863,
"margin_dpo/margin_mean": 109.89830017089844,
"margin_dpo/margin_std": 294.832275390625,
"step": 380
},
{
"KL/chosen_KL_mean": -244.27020263671875,
"KL/mean": -272.90728759765625,
"KL/rejected_KL_mean": -301.54437255859375,
"KL/std": 301.01202392578125,
"epoch": 0.5759637188208617,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 57.27418518066406,
"fcm_dpo/q_t": 0.4999944567680359,
"grad_norm": 0.0016863668570294976,
"learning_rate": 2.288697335747027e-07,
"logits/chosen": 1.2436158657073975,
"logits/rejected": 1.240373134613037,
"logps/chosen": -302.48419189453125,
"logps/ref_chosen": -58.2139892578125,
"logps/ref_rejected": -60.78669357299805,
"logps/rejected": -362.3310546875,
"loss": 1.3863,
"margin_dpo/margin_mean": 57.27417755126953,
"margin_dpo/margin_std": 403.81927490234375,
"step": 381
},
{
"KL/chosen_KL_mean": -229.37368774414062,
"KL/mean": -292.00982666015625,
"KL/rejected_KL_mean": -354.6460266113281,
"KL/std": 275.9203796386719,
"epoch": 0.5774754346182918,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 125.2723388671875,
"fcm_dpo/q_t": 0.4999878704547882,
"grad_norm": 0.0023127899039536715,
"learning_rate": 2.2755254742257706e-07,
"logits/chosen": 1.258483648300171,
"logits/rejected": 1.2473549842834473,
"logps/chosen": -291.1990051269531,
"logps/ref_chosen": -61.82532501220703,
"logps/ref_rejected": -83.0452880859375,
"logps/rejected": -437.6913146972656,
"loss": 1.3862,
"margin_dpo/margin_mean": 125.27233123779297,
"margin_dpo/margin_std": 331.326416015625,
"step": 382
},
{
"KL/chosen_KL_mean": -284.44842529296875,
"KL/mean": -303.908447265625,
"KL/rejected_KL_mean": -323.3684997558594,
"KL/std": 256.37823486328125,
"epoch": 0.5789871504157218,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 38.92008972167969,
"fcm_dpo/q_t": 0.49999624490737915,
"grad_norm": 0.002672591246664524,
"learning_rate": 2.2623598917395436e-07,
"logits/chosen": 1.2364006042480469,
"logits/rejected": 1.2648537158966064,
"logps/chosen": -365.01165771484375,
"logps/ref_chosen": -80.56326293945312,
"logps/ref_rejected": -74.62922668457031,
"logps/rejected": -397.99774169921875,
"loss": 1.3863,
"margin_dpo/margin_mean": 38.92008590698242,
"margin_dpo/margin_std": 323.5065612792969,
"step": 383
},
{
"KL/chosen_KL_mean": -215.20654296875,
"KL/mean": -270.7156677246094,
"KL/rejected_KL_mean": -326.22479248046875,
"KL/std": 235.81796264648438,
"epoch": 0.5804988662131519,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 111.01825714111328,
"fcm_dpo/q_t": 0.49998927116394043,
"grad_norm": 0.0019059223122894764,
"learning_rate": 2.2492009565579875e-07,
"logits/chosen": 1.2277659177780151,
"logits/rejected": 1.224064588546753,
"logps/chosen": -280.68170166015625,
"logps/ref_chosen": -65.47514343261719,
"logps/ref_rejected": -79.67378234863281,
"logps/rejected": -405.8985595703125,
"loss": 1.3863,
"margin_dpo/margin_mean": 111.01826477050781,
"margin_dpo/margin_std": 284.46221923828125,
"step": 384
},
{
"KL/chosen_KL_mean": -261.8004150390625,
"KL/mean": -326.974365234375,
"KL/rejected_KL_mean": -392.1483154296875,
"KL/std": 286.7763977050781,
"epoch": 0.582010582010582,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.34793090820312,
"fcm_dpo/q_t": 0.49998739361763,
"grad_norm": 0.002203842392191291,
"learning_rate": 2.2360490367648084e-07,
"logits/chosen": 1.246370792388916,
"logits/rejected": 1.2299015522003174,
"logps/chosen": -327.85693359375,
"logps/ref_chosen": -66.0565185546875,
"logps/ref_rejected": -86.68023681640625,
"logps/rejected": -478.82855224609375,
"loss": 1.3862,
"margin_dpo/margin_mean": 130.34791564941406,
"margin_dpo/margin_std": 344.5843811035156,
"step": 385
},
{
"KL/chosen_KL_mean": -280.88348388671875,
"KL/mean": -343.2861328125,
"KL/rejected_KL_mean": -405.68878173828125,
"KL/std": 325.25909423828125,
"epoch": 0.5835222978080121,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 124.80535125732422,
"fcm_dpo/q_t": 0.4999879002571106,
"grad_norm": 0.0024226726964116096,
"learning_rate": 2.2229045002474724e-07,
"logits/chosen": 1.2367877960205078,
"logits/rejected": 1.2165486812591553,
"logps/chosen": -356.50714111328125,
"logps/ref_chosen": -75.6236572265625,
"logps/ref_rejected": -92.62330627441406,
"logps/rejected": -498.3121032714844,
"loss": 1.3862,
"margin_dpo/margin_mean": 124.80534362792969,
"margin_dpo/margin_std": 393.1319580078125,
"step": 386
},
{
"KL/chosen_KL_mean": -202.7679443359375,
"KL/mean": -288.2266540527344,
"KL/rejected_KL_mean": -373.68536376953125,
"KL/std": 276.45501708984375,
"epoch": 0.5850340136054422,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 170.91744995117188,
"fcm_dpo/q_t": 0.4999834895133972,
"grad_norm": 0.0029783707577735186,
"learning_rate": 2.209767714686924e-07,
"logits/chosen": 1.2404074668884277,
"logits/rejected": 1.191530704498291,
"logps/chosen": -249.98965454101562,
"logps/ref_chosen": -47.22170639038086,
"logps/ref_rejected": -87.338134765625,
"logps/rejected": -461.02349853515625,
"loss": 1.3862,
"margin_dpo/margin_mean": 170.9174346923828,
"margin_dpo/margin_std": 304.63897705078125,
"step": 387
},
{
"KL/chosen_KL_mean": -288.9200439453125,
"KL/mean": -319.6807861328125,
"KL/rejected_KL_mean": -350.4415588378906,
"KL/std": 325.09112548828125,
"epoch": 0.5865457294028723,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 61.52149963378906,
"fcm_dpo/q_t": 0.49999403953552246,
"grad_norm": 0.00200459873303771,
"learning_rate": 2.1966390475472954e-07,
"logits/chosen": 1.2456412315368652,
"logits/rejected": 1.248414158821106,
"logps/chosen": -363.49951171875,
"logps/ref_chosen": -74.5794677734375,
"logps/ref_rejected": -79.92558288574219,
"logps/rejected": -430.36712646484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 61.52150344848633,
"margin_dpo/margin_std": 357.44195556640625,
"step": 388
},
{
"KL/chosen_KL_mean": -228.92892456054688,
"KL/mean": -281.46588134765625,
"KL/rejected_KL_mean": -334.0028381347656,
"KL/std": 260.0592041015625,
"epoch": 0.5880574452003023,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 105.07390594482422,
"fcm_dpo/q_t": 0.4999898374080658,
"grad_norm": 0.0018365130526944995,
"learning_rate": 2.1835188660656265e-07,
"logits/chosen": 1.256210446357727,
"logits/rejected": 1.2504541873931885,
"logps/chosen": -290.55328369140625,
"logps/ref_chosen": -61.624366760253906,
"logps/ref_rejected": -76.50978088378906,
"logps/rejected": -410.51263427734375,
"loss": 1.3863,
"margin_dpo/margin_mean": 105.07391357421875,
"margin_dpo/margin_std": 320.30426025390625,
"step": 389
},
{
"KL/chosen_KL_mean": -165.7669677734375,
"KL/mean": -217.87681579589844,
"KL/rejected_KL_mean": -269.9866943359375,
"KL/std": 203.6674346923828,
"epoch": 0.5895691609977324,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.21971130371094,
"fcm_dpo/q_t": 0.49998992681503296,
"grad_norm": 0.0020355647429823875,
"learning_rate": 2.170407537241599e-07,
"logits/chosen": 1.3178033828735352,
"logits/rejected": 1.2974865436553955,
"logps/chosen": -211.63882446289062,
"logps/ref_chosen": -45.871864318847656,
"logps/ref_rejected": -61.305999755859375,
"logps/rejected": -331.29266357421875,
"loss": 1.3863,
"margin_dpo/margin_mean": 104.21971130371094,
"margin_dpo/margin_std": 242.00851440429688,
"step": 390
},
{
"KL/chosen_KL_mean": -229.9234619140625,
"KL/mean": -298.4404296875,
"KL/rejected_KL_mean": -366.95745849609375,
"KL/std": 266.13519287109375,
"epoch": 0.5910808767951625,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 137.0339813232422,
"fcm_dpo/q_t": 0.4999867081642151,
"grad_norm": 0.0023842838127166033,
"learning_rate": 2.1573054278272636e-07,
"logits/chosen": 1.2694813013076782,
"logits/rejected": 1.245664119720459,
"logps/chosen": -288.1104736328125,
"logps/ref_chosen": -58.18701171875,
"logps/ref_rejected": -83.63442993164062,
"logps/rejected": -450.59185791015625,
"loss": 1.3862,
"margin_dpo/margin_mean": 137.0339813232422,
"margin_dpo/margin_std": 317.8896484375,
"step": 391
},
{
"KL/chosen_KL_mean": -249.9038848876953,
"KL/mean": -310.838623046875,
"KL/rejected_KL_mean": -371.7734069824219,
"KL/std": 360.3361511230469,
"epoch": 0.5925925925925926,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 121.86954498291016,
"fcm_dpo/q_t": 0.4999881982803345,
"grad_norm": 0.0024389002937823534,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": 1.2220652103424072,
"logits/rejected": 1.2007293701171875,
"logps/chosen": -319.6484375,
"logps/ref_chosen": -69.7445297241211,
"logps/ref_rejected": -94.05877685546875,
"logps/rejected": -465.8321838378906,
"loss": 1.3862,
"margin_dpo/margin_mean": 121.86953735351562,
"margin_dpo/margin_std": 398.31768798828125,
"step": 392
},
{
"KL/chosen_KL_mean": -212.96188354492188,
"KL/mean": -276.4071044921875,
"KL/rejected_KL_mean": -339.852294921875,
"KL/std": 308.44964599609375,
"epoch": 0.5941043083900227,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 126.89041137695312,
"fcm_dpo/q_t": 0.49998772144317627,
"grad_norm": 0.00197499874047935,
"learning_rate": 2.131130332936195e-07,
"logits/chosen": 1.2748873233795166,
"logits/rejected": 1.260786771774292,
"logps/chosen": -265.2967834472656,
"logps/ref_chosen": -52.33489990234375,
"logps/ref_rejected": -74.33809661865234,
"logps/rejected": -414.1903991699219,
"loss": 1.3862,
"margin_dpo/margin_mean": 126.89041137695312,
"margin_dpo/margin_std": 273.0295104980469,
"step": 393
},
{
"KL/chosen_KL_mean": -218.35079956054688,
"KL/mean": -262.255126953125,
"KL/rejected_KL_mean": -306.1594543457031,
"KL/std": 245.12030029296875,
"epoch": 0.5956160241874527,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 87.80865478515625,
"fcm_dpo/q_t": 0.4999915063381195,
"grad_norm": 0.0014762998325750232,
"learning_rate": 2.1180580796331323e-07,
"logits/chosen": 1.280198097229004,
"logits/rejected": 1.2725932598114014,
"logps/chosen": -279.02691650390625,
"logps/ref_chosen": -60.6761360168457,
"logps/ref_rejected": -71.36074829101562,
"logps/rejected": -377.52020263671875,
"loss": 1.3863,
"margin_dpo/margin_mean": 87.80865478515625,
"margin_dpo/margin_std": 251.21902465820312,
"step": 394
},
{
"KL/chosen_KL_mean": -204.20767211914062,
"KL/mean": -277.1258544921875,
"KL/rejected_KL_mean": -350.0440673828125,
"KL/std": 298.6371765136719,
"epoch": 0.5971277399848829,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 145.8363800048828,
"fcm_dpo/q_t": 0.49998587369918823,
"grad_norm": 0.0028106472454965115,
"learning_rate": 2.104996510066625e-07,
"logits/chosen": 1.2618656158447266,
"logits/rejected": 1.2272990942001343,
"logps/chosen": -254.81199645996094,
"logps/ref_chosen": -50.60432434082031,
"logps/ref_rejected": -77.08731079101562,
"logps/rejected": -427.13134765625,
"loss": 1.3862,
"margin_dpo/margin_mean": 145.8363800048828,
"margin_dpo/margin_std": 384.5468444824219,
"step": 395
},
{
"KL/chosen_KL_mean": -191.80307006835938,
"KL/mean": -259.073486328125,
"KL/rejected_KL_mean": -326.34393310546875,
"KL/std": 249.66404724121094,
"epoch": 0.5986394557823129,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 134.54083251953125,
"fcm_dpo/q_t": 0.49998700618743896,
"grad_norm": 0.002482072915881872,
"learning_rate": 2.0919459895968517e-07,
"logits/chosen": 1.263901948928833,
"logits/rejected": 1.2265499830245972,
"logps/chosen": -243.16268920898438,
"logps/ref_chosen": -51.35961151123047,
"logps/ref_rejected": -79.89360046386719,
"logps/rejected": -406.2375183105469,
"loss": 1.3862,
"margin_dpo/margin_mean": 134.54083251953125,
"margin_dpo/margin_std": 234.10260009765625,
"step": 396
},
{
"KL/chosen_KL_mean": -259.5587463378906,
"KL/mean": -302.8362121582031,
"KL/rejected_KL_mean": -346.1136474609375,
"KL/std": 284.9183654785156,
"epoch": 0.600151171579743,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 86.55490112304688,
"fcm_dpo/q_t": 0.49999162554740906,
"grad_norm": 0.0018756233621388674,
"learning_rate": 2.078906883274924e-07,
"logits/chosen": 1.194448709487915,
"logits/rejected": 1.1737489700317383,
"logps/chosen": -326.0149841308594,
"logps/ref_chosen": -66.45622253417969,
"logps/ref_rejected": -85.74736785888672,
"logps/rejected": -431.86102294921875,
"loss": 1.3863,
"margin_dpo/margin_mean": 86.55490112304688,
"margin_dpo/margin_std": 322.10992431640625,
"step": 397
},
{
"KL/chosen_KL_mean": -200.04078674316406,
"KL/mean": -269.54071044921875,
"KL/rejected_KL_mean": -339.0406494140625,
"KL/std": 272.4809875488281,
"epoch": 0.6016628873771731,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 138.99984741210938,
"fcm_dpo/q_t": 0.49998655915260315,
"grad_norm": 0.0020650143269449472,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": 1.2693488597869873,
"logits/rejected": 1.2469160556793213,
"logps/chosen": -249.2850341796875,
"logps/ref_chosen": -49.244239807128906,
"logps/ref_rejected": -75.18949127197266,
"logps/rejected": -414.2301025390625,
"loss": 1.3862,
"margin_dpo/margin_mean": 138.99984741210938,
"margin_dpo/margin_std": 358.0387268066406,
"step": 398
},
{
"KL/chosen_KL_mean": -245.84356689453125,
"KL/mean": -352.4913330078125,
"KL/rejected_KL_mean": -459.1390380859375,
"KL/std": 331.2461242675781,
"epoch": 0.6031746031746031,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 213.29550170898438,
"fcm_dpo/q_t": 0.4999793469905853,
"grad_norm": 0.0037792252842336893,
"learning_rate": 2.052864371672457e-07,
"logits/chosen": 1.2296762466430664,
"logits/rejected": 1.1701855659484863,
"logps/chosen": -314.15032958984375,
"logps/ref_chosen": -68.30679321289062,
"logps/ref_rejected": -113.2708511352539,
"logps/rejected": -572.409912109375,
"loss": 1.3862,
"margin_dpo/margin_mean": 213.29551696777344,
"margin_dpo/margin_std": 435.9686584472656,
"step": 399
},
{
"KL/chosen_KL_mean": -270.5350646972656,
"KL/mean": -334.7283630371094,
"KL/rejected_KL_mean": -398.9216613769531,
"KL/std": 345.2818908691406,
"epoch": 0.6046863189720333,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 128.38658142089844,
"fcm_dpo/q_t": 0.49998757243156433,
"grad_norm": 0.002531483769416809,
"learning_rate": 2.0398616948569493e-07,
"logits/chosen": 1.2615954875946045,
"logits/rejected": 1.237792730331421,
"logps/chosen": -342.16156005859375,
"logps/ref_chosen": -71.62649536132812,
"logps/ref_rejected": -90.98765563964844,
"logps/rejected": -489.9093017578125,
"loss": 1.3862,
"margin_dpo/margin_mean": 128.3865966796875,
"margin_dpo/margin_std": 375.5792541503906,
"step": 400
},
{
"KL/chosen_KL_mean": -182.42166137695312,
"KL/mean": -258.8485107421875,
"KL/rejected_KL_mean": -335.275390625,
"KL/std": 285.74456787109375,
"epoch": 0.6061980347694633,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 152.85372924804688,
"fcm_dpo/q_t": 0.4999852180480957,
"grad_norm": 0.002612342592328787,
"learning_rate": 2.0268718890989752e-07,
"logits/chosen": 1.272200584411621,
"logits/rejected": 1.2434661388397217,
"logps/chosen": -236.14662170410156,
"logps/ref_chosen": -53.72495651245117,
"logps/ref_rejected": -75.06304931640625,
"logps/rejected": -410.33843994140625,
"loss": 1.3862,
"margin_dpo/margin_mean": 152.85372924804688,
"margin_dpo/margin_std": 328.9865417480469,
"step": 401
},
{
"KL/chosen_KL_mean": -211.4462890625,
"KL/mean": -261.5096435546875,
"KL/rejected_KL_mean": -311.572998046875,
"KL/std": 243.91207885742188,
"epoch": 0.6077097505668935,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 100.12675476074219,
"fcm_dpo/q_t": 0.4999902844429016,
"grad_norm": 0.00198004231788218,
"learning_rate": 2.013895317751323e-07,
"logits/chosen": 1.2314531803131104,
"logits/rejected": 1.2497401237487793,
"logps/chosen": -273.3201904296875,
"logps/ref_chosen": -61.873931884765625,
"logps/ref_rejected": -66.15198516845703,
"logps/rejected": -377.7249755859375,
"loss": 1.3863,
"margin_dpo/margin_mean": 100.12675476074219,
"margin_dpo/margin_std": 330.08746337890625,
"step": 402
},
{
"KL/chosen_KL_mean": -185.16749572753906,
"KL/mean": -260.25628662109375,
"KL/rejected_KL_mean": -335.34503173828125,
"KL/std": 266.97735595703125,
"epoch": 0.6092214663643235,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 150.1775665283203,
"fcm_dpo/q_t": 0.4999854564666748,
"grad_norm": 0.002466583391651511,
"learning_rate": 2.0009323437965898e-07,
"logits/chosen": 1.3430713415145874,
"logits/rejected": 1.302276611328125,
"logps/chosen": -236.48899841308594,
"logps/ref_chosen": -51.321502685546875,
"logps/ref_rejected": -86.54010772705078,
"logps/rejected": -421.8851623535156,
"loss": 1.3862,
"margin_dpo/margin_mean": 150.1775665283203,
"margin_dpo/margin_std": 270.43560791015625,
"step": 403
},
{
"KL/chosen_KL_mean": -225.32257080078125,
"KL/mean": -306.13555908203125,
"KL/rejected_KL_mean": -386.9486083984375,
"KL/std": 334.2081604003906,
"epoch": 0.6107331821617535,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 161.62606811523438,
"fcm_dpo/q_t": 0.49998435378074646,
"grad_norm": 0.003134074853733182,
"learning_rate": 1.9879833298370237e-07,
"logits/chosen": 1.26384699344635,
"logits/rejected": 1.219362735748291,
"logps/chosen": -287.58544921875,
"logps/ref_chosen": -62.26288604736328,
"logps/ref_rejected": -95.19029998779297,
"logps/rejected": -482.138916015625,
"loss": 1.3862,
"margin_dpo/margin_mean": 161.6260528564453,
"margin_dpo/margin_std": 429.6391296386719,
"step": 404
},
{
"KL/chosen_KL_mean": -192.32791137695312,
"KL/mean": -249.77906799316406,
"KL/rejected_KL_mean": -307.230224609375,
"KL/std": 221.09945678710938,
"epoch": 0.6122448979591837,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 114.90231323242188,
"fcm_dpo/q_t": 0.499988853931427,
"grad_norm": 0.0018737346399575472,
"learning_rate": 1.975048638084379e-07,
"logits/chosen": 1.316763997077942,
"logits/rejected": 1.30680513381958,
"logps/chosen": -242.91226196289062,
"logps/ref_chosen": -50.5843391418457,
"logps/ref_rejected": -65.43156433105469,
"logps/rejected": -372.66180419921875,
"loss": 1.3862,
"margin_dpo/margin_mean": 114.90231323242188,
"margin_dpo/margin_std": 245.88070678710938,
"step": 405
},
{
"KL/chosen_KL_mean": -180.87625122070312,
"KL/mean": -281.7327880859375,
"KL/rejected_KL_mean": -382.58935546875,
"KL/std": 283.203369140625,
"epoch": 0.6137566137566137,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 201.71310424804688,
"fcm_dpo/q_t": 0.49998044967651367,
"grad_norm": 0.0038125943392515182,
"learning_rate": 1.9621286303497914e-07,
"logits/chosen": 1.2964931726455688,
"logits/rejected": 1.2286221981048584,
"logps/chosen": -229.87185668945312,
"logps/ref_chosen": -48.99560546875,
"logps/ref_rejected": -92.47774505615234,
"logps/rejected": -475.06707763671875,
"loss": 1.3862,
"margin_dpo/margin_mean": 201.71310424804688,
"margin_dpo/margin_std": 359.033447265625,
"step": 406
},
{
"KL/chosen_KL_mean": -312.56707763671875,
"KL/mean": -398.27764892578125,
"KL/rejected_KL_mean": -483.9881896972656,
"KL/std": 455.59527587890625,
"epoch": 0.6152683295540439,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 171.42111206054688,
"fcm_dpo/q_t": 0.49998340010643005,
"grad_norm": 0.00319691258482635,
"learning_rate": 1.9492236680336483e-07,
"logits/chosen": 1.2000561952590942,
"logits/rejected": 1.1917808055877686,
"logps/chosen": -401.9676513671875,
"logps/ref_chosen": -89.40056610107422,
"logps/ref_rejected": -99.28775024414062,
"logps/rejected": -583.2759399414062,
"loss": 1.3862,
"margin_dpo/margin_mean": 171.42111206054688,
"margin_dpo/margin_std": 497.2770080566406,
"step": 407
},
{
"KL/chosen_KL_mean": -188.9539794921875,
"KL/mean": -266.2425842285156,
"KL/rejected_KL_mean": -343.53118896484375,
"KL/std": 264.1246337890625,
"epoch": 0.6167800453514739,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 154.5772247314453,
"fcm_dpo/q_t": 0.4999850392341614,
"grad_norm": 0.0026424252428114414,
"learning_rate": 1.9363341121154895e-07,
"logits/chosen": 1.2529544830322266,
"logits/rejected": 1.2337000370025635,
"logps/chosen": -243.65789794921875,
"logps/ref_chosen": -54.70391845703125,
"logps/ref_rejected": -73.98648834228516,
"logps/rejected": -417.5177001953125,
"loss": 1.3862,
"margin_dpo/margin_mean": 154.5772247314453,
"margin_dpo/margin_std": 305.14788818359375,
"step": 408
},
{
"KL/chosen_KL_mean": -219.13699340820312,
"KL/mean": -239.82040405273438,
"KL/rejected_KL_mean": -260.5038146972656,
"KL/std": 184.755859375,
"epoch": 0.618291761148904,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 41.36683654785156,
"fcm_dpo/q_t": 0.49999600648880005,
"grad_norm": 0.0013175446074455976,
"learning_rate": 1.9234603231438994e-07,
"logits/chosen": 1.251279592514038,
"logits/rejected": 1.262905240058899,
"logps/chosen": -281.2552185058594,
"logps/ref_chosen": -62.11822509765625,
"logps/ref_rejected": -61.933509826660156,
"logps/rejected": -322.43731689453125,
"loss": 1.3863,
"margin_dpo/margin_mean": 41.36684036254883,
"margin_dpo/margin_std": 230.91448974609375,
"step": 409
},
{
"KL/chosen_KL_mean": -235.9901580810547,
"KL/mean": -287.7513122558594,
"KL/rejected_KL_mean": -339.5124206542969,
"KL/std": 274.4139404296875,
"epoch": 0.6198034769463341,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 103.52224731445312,
"fcm_dpo/q_t": 0.49998998641967773,
"grad_norm": 0.0023037977516651154,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": 1.2742689847946167,
"logits/rejected": 1.2695902585983276,
"logps/chosen": -297.7928466796875,
"logps/ref_chosen": -61.80266189575195,
"logps/ref_rejected": -76.60002136230469,
"logps/rejected": -416.1124267578125,
"loss": 1.3863,
"margin_dpo/margin_mean": 103.52224731445312,
"margin_dpo/margin_std": 323.41107177734375,
"step": 410
},
{
"KL/chosen_KL_mean": -204.3567352294922,
"KL/mean": -268.3748779296875,
"KL/rejected_KL_mean": -332.39306640625,
"KL/std": 259.58526611328125,
"epoch": 0.6213151927437641,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 128.03631591796875,
"fcm_dpo/q_t": 0.4999876022338867,
"grad_norm": 0.002165102167055011,
"learning_rate": 1.8977614860195296e-07,
"logits/chosen": 1.2589843273162842,
"logits/rejected": 1.2471109628677368,
"logps/chosen": -258.8021240234375,
"logps/ref_chosen": -54.44539260864258,
"logps/ref_rejected": -74.5650863647461,
"logps/rejected": -406.9581298828125,
"loss": 1.3862,
"margin_dpo/margin_mean": 128.03631591796875,
"margin_dpo/margin_std": 278.3114013671875,
"step": 411
},
{
"KL/chosen_KL_mean": -205.3777313232422,
"KL/mean": -259.4642028808594,
"KL/rejected_KL_mean": -313.55072021484375,
"KL/std": 227.35836791992188,
"epoch": 0.6228269085411943,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 108.17296600341797,
"fcm_dpo/q_t": 0.4999895393848419,
"grad_norm": 0.0022113437298685312,
"learning_rate": 1.8849371567184662e-07,
"logits/chosen": 1.2673554420471191,
"logits/rejected": 1.2555253505706787,
"logps/chosen": -260.6258239746094,
"logps/ref_chosen": -55.248085021972656,
"logps/ref_rejected": -68.96623229980469,
"logps/rejected": -382.51690673828125,
"loss": 1.3863,
"margin_dpo/margin_mean": 108.17295837402344,
"margin_dpo/margin_std": 254.52096557617188,
"step": 412
},
{
"KL/chosen_KL_mean": -264.1756286621094,
"KL/mean": -309.2933654785156,
"KL/rejected_KL_mean": -354.4111328125,
"KL/std": 302.36083984375,
"epoch": 0.6243386243386243,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 90.23551940917969,
"fcm_dpo/q_t": 0.4999912679195404,
"grad_norm": 0.0019386338535696268,
"learning_rate": 1.872130032047302e-07,
"logits/chosen": 1.1930580139160156,
"logits/rejected": 1.1896083354949951,
"logps/chosen": -332.8963623046875,
"logps/ref_chosen": -68.72074890136719,
"logps/ref_rejected": -78.76539611816406,
"logps/rejected": -433.176513671875,
"loss": 1.3863,
"margin_dpo/margin_mean": 90.23551940917969,
"margin_dpo/margin_std": 384.3971862792969,
"step": 413
},
{
"KL/chosen_KL_mean": -200.8385009765625,
"KL/mean": -264.8590087890625,
"KL/rejected_KL_mean": -328.8795166015625,
"KL/std": 333.3802490234375,
"epoch": 0.6258503401360545,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 128.04103088378906,
"fcm_dpo/q_t": 0.4999876022338867,
"grad_norm": 0.0021054602693766356,
"learning_rate": 1.8593404702488436e-07,
"logits/chosen": 1.2829506397247314,
"logits/rejected": 1.2689223289489746,
"logps/chosen": -254.97671508789062,
"logps/ref_chosen": -54.138214111328125,
"logps/ref_rejected": -74.65741729736328,
"logps/rejected": -403.53692626953125,
"loss": 1.3862,
"margin_dpo/margin_mean": 128.04103088378906,
"margin_dpo/margin_std": 383.669189453125,
"step": 414
},
{
"KL/chosen_KL_mean": -208.382568359375,
"KL/mean": -246.01028442382812,
"KL/rejected_KL_mean": -283.6380310058594,
"KL/std": 253.46176147460938,
"epoch": 0.6273620559334845,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 75.25543212890625,
"fcm_dpo/q_t": 0.4999927282333374,
"grad_norm": 0.00155459635425359,
"learning_rate": 1.846568829074628e-07,
"logits/chosen": 1.2656569480895996,
"logits/rejected": 1.2652559280395508,
"logps/chosen": -264.3011169433594,
"logps/ref_chosen": -55.91856002807617,
"logps/ref_rejected": -61.747703552246094,
"logps/rejected": -345.3857421875,
"loss": 1.3863,
"margin_dpo/margin_mean": 75.25544738769531,
"margin_dpo/margin_std": 311.67486572265625,
"step": 415
},
{
"KL/chosen_KL_mean": -220.16928100585938,
"KL/mean": -272.12646484375,
"KL/rejected_KL_mean": -324.0836181640625,
"KL/std": 320.10504150390625,
"epoch": 0.6288737717309146,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 103.91433715820312,
"fcm_dpo/q_t": 0.49998992681503296,
"grad_norm": 0.0020540771074593067,
"learning_rate": 1.8338154657749128e-07,
"logits/chosen": 1.2827048301696777,
"logits/rejected": 1.2663986682891846,
"logps/chosen": -274.8923645019531,
"logps/ref_chosen": -54.72308349609375,
"logps/ref_rejected": -69.17388916015625,
"logps/rejected": -393.25750732421875,
"loss": 1.3863,
"margin_dpo/margin_mean": 103.91433715820312,
"margin_dpo/margin_std": 338.7135925292969,
"step": 416
},
{
"KL/chosen_KL_mean": -224.02847290039062,
"KL/mean": -288.44708251953125,
"KL/rejected_KL_mean": -352.8656921386719,
"KL/std": 303.87835693359375,
"epoch": 0.6303854875283447,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 128.83721923828125,
"fcm_dpo/q_t": 0.49998751282691956,
"grad_norm": 0.002455303678289056,
"learning_rate": 1.8210807370886849e-07,
"logits/chosen": 1.3073334693908691,
"logits/rejected": 1.2928953170776367,
"logps/chosen": -280.8197326660156,
"logps/ref_chosen": -56.791259765625,
"logps/ref_rejected": -68.7791748046875,
"logps/rejected": -421.6448974609375,
"loss": 1.3862,
"margin_dpo/margin_mean": 128.83721923828125,
"margin_dpo/margin_std": 371.76605224609375,
"step": 417
},
{
"KL/chosen_KL_mean": -284.6827087402344,
"KL/mean": -312.7228088378906,
"KL/rejected_KL_mean": -340.7629089355469,
"KL/std": 332.7872314453125,
"epoch": 0.6318972033257747,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 56.08018112182617,
"fcm_dpo/q_t": 0.49999457597732544,
"grad_norm": 0.001611909014172852,
"learning_rate": 1.8083649992336825e-07,
"logits/chosen": 1.2963645458221436,
"logits/rejected": 1.3024256229400635,
"logps/chosen": -353.79071044921875,
"logps/ref_chosen": -69.10798645019531,
"logps/ref_rejected": -75.09132385253906,
"logps/rejected": -415.854248046875,
"loss": 1.3863,
"margin_dpo/margin_mean": 56.080177307128906,
"margin_dpo/margin_std": 397.62890625,
"step": 418
},
{
"KL/chosen_KL_mean": -200.85401916503906,
"KL/mean": -274.2613830566406,
"KL/rejected_KL_mean": -347.66876220703125,
"KL/std": 276.50970458984375,
"epoch": 0.6334089191232048,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 146.81472778320312,
"fcm_dpo/q_t": 0.49998578429222107,
"grad_norm": 0.002377911237999797,
"learning_rate": 1.7956686078964255e-07,
"logits/chosen": 1.2625758647918701,
"logits/rejected": 1.2619942426681519,
"logps/chosen": -259.0257873535156,
"logps/ref_chosen": -58.1717643737793,
"logps/ref_rejected": -71.67066955566406,
"logps/rejected": -419.33941650390625,
"loss": 1.3862,
"margin_dpo/margin_mean": 146.81472778320312,
"margin_dpo/margin_std": 333.00238037109375,
"step": 419
},
{
"KL/chosen_KL_mean": -232.24765014648438,
"KL/mean": -268.002197265625,
"KL/rejected_KL_mean": -303.7567443847656,
"KL/std": 265.8175048828125,
"epoch": 0.6349206349206349,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 71.50907897949219,
"fcm_dpo/q_t": 0.49999308586120605,
"grad_norm": 0.001728436560370028,
"learning_rate": 1.782991918222275e-07,
"logits/chosen": 1.2875854969024658,
"logits/rejected": 1.2710034847259521,
"logps/chosen": -289.3011474609375,
"logps/ref_chosen": -57.05351257324219,
"logps/ref_rejected": -62.670982360839844,
"logps/rejected": -366.427734375,
"loss": 1.3863,
"margin_dpo/margin_mean": 71.50907897949219,
"margin_dpo/margin_std": 340.4837951660156,
"step": 420
},
{
"KL/chosen_KL_mean": -230.90493774414062,
"KL/mean": -285.153076171875,
"KL/rejected_KL_mean": -339.4012145996094,
"KL/std": 287.30902099609375,
"epoch": 0.636432350718065,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 108.49628448486328,
"fcm_dpo/q_t": 0.49998950958251953,
"grad_norm": 0.002344191772863269,
"learning_rate": 1.7703352848054887e-07,
"logits/chosen": 1.237176775932312,
"logits/rejected": 1.2174639701843262,
"logps/chosen": -288.22821044921875,
"logps/ref_chosen": -57.32324981689453,
"logps/ref_rejected": -75.33782958984375,
"logps/rejected": -414.7390441894531,
"loss": 1.3863,
"margin_dpo/margin_mean": 108.49629211425781,
"margin_dpo/margin_std": 395.6588439941406,
"step": 421
},
{
"KL/chosen_KL_mean": -228.75091552734375,
"KL/mean": -281.6344909667969,
"KL/rejected_KL_mean": -334.51806640625,
"KL/std": 274.93902587890625,
"epoch": 0.6379440665154951,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 105.76715087890625,
"fcm_dpo/q_t": 0.49998974800109863,
"grad_norm": 0.0021835805382579565,
"learning_rate": 1.7576990616793137e-07,
"logits/chosen": 1.231833577156067,
"logits/rejected": 1.2441257238388062,
"logps/chosen": -295.8084716796875,
"logps/ref_chosen": -67.05757141113281,
"logps/ref_rejected": -72.12803649902344,
"logps/rejected": -406.6460876464844,
"loss": 1.3863,
"margin_dpo/margin_mean": 105.76715087890625,
"margin_dpo/margin_std": 315.6912841796875,
"step": 422
},
{
"KL/chosen_KL_mean": -189.46343994140625,
"KL/mean": -263.8796691894531,
"KL/rejected_KL_mean": -338.2958679199219,
"KL/std": 262.6513671875,
"epoch": 0.6394557823129252,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 148.83242797851562,
"fcm_dpo/q_t": 0.49998557567596436,
"grad_norm": 0.0025841707829385996,
"learning_rate": 1.745083602306071e-07,
"logits/chosen": 1.2950785160064697,
"logits/rejected": 1.2744381427764893,
"logps/chosen": -243.52511596679688,
"logps/ref_chosen": -54.06167221069336,
"logps/ref_rejected": -76.64092254638672,
"logps/rejected": -414.9367980957031,
"loss": 1.3862,
"margin_dpo/margin_mean": 148.83242797851562,
"margin_dpo/margin_std": 266.20477294921875,
"step": 423
},
{
"KL/chosen_KL_mean": -204.17025756835938,
"KL/mean": -266.5606384277344,
"KL/rejected_KL_mean": -328.95098876953125,
"KL/std": 212.84597778320312,
"epoch": 0.6409674981103552,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 124.78072357177734,
"fcm_dpo/q_t": 0.499987930059433,
"grad_norm": 0.002034224336966872,
"learning_rate": 1.7324892595672804e-07,
"logits/chosen": 1.2100038528442383,
"logits/rejected": 1.197531819343567,
"logps/chosen": -257.7791442871094,
"logps/ref_chosen": -53.60887145996094,
"logps/ref_rejected": -79.2139892578125,
"logps/rejected": -408.16497802734375,
"loss": 1.3862,
"margin_dpo/margin_mean": 124.78071594238281,
"margin_dpo/margin_std": 222.72096252441406,
"step": 424
},
{
"KL/chosen_KL_mean": -202.778564453125,
"KL/mean": -247.59307861328125,
"KL/rejected_KL_mean": -292.4075927734375,
"KL/std": 240.92269897460938,
"epoch": 0.6424792139077853,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.62903594970703,
"fcm_dpo/q_t": 0.4999913275241852,
"grad_norm": 0.0015491149388253689,
"learning_rate": 1.7199163857537824e-07,
"logits/chosen": 1.2798230648040771,
"logits/rejected": 1.2812039852142334,
"logps/chosen": -261.1932373046875,
"logps/ref_chosen": -58.41468048095703,
"logps/ref_rejected": -66.59054565429688,
"logps/rejected": -358.9981384277344,
"loss": 1.3863,
"margin_dpo/margin_mean": 89.62903594970703,
"margin_dpo/margin_std": 312.6791687011719,
"step": 425
},
{
"KL/chosen_KL_mean": -276.497314453125,
"KL/mean": -286.79400634765625,
"KL/rejected_KL_mean": -297.0906677246094,
"KL/std": 287.28729248046875,
"epoch": 0.6439909297052154,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 20.59334373474121,
"fcm_dpo/q_t": 0.4999980032444,
"grad_norm": 0.0016692699864506721,
"learning_rate": 1.7073653325558828e-07,
"logits/chosen": 1.2584002017974854,
"logits/rejected": 1.2575640678405762,
"logps/chosen": -348.20550537109375,
"logps/ref_chosen": -71.70822143554688,
"logps/ref_rejected": -73.57725524902344,
"logps/rejected": -370.66790771484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 20.593345642089844,
"margin_dpo/margin_std": 402.493408203125,
"step": 426
},
{
"KL/chosen_KL_mean": -244.16412353515625,
"KL/mean": -299.6019287109375,
"KL/rejected_KL_mean": -355.03973388671875,
"KL/std": 328.6190185546875,
"epoch": 0.6455026455026455,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 110.87562561035156,
"fcm_dpo/q_t": 0.49998927116394043,
"grad_norm": 0.002250807359814644,
"learning_rate": 1.6948364510535218e-07,
"logits/chosen": 1.2549666166305542,
"logits/rejected": 1.2232038974761963,
"logps/chosen": -302.806884765625,
"logps/ref_chosen": -58.64276885986328,
"logps/ref_rejected": -86.25437927246094,
"logps/rejected": -441.29412841796875,
"loss": 1.3863,
"margin_dpo/margin_mean": 110.87562561035156,
"margin_dpo/margin_std": 329.8369140625,
"step": 427
},
{
"KL/chosen_KL_mean": -276.95233154296875,
"KL/mean": -329.34759521484375,
"KL/rejected_KL_mean": -381.74285888671875,
"KL/std": 323.8177185058594,
"epoch": 0.6470143613000756,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.79049682617188,
"fcm_dpo/q_t": 0.4999898374080658,
"grad_norm": 0.0020647216588258743,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": 1.2424485683441162,
"logits/rejected": 1.2366441488265991,
"logps/chosen": -343.54840087890625,
"logps/ref_chosen": -66.5960464477539,
"logps/ref_rejected": -82.3941650390625,
"logps/rejected": -464.13702392578125,
"loss": 1.3863,
"margin_dpo/margin_mean": 104.79049682617188,
"margin_dpo/margin_std": 409.09844970703125,
"step": 428
},
{
"KL/chosen_KL_mean": -245.20016479492188,
"KL/mean": -263.1362609863281,
"KL/rejected_KL_mean": -281.0723571777344,
"KL/std": 223.36959838867188,
"epoch": 0.6485260770975056,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 35.87220764160156,
"fcm_dpo/q_t": 0.499996542930603,
"grad_norm": 0.0011962910648435354,
"learning_rate": 1.669846604344412e-07,
"logits/chosen": 1.259268879890442,
"logits/rejected": 1.2742395401000977,
"logps/chosen": -302.2098693847656,
"logps/ref_chosen": -57.00970458984375,
"logps/ref_rejected": -59.86549377441406,
"logps/rejected": -340.9378662109375,
"loss": 1.3863,
"margin_dpo/margin_mean": 35.8722038269043,
"margin_dpo/margin_std": 258.54791259765625,
"step": 429
},
{
"KL/chosen_KL_mean": -219.3946533203125,
"KL/mean": -255.57508850097656,
"KL/rejected_KL_mean": -291.75555419921875,
"KL/std": 203.72706604003906,
"epoch": 0.6500377928949358,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 72.36093139648438,
"fcm_dpo/q_t": 0.4999930262565613,
"grad_norm": 0.0014631770318374038,
"learning_rate": 1.6573863381573954e-07,
"logits/chosen": 1.1872947216033936,
"logits/rejected": 1.187971591949463,
"logps/chosen": -278.95782470703125,
"logps/ref_chosen": -59.563194274902344,
"logps/ref_rejected": -70.52289581298828,
"logps/rejected": -362.2784423828125,
"loss": 1.3863,
"margin_dpo/margin_mean": 72.36092376708984,
"margin_dpo/margin_std": 281.1922302246094,
"step": 430
},
{
"KL/chosen_KL_mean": -191.01605224609375,
"KL/mean": -245.7452850341797,
"KL/rejected_KL_mean": -300.47454833984375,
"KL/std": 234.70254516601562,
"epoch": 0.6515495086923658,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 109.45848083496094,
"fcm_dpo/q_t": 0.49998942017555237,
"grad_norm": 0.0020323162898421288,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": 1.2493000030517578,
"logits/rejected": 1.2200860977172852,
"logps/chosen": -241.21636962890625,
"logps/ref_chosen": -50.20032501220703,
"logps/ref_rejected": -77.81680297851562,
"logps/rejected": -378.29132080078125,
"loss": 1.3863,
"margin_dpo/margin_mean": 109.45848846435547,
"margin_dpo/margin_std": 307.2296142578125,
"step": 431
},
{
"KL/chosen_KL_mean": -256.33648681640625,
"KL/mean": -303.86322021484375,
"KL/rejected_KL_mean": -351.3900146484375,
"KL/std": 295.88214111328125,
"epoch": 0.6530612244897959,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 95.05351257324219,
"fcm_dpo/q_t": 0.4999908208847046,
"grad_norm": 0.0019163392717018723,
"learning_rate": 1.632536862810844e-07,
"logits/chosen": 1.286834716796875,
"logits/rejected": 1.2581818103790283,
"logps/chosen": -317.999267578125,
"logps/ref_chosen": -61.662757873535156,
"logps/ref_rejected": -83.94496154785156,
"logps/rejected": -435.3349609375,
"loss": 1.3863,
"margin_dpo/margin_mean": 95.05350494384766,
"margin_dpo/margin_std": 400.80767822265625,
"step": 432
},
{
"KL/chosen_KL_mean": -221.035400390625,
"KL/mean": -255.6205291748047,
"KL/rejected_KL_mean": -290.2056579589844,
"KL/std": 240.65435791015625,
"epoch": 0.654572940287226,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 69.17024993896484,
"fcm_dpo/q_t": 0.49999332427978516,
"grad_norm": 0.0015163794159889221,
"learning_rate": 1.6201483487445515e-07,
"logits/chosen": 1.273536205291748,
"logits/rejected": 1.2836341857910156,
"logps/chosen": -284.76458740234375,
"logps/ref_chosen": -63.72917938232422,
"logps/ref_rejected": -65.8391342163086,
"logps/rejected": -356.0447998046875,
"loss": 1.3863,
"margin_dpo/margin_mean": 69.17024993896484,
"margin_dpo/margin_std": 295.4766845703125,
"step": 433
},
{
"KL/chosen_KL_mean": -177.51564025878906,
"KL/mean": -254.1595458984375,
"KL/rejected_KL_mean": -330.803466796875,
"KL/std": 256.6459655761719,
"epoch": 0.656084656084656,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 153.2877960205078,
"fcm_dpo/q_t": 0.4999851584434509,
"grad_norm": 0.0024449967313557863,
"learning_rate": 1.6077844460203204e-07,
"logits/chosen": 1.3231778144836426,
"logits/rejected": 1.300271987915039,
"logps/chosen": -225.48895263671875,
"logps/ref_chosen": -47.97331619262695,
"logps/ref_rejected": -72.51132202148438,
"logps/rejected": -403.31475830078125,
"loss": 1.3862,
"margin_dpo/margin_mean": 153.28781127929688,
"margin_dpo/margin_std": 307.38153076171875,
"step": 434
},
{
"KL/chosen_KL_mean": -212.97402954101562,
"KL/mean": -253.61410522460938,
"KL/rejected_KL_mean": -294.25421142578125,
"KL/std": 250.4377899169922,
"epoch": 0.6575963718820862,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 81.28021240234375,
"fcm_dpo/q_t": 0.49999213218688965,
"grad_norm": 0.0016701404238119721,
"learning_rate": 1.5954455004830878e-07,
"logits/chosen": 1.3001594543457031,
"logits/rejected": 1.285398244857788,
"logps/chosen": -270.0342712402344,
"logps/ref_chosen": -57.06024932861328,
"logps/ref_rejected": -71.69146728515625,
"logps/rejected": -365.9456787109375,
"loss": 1.3863,
"margin_dpo/margin_mean": 81.28020477294922,
"margin_dpo/margin_std": 294.3331298828125,
"step": 435
},
{
"KL/chosen_KL_mean": -216.45889282226562,
"KL/mean": -261.227294921875,
"KL/rejected_KL_mean": -305.9957275390625,
"KL/std": 265.82232666015625,
"epoch": 0.6591080876795162,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.53681182861328,
"fcm_dpo/q_t": 0.49999135732650757,
"grad_norm": 0.0022236332297325134,
"learning_rate": 1.5831318572796847e-07,
"logits/chosen": 1.289149284362793,
"logits/rejected": 1.2721104621887207,
"logps/chosen": -272.616943359375,
"logps/ref_chosen": -56.158050537109375,
"logps/ref_rejected": -67.63787841796875,
"logps/rejected": -373.63360595703125,
"loss": 1.3863,
"margin_dpo/margin_mean": 89.53681182861328,
"margin_dpo/margin_std": 242.168701171875,
"step": 436
},
{
"KL/chosen_KL_mean": -215.29190063476562,
"KL/mean": -292.97491455078125,
"KL/rejected_KL_mean": -370.65789794921875,
"KL/std": 285.3720397949219,
"epoch": 0.6606198034769464,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 155.36599731445312,
"fcm_dpo/q_t": 0.4999849498271942,
"grad_norm": 0.0029099148232489824,
"learning_rate": 1.5708438608491815e-07,
"logits/chosen": 1.2747769355773926,
"logits/rejected": 1.217381238937378,
"logps/chosen": -272.2777099609375,
"logps/ref_chosen": -56.98578643798828,
"logps/ref_rejected": -85.61524963378906,
"logps/rejected": -456.2731628417969,
"loss": 1.3862,
"margin_dpo/margin_mean": 155.36599731445312,
"margin_dpo/margin_std": 385.7756042480469,
"step": 437
},
{
"KL/chosen_KL_mean": -179.04933166503906,
"KL/mean": -239.98414611816406,
"KL/rejected_KL_mean": -300.9189758300781,
"KL/std": 274.84893798828125,
"epoch": 0.6621315192743764,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 121.86964416503906,
"fcm_dpo/q_t": 0.4999881982803345,
"grad_norm": 0.002175640081986785,
"learning_rate": 1.558581854913253e-07,
"logits/chosen": 1.3229100704193115,
"logits/rejected": 1.2967147827148438,
"logps/chosen": -220.32711791992188,
"logps/ref_chosen": -41.27777862548828,
"logps/ref_rejected": -65.33840942382812,
"logps/rejected": -366.25738525390625,
"loss": 1.3862,
"margin_dpo/margin_mean": 121.86964416503906,
"margin_dpo/margin_std": 330.1829528808594,
"step": 438
},
{
"KL/chosen_KL_mean": -285.81622314453125,
"KL/mean": -340.0804443359375,
"KL/rejected_KL_mean": -394.3446350097656,
"KL/std": 310.67144775390625,
"epoch": 0.6636432350718064,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 108.52841186523438,
"fcm_dpo/q_t": 0.49998950958251953,
"grad_norm": 0.0019142036326229572,
"learning_rate": 1.5463461824665658e-07,
"logits/chosen": 1.188014268875122,
"logits/rejected": 1.1888413429260254,
"logps/chosen": -367.23388671875,
"logps/ref_chosen": -81.41764831542969,
"logps/ref_rejected": -94.72309875488281,
"logps/rejected": -489.0677185058594,
"loss": 1.3863,
"margin_dpo/margin_mean": 108.52839660644531,
"margin_dpo/margin_std": 330.1500549316406,
"step": 439
},
{
"KL/chosen_KL_mean": -170.18858337402344,
"KL/mean": -235.81759643554688,
"KL/rejected_KL_mean": -301.44659423828125,
"KL/std": 246.02532958984375,
"epoch": 0.6651549508692366,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 131.25802612304688,
"fcm_dpo/q_t": 0.49998730421066284,
"grad_norm": 0.002329205861315131,
"learning_rate": 1.534137185767178e-07,
"logits/chosen": 1.2724201679229736,
"logits/rejected": 1.230487585067749,
"logps/chosen": -212.72677612304688,
"logps/ref_chosen": -42.538185119628906,
"logps/ref_rejected": -69.78813934326172,
"logps/rejected": -371.2347412109375,
"loss": 1.3862,
"margin_dpo/margin_mean": 131.25802612304688,
"margin_dpo/margin_std": 252.58546447753906,
"step": 440
},
{
"KL/chosen_KL_mean": -212.27427673339844,
"KL/mean": -289.75164794921875,
"KL/rejected_KL_mean": -367.22900390625,
"KL/std": 316.7421875,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 154.95472717285156,
"fcm_dpo/q_t": 0.499985009431839,
"grad_norm": 0.0029717073775827885,
"learning_rate": 1.521955206326976e-07,
"logits/chosen": 1.239859938621521,
"logits/rejected": 1.210754156112671,
"logps/chosen": -269.86749267578125,
"logps/ref_chosen": -57.593223571777344,
"logps/ref_rejected": -84.82878875732422,
"logps/rejected": -452.05780029296875,
"loss": 1.3862,
"margin_dpo/margin_mean": 154.95472717285156,
"margin_dpo/margin_std": 341.307861328125,
"step": 441
},
{
"KL/chosen_KL_mean": -266.17144775390625,
"KL/mean": -336.0986328125,
"KL/rejected_KL_mean": -406.02587890625,
"KL/std": 319.74578857421875,
"epoch": 0.6681783824640968,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 139.8543701171875,
"fcm_dpo/q_t": 0.499986469745636,
"grad_norm": 0.0025126931723207235,
"learning_rate": 1.5098005849021078e-07,
"logits/chosen": 1.2591302394866943,
"logits/rejected": 1.2514472007751465,
"logps/chosen": -333.6326599121094,
"logps/ref_chosen": -67.46121978759766,
"logps/ref_rejected": -89.0693588256836,
"logps/rejected": -495.09521484375,
"loss": 1.3862,
"margin_dpo/margin_mean": 139.85438537597656,
"margin_dpo/margin_std": 342.5337219238281,
"step": 442
},
{
"KL/chosen_KL_mean": -193.10202026367188,
"KL/mean": -278.1657409667969,
"KL/rejected_KL_mean": -363.22943115234375,
"KL/std": 328.15521240234375,
"epoch": 0.6696900982615268,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 170.12744140625,
"fcm_dpo/q_t": 0.499983549118042,
"grad_norm": 0.0025645981077104807,
"learning_rate": 1.4976736614834662e-07,
"logits/chosen": 1.2513320446014404,
"logits/rejected": 1.2338635921478271,
"logps/chosen": -247.8981170654297,
"logps/ref_chosen": -54.79610061645508,
"logps/ref_rejected": -77.80781555175781,
"logps/rejected": -441.0372619628906,
"loss": 1.3862,
"margin_dpo/margin_mean": 170.12744140625,
"margin_dpo/margin_std": 411.92755126953125,
"step": 443
},
{
"KL/chosen_KL_mean": -238.94351196289062,
"KL/mean": -313.384521484375,
"KL/rejected_KL_mean": -387.8255615234375,
"KL/std": 340.74688720703125,
"epoch": 0.671201814058957,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 148.88204956054688,
"fcm_dpo/q_t": 0.49998557567596436,
"grad_norm": 0.0032997538801282644,
"learning_rate": 1.4855747752871654e-07,
"logits/chosen": 1.279803991317749,
"logits/rejected": 1.249694585800171,
"logps/chosen": -297.69256591796875,
"logps/ref_chosen": -58.749061584472656,
"logps/ref_rejected": -86.87396240234375,
"logps/rejected": -474.69952392578125,
"loss": 1.3862,
"margin_dpo/margin_mean": 148.88204956054688,
"margin_dpo/margin_std": 396.679443359375,
"step": 444
},
{
"KL/chosen_KL_mean": -241.49159240722656,
"KL/mean": -290.87945556640625,
"KL/rejected_KL_mean": -340.2673645019531,
"KL/std": 260.7337341308594,
"epoch": 0.672713529856387,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 98.77577209472656,
"fcm_dpo/q_t": 0.49999046325683594,
"grad_norm": 0.0014876207569614053,
"learning_rate": 1.473504264745062e-07,
"logits/chosen": 1.2597832679748535,
"logits/rejected": 1.2682100534439087,
"logps/chosen": -302.4090270996094,
"logps/ref_chosen": -60.91743850708008,
"logps/ref_rejected": -71.5637435913086,
"logps/rejected": -411.83111572265625,
"loss": 1.3863,
"margin_dpo/margin_mean": 98.77577209472656,
"margin_dpo/margin_std": 304.1396484375,
"step": 445
},
{
"KL/chosen_KL_mean": -183.92437744140625,
"KL/mean": -257.93829345703125,
"KL/rejected_KL_mean": -331.9521484375,
"KL/std": 250.339111328125,
"epoch": 0.674225245653817,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 148.02777099609375,
"fcm_dpo/q_t": 0.4999856650829315,
"grad_norm": 0.002402180340141058,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": 1.2483906745910645,
"logits/rejected": 1.225560188293457,
"logps/chosen": -232.7236328125,
"logps/ref_chosen": -48.79924774169922,
"logps/ref_rejected": -71.8719482421875,
"logps/rejected": -403.8240966796875,
"loss": 1.3862,
"margin_dpo/margin_mean": 148.02777099609375,
"margin_dpo/margin_std": 283.2513427734375,
"step": 446
},
{
"KL/chosen_KL_mean": -195.73277282714844,
"KL/mean": -288.6422424316406,
"KL/rejected_KL_mean": -381.55169677734375,
"KL/std": 270.0787353515625,
"epoch": 0.6757369614512472,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 185.81895446777344,
"fcm_dpo/q_t": 0.49998199939727783,
"grad_norm": 0.0031413903925567865,
"learning_rate": 1.4494497203727843e-07,
"logits/chosen": 1.2971501350402832,
"logits/rejected": 1.2553091049194336,
"logps/chosen": -249.41549682617188,
"logps/ref_chosen": -53.682716369628906,
"logps/ref_rejected": -88.17315673828125,
"logps/rejected": -469.724853515625,
"loss": 1.3862,
"margin_dpo/margin_mean": 185.81893920898438,
"margin_dpo/margin_std": 336.22564697265625,
"step": 447
},
{
"KL/chosen_KL_mean": -236.30259704589844,
"KL/mean": -296.205078125,
"KL/rejected_KL_mean": -356.1075439453125,
"KL/std": 286.213623046875,
"epoch": 0.6772486772486772,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 119.80496215820312,
"fcm_dpo/q_t": 0.4999884366989136,
"grad_norm": 0.002245934447273612,
"learning_rate": 1.4374663593999256e-07,
"logits/chosen": 1.302297830581665,
"logits/rejected": 1.282482624053955,
"logps/chosen": -290.0538330078125,
"logps/ref_chosen": -53.75125503540039,
"logps/ref_rejected": -77.17623901367188,
"logps/rejected": -433.2837829589844,
"loss": 1.3862,
"margin_dpo/margin_mean": 119.80496215820312,
"margin_dpo/margin_std": 352.40093994140625,
"step": 448
},
{
"KL/chosen_KL_mean": -323.4554443359375,
"KL/mean": -345.5867919921875,
"KL/rejected_KL_mean": -367.7181701660156,
"KL/std": 359.06500244140625,
"epoch": 0.6787603930461074,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 44.262718200683594,
"fcm_dpo/q_t": 0.49999570846557617,
"grad_norm": 0.0019348947098478675,
"learning_rate": 1.4255127197770707e-07,
"logits/chosen": 1.22648024559021,
"logits/rejected": 1.2325232028961182,
"logps/chosen": -399.2828369140625,
"logps/ref_chosen": -75.82737731933594,
"logps/ref_rejected": -82.20687866210938,
"logps/rejected": -449.925048828125,
"loss": 1.3863,
"margin_dpo/margin_mean": 44.262718200683594,
"margin_dpo/margin_std": 400.443359375,
"step": 449
},
{
"KL/chosen_KL_mean": -181.9138946533203,
"KL/mean": -270.99798583984375,
"KL/rejected_KL_mean": -360.08209228515625,
"KL/std": 310.10675048828125,
"epoch": 0.6802721088435374,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 178.16824340820312,
"fcm_dpo/q_t": 0.4999827742576599,
"grad_norm": 0.0035590401384979486,
"learning_rate": 1.4135891358732205e-07,
"logits/chosen": 1.3165409564971924,
"logits/rejected": 1.2763895988464355,
"logps/chosen": -229.02960205078125,
"logps/ref_chosen": -47.11572265625,
"logps/ref_rejected": -78.7546615600586,
"logps/rejected": -438.8367919921875,
"loss": 1.3862,
"margin_dpo/margin_mean": 178.16824340820312,
"margin_dpo/margin_std": 364.7618408203125,
"step": 450
},
{
"KL/chosen_KL_mean": -227.50653076171875,
"KL/mean": -281.16180419921875,
"KL/rejected_KL_mean": -334.8171081542969,
"KL/std": 289.9800720214844,
"epoch": 0.6817838246409675,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 107.31055450439453,
"fcm_dpo/q_t": 0.4999896287918091,
"grad_norm": 0.002129608066752553,
"learning_rate": 1.4016959412166437e-07,
"logits/chosen": 1.2388477325439453,
"logits/rejected": 1.225501298904419,
"logps/chosen": -290.85699462890625,
"logps/ref_chosen": -63.350440979003906,
"logps/ref_rejected": -76.28530883789062,
"logps/rejected": -411.1024169921875,
"loss": 1.3863,
"margin_dpo/margin_mean": 107.31056213378906,
"margin_dpo/margin_std": 304.1910400390625,
"step": 451
},
{
"KL/chosen_KL_mean": -224.90155029296875,
"KL/mean": -285.91082763671875,
"KL/rejected_KL_mean": -346.92010498046875,
"KL/std": 273.02587890625,
"epoch": 0.6832955404383976,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 122.01853942871094,
"fcm_dpo/q_t": 0.4999881982803345,
"grad_norm": 0.002451541367918253,
"learning_rate": 1.3898334684855645e-07,
"logits/chosen": 1.2452630996704102,
"logits/rejected": 1.216700553894043,
"logps/chosen": -280.4873962402344,
"logps/ref_chosen": -55.58583450317383,
"logps/ref_rejected": -77.68738555908203,
"logps/rejected": -424.60748291015625,
"loss": 1.3862,
"margin_dpo/margin_mean": 122.01854705810547,
"margin_dpo/margin_std": 345.8335266113281,
"step": 452
},
{
"KL/chosen_KL_mean": -220.40438842773438,
"KL/mean": -267.64581298828125,
"KL/rejected_KL_mean": -314.8872375488281,
"KL/std": 251.55581665039062,
"epoch": 0.6848072562358276,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 94.48284149169922,
"fcm_dpo/q_t": 0.49999088048934937,
"grad_norm": 0.0015818747924640775,
"learning_rate": 1.3780020494988445e-07,
"logits/chosen": 1.2517281770706177,
"logits/rejected": 1.2492544651031494,
"logps/chosen": -282.1825866699219,
"logps/ref_chosen": -61.778202056884766,
"logps/ref_rejected": -71.51403045654297,
"logps/rejected": -386.4012451171875,
"loss": 1.3863,
"margin_dpo/margin_mean": 94.48284149169922,
"margin_dpo/margin_std": 269.7317810058594,
"step": 453
},
{
"KL/chosen_KL_mean": -197.87136840820312,
"KL/mean": -252.01133728027344,
"KL/rejected_KL_mean": -306.15130615234375,
"KL/std": 242.91844177246094,
"epoch": 0.6863189720332578,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 108.2799301147461,
"fcm_dpo/q_t": 0.49998950958251953,
"grad_norm": 0.001828188425861299,
"learning_rate": 1.366202015206706e-07,
"logits/chosen": 1.2671639919281006,
"logits/rejected": 1.2640118598937988,
"logps/chosen": -249.46652221679688,
"logps/ref_chosen": -51.59515380859375,
"logps/ref_rejected": -63.96732711791992,
"logps/rejected": -370.11865234375,
"loss": 1.3863,
"margin_dpo/margin_mean": 108.27993774414062,
"margin_dpo/margin_std": 284.8143615722656,
"step": 454
},
{
"KL/chosen_KL_mean": -279.669677734375,
"KL/mean": -327.62640380859375,
"KL/rejected_KL_mean": -375.5830993652344,
"KL/std": 286.3306884765625,
"epoch": 0.6878306878306878,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 95.91340637207031,
"fcm_dpo/q_t": 0.49999070167541504,
"grad_norm": 0.0017985355807468295,
"learning_rate": 1.354433695681474e-07,
"logits/chosen": 1.2243902683258057,
"logits/rejected": 1.225219964981079,
"logps/chosen": -350.3214111328125,
"logps/ref_chosen": -70.65170288085938,
"logps/ref_rejected": -77.44276428222656,
"logps/rejected": -453.02587890625,
"loss": 1.3863,
"margin_dpo/margin_mean": 95.91340637207031,
"margin_dpo/margin_std": 302.83038330078125,
"step": 455
},
{
"KL/chosen_KL_mean": -215.04669189453125,
"KL/mean": -274.91461181640625,
"KL/rejected_KL_mean": -334.78253173828125,
"KL/std": 239.27294921875,
"epoch": 0.6893424036281179,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 119.73584747314453,
"fcm_dpo/q_t": 0.4999884366989136,
"grad_norm": 0.0024633186403661966,
"learning_rate": 1.3426974201083439e-07,
"logits/chosen": 1.2097394466400146,
"logits/rejected": 1.1886322498321533,
"logps/chosen": -271.4449768066406,
"logps/ref_chosen": -56.398284912109375,
"logps/ref_rejected": -82.61642456054688,
"logps/rejected": -417.3989562988281,
"loss": 1.3862,
"margin_dpo/margin_mean": 119.73583984375,
"margin_dpo/margin_std": 301.7488098144531,
"step": 456
},
{
"KL/chosen_KL_mean": -213.508544921875,
"KL/mean": -266.53350830078125,
"KL/rejected_KL_mean": -319.5584411621094,
"KL/std": 283.9473876953125,
"epoch": 0.690854119425548,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 106.04988098144531,
"fcm_dpo/q_t": 0.49998974800109863,
"grad_norm": 0.0022488024551421404,
"learning_rate": 1.3309935167761717e-07,
"logits/chosen": 1.3623318672180176,
"logits/rejected": 1.318712592124939,
"logps/chosen": -258.2291259765625,
"logps/ref_chosen": -44.72057342529297,
"logps/ref_rejected": -68.1158676147461,
"logps/rejected": -387.67431640625,
"loss": 1.3863,
"margin_dpo/margin_mean": 106.04988861083984,
"margin_dpo/margin_std": 333.2089538574219,
"step": 457
},
{
"KL/chosen_KL_mean": -188.3619384765625,
"KL/mean": -288.2777404785156,
"KL/rejected_KL_mean": -388.19354248046875,
"KL/std": 289.7039489746094,
"epoch": 0.6923658352229781,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 199.83157348632812,
"fcm_dpo/q_t": 0.4999806582927704,
"grad_norm": 0.00349022657610476,
"learning_rate": 1.3193223130682936e-07,
"logits/chosen": 1.2601938247680664,
"logits/rejected": 1.2150036096572876,
"logps/chosen": -238.36764526367188,
"logps/ref_chosen": -50.00569152832031,
"logps/ref_rejected": -87.50015258789062,
"logps/rejected": -475.69366455078125,
"loss": 1.3862,
"margin_dpo/margin_mean": 199.83157348632812,
"margin_dpo/margin_std": 349.6432800292969,
"step": 458
},
{
"KL/chosen_KL_mean": -256.993896484375,
"KL/mean": -320.4532470703125,
"KL/rejected_KL_mean": -383.91259765625,
"KL/std": 375.80279541015625,
"epoch": 0.6938775510204082,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 126.91870880126953,
"fcm_dpo/q_t": 0.49998772144317627,
"grad_norm": 0.002192882588133216,
"learning_rate": 1.3076841354533658e-07,
"logits/chosen": 1.2736434936523438,
"logits/rejected": 1.2665081024169922,
"logps/chosen": -322.371826171875,
"logps/ref_chosen": -65.37794494628906,
"logps/ref_rejected": -88.19244384765625,
"logps/rejected": -472.10504150390625,
"loss": 1.3862,
"margin_dpo/margin_mean": 126.918701171875,
"margin_dpo/margin_std": 348.869140625,
"step": 459
},
{
"KL/chosen_KL_mean": -235.994384765625,
"KL/mean": -324.6895751953125,
"KL/rejected_KL_mean": -413.38482666015625,
"KL/std": 302.2080993652344,
"epoch": 0.6953892668178382,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 177.39044189453125,
"fcm_dpo/q_t": 0.4999828338623047,
"grad_norm": 0.00325951655395329,
"learning_rate": 1.2960793094762345e-07,
"logits/chosen": 1.2858896255493164,
"logits/rejected": 1.2416361570358276,
"logps/chosen": -300.5560302734375,
"logps/ref_chosen": -64.5616683959961,
"logps/ref_rejected": -88.67890167236328,
"logps/rejected": -502.063720703125,
"loss": 1.3862,
"margin_dpo/margin_mean": 177.39044189453125,
"margin_dpo/margin_std": 368.12908935546875,
"step": 460
},
{
"KL/chosen_KL_mean": -190.54605102539062,
"KL/mean": -272.08990478515625,
"KL/rejected_KL_mean": -353.6337585449219,
"KL/std": 280.8585510253906,
"epoch": 0.6969009826152683,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 163.08770751953125,
"fcm_dpo/q_t": 0.4999842047691345,
"grad_norm": 0.0028130814898759127,
"learning_rate": 1.2845081597488286e-07,
"logits/chosen": 1.2972333431243896,
"logits/rejected": 1.270198106765747,
"logps/chosen": -240.02395629882812,
"logps/ref_chosen": -49.4779167175293,
"logps/ref_rejected": -72.65262603759766,
"logps/rejected": -426.286376953125,
"loss": 1.3862,
"margin_dpo/margin_mean": 163.08770751953125,
"margin_dpo/margin_std": 284.0039978027344,
"step": 461
},
{
"KL/chosen_KL_mean": -188.55178833007812,
"KL/mean": -252.95437622070312,
"KL/rejected_KL_mean": -317.35693359375,
"KL/std": 175.87823486328125,
"epoch": 0.6984126984126984,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 128.80517578125,
"fcm_dpo/q_t": 0.49998754262924194,
"grad_norm": 0.002148764207959175,
"learning_rate": 1.27297100994108e-07,
"logits/chosen": 1.276878833770752,
"logits/rejected": 1.2633922100067139,
"logps/chosen": -249.04689025878906,
"logps/ref_chosen": -60.4951171875,
"logps/ref_rejected": -74.82136535644531,
"logps/rejected": -392.1783142089844,
"loss": 1.3862,
"margin_dpo/margin_mean": 128.80517578125,
"margin_dpo/margin_std": 224.19122314453125,
"step": 462
},
{
"KL/chosen_KL_mean": -273.58941650390625,
"KL/mean": -293.7682800292969,
"KL/rejected_KL_mean": -313.9471435546875,
"KL/std": 260.3634033203125,
"epoch": 0.6999244142101285,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 40.35773468017578,
"fcm_dpo/q_t": 0.4999960660934448,
"grad_norm": 0.0018327016150578856,
"learning_rate": 1.2614681827718695e-07,
"logits/chosen": 1.2572622299194336,
"logits/rejected": 1.2583889961242676,
"logps/chosen": -341.2745361328125,
"logps/ref_chosen": -67.68511962890625,
"logps/ref_rejected": -71.32196044921875,
"logps/rejected": -385.26910400390625,
"loss": 1.3863,
"margin_dpo/margin_mean": 40.35773468017578,
"margin_dpo/margin_std": 348.34228515625,
"step": 463
},
{
"KL/chosen_KL_mean": -230.39385986328125,
"KL/mean": -282.303466796875,
"KL/rejected_KL_mean": -334.21307373046875,
"KL/std": 285.50360107421875,
"epoch": 0.7014361300075586,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 103.8191909790039,
"fcm_dpo/q_t": 0.49998992681503296,
"grad_norm": 0.0019776602275669575,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": 1.2877094745635986,
"logits/rejected": 1.2803895473480225,
"logps/chosen": -289.55950927734375,
"logps/ref_chosen": -59.16564178466797,
"logps/ref_rejected": -69.56146240234375,
"logps/rejected": -403.7745361328125,
"loss": 1.3863,
"margin_dpo/margin_mean": 103.81919860839844,
"margin_dpo/margin_std": 348.489501953125,
"step": 464
},
{
"KL/chosen_KL_mean": -232.89459228515625,
"KL/mean": -298.0108337402344,
"KL/rejected_KL_mean": -363.1270751953125,
"KL/std": 265.3921813964844,
"epoch": 0.7029478458049887,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.23245239257812,
"fcm_dpo/q_t": 0.4999874234199524,
"grad_norm": 0.0023306291550397873,
"learning_rate": 1.238566782415197e-07,
"logits/chosen": 1.3072185516357422,
"logits/rejected": 1.2831568717956543,
"logps/chosen": -291.40826416015625,
"logps/ref_chosen": -58.513671875,
"logps/ref_rejected": -84.31745910644531,
"logps/rejected": -447.44451904296875,
"loss": 1.3862,
"margin_dpo/margin_mean": 130.23245239257812,
"margin_dpo/margin_std": 334.58380126953125,
"step": 465
},
{
"KL/chosen_KL_mean": -299.5732421875,
"KL/mean": -299.2271728515625,
"KL/rejected_KL_mean": -298.881103515625,
"KL/std": 311.90045166015625,
"epoch": 0.7044595616024187,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.6921577453613281,
"fcm_dpo/q_t": 0.5000000596046448,
"grad_norm": 0.001482538296841085,
"learning_rate": 1.2271688498291334e-07,
"logits/chosen": 1.261409878730774,
"logits/rejected": 1.2607641220092773,
"logps/chosen": -372.83905029296875,
"logps/ref_chosen": -73.26580810546875,
"logps/ref_rejected": -74.83621215820312,
"logps/rejected": -373.71728515625,
"loss": 1.3863,
"margin_dpo/margin_mean": -0.6921501159667969,
"margin_dpo/margin_std": 345.80474853515625,
"step": 466
},
{
"KL/chosen_KL_mean": -198.47096252441406,
"KL/mean": -302.1253356933594,
"KL/rejected_KL_mean": -405.77972412109375,
"KL/std": 344.73516845703125,
"epoch": 0.7059712773998488,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 207.30874633789062,
"fcm_dpo/q_t": 0.4999799430370331,
"grad_norm": 0.004299922846257687,
"learning_rate": 1.2158065210664848e-07,
"logits/chosen": 1.2816591262817383,
"logits/rejected": 1.2331783771514893,
"logps/chosen": -246.05044555664062,
"logps/ref_chosen": -47.57947540283203,
"logps/ref_rejected": -78.68522644042969,
"logps/rejected": -484.4649353027344,
"loss": 1.3862,
"margin_dpo/margin_mean": 207.30874633789062,
"margin_dpo/margin_std": 427.2309265136719,
"step": 467
},
{
"KL/chosen_KL_mean": -241.93890380859375,
"KL/mean": -287.9136962890625,
"KL/rejected_KL_mean": -333.88848876953125,
"KL/std": 330.4139404296875,
"epoch": 0.7074829931972789,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 91.94956970214844,
"fcm_dpo/q_t": 0.4999910891056061,
"grad_norm": 0.0016830574022606015,
"learning_rate": 1.204480113956011e-07,
"logits/chosen": 1.2580692768096924,
"logits/rejected": 1.2581918239593506,
"logps/chosen": -305.86669921875,
"logps/ref_chosen": -63.92778778076172,
"logps/ref_rejected": -76.51626586914062,
"logps/rejected": -410.40472412109375,
"loss": 1.3863,
"margin_dpo/margin_mean": 91.94956970214844,
"margin_dpo/margin_std": 405.082275390625,
"step": 468
},
{
"KL/chosen_KL_mean": -257.592529296875,
"KL/mean": -292.6340026855469,
"KL/rejected_KL_mean": -327.6754455566406,
"KL/std": 307.759521484375,
"epoch": 0.708994708994709,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 70.08292388916016,
"fcm_dpo/q_t": 0.4999932050704956,
"grad_norm": 0.0019712212961167097,
"learning_rate": 1.1931899453216697e-07,
"logits/chosen": 1.2918429374694824,
"logits/rejected": 1.2837411165237427,
"logps/chosen": -316.65069580078125,
"logps/ref_chosen": -59.05818176269531,
"logps/ref_rejected": -75.67672729492188,
"logps/rejected": -403.3521728515625,
"loss": 1.3863,
"margin_dpo/margin_mean": 70.08292388916016,
"margin_dpo/margin_std": 339.2579345703125,
"step": 469
},
{
"KL/chosen_KL_mean": -186.16371154785156,
"KL/mean": -246.60861206054688,
"KL/rejected_KL_mean": -307.05352783203125,
"KL/std": 228.65818786621094,
"epoch": 0.7105064247921391,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 120.88981628417969,
"fcm_dpo/q_t": 0.499988317489624,
"grad_norm": 0.002206693636253476,
"learning_rate": 1.1819363309737438e-07,
"logits/chosen": 1.2765345573425293,
"logits/rejected": 1.256923794746399,
"logps/chosen": -234.03114318847656,
"logps/ref_chosen": -47.86743927001953,
"logps/ref_rejected": -65.96859741210938,
"logps/rejected": -373.0220947265625,
"loss": 1.3862,
"margin_dpo/margin_mean": 120.88981628417969,
"margin_dpo/margin_std": 278.8352966308594,
"step": 470
},
{
"KL/chosen_KL_mean": -218.08656311035156,
"KL/mean": -278.8359680175781,
"KL/rejected_KL_mean": -339.5853576660156,
"KL/std": 286.42633056640625,
"epoch": 0.7120181405895691,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 121.498779296875,
"fcm_dpo/q_t": 0.49998825788497925,
"grad_norm": 0.0023722737096250057,
"learning_rate": 1.1707195857000215e-07,
"logits/chosen": 1.238015055656433,
"logits/rejected": 1.219024658203125,
"logps/chosen": -275.8644104003906,
"logps/ref_chosen": -57.777854919433594,
"logps/ref_rejected": -73.81172180175781,
"logps/rejected": -413.3970947265625,
"loss": 1.3862,
"margin_dpo/margin_mean": 121.498779296875,
"margin_dpo/margin_std": 337.14422607421875,
"step": 471
},
{
"KL/chosen_KL_mean": -214.7991485595703,
"KL/mean": -261.65557861328125,
"KL/rejected_KL_mean": -308.5119934082031,
"KL/std": 277.4007568359375,
"epoch": 0.7135298563869993,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 93.71286010742188,
"fcm_dpo/q_t": 0.49999094009399414,
"grad_norm": 0.0018467764602974057,
"learning_rate": 1.1595400232569768e-07,
"logits/chosen": 1.2192950248718262,
"logits/rejected": 1.2010465860366821,
"logps/chosen": -270.70782470703125,
"logps/ref_chosen": -55.908668518066406,
"logps/ref_rejected": -74.70294189453125,
"logps/rejected": -383.2149353027344,
"loss": 1.3863,
"margin_dpo/margin_mean": 93.71286010742188,
"margin_dpo/margin_std": 368.9359130859375,
"step": 472
},
{
"KL/chosen_KL_mean": -204.37344360351562,
"KL/mean": -304.8914489746094,
"KL/rejected_KL_mean": -405.40948486328125,
"KL/std": 354.4217529296875,
"epoch": 0.7150415721844293,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 201.0360107421875,
"fcm_dpo/q_t": 0.49998053908348083,
"grad_norm": 0.003650173544883728,
"learning_rate": 1.1483979563610069e-07,
"logits/chosen": 1.3033931255340576,
"logits/rejected": 1.2516118288040161,
"logps/chosen": -258.5343322753906,
"logps/ref_chosen": -54.16088104248047,
"logps/ref_rejected": -92.76789855957031,
"logps/rejected": -498.1773681640625,
"loss": 1.3862,
"margin_dpo/margin_mean": 201.03599548339844,
"margin_dpo/margin_std": 407.78076171875,
"step": 473
},
{
"KL/chosen_KL_mean": -185.13568115234375,
"KL/mean": -263.2508544921875,
"KL/rejected_KL_mean": -341.36602783203125,
"KL/std": 283.1648254394531,
"epoch": 0.7165532879818595,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 156.2303466796875,
"fcm_dpo/q_t": 0.49998486042022705,
"grad_norm": 0.0029000849463045597,
"learning_rate": 1.1372936966796709e-07,
"logits/chosen": 1.295190691947937,
"logits/rejected": 1.2742294073104858,
"logps/chosen": -231.82138061523438,
"logps/ref_chosen": -46.685707092285156,
"logps/ref_rejected": -71.44731903076172,
"logps/rejected": -412.8133544921875,
"loss": 1.3862,
"margin_dpo/margin_mean": 156.2303466796875,
"margin_dpo/margin_std": 273.30877685546875,
"step": 474
},
{
"KL/chosen_KL_mean": -244.22735595703125,
"KL/mean": -307.16156005859375,
"KL/rejected_KL_mean": -370.0957946777344,
"KL/std": 284.45855712890625,
"epoch": 0.7180650037792895,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 125.86846160888672,
"fcm_dpo/q_t": 0.4999878406524658,
"grad_norm": 0.002193914959207177,
"learning_rate": 1.126227554822985e-07,
"logits/chosen": 1.2467389106750488,
"logits/rejected": 1.2255311012268066,
"logps/chosen": -302.71466064453125,
"logps/ref_chosen": -58.4873046875,
"logps/ref_rejected": -87.00187683105469,
"logps/rejected": -457.0976867675781,
"loss": 1.3862,
"margin_dpo/margin_mean": 125.86845397949219,
"margin_dpo/margin_std": 340.77777099609375,
"step": 475
},
{
"KL/chosen_KL_mean": -278.6739501953125,
"KL/mean": -314.0935363769531,
"KL/rejected_KL_mean": -349.51312255859375,
"KL/std": 314.4804382324219,
"epoch": 0.7195767195767195,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 70.83914184570312,
"fcm_dpo/q_t": 0.49999314546585083,
"grad_norm": 0.001390301389619708,
"learning_rate": 1.1151998403347243e-07,
"logits/chosen": 1.2042824029922485,
"logits/rejected": 1.2116657495498657,
"logps/chosen": -354.05560302734375,
"logps/ref_chosen": -75.38162231445312,
"logps/ref_rejected": -76.99822235107422,
"logps/rejected": -426.5113525390625,
"loss": 1.3863,
"margin_dpo/margin_mean": 70.83914184570312,
"margin_dpo/margin_std": 338.26055908203125,
"step": 476
},
{
"KL/chosen_KL_mean": -237.08114624023438,
"KL/mean": -295.21539306640625,
"KL/rejected_KL_mean": -353.34967041015625,
"KL/std": 313.53399658203125,
"epoch": 0.7210884353741497,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 116.26850891113281,
"fcm_dpo/q_t": 0.49998873472213745,
"grad_norm": 0.0024357279762625694,
"learning_rate": 1.1042108616837692e-07,
"logits/chosen": 1.2554326057434082,
"logits/rejected": 1.2339025735855103,
"logps/chosen": -298.154541015625,
"logps/ref_chosen": -61.073387145996094,
"logps/ref_rejected": -81.34375,
"logps/rejected": -434.69342041015625,
"loss": 1.3862,
"margin_dpo/margin_mean": 116.26850891113281,
"margin_dpo/margin_std": 433.21453857421875,
"step": 477
},
{
"KL/chosen_KL_mean": -206.6198272705078,
"KL/mean": -230.17007446289062,
"KL/rejected_KL_mean": -253.7202911376953,
"KL/std": 195.4527130126953,
"epoch": 0.7226001511715797,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 47.100467681884766,
"fcm_dpo/q_t": 0.4999954402446747,
"grad_norm": 0.0013403815682977438,
"learning_rate": 1.0932609262554746e-07,
"logits/chosen": 1.2230312824249268,
"logits/rejected": 1.2336747646331787,
"logps/chosen": -263.7871398925781,
"logps/ref_chosen": -57.16731643676758,
"logps/ref_rejected": -53.30917739868164,
"logps/rejected": -307.02947998046875,
"loss": 1.3863,
"margin_dpo/margin_mean": 47.1004638671875,
"margin_dpo/margin_std": 252.40872192382812,
"step": 478
},
{
"KL/chosen_KL_mean": -214.67568969726562,
"KL/mean": -258.00732421875,
"KL/rejected_KL_mean": -301.3389892578125,
"KL/std": 248.02645874023438,
"epoch": 0.7241118669690099,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 86.66331481933594,
"fcm_dpo/q_t": 0.49999159574508667,
"grad_norm": 0.0024421438574790955,
"learning_rate": 1.0823503403430734e-07,
"logits/chosen": 1.2256994247436523,
"logits/rejected": 1.2227530479431152,
"logps/chosen": -273.5889892578125,
"logps/ref_chosen": -58.91331481933594,
"logps/ref_rejected": -63.7403450012207,
"logps/rejected": -365.079345703125,
"loss": 1.3863,
"margin_dpo/margin_mean": 86.66331481933594,
"margin_dpo/margin_std": 288.06201171875,
"step": 479
},
{
"KL/chosen_KL_mean": -232.02232360839844,
"KL/mean": -267.8770446777344,
"KL/rejected_KL_mean": -303.73175048828125,
"KL/std": 240.46432495117188,
"epoch": 0.7256235827664399,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 71.70942687988281,
"fcm_dpo/q_t": 0.49999308586120605,
"grad_norm": 0.0013092632871121168,
"learning_rate": 1.0714794091391072e-07,
"logits/chosen": 1.234184741973877,
"logits/rejected": 1.2392432689666748,
"logps/chosen": -294.82293701171875,
"logps/ref_chosen": -62.80061340332031,
"logps/ref_rejected": -67.58859252929688,
"logps/rejected": -371.3203430175781,
"loss": 1.3863,
"margin_dpo/margin_mean": 71.70941925048828,
"margin_dpo/margin_std": 270.3521728515625,
"step": 480
},
{
"KL/chosen_KL_mean": -257.1699523925781,
"KL/mean": -295.2996826171875,
"KL/rejected_KL_mean": -333.42950439453125,
"KL/std": 286.9197082519531,
"epoch": 0.72713529856387,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 76.25953674316406,
"fcm_dpo/q_t": 0.49999263882637024,
"grad_norm": 0.0014908494194969535,
"learning_rate": 1.0606484367268906e-07,
"logits/chosen": 1.2494676113128662,
"logits/rejected": 1.257669448852539,
"logps/chosen": -322.4564208984375,
"logps/ref_chosen": -65.28649139404297,
"logps/ref_rejected": -70.78668212890625,
"logps/rejected": -404.2161865234375,
"loss": 1.3863,
"margin_dpo/margin_mean": 76.2595443725586,
"margin_dpo/margin_std": 337.06689453125,
"step": 481
},
{
"KL/chosen_KL_mean": -263.18341064453125,
"KL/mean": -354.302978515625,
"KL/rejected_KL_mean": -445.4225158691406,
"KL/std": 386.18914794921875,
"epoch": 0.7286470143613001,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 182.2390594482422,
"fcm_dpo/q_t": 0.4999823570251465,
"grad_norm": 0.003877797396853566,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": 1.230767011642456,
"logits/rejected": 1.1534785032272339,
"logps/chosen": -324.089599609375,
"logps/ref_chosen": -60.906185150146484,
"logps/ref_rejected": -103.44656372070312,
"logps/rejected": -548.8690795898438,
"loss": 1.3862,
"margin_dpo/margin_mean": 182.23907470703125,
"margin_dpo/margin_std": 488.2938232421875,
"step": 482
},
{
"KL/chosen_KL_mean": -213.94471740722656,
"KL/mean": -296.33514404296875,
"KL/rejected_KL_mean": -378.7255859375,
"KL/std": 284.4780578613281,
"epoch": 0.7301587301587301,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 164.78086853027344,
"fcm_dpo/q_t": 0.4999840557575226,
"grad_norm": 0.0031552868895232677,
"learning_rate": 1.0391075790138232e-07,
"logits/chosen": 1.2734465599060059,
"logits/rejected": 1.2415012121200562,
"logps/chosen": -267.13671875,
"logps/ref_chosen": -53.192012786865234,
"logps/ref_rejected": -81.83927154541016,
"logps/rejected": -460.56488037109375,
"loss": 1.3862,
"margin_dpo/margin_mean": 164.78086853027344,
"margin_dpo/margin_std": 341.28466796875,
"step": 483
},
{
"KL/chosen_KL_mean": -244.69627380371094,
"KL/mean": -283.5141296386719,
"KL/rejected_KL_mean": -322.3320007324219,
"KL/std": 284.9932861328125,
"epoch": 0.7316704459561603,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 77.6357421875,
"fcm_dpo/q_t": 0.4999924898147583,
"grad_norm": 0.0018454925157129765,
"learning_rate": 1.0283982962570681e-07,
"logits/chosen": 1.2972228527069092,
"logits/rejected": 1.2862370014190674,
"logps/chosen": -302.4657287597656,
"logps/ref_chosen": -57.76945877075195,
"logps/ref_rejected": -71.6829833984375,
"logps/rejected": -394.0149841308594,
"loss": 1.3863,
"margin_dpo/margin_mean": 77.6357421875,
"margin_dpo/margin_std": 290.02801513671875,
"step": 484
},
{
"KL/chosen_KL_mean": -232.70870971679688,
"KL/mean": -280.1761779785156,
"KL/rejected_KL_mean": -327.6436462402344,
"KL/std": 277.9726257324219,
"epoch": 0.7331821617535903,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 94.93495178222656,
"fcm_dpo/q_t": 0.4999908208847046,
"grad_norm": 0.0017475350759923458,
"learning_rate": 1.0177301773633992e-07,
"logits/chosen": 1.2424696683883667,
"logits/rejected": 1.2406383752822876,
"logps/chosen": -289.34454345703125,
"logps/ref_chosen": -56.63584899902344,
"logps/ref_rejected": -70.85614013671875,
"logps/rejected": -398.4997863769531,
"loss": 1.3863,
"margin_dpo/margin_mean": 94.9349594116211,
"margin_dpo/margin_std": 307.6720275878906,
"step": 485
},
{
"KL/chosen_KL_mean": -219.482666015625,
"KL/mean": -304.8910827636719,
"KL/rejected_KL_mean": -390.29949951171875,
"KL/std": 327.8187255859375,
"epoch": 0.7346938775510204,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 170.81683349609375,
"fcm_dpo/q_t": 0.49998345971107483,
"grad_norm": 0.00316282594576478,
"learning_rate": 1.007103520743035e-07,
"logits/chosen": 1.260047435760498,
"logits/rejected": 1.2206038236618042,
"logps/chosen": -275.8296813964844,
"logps/ref_chosen": -56.347023010253906,
"logps/ref_rejected": -85.97221374511719,
"logps/rejected": -476.271728515625,
"loss": 1.3862,
"margin_dpo/margin_mean": 170.8168487548828,
"margin_dpo/margin_std": 370.9302978515625,
"step": 486
},
{
"KL/chosen_KL_mean": -245.201416015625,
"KL/mean": -311.9969787597656,
"KL/rejected_KL_mean": -378.79254150390625,
"KL/std": 356.8734130859375,
"epoch": 0.7362055933484505,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 133.5911407470703,
"fcm_dpo/q_t": 0.49998706579208374,
"grad_norm": 0.0026520034298300743,
"learning_rate": 9.965186236464046e-08,
"logits/chosen": 1.3191529512405396,
"logits/rejected": 1.297263503074646,
"logps/chosen": -305.8186340332031,
"logps/ref_chosen": -60.617218017578125,
"logps/ref_rejected": -82.50975036621094,
"logps/rejected": -461.3022766113281,
"loss": 1.3862,
"margin_dpo/margin_mean": 133.59112548828125,
"margin_dpo/margin_std": 416.29364013671875,
"step": 487
},
{
"KL/chosen_KL_mean": -238.37918090820312,
"KL/mean": -309.9017333984375,
"KL/rejected_KL_mean": -381.42431640625,
"KL/std": 288.9468078613281,
"epoch": 0.7377173091458806,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 143.04513549804688,
"fcm_dpo/q_t": 0.4999861717224121,
"grad_norm": 0.0026024070102721453,
"learning_rate": 9.859757821558337e-08,
"logits/chosen": 1.2986162900924683,
"logits/rejected": 1.2784430980682373,
"logps/chosen": -301.48822021484375,
"logps/ref_chosen": -63.10905075073242,
"logps/ref_rejected": -82.49348449707031,
"logps/rejected": -463.91778564453125,
"loss": 1.3862,
"margin_dpo/margin_mean": 143.04513549804688,
"margin_dpo/margin_std": 348.89385986328125,
"step": 488
},
{
"KL/chosen_KL_mean": -275.52362060546875,
"KL/mean": -324.05682373046875,
"KL/rejected_KL_mean": -372.5899658203125,
"KL/std": 305.4133605957031,
"epoch": 0.7392290249433107,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 97.06634521484375,
"fcm_dpo/q_t": 0.4999906122684479,
"grad_norm": 0.0021042677108198404,
"learning_rate": 9.754752911772615e-08,
"logits/chosen": 1.2624127864837646,
"logits/rejected": 1.245917797088623,
"logps/chosen": -340.5125732421875,
"logps/ref_chosen": -64.98896026611328,
"logps/ref_rejected": -84.39607238769531,
"logps/rejected": -456.98602294921875,
"loss": 1.3863,
"margin_dpo/margin_mean": 97.06634521484375,
"margin_dpo/margin_std": 349.16461181640625,
"step": 489
},
{
"KL/chosen_KL_mean": -234.01315307617188,
"KL/mean": -281.6185607910156,
"KL/rejected_KL_mean": -329.2239990234375,
"KL/std": 286.2935791015625,
"epoch": 0.7407407407407407,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 95.21084594726562,
"fcm_dpo/q_t": 0.4999907612800598,
"grad_norm": 0.001769617898389697,
"learning_rate": 9.650174444319956e-08,
"logits/chosen": 1.2830909490585327,
"logits/rejected": 1.288257360458374,
"logps/chosen": -295.921875,
"logps/ref_chosen": -61.90874481201172,
"logps/ref_rejected": -70.58566284179688,
"logps/rejected": -399.80963134765625,
"loss": 1.3863,
"margin_dpo/margin_mean": 95.21085357666016,
"margin_dpo/margin_std": 358.0466613769531,
"step": 490
},
{
"KL/chosen_KL_mean": -204.4666748046875,
"KL/mean": -257.9485778808594,
"KL/rejected_KL_mean": -311.43048095703125,
"KL/std": 246.65692138671875,
"epoch": 0.7422524565381708,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 106.96379852294922,
"fcm_dpo/q_t": 0.4999896287918091,
"grad_norm": 0.0019353864481672645,
"learning_rate": 9.546025344484868e-08,
"logits/chosen": 1.2297401428222656,
"logits/rejected": 1.202380895614624,
"logps/chosen": -259.9423828125,
"logps/ref_chosen": -55.47570037841797,
"logps/ref_rejected": -78.70318603515625,
"logps/rejected": -390.1336669921875,
"loss": 1.3863,
"margin_dpo/margin_mean": 106.96379852294922,
"margin_dpo/margin_std": 309.4108581542969,
"step": 491
},
{
"KL/chosen_KL_mean": -274.214599609375,
"KL/mean": -337.2662353515625,
"KL/rejected_KL_mean": -400.31787109375,
"KL/std": 328.02960205078125,
"epoch": 0.7437641723356009,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 126.1032943725586,
"fcm_dpo/q_t": 0.49998778104782104,
"grad_norm": 0.002519700676202774,
"learning_rate": 9.442308525541589e-08,
"logits/chosen": 1.2626538276672363,
"logits/rejected": 1.2385413646697998,
"logps/chosen": -341.5009765625,
"logps/ref_chosen": -67.28638458251953,
"logps/ref_rejected": -82.78628540039062,
"logps/rejected": -483.10418701171875,
"loss": 1.3862,
"margin_dpo/margin_mean": 126.1032943725586,
"margin_dpo/margin_std": 430.36181640625,
"step": 492
},
{
"KL/chosen_KL_mean": -193.55007934570312,
"KL/mean": -267.8668212890625,
"KL/rejected_KL_mean": -342.18353271484375,
"KL/std": 282.44659423828125,
"epoch": 0.745275888133031,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 148.63345336914062,
"fcm_dpo/q_t": 0.49998563528060913,
"grad_norm": 0.0027879553381353617,
"learning_rate": 9.339026888672468e-08,
"logits/chosen": 1.2619478702545166,
"logits/rejected": 1.2343031167984009,
"logps/chosen": -249.47760009765625,
"logps/ref_chosen": -55.92750549316406,
"logps/ref_rejected": -79.12149810791016,
"logps/rejected": -421.3050537109375,
"loss": 1.3862,
"margin_dpo/margin_mean": 148.63343811035156,
"margin_dpo/margin_std": 357.33514404296875,
"step": 493
},
{
"KL/chosen_KL_mean": -263.4035949707031,
"KL/mean": -321.43548583984375,
"KL/rejected_KL_mean": -379.4674072265625,
"KL/std": 349.48931884765625,
"epoch": 0.7467876039304611,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 116.06379699707031,
"fcm_dpo/q_t": 0.4999887943267822,
"grad_norm": 0.002763553289696574,
"learning_rate": 9.236183322886945e-08,
"logits/chosen": 1.210706114768982,
"logits/rejected": 1.1937711238861084,
"logps/chosen": -331.3576965332031,
"logps/ref_chosen": -67.95410919189453,
"logps/ref_rejected": -90.50865173339844,
"logps/rejected": -469.9760437011719,
"loss": 1.3862,
"margin_dpo/margin_mean": 116.06378936767578,
"margin_dpo/margin_std": 462.98712158203125,
"step": 494
},
{
"KL/chosen_KL_mean": -195.76187133789062,
"KL/mean": -256.2266540527344,
"KL/rejected_KL_mean": -316.69140625,
"KL/std": 313.70343017578125,
"epoch": 0.7482993197278912,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 120.9295654296875,
"fcm_dpo/q_t": 0.49998828768730164,
"grad_norm": 0.002702318364754319,
"learning_rate": 9.133780704940594e-08,
"logits/chosen": 1.2848858833312988,
"logits/rejected": 1.2627146244049072,
"logps/chosen": -248.3873291015625,
"logps/ref_chosen": -52.62546157836914,
"logps/ref_rejected": -72.06781005859375,
"logps/rejected": -388.75921630859375,
"loss": 1.3862,
"margin_dpo/margin_mean": 120.92955780029297,
"margin_dpo/margin_std": 426.9603271484375,
"step": 495
},
{
"KL/chosen_KL_mean": -217.66781616210938,
"KL/mean": -307.02685546875,
"KL/rejected_KL_mean": -396.3858642578125,
"KL/std": 350.0010070800781,
"epoch": 0.7498110355253212,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 178.71804809570312,
"fcm_dpo/q_t": 0.49998271465301514,
"grad_norm": 0.003732392331585288,
"learning_rate": 9.031821899254797e-08,
"logits/chosen": 1.2494525909423828,
"logits/rejected": 1.1938560009002686,
"logps/chosen": -275.26513671875,
"logps/ref_chosen": -57.597320556640625,
"logps/ref_rejected": -94.36127471923828,
"logps/rejected": -490.7471618652344,
"loss": 1.3862,
"margin_dpo/margin_mean": 178.71804809570312,
"margin_dpo/margin_std": 492.61053466796875,
"step": 496
},
{
"KL/chosen_KL_mean": -261.3085021972656,
"KL/mean": -313.65008544921875,
"KL/rejected_KL_mean": -365.99163818359375,
"KL/std": 278.7204284667969,
"epoch": 0.7513227513227513,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.68314361572266,
"fcm_dpo/q_t": 0.4999898672103882,
"grad_norm": 0.002317574806511402,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": 1.2523890733718872,
"logits/rejected": 1.24021315574646,
"logps/chosen": -334.09844970703125,
"logps/ref_chosen": -72.78994750976562,
"logps/ref_rejected": -89.48483276367188,
"logps/rejected": -455.4764709472656,
"loss": 1.3863,
"margin_dpo/margin_mean": 104.68313598632812,
"margin_dpo/margin_std": 338.248291015625,
"step": 497
},
{
"KL/chosen_KL_mean": -224.85238647460938,
"KL/mean": -278.84197998046875,
"KL/rejected_KL_mean": -332.83160400390625,
"KL/std": 270.00634765625,
"epoch": 0.7528344671201814,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 107.97920989990234,
"fcm_dpo/q_t": 0.4999895691871643,
"grad_norm": 0.0020106714218854904,
"learning_rate": 8.829247120198563e-08,
"logits/chosen": 1.2464903593063354,
"logits/rejected": 1.2533658742904663,
"logps/chosen": -293.21807861328125,
"logps/ref_chosen": -68.36572265625,
"logps/ref_rejected": -71.28846740722656,
"logps/rejected": -404.1200866699219,
"loss": 1.3863,
"margin_dpo/margin_mean": 107.97921752929688,
"margin_dpo/margin_std": 352.33111572265625,
"step": 498
},
{
"KL/chosen_KL_mean": -242.342529296875,
"KL/mean": -309.7532653808594,
"KL/rejected_KL_mean": -377.1640319824219,
"KL/std": 302.4976806640625,
"epoch": 0.7543461829176115,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 134.82147216796875,
"fcm_dpo/q_t": 0.4999869465827942,
"grad_norm": 0.0024030758067965508,
"learning_rate": 8.728636813280163e-08,
"logits/chosen": 1.269078016281128,
"logits/rejected": 1.2355639934539795,
"logps/chosen": -304.2513427734375,
"logps/ref_chosen": -61.90882873535156,
"logps/ref_rejected": -91.9411392211914,
"logps/rejected": -469.10516357421875,
"loss": 1.3862,
"margin_dpo/margin_mean": 134.82147216796875,
"margin_dpo/margin_std": 416.40582275390625,
"step": 499
},
{
"KL/chosen_KL_mean": -253.39688110351562,
"KL/mean": -294.63592529296875,
"KL/rejected_KL_mean": -335.87493896484375,
"KL/std": 297.90802001953125,
"epoch": 0.7558578987150416,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 82.47804260253906,
"fcm_dpo/q_t": 0.4999920129776001,
"grad_norm": 0.0017515573417767882,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": 1.25115966796875,
"logits/rejected": 1.267160415649414,
"logps/chosen": -323.62274169921875,
"logps/ref_chosen": -70.225830078125,
"logps/ref_rejected": -71.72203063964844,
"logps/rejected": -407.59698486328125,
"loss": 1.3863,
"margin_dpo/margin_mean": 82.47804260253906,
"margin_dpo/margin_std": 377.62408447265625,
"step": 500
},
{
"KL/chosen_KL_mean": -261.0697326660156,
"KL/mean": -288.06939697265625,
"KL/rejected_KL_mean": -315.069091796875,
"KL/std": 263.5559387207031,
"epoch": 0.7573696145124716,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 53.99939727783203,
"fcm_dpo/q_t": 0.49999478459358215,
"grad_norm": 0.0011944005964323878,
"learning_rate": 8.528784436016878e-08,
"logits/chosen": 1.2886323928833008,
"logits/rejected": 1.289060354232788,
"logps/chosen": -325.66851806640625,
"logps/ref_chosen": -64.59880828857422,
"logps/ref_rejected": -70.59329223632812,
"logps/rejected": -385.66241455078125,
"loss": 1.3863,
"margin_dpo/margin_mean": 53.99940490722656,
"margin_dpo/margin_std": 239.36679077148438,
"step": 501
},
{
"KL/chosen_KL_mean": -269.3963317871094,
"KL/mean": -318.6138916015625,
"KL/rejected_KL_mean": -367.83148193359375,
"KL/std": 324.662841796875,
"epoch": 0.7588813303099018,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 98.43511962890625,
"fcm_dpo/q_t": 0.49999046325683594,
"grad_norm": 0.0020151876378804445,
"learning_rate": 8.4295479559726e-08,
"logits/chosen": 1.2497098445892334,
"logits/rejected": 1.2299872636795044,
"logps/chosen": -334.86297607421875,
"logps/ref_chosen": -65.46662902832031,
"logps/ref_rejected": -90.22233581542969,
"logps/rejected": -458.0538024902344,
"loss": 1.3863,
"margin_dpo/margin_mean": 98.43512725830078,
"margin_dpo/margin_std": 369.150634765625,
"step": 502
},
{
"KL/chosen_KL_mean": -190.64247131347656,
"KL/mean": -217.7227783203125,
"KL/rejected_KL_mean": -244.80307006835938,
"KL/std": 184.62762451171875,
"epoch": 0.7603930461073318,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 54.16061019897461,
"fcm_dpo/q_t": 0.49999475479125977,
"grad_norm": 0.0011236389400437474,
"learning_rate": 8.330774987092712e-08,
"logits/chosen": 1.2540428638458252,
"logits/rejected": 1.2642710208892822,
"logps/chosen": -242.47723388671875,
"logps/ref_chosen": -51.83476257324219,
"logps/ref_rejected": -57.62522506713867,
"logps/rejected": -302.4283142089844,
"loss": 1.3863,
"margin_dpo/margin_mean": 54.160614013671875,
"margin_dpo/margin_std": 232.08360290527344,
"step": 503
},
{
"KL/chosen_KL_mean": -230.22885131835938,
"KL/mean": -277.79840087890625,
"KL/rejected_KL_mean": -325.367919921875,
"KL/std": 244.9968719482422,
"epoch": 0.7619047619047619,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 95.13908386230469,
"fcm_dpo/q_t": 0.4999907910823822,
"grad_norm": 0.0015835659578442574,
"learning_rate": 8.232468292269479e-08,
"logits/chosen": 1.2332806587219238,
"logits/rejected": 1.2388975620269775,
"logps/chosen": -298.88006591796875,
"logps/ref_chosen": -68.65119934082031,
"logps/ref_rejected": -77.91394805908203,
"logps/rejected": -403.2818908691406,
"loss": 1.3863,
"margin_dpo/margin_mean": 95.13908386230469,
"margin_dpo/margin_std": 298.76800537109375,
"step": 504
},
{
"KL/chosen_KL_mean": -239.896484375,
"KL/mean": -291.92791748046875,
"KL/rejected_KL_mean": -343.9593200683594,
"KL/std": 342.92205810546875,
"epoch": 0.763416477702192,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.06285095214844,
"fcm_dpo/q_t": 0.49998992681503296,
"grad_norm": 0.002278943546116352,
"learning_rate": 8.134630621352483e-08,
"logits/chosen": 1.2496414184570312,
"logits/rejected": 1.2356963157653809,
"logps/chosen": -299.89532470703125,
"logps/ref_chosen": -59.99884796142578,
"logps/ref_rejected": -76.88048553466797,
"logps/rejected": -420.8398132324219,
"loss": 1.3863,
"margin_dpo/margin_mean": 104.06285095214844,
"margin_dpo/margin_std": 399.6540222167969,
"step": 505
},
{
"KL/chosen_KL_mean": -271.675048828125,
"KL/mean": -315.27374267578125,
"KL/rejected_KL_mean": -358.87237548828125,
"KL/std": 294.46759033203125,
"epoch": 0.764928193499622,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 87.19731903076172,
"fcm_dpo/q_t": 0.4999915361404419,
"grad_norm": 0.0017523845890536904,
"learning_rate": 8.037264711071698e-08,
"logits/chosen": 1.2550417184829712,
"logits/rejected": 1.2555642127990723,
"logps/chosen": -341.746337890625,
"logps/ref_chosen": -70.07130432128906,
"logps/ref_rejected": -82.03775024414062,
"logps/rejected": -440.9101257324219,
"loss": 1.3863,
"margin_dpo/margin_mean": 87.19731903076172,
"margin_dpo/margin_std": 383.368896484375,
"step": 506
},
{
"KL/chosen_KL_mean": -286.790283203125,
"KL/mean": -347.9385986328125,
"KL/rejected_KL_mean": -409.0868835449219,
"KL/std": 353.75408935546875,
"epoch": 0.7664399092970522,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 122.29658508300781,
"fcm_dpo/q_t": 0.4999881684780121,
"grad_norm": 0.0025757874827831984,
"learning_rate": 7.940373284960933e-08,
"logits/chosen": 1.2367452383041382,
"logits/rejected": 1.2185437679290771,
"logps/chosen": -358.79730224609375,
"logps/ref_chosen": -72.00703430175781,
"logps/ref_rejected": -93.94987487792969,
"logps/rejected": -503.0367431640625,
"loss": 1.3862,
"margin_dpo/margin_mean": 122.29658508300781,
"margin_dpo/margin_std": 428.36688232421875,
"step": 507
},
{
"KL/chosen_KL_mean": -234.4129638671875,
"KL/mean": -341.6184387207031,
"KL/rejected_KL_mean": -448.82391357421875,
"KL/std": 360.1383056640625,
"epoch": 0.7679516250944822,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 214.41094970703125,
"fcm_dpo/q_t": 0.49997925758361816,
"grad_norm": 0.004092069808393717,
"learning_rate": 7.843959053281663e-08,
"logits/chosen": 1.2509568929672241,
"logits/rejected": 1.2035400867462158,
"logps/chosen": -294.63287353515625,
"logps/ref_chosen": -60.21992492675781,
"logps/ref_rejected": -95.9200668334961,
"logps/rejected": -544.7439575195312,
"loss": 1.3862,
"margin_dpo/margin_mean": 214.41094970703125,
"margin_dpo/margin_std": 459.3138122558594,
"step": 508
},
{
"KL/chosen_KL_mean": -234.1763458251953,
"KL/mean": -279.1103210449219,
"KL/rejected_KL_mean": -324.0443115234375,
"KL/std": 251.64889526367188,
"epoch": 0.7694633408919124,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.86798858642578,
"fcm_dpo/q_t": 0.4999912977218628,
"grad_norm": 0.001714211655780673,
"learning_rate": 7.748024712947204e-08,
"logits/chosen": 1.2467401027679443,
"logits/rejected": 1.2466963529586792,
"logps/chosen": -300.4465026855469,
"logps/ref_chosen": -66.27017211914062,
"logps/ref_rejected": -71.73065185546875,
"logps/rejected": -395.77496337890625,
"loss": 1.3863,
"margin_dpo/margin_mean": 89.86798858642578,
"margin_dpo/margin_std": 300.79638671875,
"step": 509
},
{
"KL/chosen_KL_mean": -217.1124725341797,
"KL/mean": -314.4279479980469,
"KL/rejected_KL_mean": -411.743408203125,
"KL/std": 359.33087158203125,
"epoch": 0.7709750566893424,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 194.6309814453125,
"fcm_dpo/q_t": 0.499981164932251,
"grad_norm": 0.0035733687691390514,
"learning_rate": 7.652572947447272e-08,
"logits/chosen": 1.3153700828552246,
"logits/rejected": 1.2759437561035156,
"logps/chosen": -270.6573486328125,
"logps/ref_chosen": -53.54487609863281,
"logps/ref_rejected": -91.36648559570312,
"logps/rejected": -503.10992431640625,
"loss": 1.3862,
"margin_dpo/margin_mean": 194.6309814453125,
"margin_dpo/margin_std": 484.9547119140625,
"step": 510
},
{
"KL/chosen_KL_mean": -246.47100830078125,
"KL/mean": -316.29608154296875,
"KL/rejected_KL_mean": -386.12115478515625,
"KL/std": 318.93328857421875,
"epoch": 0.7724867724867724,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 139.65016174316406,
"fcm_dpo/q_t": 0.499986469745636,
"grad_norm": 0.0027404821012169123,
"learning_rate": 7.557606426772961e-08,
"logits/chosen": 1.2811682224273682,
"logits/rejected": 1.257398009300232,
"logps/chosen": -302.31536865234375,
"logps/ref_chosen": -55.844383239746094,
"logps/ref_rejected": -86.49819946289062,
"logps/rejected": -472.6193542480469,
"loss": 1.3862,
"margin_dpo/margin_mean": 139.65016174316406,
"margin_dpo/margin_std": 339.78717041015625,
"step": 511
},
{
"KL/chosen_KL_mean": -258.85089111328125,
"KL/mean": -287.54278564453125,
"KL/rejected_KL_mean": -316.2346496582031,
"KL/std": 276.65423583984375,
"epoch": 0.7739984882842026,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 57.38376998901367,
"fcm_dpo/q_t": 0.4999944269657135,
"grad_norm": 0.0019444817444309592,
"learning_rate": 7.463127807341966e-08,
"logits/chosen": 1.254358172416687,
"logits/rejected": 1.2565571069717407,
"logps/chosen": -320.5039367675781,
"logps/ref_chosen": -61.653038024902344,
"logps/ref_rejected": -72.83148193359375,
"logps/rejected": -389.066162109375,
"loss": 1.3863,
"margin_dpo/margin_mean": 57.383766174316406,
"margin_dpo/margin_std": 347.9620361328125,
"step": 512
},
{
"KL/chosen_KL_mean": -200.0421142578125,
"KL/mean": -261.5788269042969,
"KL/rejected_KL_mean": -323.11553955078125,
"KL/std": 258.5805969238281,
"epoch": 0.7755102040816326,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 123.07341003417969,
"fcm_dpo/q_t": 0.4999880790710449,
"grad_norm": 0.002271485747769475,
"learning_rate": 7.369139731924401e-08,
"logits/chosen": 1.3097870349884033,
"logits/rejected": 1.2971343994140625,
"logps/chosen": -250.89468383789062,
"logps/ref_chosen": -50.85256576538086,
"logps/ref_rejected": -69.21754455566406,
"logps/rejected": -392.33306884765625,
"loss": 1.3862,
"margin_dpo/margin_mean": 123.07341003417969,
"margin_dpo/margin_std": 297.73004150390625,
"step": 513
},
{
"KL/chosen_KL_mean": -270.1363830566406,
"KL/mean": -320.6453552246094,
"KL/rejected_KL_mean": -371.15435791015625,
"KL/std": 278.06121826171875,
"epoch": 0.7770219198790628,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 101.01797485351562,
"fcm_dpo/q_t": 0.49999022483825684,
"grad_norm": 0.0019997938070446253,
"learning_rate": 7.275644829568747e-08,
"logits/chosen": 1.2445127964019775,
"logits/rejected": 1.241532802581787,
"logps/chosen": -339.52130126953125,
"logps/ref_chosen": -69.38493347167969,
"logps/ref_rejected": -83.32447814941406,
"logps/rejected": -454.47882080078125,
"loss": 1.3863,
"margin_dpo/margin_mean": 101.01797485351562,
"margin_dpo/margin_std": 324.2949523925781,
"step": 514
},
{
"KL/chosen_KL_mean": -241.34022521972656,
"KL/mean": -301.35577392578125,
"KL/rejected_KL_mean": -361.371337890625,
"KL/std": 278.6277160644531,
"epoch": 0.7785336356764928,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 120.03112030029297,
"fcm_dpo/q_t": 0.4999883770942688,
"grad_norm": 0.0023619269486516714,
"learning_rate": 7.182645715528435e-08,
"logits/chosen": 1.2922251224517822,
"logits/rejected": 1.2539401054382324,
"logps/chosen": -295.02728271484375,
"logps/ref_chosen": -53.687034606933594,
"logps/ref_rejected": -83.59614562988281,
"logps/rejected": -444.96746826171875,
"loss": 1.3862,
"margin_dpo/margin_mean": 120.03111267089844,
"margin_dpo/margin_std": 330.4995422363281,
"step": 515
},
{
"KL/chosen_KL_mean": -212.0453643798828,
"KL/mean": -263.1299133300781,
"KL/rejected_KL_mean": -314.2144470214844,
"KL/std": 261.9797668457031,
"epoch": 0.780045351473923,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.16908264160156,
"fcm_dpo/q_t": 0.4999901056289673,
"grad_norm": 0.0018633684376254678,
"learning_rate": 7.090144991188568e-08,
"logits/chosen": 1.2701607942581177,
"logits/rejected": 1.2603992223739624,
"logps/chosen": -268.94708251953125,
"logps/ref_chosen": -56.9017219543457,
"logps/ref_rejected": -67.83477783203125,
"logps/rejected": -382.0492248535156,
"loss": 1.3863,
"margin_dpo/margin_mean": 102.16908264160156,
"margin_dpo/margin_std": 321.8263244628906,
"step": 516
},
{
"KL/chosen_KL_mean": -254.77786254882812,
"KL/mean": -277.159423828125,
"KL/rejected_KL_mean": -299.541015625,
"KL/std": 273.0831298828125,
"epoch": 0.781557067271353,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 44.76313781738281,
"fcm_dpo/q_t": 0.4999956488609314,
"grad_norm": 0.001817610813304782,
"learning_rate": 6.998145243993284e-08,
"logits/chosen": 1.2920082807540894,
"logits/rejected": 1.2981328964233398,
"logps/chosen": -316.5530090332031,
"logps/ref_chosen": -61.775142669677734,
"logps/ref_rejected": -62.88270950317383,
"logps/rejected": -362.4237365722656,
"loss": 1.3863,
"margin_dpo/margin_mean": 44.76313400268555,
"margin_dpo/margin_std": 304.3832702636719,
"step": 517
},
{
"KL/chosen_KL_mean": -219.64892578125,
"KL/mean": -284.0128173828125,
"KL/rejected_KL_mean": -348.376708984375,
"KL/std": 307.4132385253906,
"epoch": 0.783068783068783,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 128.727783203125,
"fcm_dpo/q_t": 0.49998754262924194,
"grad_norm": 0.002293572761118412,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": 1.2533849477767944,
"logits/rejected": 1.2436450719833374,
"logps/chosen": -281.6741638183594,
"logps/ref_chosen": -62.02523422241211,
"logps/ref_rejected": -79.06085205078125,
"logps/rejected": -427.43756103515625,
"loss": 1.3862,
"margin_dpo/margin_mean": 128.72779846191406,
"margin_dpo/margin_std": 379.77642822265625,
"step": 518
},
{
"KL/chosen_KL_mean": -248.82159423828125,
"KL/mean": -290.4149169921875,
"KL/rejected_KL_mean": -332.00823974609375,
"KL/std": 269.19866943359375,
"epoch": 0.7845804988662132,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 83.18663024902344,
"fcm_dpo/q_t": 0.4999919533729553,
"grad_norm": 0.002268343698233366,
"learning_rate": 6.815658960673781e-08,
"logits/chosen": 1.260763168334961,
"logits/rejected": 1.2478408813476562,
"logps/chosen": -310.427978515625,
"logps/ref_chosen": -61.60636901855469,
"logps/ref_rejected": -74.50727844238281,
"logps/rejected": -406.5155029296875,
"loss": 1.3863,
"margin_dpo/margin_mean": 83.18663024902344,
"margin_dpo/margin_std": 341.28594970703125,
"step": 519
},
{
"KL/chosen_KL_mean": -225.19485473632812,
"KL/mean": -291.09344482421875,
"KL/rejected_KL_mean": -356.9920349121094,
"KL/std": 300.70770263671875,
"epoch": 0.7860922146636432,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 131.7971649169922,
"fcm_dpo/q_t": 0.49998724460601807,
"grad_norm": 0.002785892691463232,
"learning_rate": 6.725177529083209e-08,
"logits/chosen": 1.3261933326721191,
"logits/rejected": 1.3110194206237793,
"logps/chosen": -288.06829833984375,
"logps/ref_chosen": -62.87343215942383,
"logps/ref_rejected": -76.505615234375,
"logps/rejected": -433.4976501464844,
"loss": 1.3862,
"margin_dpo/margin_mean": 131.7971649169922,
"margin_dpo/margin_std": 311.9014892578125,
"step": 520
},
{
"KL/chosen_KL_mean": -233.61395263671875,
"KL/mean": -314.4349060058594,
"KL/rejected_KL_mean": -395.255859375,
"KL/std": 295.2518310546875,
"epoch": 0.7876039304610734,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 161.64187622070312,
"fcm_dpo/q_t": 0.49998435378074646,
"grad_norm": 0.0028386113699525595,
"learning_rate": 6.63520728356167e-08,
"logits/chosen": 1.244354486465454,
"logits/rejected": 1.2204021215438843,
"logps/chosen": -297.82061767578125,
"logps/ref_chosen": -64.20668029785156,
"logps/ref_rejected": -92.28083038330078,
"logps/rejected": -487.53668212890625,
"loss": 1.3862,
"margin_dpo/margin_mean": 161.6418914794922,
"margin_dpo/margin_std": 372.21527099609375,
"step": 521
},
{
"KL/chosen_KL_mean": -254.05210876464844,
"KL/mean": -273.76922607421875,
"KL/rejected_KL_mean": -293.48638916015625,
"KL/std": 242.01705932617188,
"epoch": 0.7891156462585034,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 39.43425750732422,
"fcm_dpo/q_t": 0.4999961853027344,
"grad_norm": 0.001333137508481741,
"learning_rate": 6.545750740770336e-08,
"logits/chosen": 1.263893723487854,
"logits/rejected": 1.2616896629333496,
"logps/chosen": -312.42181396484375,
"logps/ref_chosen": -58.369720458984375,
"logps/ref_rejected": -68.79248046875,
"logps/rejected": -362.27886962890625,
"loss": 1.3863,
"margin_dpo/margin_mean": 39.434261322021484,
"margin_dpo/margin_std": 283.5724792480469,
"step": 522
},
{
"KL/chosen_KL_mean": -269.55242919921875,
"KL/mean": -339.4023132324219,
"KL/rejected_KL_mean": -409.252197265625,
"KL/std": 313.99249267578125,
"epoch": 0.7906273620559335,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 139.69979858398438,
"fcm_dpo/q_t": 0.499986469745636,
"grad_norm": 0.0031393070239573717,
"learning_rate": 6.456810403001012e-08,
"logits/chosen": 1.2486999034881592,
"logits/rejected": 1.2007946968078613,
"logps/chosen": -335.2656555175781,
"logps/ref_chosen": -65.71324157714844,
"logps/ref_rejected": -91.98896789550781,
"logps/rejected": -501.2411804199219,
"loss": 1.3862,
"margin_dpo/margin_mean": 139.69979858398438,
"margin_dpo/margin_std": 364.4853515625,
"step": 523
},
{
"KL/chosen_KL_mean": -258.45904541015625,
"KL/mean": -302.37017822265625,
"KL/rejected_KL_mean": -346.28131103515625,
"KL/std": 282.6915283203125,
"epoch": 0.7921390778533636,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 87.82229614257812,
"fcm_dpo/q_t": 0.4999915063381195,
"grad_norm": 0.0017077861120924354,
"learning_rate": 6.368388758106134e-08,
"logits/chosen": 1.2249318361282349,
"logits/rejected": 1.2157388925552368,
"logps/chosen": -334.8102722167969,
"logps/ref_chosen": -76.35124969482422,
"logps/ref_rejected": -89.96072387695312,
"logps/rejected": -436.2420349121094,
"loss": 1.3863,
"margin_dpo/margin_mean": 87.82229614257812,
"margin_dpo/margin_std": 297.6631774902344,
"step": 524
},
{
"KL/chosen_KL_mean": -292.2108154296875,
"KL/mean": -318.42254638671875,
"KL/rejected_KL_mean": -344.63427734375,
"KL/std": 277.3336486816406,
"epoch": 0.7936507936507936,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 52.42350387573242,
"fcm_dpo/q_t": 0.4999949336051941,
"grad_norm": 0.0015537795843556523,
"learning_rate": 6.280488279429185e-08,
"logits/chosen": 1.1935780048370361,
"logits/rejected": 1.1954379081726074,
"logps/chosen": -367.70660400390625,
"logps/ref_chosen": -75.49578857421875,
"logps/ref_rejected": -84.04852294921875,
"logps/rejected": -428.68280029296875,
"loss": 1.3863,
"margin_dpo/margin_mean": 52.42350769042969,
"margin_dpo/margin_std": 347.2845458984375,
"step": 525
},
{
"KL/chosen_KL_mean": -247.38699340820312,
"KL/mean": -307.152587890625,
"KL/rejected_KL_mean": -366.91815185546875,
"KL/std": 318.8529968261719,
"epoch": 0.7951625094482238,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 119.53118896484375,
"fcm_dpo/q_t": 0.4999884366989136,
"grad_norm": 0.0027431962080299854,
"learning_rate": 6.193111425735515e-08,
"logits/chosen": 1.2542002201080322,
"logits/rejected": 1.226431131362915,
"logps/chosen": -308.67938232421875,
"logps/ref_chosen": -61.29241943359375,
"logps/ref_rejected": -82.47763061523438,
"logps/rejected": -449.39581298828125,
"loss": 1.3862,
"margin_dpo/margin_mean": 119.53117370605469,
"margin_dpo/margin_std": 372.6949768066406,
"step": 526
},
{
"KL/chosen_KL_mean": -241.8243408203125,
"KL/mean": -305.8216247558594,
"KL/rejected_KL_mean": -369.81884765625,
"KL/std": 304.98187255859375,
"epoch": 0.7966742252456538,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 127.9945068359375,
"fcm_dpo/q_t": 0.4999876022338867,
"grad_norm": 0.0031173720490187407,
"learning_rate": 6.106260641143546e-08,
"logits/chosen": 1.2762367725372314,
"logits/rejected": 1.2428832054138184,
"logps/chosen": -303.2969970703125,
"logps/ref_chosen": -61.472625732421875,
"logps/ref_rejected": -90.52831268310547,
"logps/rejected": -460.34716796875,
"loss": 1.3862,
"margin_dpo/margin_mean": 127.9945068359375,
"margin_dpo/margin_std": 330.25439453125,
"step": 527
},
{
"KL/chosen_KL_mean": -213.7490234375,
"KL/mean": -268.8468017578125,
"KL/rejected_KL_mean": -323.944580078125,
"KL/std": 265.69378662109375,
"epoch": 0.7981859410430839,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 110.19552612304688,
"fcm_dpo/q_t": 0.4999893605709076,
"grad_norm": 0.0022039057221263647,
"learning_rate": 6.019938355056422e-08,
"logits/chosen": 1.2133557796478271,
"logits/rejected": 1.192253589630127,
"logps/chosen": -272.5410461425781,
"logps/ref_chosen": -58.792015075683594,
"logps/ref_rejected": -71.82516479492188,
"logps/rejected": -395.76971435546875,
"loss": 1.3863,
"margin_dpo/margin_mean": 110.1955337524414,
"margin_dpo/margin_std": 350.6766357421875,
"step": 528
},
{
"KL/chosen_KL_mean": -207.93161010742188,
"KL/mean": -281.9962158203125,
"KL/rejected_KL_mean": -356.06085205078125,
"KL/std": 229.11720275878906,
"epoch": 0.799697656840514,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 148.1292266845703,
"fcm_dpo/q_t": 0.4999856650829315,
"grad_norm": 0.0023845250252634287,
"learning_rate": 5.934146982094049e-08,
"logits/chosen": 1.2498539686203003,
"logits/rejected": 1.2348082065582275,
"logps/chosen": -263.0025634765625,
"logps/ref_chosen": -55.070960998535156,
"logps/ref_rejected": -75.44007873535156,
"logps/rejected": -431.50091552734375,
"loss": 1.3862,
"margin_dpo/margin_mean": 148.1292266845703,
"margin_dpo/margin_std": 242.88185119628906,
"step": 529
},
{
"KL/chosen_KL_mean": -232.45703125,
"KL/mean": -286.29730224609375,
"KL/rejected_KL_mean": -340.1375732421875,
"KL/std": 313.3363037109375,
"epoch": 0.8012093726379441,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 107.68051147460938,
"fcm_dpo/q_t": 0.4999895691871643,
"grad_norm": 0.002098920289427042,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": 1.2603845596313477,
"logits/rejected": 1.2442349195480347,
"logps/chosen": -289.20086669921875,
"logps/ref_chosen": -56.743812561035156,
"logps/ref_rejected": -76.6692123413086,
"logps/rejected": -416.8067626953125,
"loss": 1.3863,
"margin_dpo/margin_mean": 107.68052673339844,
"margin_dpo/margin_std": 359.12908935546875,
"step": 530
},
{
"KL/chosen_KL_mean": -207.04556274414062,
"KL/mean": -264.3824462890625,
"KL/rejected_KL_mean": -321.7193603515625,
"KL/std": 270.8686218261719,
"epoch": 0.8027210884353742,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 114.67379760742188,
"fcm_dpo/q_t": 0.4999889135360718,
"grad_norm": 0.0024485730100423098,
"learning_rate": 5.7641665597021435e-08,
"logits/chosen": 1.262247085571289,
"logits/rejected": 1.2219009399414062,
"logps/chosen": -258.1620178222656,
"logps/ref_chosen": -51.116455078125,
"logps/ref_rejected": -79.52884674072266,
"logps/rejected": -401.2481689453125,
"loss": 1.3863,
"margin_dpo/margin_mean": 114.67379760742188,
"margin_dpo/margin_std": 330.855712890625,
"step": 531
},
{
"KL/chosen_KL_mean": -252.53512573242188,
"KL/mean": -311.3507080078125,
"KL/rejected_KL_mean": -370.16632080078125,
"KL/std": 310.71435546875,
"epoch": 0.8042328042328042,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 117.63117980957031,
"fcm_dpo/q_t": 0.4999886155128479,
"grad_norm": 0.0020603465382009745,
"learning_rate": 5.679982264990424e-08,
"logits/chosen": 1.233213186264038,
"logits/rejected": 1.2175933122634888,
"logps/chosen": -310.8150634765625,
"logps/ref_chosen": -58.279945373535156,
"logps/ref_rejected": -78.05426788330078,
"logps/rejected": -448.2205810546875,
"loss": 1.3862,
"margin_dpo/margin_mean": 117.63117980957031,
"margin_dpo/margin_std": 353.5859375,
"step": 532
},
{
"KL/chosen_KL_mean": -186.49844360351562,
"KL/mean": -257.16241455078125,
"KL/rejected_KL_mean": -327.8263854980469,
"KL/std": 312.2849426269531,
"epoch": 0.8057445200302343,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 141.3279571533203,
"fcm_dpo/q_t": 0.49998632073402405,
"grad_norm": 0.0025260313414037228,
"learning_rate": 5.596338392706076e-08,
"logits/chosen": 1.2709081172943115,
"logits/rejected": 1.2562451362609863,
"logps/chosen": -242.9164581298828,
"logps/ref_chosen": -56.41801071166992,
"logps/ref_rejected": -73.89324951171875,
"logps/rejected": -401.7196350097656,
"loss": 1.3862,
"margin_dpo/margin_mean": 141.3279571533203,
"margin_dpo/margin_std": 353.8362121582031,
"step": 533
},
{
"KL/chosen_KL_mean": -237.41729736328125,
"KL/mean": -282.9198913574219,
"KL/rejected_KL_mean": -328.4224853515625,
"KL/std": 276.8403625488281,
"epoch": 0.8072562358276644,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 91.00517272949219,
"fcm_dpo/q_t": 0.49999117851257324,
"grad_norm": 0.002167914994060993,
"learning_rate": 5.513237282548033e-08,
"logits/chosen": 1.2200748920440674,
"logits/rejected": 1.2163264751434326,
"logps/chosen": -298.1659851074219,
"logps/ref_chosen": -60.748687744140625,
"logps/ref_rejected": -73.8623046875,
"logps/rejected": -402.2847900390625,
"loss": 1.3863,
"margin_dpo/margin_mean": 91.00517272949219,
"margin_dpo/margin_std": 360.96624755859375,
"step": 534
},
{
"KL/chosen_KL_mean": -243.32843017578125,
"KL/mean": -293.2642822265625,
"KL/rejected_KL_mean": -343.20013427734375,
"KL/std": 254.28857421875,
"epoch": 0.8087679516250945,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.87169647216797,
"fcm_dpo/q_t": 0.4999903440475464,
"grad_norm": 0.0023620380088686943,
"learning_rate": 5.430681259032957e-08,
"logits/chosen": 1.2484328746795654,
"logits/rejected": 1.2265081405639648,
"logps/chosen": -304.9658508300781,
"logps/ref_chosen": -61.637413024902344,
"logps/ref_rejected": -80.93138885498047,
"logps/rejected": -424.13153076171875,
"loss": 1.3863,
"margin_dpo/margin_mean": 99.8717041015625,
"margin_dpo/margin_std": 285.54364013671875,
"step": 535
},
{
"KL/chosen_KL_mean": -181.73826599121094,
"KL/mean": -265.3481140136719,
"KL/rejected_KL_mean": -348.95794677734375,
"KL/std": 246.2275390625,
"epoch": 0.8102796674225246,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 167.21969604492188,
"fcm_dpo/q_t": 0.4999838173389435,
"grad_norm": 0.003139457432553172,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": 1.2669060230255127,
"logits/rejected": 1.2405221462249756,
"logps/chosen": -233.6272430419922,
"logps/ref_chosen": -51.88897705078125,
"logps/ref_rejected": -73.34864044189453,
"logps/rejected": -422.30657958984375,
"loss": 1.3862,
"margin_dpo/margin_mean": 167.21969604492188,
"margin_dpo/margin_std": 304.58843994140625,
"step": 536
},
{
"KL/chosen_KL_mean": -222.4708251953125,
"KL/mean": -305.9654541015625,
"KL/rejected_KL_mean": -389.4600830078125,
"KL/std": 298.34405517578125,
"epoch": 0.8117913832199547,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 166.9892578125,
"fcm_dpo/q_t": 0.49998384714126587,
"grad_norm": 0.0030098857823759317,
"learning_rate": 5.267213693697695e-08,
"logits/chosen": 1.298835277557373,
"logits/rejected": 1.2577811479568481,
"logps/chosen": -276.7194519042969,
"logps/ref_chosen": -54.248619079589844,
"logps/ref_rejected": -94.94343566894531,
"logps/rejected": -484.40350341796875,
"loss": 1.3862,
"margin_dpo/margin_mean": 166.9892578125,
"margin_dpo/margin_std": 360.7626953125,
"step": 537
},
{
"KL/chosen_KL_mean": -288.58770751953125,
"KL/mean": -338.69049072265625,
"KL/rejected_KL_mean": -388.79327392578125,
"KL/std": 344.57171630859375,
"epoch": 0.8133030990173847,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 100.20556640625,
"fcm_dpo/q_t": 0.499990314245224,
"grad_norm": 0.002249750541523099,
"learning_rate": 5.1863067244167144e-08,
"logits/chosen": 1.2919939756393433,
"logits/rejected": 1.285117268562317,
"logps/chosen": -358.6812438964844,
"logps/ref_chosen": -70.09353637695312,
"logps/ref_rejected": -79.49833679199219,
"logps/rejected": -468.2916259765625,
"loss": 1.3863,
"margin_dpo/margin_mean": 100.20558166503906,
"margin_dpo/margin_std": 351.8709411621094,
"step": 538
},
{
"KL/chosen_KL_mean": -225.29541015625,
"KL/mean": -294.580078125,
"KL/rejected_KL_mean": -363.8647766113281,
"KL/std": 290.5321044921875,
"epoch": 0.8148148148148148,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 138.56936645507812,
"fcm_dpo/q_t": 0.49998658895492554,
"grad_norm": 0.0030424855649471283,
"learning_rate": 5.105953986729195e-08,
"logits/chosen": 1.2382125854492188,
"logits/rejected": 1.2072124481201172,
"logps/chosen": -287.22711181640625,
"logps/ref_chosen": -61.93169403076172,
"logps/ref_rejected": -84.08946228027344,
"logps/rejected": -447.9542236328125,
"loss": 1.3862,
"margin_dpo/margin_mean": 138.56936645507812,
"margin_dpo/margin_std": 335.16851806640625,
"step": 539
},
{
"KL/chosen_KL_mean": -242.7515869140625,
"KL/mean": -336.0493469238281,
"KL/rejected_KL_mean": -429.34710693359375,
"KL/std": 340.7681884765625,
"epoch": 0.8163265306122449,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 186.59555053710938,
"fcm_dpo/q_t": 0.49998193979263306,
"grad_norm": 0.003513064468279481,
"learning_rate": 5.026157728273966e-08,
"logits/chosen": 1.3039121627807617,
"logits/rejected": 1.2665743827819824,
"logps/chosen": -305.455810546875,
"logps/ref_chosen": -62.704254150390625,
"logps/ref_rejected": -95.63597106933594,
"logps/rejected": -524.9830932617188,
"loss": 1.3862,
"margin_dpo/margin_mean": 186.59555053710938,
"margin_dpo/margin_std": 353.68701171875,
"step": 540
},
{
"KL/chosen_KL_mean": -221.798095703125,
"KL/mean": -250.46072387695312,
"KL/rejected_KL_mean": -279.1233825683594,
"KL/std": 239.862548828125,
"epoch": 0.817838246409675,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 57.32527160644531,
"fcm_dpo/q_t": 0.4999944567680359,
"grad_norm": 0.0014312553685158491,
"learning_rate": 4.9469201811239035e-08,
"logits/chosen": 1.290186882019043,
"logits/rejected": 1.3193426132202148,
"logps/chosen": -284.2789306640625,
"logps/ref_chosen": -62.48084259033203,
"logps/ref_rejected": -57.55541229248047,
"logps/rejected": -336.67877197265625,
"loss": 1.3863,
"margin_dpo/margin_mean": 57.32527160644531,
"margin_dpo/margin_std": 238.53594970703125,
"step": 541
},
{
"KL/chosen_KL_mean": -184.5562744140625,
"KL/mean": -236.01080322265625,
"KL/rejected_KL_mean": -287.46533203125,
"KL/std": 224.02711486816406,
"epoch": 0.8193499622071051,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.90904235839844,
"fcm_dpo/q_t": 0.4999900460243225,
"grad_norm": 0.0018258477794006467,
"learning_rate": 4.868243561723534e-08,
"logits/chosen": 1.2936313152313232,
"logits/rejected": 1.2742774486541748,
"logps/chosen": -234.01116943359375,
"logps/ref_chosen": -49.454891204833984,
"logps/ref_rejected": -65.33275604248047,
"logps/rejected": -352.798095703125,
"loss": 1.3863,
"margin_dpo/margin_mean": 102.90904998779297,
"margin_dpo/margin_std": 270.83367919921875,
"step": 542
},
{
"KL/chosen_KL_mean": -195.629638671875,
"KL/mean": -286.21685791015625,
"KL/rejected_KL_mean": -376.8040771484375,
"KL/std": 335.13543701171875,
"epoch": 0.8208616780045351,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 181.17446899414062,
"fcm_dpo/q_t": 0.49998247623443604,
"grad_norm": 0.003246963955461979,
"learning_rate": 4.790130070827028e-08,
"logits/chosen": 1.257094383239746,
"logits/rejected": 1.228837490081787,
"logps/chosen": -246.73049926757812,
"logps/ref_chosen": -51.100860595703125,
"logps/ref_rejected": -76.06130981445312,
"logps/rejected": -452.86541748046875,
"loss": 1.3862,
"margin_dpo/margin_mean": 181.1744842529297,
"margin_dpo/margin_std": 412.4785461425781,
"step": 543
},
{
"KL/chosen_KL_mean": -238.99652099609375,
"KL/mean": -313.9105529785156,
"KL/rejected_KL_mean": -388.8245544433594,
"KL/std": 292.81646728515625,
"epoch": 0.8223733938019653,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 149.82806396484375,
"fcm_dpo/q_t": 0.4999855160713196,
"grad_norm": 0.003041402902454138,
"learning_rate": 4.7125818934366454e-08,
"logits/chosen": 1.2706331014633179,
"logits/rejected": 1.2441833019256592,
"logps/chosen": -299.27374267578125,
"logps/ref_chosen": -60.2772331237793,
"logps/ref_rejected": -88.40553283691406,
"logps/rejected": -477.2301025390625,
"loss": 1.3862,
"margin_dpo/margin_mean": 149.8280487060547,
"margin_dpo/margin_std": 375.4241943359375,
"step": 544
},
{
"KL/chosen_KL_mean": -241.99879455566406,
"KL/mean": -287.8915100097656,
"KL/rejected_KL_mean": -333.7841796875,
"KL/std": 267.11920166015625,
"epoch": 0.8238851095993953,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 91.78538513183594,
"fcm_dpo/q_t": 0.49999111890792847,
"grad_norm": 0.0022016982547938824,
"learning_rate": 4.635601198741607e-08,
"logits/chosen": 1.2439241409301758,
"logits/rejected": 1.217350959777832,
"logps/chosen": -303.6140441894531,
"logps/ref_chosen": -61.61524963378906,
"logps/ref_rejected": -78.71266174316406,
"logps/rejected": -412.496826171875,
"loss": 1.3863,
"margin_dpo/margin_mean": 91.78538513183594,
"margin_dpo/margin_std": 318.565673828125,
"step": 545
},
{
"KL/chosen_KL_mean": -221.55914306640625,
"KL/mean": -255.8831787109375,
"KL/rejected_KL_mean": -290.20721435546875,
"KL/std": 262.83538818359375,
"epoch": 0.8253968253968254,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 68.64806365966797,
"fcm_dpo/q_t": 0.49999332427978516,
"grad_norm": 0.0015192232094705105,
"learning_rate": 4.559190140057428e-08,
"logits/chosen": 1.302754282951355,
"logits/rejected": 1.3086268901824951,
"logps/chosen": -280.8724060058594,
"logps/ref_chosen": -59.313262939453125,
"logps/ref_rejected": -64.73631286621094,
"logps/rejected": -354.9435119628906,
"loss": 1.3863,
"margin_dpo/margin_mean": 68.6480712890625,
"margin_dpo/margin_std": 304.9132080078125,
"step": 546
},
{
"KL/chosen_KL_mean": -201.32754516601562,
"KL/mean": -267.2884826660156,
"KL/rejected_KL_mean": -333.24945068359375,
"KL/std": 234.06785583496094,
"epoch": 0.8269085411942555,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 131.92190551757812,
"fcm_dpo/q_t": 0.49998724460601807,
"grad_norm": 0.00217438698746264,
"learning_rate": 4.483350854765672e-08,
"logits/chosen": 1.2100231647491455,
"logits/rejected": 1.1951963901519775,
"logps/chosen": -256.3042907714844,
"logps/ref_chosen": -54.97674560546875,
"logps/ref_rejected": -75.35922241210938,
"logps/rejected": -408.608642578125,
"loss": 1.3862,
"margin_dpo/margin_mean": 131.92190551757812,
"margin_dpo/margin_std": 245.69558715820312,
"step": 547
},
{
"KL/chosen_KL_mean": -246.73431396484375,
"KL/mean": -302.9718017578125,
"KL/rejected_KL_mean": -359.20928955078125,
"KL/std": 298.39251708984375,
"epoch": 0.8284202569916855,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 112.47500610351562,
"fcm_dpo/q_t": 0.4999891221523285,
"grad_norm": 0.0023801657371222973,
"learning_rate": 4.4080854642541826e-08,
"logits/chosen": 1.2188358306884766,
"logits/rejected": 1.203696846961975,
"logps/chosen": -309.9449768066406,
"logps/ref_chosen": -63.21067428588867,
"logps/ref_rejected": -81.23347473144531,
"logps/rejected": -440.4427795410156,
"loss": 1.3863,
"margin_dpo/margin_mean": 112.47501373291016,
"margin_dpo/margin_std": 335.791015625,
"step": 548
},
{
"KL/chosen_KL_mean": -246.50332641601562,
"KL/mean": -307.3105773925781,
"KL/rejected_KL_mean": -368.1178283691406,
"KL/std": 339.2923583984375,
"epoch": 0.8299319727891157,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 121.61450958251953,
"fcm_dpo/q_t": 0.49998822808265686,
"grad_norm": 0.00257733091711998,
"learning_rate": 4.333396073857723e-08,
"logits/chosen": 1.2848644256591797,
"logits/rejected": 1.2579424381256104,
"logps/chosen": -310.77685546875,
"logps/ref_chosen": -64.27351379394531,
"logps/ref_rejected": -92.31663513183594,
"logps/rejected": -460.4344482421875,
"loss": 1.3862,
"margin_dpo/margin_mean": 121.61450958251953,
"margin_dpo/margin_std": 392.3224792480469,
"step": 549
},
{
"KL/chosen_KL_mean": -218.34458923339844,
"KL/mean": -256.52752685546875,
"KL/rejected_KL_mean": -294.71044921875,
"KL/std": 242.93511962890625,
"epoch": 0.8314436885865457,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 76.36582946777344,
"fcm_dpo/q_t": 0.49999260902404785,
"grad_norm": 0.0018148054368793964,
"learning_rate": 4.259284772799099e-08,
"logits/chosen": 1.2855148315429688,
"logits/rejected": 1.2823785543441772,
"logps/chosen": -274.57501220703125,
"logps/ref_chosen": -56.230438232421875,
"logps/ref_rejected": -62.59788513183594,
"logps/rejected": -357.3083190917969,
"loss": 1.3863,
"margin_dpo/margin_mean": 76.3658218383789,
"margin_dpo/margin_std": 288.6811218261719,
"step": 550
},
{
"KL/chosen_KL_mean": -248.3517303466797,
"KL/mean": -309.2677917480469,
"KL/rejected_KL_mean": -370.1838684082031,
"KL/std": 306.270751953125,
"epoch": 0.8329554043839759,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 121.83216094970703,
"fcm_dpo/q_t": 0.4999881982803345,
"grad_norm": 0.0023011192679405212,
"learning_rate": 4.1857536341307176e-08,
"logits/chosen": 1.2524220943450928,
"logits/rejected": 1.2448160648345947,
"logps/chosen": -316.09893798828125,
"logps/ref_chosen": -67.74720764160156,
"logps/ref_rejected": -87.04285430908203,
"logps/rejected": -457.22674560546875,
"loss": 1.3862,
"margin_dpo/margin_mean": 121.83216094970703,
"margin_dpo/margin_std": 341.08929443359375,
"step": 551
},
{
"KL/chosen_KL_mean": -232.7744140625,
"KL/mean": -290.295166015625,
"KL/rejected_KL_mean": -347.8158874511719,
"KL/std": 296.06097412109375,
"epoch": 0.8344671201814059,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 115.04149627685547,
"fcm_dpo/q_t": 0.4999888837337494,
"grad_norm": 0.0021020383574068546,
"learning_rate": 4.112804714676593e-08,
"logits/chosen": 1.2442529201507568,
"logits/rejected": 1.2225983142852783,
"logps/chosen": -295.70068359375,
"logps/ref_chosen": -62.92625427246094,
"logps/ref_rejected": -82.98365783691406,
"logps/rejected": -430.799560546875,
"loss": 1.3862,
"margin_dpo/margin_mean": 115.04149627685547,
"margin_dpo/margin_std": 302.3883361816406,
"step": 552
},
{
"KL/chosen_KL_mean": -223.17971801757812,
"KL/mean": -299.51727294921875,
"KL/rejected_KL_mean": -375.8548583984375,
"KL/std": 325.5655517578125,
"epoch": 0.8359788359788359,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 152.67514038085938,
"fcm_dpo/q_t": 0.4999852180480957,
"grad_norm": 0.0032611123751848936,
"learning_rate": 4.0404400549748144e-08,
"logits/chosen": 1.2355235815048218,
"logits/rejected": 1.1960480213165283,
"logps/chosen": -279.21820068359375,
"logps/ref_chosen": -56.038490295410156,
"logps/ref_rejected": -84.48454284667969,
"logps/rejected": -460.3393859863281,
"loss": 1.3862,
"margin_dpo/margin_mean": 152.6751251220703,
"margin_dpo/margin_std": 377.767822265625,
"step": 553
},
{
"KL/chosen_KL_mean": -241.2162628173828,
"KL/mean": -287.3515625,
"KL/rejected_KL_mean": -333.48687744140625,
"KL/std": 261.48486328125,
"epoch": 0.8374905517762661,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 92.27063751220703,
"fcm_dpo/q_t": 0.4999910593032837,
"grad_norm": 0.0016775699332356453,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": 1.247375726699829,
"logits/rejected": 1.2545955181121826,
"logps/chosen": -305.7468566894531,
"logps/ref_chosen": -64.53059387207031,
"logps/ref_rejected": -71.2155990600586,
"logps/rejected": -404.7025146484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 92.27063751220703,
"margin_dpo/margin_std": 262.7148742675781,
"step": 554
},
{
"KL/chosen_KL_mean": -242.61669921875,
"KL/mean": -278.1309814453125,
"KL/rejected_KL_mean": -313.645263671875,
"KL/std": 235.6993408203125,
"epoch": 0.8390022675736961,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 71.02857971191406,
"fcm_dpo/q_t": 0.49999311566352844,
"grad_norm": 0.0012565868673846126,
"learning_rate": 3.89747159520904e-08,
"logits/chosen": 1.2524785995483398,
"logits/rejected": 1.2564916610717773,
"logps/chosen": -309.26861572265625,
"logps/ref_chosen": -66.65191650390625,
"logps/ref_rejected": -68.6667251586914,
"logps/rejected": -382.3119812011719,
"loss": 1.3863,
"margin_dpo/margin_mean": 71.02857971191406,
"margin_dpo/margin_std": 269.95416259765625,
"step": 555
},
{
"KL/chosen_KL_mean": -205.25283813476562,
"KL/mean": -254.348876953125,
"KL/rejected_KL_mean": -303.4449462890625,
"KL/std": 214.45046997070312,
"epoch": 0.8405139833711263,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 98.19207763671875,
"fcm_dpo/q_t": 0.49999046325683594,
"grad_norm": 0.001899282680824399,
"learning_rate": 3.826871794280192e-08,
"logits/chosen": 1.2756855487823486,
"logits/rejected": 1.2640533447265625,
"logps/chosen": -258.085205078125,
"logps/ref_chosen": -52.832366943359375,
"logps/ref_rejected": -64.49044036865234,
"logps/rejected": -367.93536376953125,
"loss": 1.3863,
"margin_dpo/margin_mean": 98.19209289550781,
"margin_dpo/margin_std": 222.29449462890625,
"step": 556
},
{
"KL/chosen_KL_mean": -227.02926635742188,
"KL/mean": -313.59149169921875,
"KL/rejected_KL_mean": -400.1536865234375,
"KL/std": 321.18719482421875,
"epoch": 0.8420256991685563,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 173.12440490722656,
"fcm_dpo/q_t": 0.4999832510948181,
"grad_norm": 0.003363779978826642,
"learning_rate": 3.756864251262143e-08,
"logits/chosen": 1.310492992401123,
"logits/rejected": 1.288170576095581,
"logps/chosen": -282.06524658203125,
"logps/ref_chosen": -55.03598403930664,
"logps/ref_rejected": -75.80644989013672,
"logps/rejected": -475.96014404296875,
"loss": 1.3862,
"margin_dpo/margin_mean": 173.12442016601562,
"margin_dpo/margin_std": 381.8446350097656,
"step": 557
},
{
"KL/chosen_KL_mean": -248.1762237548828,
"KL/mean": -312.669189453125,
"KL/rejected_KL_mean": -377.16217041015625,
"KL/std": 295.711669921875,
"epoch": 0.8435374149659864,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 128.98593139648438,
"fcm_dpo/q_t": 0.49998751282691956,
"grad_norm": 0.0022693185601383448,
"learning_rate": 3.687450924416341e-08,
"logits/chosen": 1.2714734077453613,
"logits/rejected": 1.2482508420944214,
"logps/chosen": -311.402587890625,
"logps/ref_chosen": -63.226348876953125,
"logps/ref_rejected": -91.46881866455078,
"logps/rejected": -468.6309814453125,
"loss": 1.3862,
"margin_dpo/margin_mean": 128.98593139648438,
"margin_dpo/margin_std": 365.5770263671875,
"step": 558
},
{
"KL/chosen_KL_mean": -236.52352905273438,
"KL/mean": -312.6187744140625,
"KL/rejected_KL_mean": -388.71405029296875,
"KL/std": 306.2005615234375,
"epoch": 0.8450491307634165,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 152.19052124023438,
"fcm_dpo/q_t": 0.4999852776527405,
"grad_norm": 0.0024620580952614546,
"learning_rate": 3.6186337553827743e-08,
"logits/chosen": 1.2394496202468872,
"logits/rejected": 1.2185451984405518,
"logps/chosen": -298.045166015625,
"logps/ref_chosen": -61.521644592285156,
"logps/ref_rejected": -82.83859252929688,
"logps/rejected": -471.5526428222656,
"loss": 1.3862,
"margin_dpo/margin_mean": 152.1905059814453,
"margin_dpo/margin_std": 371.47833251953125,
"step": 559
},
{
"KL/chosen_KL_mean": -240.15634155273438,
"KL/mean": -291.6183776855469,
"KL/rejected_KL_mean": -343.08038330078125,
"KL/std": 260.51849365234375,
"epoch": 0.8465608465608465,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.924072265625,
"fcm_dpo/q_t": 0.4999900460243225,
"grad_norm": 0.002105724299326539,
"learning_rate": 3.550414669125573e-08,
"logits/chosen": 1.3063299655914307,
"logits/rejected": 1.2882559299468994,
"logps/chosen": -300.79754638671875,
"logps/ref_chosen": -60.64122009277344,
"logps/ref_rejected": -78.75474548339844,
"logps/rejected": -421.83514404296875,
"loss": 1.3863,
"margin_dpo/margin_mean": 102.924072265625,
"margin_dpo/margin_std": 279.7295227050781,
"step": 560
},
{
"KL/chosen_KL_mean": -231.18161010742188,
"KL/mean": -277.0013427734375,
"KL/rejected_KL_mean": -322.821044921875,
"KL/std": 236.68751525878906,
"epoch": 0.8480725623582767,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 91.63945770263672,
"fcm_dpo/q_t": 0.49999114871025085,
"grad_norm": 0.0019574423786252737,
"learning_rate": 3.482795573879241e-08,
"logits/chosen": 1.2543569803237915,
"logits/rejected": 1.2415952682495117,
"logps/chosen": -293.68023681640625,
"logps/ref_chosen": -62.49859619140625,
"logps/ref_rejected": -78.72064208984375,
"logps/rejected": -401.54168701171875,
"loss": 1.3863,
"margin_dpo/margin_mean": 91.63945770263672,
"margin_dpo/margin_std": 286.7827453613281,
"step": 561
},
{
"KL/chosen_KL_mean": -279.95501708984375,
"KL/mean": -352.92193603515625,
"KL/rejected_KL_mean": -425.88885498046875,
"KL/std": 363.0885009765625,
"epoch": 0.8495842781557067,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 145.93382263183594,
"fcm_dpo/q_t": 0.49998587369918823,
"grad_norm": 0.002555001061409712,
"learning_rate": 3.415778361095226e-08,
"logits/chosen": 1.256950855255127,
"logits/rejected": 1.2576011419296265,
"logps/chosen": -354.73675537109375,
"logps/ref_chosen": -74.78173828125,
"logps/ref_rejected": -92.63499450683594,
"logps/rejected": -518.5238037109375,
"loss": 1.3862,
"margin_dpo/margin_mean": 145.93382263183594,
"margin_dpo/margin_std": 366.14715576171875,
"step": 562
},
{
"KL/chosen_KL_mean": -186.45376586914062,
"KL/mean": -236.38587951660156,
"KL/rejected_KL_mean": -286.3179931640625,
"KL/std": 214.30816650390625,
"epoch": 0.8510959939531368,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.86424255371094,
"fcm_dpo/q_t": 0.4999903440475464,
"grad_norm": 0.0017422273522242904,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": 1.329614281654358,
"logits/rejected": 1.3034515380859375,
"logps/chosen": -236.6522674560547,
"logps/ref_chosen": -50.19850158691406,
"logps/ref_rejected": -66.76687622070312,
"logps/rejected": -353.0848693847656,
"loss": 1.3863,
"margin_dpo/margin_mean": 99.86424255371094,
"margin_dpo/margin_std": 297.21343994140625,
"step": 563
},
{
"KL/chosen_KL_mean": -198.27935791015625,
"KL/mean": -253.13412475585938,
"KL/rejected_KL_mean": -307.9888610839844,
"KL/std": 239.6055145263672,
"epoch": 0.8526077097505669,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 109.70948028564453,
"fcm_dpo/q_t": 0.49998939037323,
"grad_norm": 0.002160602016374469,
"learning_rate": 3.283557064487785e-08,
"logits/chosen": 1.2637401819229126,
"logits/rejected": 1.250982642173767,
"logps/chosen": -254.0202178955078,
"logps/ref_chosen": -55.7408447265625,
"logps/ref_rejected": -74.82323455810547,
"logps/rejected": -382.81207275390625,
"loss": 1.3863,
"margin_dpo/margin_mean": 109.70947265625,
"margin_dpo/margin_std": 332.83740234375,
"step": 564
},
{
"KL/chosen_KL_mean": -264.5958557128906,
"KL/mean": -321.555908203125,
"KL/rejected_KL_mean": -378.5159912109375,
"KL/std": 297.69415283203125,
"epoch": 0.854119425547997,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 113.92015075683594,
"fcm_dpo/q_t": 0.49998897314071655,
"grad_norm": 0.002675557043403387,
"learning_rate": 3.218356679178252e-08,
"logits/chosen": 1.3032476902008057,
"logits/rejected": 1.2836694717407227,
"logps/chosen": -322.9332275390625,
"logps/ref_chosen": -58.33738327026367,
"logps/ref_rejected": -78.31776428222656,
"logps/rejected": -456.833740234375,
"loss": 1.3863,
"margin_dpo/margin_mean": 113.92015075683594,
"margin_dpo/margin_std": 335.97607421875,
"step": 565
},
{
"KL/chosen_KL_mean": -251.61575317382812,
"KL/mean": -298.7529296875,
"KL/rejected_KL_mean": -345.89007568359375,
"KL/std": 310.39801025390625,
"epoch": 0.8556311413454271,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 94.27432250976562,
"fcm_dpo/q_t": 0.49999088048934937,
"grad_norm": 0.0015177810564637184,
"learning_rate": 3.1537655732553764e-08,
"logits/chosen": 1.2642093896865845,
"logits/rejected": 1.2712171077728271,
"logps/chosen": -322.8394775390625,
"logps/ref_chosen": -71.22373962402344,
"logps/ref_rejected": -71.11601257324219,
"logps/rejected": -417.006103515625,
"loss": 1.3863,
"margin_dpo/margin_mean": 94.27432250976562,
"margin_dpo/margin_std": 391.6617431640625,
"step": 566
},
{
"KL/chosen_KL_mean": -201.491943359375,
"KL/mean": -286.70196533203125,
"KL/rejected_KL_mean": -371.9119873046875,
"KL/std": 300.2903747558594,
"epoch": 0.8571428571428571,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 170.42001342773438,
"fcm_dpo/q_t": 0.4999834895133972,
"grad_norm": 0.002934924094006419,
"learning_rate": 3.089785553471233e-08,
"logits/chosen": 1.2649750709533691,
"logits/rejected": 1.2357361316680908,
"logps/chosen": -254.1612091064453,
"logps/ref_chosen": -52.669273376464844,
"logps/ref_rejected": -74.34785461425781,
"logps/rejected": -446.25982666015625,
"loss": 1.3862,
"margin_dpo/margin_mean": 170.42002868652344,
"margin_dpo/margin_std": 392.0222473144531,
"step": 567
},
{
"KL/chosen_KL_mean": -201.22650146484375,
"KL/mean": -296.65826416015625,
"KL/rejected_KL_mean": -392.0899963378906,
"KL/std": 302.1213073730469,
"epoch": 0.8586545729402872,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 190.86349487304688,
"fcm_dpo/q_t": 0.49998152256011963,
"grad_norm": 0.003850526176393032,
"learning_rate": 3.026418409484513e-08,
"logits/chosen": 1.2919840812683105,
"logits/rejected": 1.2600350379943848,
"logps/chosen": -253.4044952392578,
"logps/ref_chosen": -52.178001403808594,
"logps/ref_rejected": -85.8277587890625,
"logps/rejected": -477.9177551269531,
"loss": 1.3862,
"margin_dpo/margin_mean": 190.86349487304688,
"margin_dpo/margin_std": 386.70355224609375,
"step": 568
},
{
"KL/chosen_KL_mean": -245.7794189453125,
"KL/mean": -283.582275390625,
"KL/rejected_KL_mean": -321.3851318359375,
"KL/std": 323.57220458984375,
"epoch": 0.8601662887377173,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 75.60567474365234,
"fcm_dpo/q_t": 0.4999926686286926,
"grad_norm": 0.0020971940830349922,
"learning_rate": 2.963665913810451e-08,
"logits/chosen": 1.2494065761566162,
"logits/rejected": 1.2419638633728027,
"logps/chosen": -308.4286804199219,
"logps/ref_chosen": -62.649261474609375,
"logps/ref_rejected": -75.4298324584961,
"logps/rejected": -396.81494140625,
"loss": 1.3863,
"margin_dpo/margin_mean": 75.60567474365234,
"margin_dpo/margin_std": 389.1278381347656,
"step": 569
},
{
"KL/chosen_KL_mean": -213.00286865234375,
"KL/mean": -280.4457702636719,
"KL/rejected_KL_mean": -347.888671875,
"KL/std": 267.0771789550781,
"epoch": 0.8616780045351474,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 134.8858184814453,
"fcm_dpo/q_t": 0.4999869465827942,
"grad_norm": 0.0022408810909837484,
"learning_rate": 2.9015298217712453e-08,
"logits/chosen": 1.2825806140899658,
"logits/rejected": 1.2437119483947754,
"logps/chosen": -263.044677734375,
"logps/ref_chosen": -50.04179382324219,
"logps/ref_rejected": -78.27146911621094,
"logps/rejected": -426.16015625,
"loss": 1.3862,
"margin_dpo/margin_mean": 134.88583374023438,
"margin_dpo/margin_std": 310.875,
"step": 570
},
{
"KL/chosen_KL_mean": -230.393310546875,
"KL/mean": -274.6932678222656,
"KL/rejected_KL_mean": -318.99322509765625,
"KL/std": 289.2368469238281,
"epoch": 0.8631897203325775,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 88.59991455078125,
"fcm_dpo/q_t": 0.49999141693115234,
"grad_norm": 0.0022626707796007395,
"learning_rate": 2.840011871446962e-08,
"logits/chosen": 1.280132532119751,
"logits/rejected": 1.2787832021713257,
"logps/chosen": -284.05010986328125,
"logps/ref_chosen": -53.65681457519531,
"logps/ref_rejected": -66.13298034667969,
"logps/rejected": -385.126220703125,
"loss": 1.3863,
"margin_dpo/margin_mean": 88.59992980957031,
"margin_dpo/margin_std": 364.80145263671875,
"step": 571
},
{
"KL/chosen_KL_mean": -287.53131103515625,
"KL/mean": -291.213623046875,
"KL/rejected_KL_mean": -294.89599609375,
"KL/std": 277.07366943359375,
"epoch": 0.8647014361300076,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.364675521850586,
"fcm_dpo/q_t": 0.4999992847442627,
"grad_norm": 0.001692647230811417,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": 1.258278250694275,
"logits/rejected": 1.2805957794189453,
"logps/chosen": -362.3492431640625,
"logps/ref_chosen": -74.81792449951172,
"logps/ref_rejected": -65.88681030273438,
"logps/rejected": -360.7828063964844,
"loss": 1.3863,
"margin_dpo/margin_mean": 7.364673614501953,
"margin_dpo/margin_std": 344.2757568359375,
"step": 572
},
{
"KL/chosen_KL_mean": -281.252685546875,
"KL/mean": -336.67999267578125,
"KL/rejected_KL_mean": -392.10736083984375,
"KL/std": 340.51861572265625,
"epoch": 0.8662131519274376,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 110.85468292236328,
"fcm_dpo/q_t": 0.4999893009662628,
"grad_norm": 0.002122138626873493,
"learning_rate": 2.718837261761528e-08,
"logits/chosen": 1.2949762344360352,
"logits/rejected": 1.2791988849639893,
"logps/chosen": -349.97833251953125,
"logps/ref_chosen": -68.72564697265625,
"logps/ref_rejected": -88.16201782226562,
"logps/rejected": -480.26934814453125,
"loss": 1.3863,
"margin_dpo/margin_mean": 110.85468292236328,
"margin_dpo/margin_std": 395.3820495605469,
"step": 573
},
{
"KL/chosen_KL_mean": -224.64120483398438,
"KL/mean": -303.6715087890625,
"KL/rejected_KL_mean": -382.7017822265625,
"KL/std": 301.89447021484375,
"epoch": 0.8677248677248677,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 158.06060791015625,
"fcm_dpo/q_t": 0.4999846816062927,
"grad_norm": 0.0023631516378372908,
"learning_rate": 2.659183991914696e-08,
"logits/chosen": 1.285936713218689,
"logits/rejected": 1.2656428813934326,
"logps/chosen": -280.95458984375,
"logps/ref_chosen": -56.31340026855469,
"logps/ref_rejected": -83.91553497314453,
"logps/rejected": -466.6173095703125,
"loss": 1.3862,
"margin_dpo/margin_mean": 158.0605926513672,
"margin_dpo/margin_std": 359.18701171875,
"step": 574
},
{
"KL/chosen_KL_mean": -240.16212463378906,
"KL/mean": -306.2142333984375,
"KL/rejected_KL_mean": -372.2663269042969,
"KL/std": 327.98846435546875,
"epoch": 0.8692365835222978,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 132.1042022705078,
"fcm_dpo/q_t": 0.4999872148036957,
"grad_norm": 0.00271924352273345,
"learning_rate": 2.600155642716606e-08,
"logits/chosen": 1.2739206552505493,
"logits/rejected": 1.2386579513549805,
"logps/chosen": -304.74627685546875,
"logps/ref_chosen": -64.5841293334961,
"logps/ref_rejected": -93.47034454345703,
"logps/rejected": -465.7366638183594,
"loss": 1.3862,
"margin_dpo/margin_mean": 132.1042022705078,
"margin_dpo/margin_std": 406.8048095703125,
"step": 575
},
{
"KL/chosen_KL_mean": -185.20242309570312,
"KL/mean": -287.28302001953125,
"KL/rejected_KL_mean": -389.36358642578125,
"KL/std": 299.2473449707031,
"epoch": 0.8707482993197279,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 204.16119384765625,
"fcm_dpo/q_t": 0.49998024106025696,
"grad_norm": 0.0033409777097404003,
"learning_rate": 2.5417538653170754e-08,
"logits/chosen": 1.2817810773849487,
"logits/rejected": 1.2451581954956055,
"logps/chosen": -238.48294067382812,
"logps/ref_chosen": -53.28052520751953,
"logps/ref_rejected": -84.2000503540039,
"logps/rejected": -473.56365966796875,
"loss": 1.3862,
"margin_dpo/margin_mean": 204.16119384765625,
"margin_dpo/margin_std": 395.5772705078125,
"step": 576
},
{
"KL/chosen_KL_mean": -230.99880981445312,
"KL/mean": -275.36846923828125,
"KL/rejected_KL_mean": -319.7381591796875,
"KL/std": 301.64703369140625,
"epoch": 0.872260015117158,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 88.7393569946289,
"fcm_dpo/q_t": 0.49999138712882996,
"grad_norm": 0.002223484218120575,
"learning_rate": 2.4839802933393607e-08,
"logits/chosen": 1.2827671766281128,
"logits/rejected": 1.2928658723831177,
"logps/chosen": -293.323486328125,
"logps/ref_chosen": -62.32468795776367,
"logps/ref_rejected": -67.300537109375,
"logps/rejected": -387.0386962890625,
"loss": 1.3863,
"margin_dpo/margin_mean": 88.73936462402344,
"margin_dpo/margin_std": 334.8846435546875,
"step": 577
},
{
"KL/chosen_KL_mean": -238.35610961914062,
"KL/mean": -270.2100830078125,
"KL/rejected_KL_mean": -302.0640869140625,
"KL/std": 326.62493896484375,
"epoch": 0.873771730914588,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 63.7080078125,
"fcm_dpo/q_t": 0.49999380111694336,
"grad_norm": 0.0015426744939759374,
"learning_rate": 2.4268365428344733e-08,
"logits/chosen": 1.3032362461090088,
"logits/rejected": 1.3008297681808472,
"logps/chosen": -295.0116882324219,
"logps/ref_chosen": -56.65557861328125,
"logps/ref_rejected": -68.21835327148438,
"logps/rejected": -370.282470703125,
"loss": 1.3863,
"margin_dpo/margin_mean": 63.7080078125,
"margin_dpo/margin_std": 349.22344970703125,
"step": 578
},
{
"KL/chosen_KL_mean": -219.2360076904297,
"KL/mean": -270.0174255371094,
"KL/rejected_KL_mean": -320.7988586425781,
"KL/std": 243.76840209960938,
"epoch": 0.8752834467120182,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 101.56283569335938,
"fcm_dpo/q_t": 0.49999016523361206,
"grad_norm": 0.001675861538387835,
"learning_rate": 2.3703242122359357e-08,
"logits/chosen": 1.258755087852478,
"logits/rejected": 1.2550430297851562,
"logps/chosen": -276.0456848144531,
"logps/ref_chosen": -56.809661865234375,
"logps/ref_rejected": -68.09613037109375,
"logps/rejected": -388.8949890136719,
"loss": 1.3863,
"margin_dpo/margin_mean": 101.56283569335938,
"margin_dpo/margin_std": 307.77728271484375,
"step": 579
},
{
"KL/chosen_KL_mean": -229.52764892578125,
"KL/mean": -310.90203857421875,
"KL/rejected_KL_mean": -392.2764587402344,
"KL/std": 354.71600341796875,
"epoch": 0.8767951625094482,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 162.74879455566406,
"fcm_dpo/q_t": 0.4999842643737793,
"grad_norm": 0.0027315111365169287,
"learning_rate": 2.3144448823151392e-08,
"logits/chosen": 1.252476453781128,
"logits/rejected": 1.2458863258361816,
"logps/chosen": -287.227783203125,
"logps/ref_chosen": -57.70011520385742,
"logps/ref_rejected": -77.90664672851562,
"logps/rejected": -470.18310546875,
"loss": 1.3862,
"margin_dpo/margin_mean": 162.74879455566406,
"margin_dpo/margin_std": 423.9322204589844,
"step": 580
},
{
"KL/chosen_KL_mean": -243.62631225585938,
"KL/mean": -305.4908447265625,
"KL/rejected_KL_mean": -367.3553466796875,
"KL/std": 283.1879577636719,
"epoch": 0.8783068783068783,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 123.72905731201172,
"fcm_dpo/q_t": 0.49998801946640015,
"grad_norm": 0.002351470058783889,
"learning_rate": 2.259200116137039e-08,
"logits/chosen": 1.2822654247283936,
"logits/rejected": 1.2544565200805664,
"logps/chosen": -302.95867919921875,
"logps/ref_chosen": -59.332359313964844,
"logps/ref_rejected": -83.64482116699219,
"logps/rejected": -451.00018310546875,
"loss": 1.3862,
"margin_dpo/margin_mean": 123.72904968261719,
"margin_dpo/margin_std": 322.47772216796875,
"step": 581
},
{
"KL/chosen_KL_mean": -234.3776092529297,
"KL/mean": -258.0681457519531,
"KL/rejected_KL_mean": -281.7586669921875,
"KL/std": 228.97186279296875,
"epoch": 0.8798185941043084,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 47.381065368652344,
"fcm_dpo/q_t": 0.4999954104423523,
"grad_norm": 0.0013384189223870635,
"learning_rate": 2.204591459016525e-08,
"logits/chosen": 1.285982370376587,
"logits/rejected": 1.3120609521865845,
"logps/chosen": -298.54046630859375,
"logps/ref_chosen": -64.16285705566406,
"logps/ref_rejected": -58.632896423339844,
"logps/rejected": -340.3916015625,
"loss": 1.3863,
"margin_dpo/margin_mean": 47.38106918334961,
"margin_dpo/margin_std": 315.0589904785156,
"step": 582
},
{
"KL/chosen_KL_mean": -187.56515502929688,
"KL/mean": -304.24945068359375,
"KL/rejected_KL_mean": -420.93377685546875,
"KL/std": 375.9952087402344,
"epoch": 0.8813303099017384,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 233.36862182617188,
"fcm_dpo/q_t": 0.4999774098396301,
"grad_norm": 0.0039016844239085913,
"learning_rate": 2.1506204384751064e-08,
"logits/chosen": 1.322705626487732,
"logits/rejected": 1.2900676727294922,
"logps/chosen": -239.4375457763672,
"logps/ref_chosen": -51.87239456176758,
"logps/ref_rejected": -83.86331176757812,
"logps/rejected": -504.7970886230469,
"loss": 1.3862,
"margin_dpo/margin_mean": 233.36859130859375,
"margin_dpo/margin_std": 436.15399169921875,
"step": 583
},
{
"KL/chosen_KL_mean": -171.99374389648438,
"KL/mean": -252.2381591796875,
"KL/rejected_KL_mean": -332.48260498046875,
"KL/std": 261.9239807128906,
"epoch": 0.8828420256991686,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 160.48883056640625,
"fcm_dpo/q_t": 0.499984472990036,
"grad_norm": 0.003471849486231804,
"learning_rate": 2.09728856419826e-08,
"logits/chosen": 1.3081614971160889,
"logits/rejected": 1.2692360877990723,
"logps/chosen": -218.5651397705078,
"logps/ref_chosen": -46.571388244628906,
"logps/ref_rejected": -80.67969512939453,
"logps/rejected": -413.16229248046875,
"loss": 1.3862,
"margin_dpo/margin_mean": 160.48883056640625,
"margin_dpo/margin_std": 291.8946228027344,
"step": 584
},
{
"KL/chosen_KL_mean": -242.6631622314453,
"KL/mean": -276.41943359375,
"KL/rejected_KL_mean": -310.1756896972656,
"KL/std": 275.22735595703125,
"epoch": 0.8843537414965986,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 67.51254272460938,
"fcm_dpo/q_t": 0.4999934732913971,
"grad_norm": 0.001750921132043004,
"learning_rate": 2.044597327993153e-08,
"logits/chosen": 1.2745296955108643,
"logits/rejected": 1.2461663484573364,
"logps/chosen": -300.7877197265625,
"logps/ref_chosen": -58.124534606933594,
"logps/ref_rejected": -79.00538635253906,
"logps/rejected": -389.18109130859375,
"loss": 1.3863,
"margin_dpo/margin_mean": 67.51254272460938,
"margin_dpo/margin_std": 317.5940246582031,
"step": 585
},
{
"KL/chosen_KL_mean": -195.4407958984375,
"KL/mean": -253.28018188476562,
"KL/rejected_KL_mean": -311.11956787109375,
"KL/std": 251.697021484375,
"epoch": 0.8858654572940288,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 115.67878723144531,
"fcm_dpo/q_t": 0.4999887943267822,
"grad_norm": 0.0022813198156654835,
"learning_rate": 1.9925482037469187e-08,
"logits/chosen": 1.2979401350021362,
"logits/rejected": 1.2939859628677368,
"logps/chosen": -249.54241943359375,
"logps/ref_chosen": -54.10163879394531,
"logps/ref_rejected": -63.72113037109375,
"logps/rejected": -374.8406982421875,
"loss": 1.3862,
"margin_dpo/margin_mean": 115.67877960205078,
"margin_dpo/margin_std": 308.5834655761719,
"step": 586
},
{
"KL/chosen_KL_mean": -257.64990234375,
"KL/mean": -289.8480224609375,
"KL/rejected_KL_mean": -322.046142578125,
"KL/std": 334.7420959472656,
"epoch": 0.8873771730914588,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 64.39622497558594,
"fcm_dpo/q_t": 0.4999937415122986,
"grad_norm": 0.0018675602041184902,
"learning_rate": 1.9411426473854687e-08,
"logits/chosen": 1.2932814359664917,
"logits/rejected": 1.2983059883117676,
"logps/chosen": -321.0671081542969,
"logps/ref_chosen": -63.41719436645508,
"logps/ref_rejected": -63.47003936767578,
"logps/rejected": -385.5162048339844,
"loss": 1.3863,
"margin_dpo/margin_mean": 64.39622497558594,
"margin_dpo/margin_std": 453.559814453125,
"step": 587
},
{
"KL/chosen_KL_mean": -256.47723388671875,
"KL/mean": -315.72064208984375,
"KL/rejected_KL_mean": -374.96405029296875,
"KL/std": 312.8167724609375,
"epoch": 0.8888888888888888,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 118.48678588867188,
"fcm_dpo/q_t": 0.4999885559082031,
"grad_norm": 0.0022324968595057726,
"learning_rate": 1.890382096832699e-08,
"logits/chosen": 1.307366132736206,
"logits/rejected": 1.2943463325500488,
"logps/chosen": -318.67828369140625,
"logps/ref_chosen": -62.20103454589844,
"logps/ref_rejected": -82.10249328613281,
"logps/rejected": -457.0665283203125,
"loss": 1.3862,
"margin_dpo/margin_mean": 118.48678588867188,
"margin_dpo/margin_std": 407.2249755859375,
"step": 588
},
{
"KL/chosen_KL_mean": -240.05502319335938,
"KL/mean": -276.25872802734375,
"KL/rejected_KL_mean": -312.46246337890625,
"KL/std": 287.0700378417969,
"epoch": 0.890400604686319,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 72.40746307373047,
"fcm_dpo/q_t": 0.4999929666519165,
"grad_norm": 0.001997460378333926,
"learning_rate": 1.840267971970344e-08,
"logits/chosen": 1.2659380435943604,
"logits/rejected": 1.2496306896209717,
"logps/chosen": -296.76861572265625,
"logps/ref_chosen": -56.71361541748047,
"logps/ref_rejected": -76.7366943359375,
"logps/rejected": -389.19915771484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 72.40746307373047,
"margin_dpo/margin_std": 305.57440185546875,
"step": 589
},
{
"KL/chosen_KL_mean": -246.395263671875,
"KL/mean": -291.65802001953125,
"KL/rejected_KL_mean": -336.9208068847656,
"KL/std": 262.71783447265625,
"epoch": 0.891912320483749,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 90.52552032470703,
"fcm_dpo/q_t": 0.499991238117218,
"grad_norm": 0.002160015283152461,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": 1.2421910762786865,
"logits/rejected": 1.2292957305908203,
"logps/chosen": -312.9090881347656,
"logps/ref_chosen": -66.5138168334961,
"logps/ref_rejected": -85.70820617675781,
"logps/rejected": -422.6290283203125,
"loss": 1.3863,
"margin_dpo/margin_mean": 90.52552795410156,
"margin_dpo/margin_std": 272.5157470703125,
"step": 590
},
{
"KL/chosen_KL_mean": -197.66827392578125,
"KL/mean": -279.355224609375,
"KL/rejected_KL_mean": -361.042236328125,
"KL/std": 297.4459228515625,
"epoch": 0.8934240362811792,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 163.37396240234375,
"fcm_dpo/q_t": 0.4999842047691345,
"grad_norm": 0.0024286925327032804,
"learning_rate": 1.7419845883949098e-08,
"logits/chosen": 1.2815182209014893,
"logits/rejected": 1.266688585281372,
"logps/chosen": -258.3654479980469,
"logps/ref_chosen": -60.697181701660156,
"logps/ref_rejected": -86.12278747558594,
"logps/rejected": -447.1650390625,
"loss": 1.3862,
"margin_dpo/margin_mean": 163.37396240234375,
"margin_dpo/margin_std": 392.95819091796875,
"step": 591
},
{
"KL/chosen_KL_mean": -201.080322265625,
"KL/mean": -273.27728271484375,
"KL/rejected_KL_mean": -345.4742736816406,
"KL/std": 250.97198486328125,
"epoch": 0.8949357520786092,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 144.39395141601562,
"fcm_dpo/q_t": 0.4999859929084778,
"grad_norm": 0.0026749190874397755,
"learning_rate": 1.6938180788793556e-08,
"logits/chosen": 1.2822060585021973,
"logits/rejected": 1.2332913875579834,
"logps/chosen": -252.31764221191406,
"logps/ref_chosen": -51.237327575683594,
"logps/ref_rejected": -81.60242462158203,
"logps/rejected": -427.0766906738281,
"loss": 1.3862,
"margin_dpo/margin_mean": 144.39395141601562,
"margin_dpo/margin_std": 278.5047607421875,
"step": 592
},
{
"KL/chosen_KL_mean": -165.1071319580078,
"KL/mean": -229.7209014892578,
"KL/rejected_KL_mean": -294.33465576171875,
"KL/std": 204.59747314453125,
"epoch": 0.8964474678760394,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 129.22750854492188,
"fcm_dpo/q_t": 0.49998748302459717,
"grad_norm": 0.002245598705485463,
"learning_rate": 1.6463034933723336e-08,
"logits/chosen": 1.2915213108062744,
"logits/rejected": 1.2537872791290283,
"logps/chosen": -207.1871337890625,
"logps/ref_chosen": -42.08000183105469,
"logps/ref_rejected": -68.47499084472656,
"logps/rejected": -362.80963134765625,
"loss": 1.3862,
"margin_dpo/margin_mean": 129.22750854492188,
"margin_dpo/margin_std": 228.52536010742188,
"step": 593
},
{
"KL/chosen_KL_mean": -243.26272583007812,
"KL/mean": -281.5732421875,
"KL/rejected_KL_mean": -319.88372802734375,
"KL/std": 282.73321533203125,
"epoch": 0.8979591836734694,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 76.62098693847656,
"fcm_dpo/q_t": 0.49999260902404785,
"grad_norm": 0.0013759155990555882,
"learning_rate": 1.5994421609589385e-08,
"logits/chosen": 1.2534310817718506,
"logits/rejected": 1.2528616189956665,
"logps/chosen": -306.92138671875,
"logps/ref_chosen": -63.658668518066406,
"logps/ref_rejected": -70.35597229003906,
"logps/rejected": -390.23968505859375,
"loss": 1.3863,
"margin_dpo/margin_mean": 76.62098693847656,
"margin_dpo/margin_std": 327.9439697265625,
"step": 594
},
{
"KL/chosen_KL_mean": -198.01931762695312,
"KL/mean": -272.7323913574219,
"KL/rejected_KL_mean": -347.44549560546875,
"KL/std": 273.14923095703125,
"epoch": 0.8994708994708994,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 149.4261474609375,
"fcm_dpo/q_t": 0.4999855160713196,
"grad_norm": 0.0027599758468568325,
"learning_rate": 1.553235392451377e-08,
"logits/chosen": 1.3034560680389404,
"logits/rejected": 1.2654231786727905,
"logps/chosen": -254.2380828857422,
"logps/ref_chosen": -56.21875762939453,
"logps/ref_rejected": -83.95773315429688,
"logps/rejected": -431.4031982421875,
"loss": 1.3862,
"margin_dpo/margin_mean": 149.4261474609375,
"margin_dpo/margin_std": 291.680908203125,
"step": 595
},
{
"KL/chosen_KL_mean": -268.5811767578125,
"KL/mean": -279.02777099609375,
"KL/rejected_KL_mean": -289.474365234375,
"KL/std": 269.3888244628906,
"epoch": 0.9009826152683296,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 20.893169403076172,
"fcm_dpo/q_t": 0.49999797344207764,
"grad_norm": 0.001436872873455286,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": 1.215804100036621,
"logits/rejected": 1.2338910102844238,
"logps/chosen": -337.06207275390625,
"logps/ref_chosen": -68.48088073730469,
"logps/ref_rejected": -61.732967376708984,
"logps/rejected": -351.20733642578125,
"loss": 1.3863,
"margin_dpo/margin_mean": 20.893165588378906,
"margin_dpo/margin_std": 239.36436462402344,
"step": 596
},
{
"KL/chosen_KL_mean": -190.21514892578125,
"KL/mean": -211.99703979492188,
"KL/rejected_KL_mean": -233.7789306640625,
"KL/std": 190.4124755859375,
"epoch": 0.9024943310657596,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 43.563785552978516,
"fcm_dpo/q_t": 0.49999576807022095,
"grad_norm": 0.0016540532233193517,
"learning_rate": 1.4627906988186111e-08,
"logits/chosen": 1.276237964630127,
"logits/rejected": 1.267054557800293,
"logps/chosen": -239.07266235351562,
"logps/ref_chosen": -48.85750961303711,
"logps/ref_rejected": -55.068084716796875,
"logps/rejected": -288.8470153808594,
"loss": 1.3863,
"margin_dpo/margin_mean": 43.56378173828125,
"margin_dpo/margin_std": 183.7689208984375,
"step": 597
},
{
"KL/chosen_KL_mean": -242.9158935546875,
"KL/mean": -291.18408203125,
"KL/rejected_KL_mean": -339.4522705078125,
"KL/std": 272.9280090332031,
"epoch": 0.9040060468631897,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 96.53633880615234,
"fcm_dpo/q_t": 0.49999064207077026,
"grad_norm": 0.0021814981009811163,
"learning_rate": 1.4185553036259095e-08,
"logits/chosen": 1.2822654247283936,
"logits/rejected": 1.2468507289886475,
"logps/chosen": -301.80303955078125,
"logps/ref_chosen": -58.88715362548828,
"logps/ref_rejected": -81.43145751953125,
"logps/rejected": -420.88372802734375,
"loss": 1.3863,
"margin_dpo/margin_mean": 96.53633117675781,
"margin_dpo/margin_std": 309.15411376953125,
"step": 598
},
{
"KL/chosen_KL_mean": -241.9964599609375,
"KL/mean": -287.59735107421875,
"KL/rejected_KL_mean": -333.1982421875,
"KL/std": 273.47198486328125,
"epoch": 0.9055177626606198,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 91.2018051147461,
"fcm_dpo/q_t": 0.49999117851257324,
"grad_norm": 0.001978973625227809,
"learning_rate": 1.3749795321332885e-08,
"logits/chosen": 1.3052904605865479,
"logits/rejected": 1.2893433570861816,
"logps/chosen": -299.6036376953125,
"logps/ref_chosen": -57.60719299316406,
"logps/ref_rejected": -71.80469512939453,
"logps/rejected": -405.0029296875,
"loss": 1.3863,
"margin_dpo/margin_mean": 91.20179748535156,
"margin_dpo/margin_std": 326.43243408203125,
"step": 599
},
{
"KL/chosen_KL_mean": -229.87130737304688,
"KL/mean": -281.87884521484375,
"KL/rejected_KL_mean": -333.8863525390625,
"KL/std": 281.9407958984375,
"epoch": 0.9070294784580499,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.01502227783203,
"fcm_dpo/q_t": 0.49998992681503296,
"grad_norm": 0.00212049949914217,
"learning_rate": 1.3320646032487393e-08,
"logits/chosen": 1.2342650890350342,
"logits/rejected": 1.2097012996673584,
"logps/chosen": -288.3136291503906,
"logps/ref_chosen": -58.44231414794922,
"logps/ref_rejected": -83.64639282226562,
"logps/rejected": -417.53271484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 104.01502227783203,
"margin_dpo/margin_std": 371.31842041015625,
"step": 600
},
{
"KL/chosen_KL_mean": -204.21005249023438,
"KL/mean": -272.1788635253906,
"KL/rejected_KL_mean": -340.147705078125,
"KL/std": 289.89752197265625,
"epoch": 0.90854119425548,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 135.9376220703125,
"fcm_dpo/q_t": 0.499986857175827,
"grad_norm": 0.002290458418428898,
"learning_rate": 1.2898117173950868e-08,
"logits/chosen": 1.2456085681915283,
"logits/rejected": 1.209405541419983,
"logps/chosen": -259.80438232421875,
"logps/ref_chosen": -55.59432601928711,
"logps/ref_rejected": -83.68630981445312,
"logps/rejected": -423.833984375,
"loss": 1.3862,
"margin_dpo/margin_mean": 135.9376220703125,
"margin_dpo/margin_std": 325.400634765625,
"step": 601
},
{
"KL/chosen_KL_mean": -210.0914764404297,
"KL/mean": -256.0052185058594,
"KL/rejected_KL_mean": -301.9189453125,
"KL/std": 259.4412841796875,
"epoch": 0.91005291005291,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 91.82748413085938,
"fcm_dpo/q_t": 0.49999111890792847,
"grad_norm": 0.002178723691031337,
"learning_rate": 1.2482220564763667e-08,
"logits/chosen": 1.2279410362243652,
"logits/rejected": 1.2131808996200562,
"logps/chosen": -266.440673828125,
"logps/ref_chosen": -56.349185943603516,
"logps/ref_rejected": -71.9959716796875,
"logps/rejected": -373.9149169921875,
"loss": 1.3863,
"margin_dpo/margin_mean": 91.82748413085938,
"margin_dpo/margin_std": 305.39068603515625,
"step": 602
},
{
"KL/chosen_KL_mean": -200.24911499023438,
"KL/mean": -269.5469665527344,
"KL/rejected_KL_mean": -338.8448486328125,
"KL/std": 285.01885986328125,
"epoch": 0.9115646258503401,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 138.59573364257812,
"fcm_dpo/q_t": 0.49998658895492554,
"grad_norm": 0.0024140651803463697,
"learning_rate": 1.2072967838448051e-08,
"logits/chosen": 1.2549041509628296,
"logits/rejected": 1.244372844696045,
"logps/chosen": -253.41749572753906,
"logps/ref_chosen": -53.16838836669922,
"logps/ref_rejected": -73.8604736328125,
"logps/rejected": -412.705322265625,
"loss": 1.3862,
"margin_dpo/margin_mean": 138.59573364257812,
"margin_dpo/margin_std": 345.45501708984375,
"step": 603
},
{
"KL/chosen_KL_mean": -227.04763793945312,
"KL/mean": -257.6112365722656,
"KL/rejected_KL_mean": -288.1748046875,
"KL/std": 230.52130126953125,
"epoch": 0.9130763416477702,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 61.12716293334961,
"fcm_dpo/q_t": 0.49999406933784485,
"grad_norm": 0.0016172666801139712,
"learning_rate": 1.1670370442682459e-08,
"logits/chosen": 1.210902214050293,
"logits/rejected": 1.2236418724060059,
"logps/chosen": -299.69708251953125,
"logps/ref_chosen": -72.64942169189453,
"logps/ref_rejected": -69.8792724609375,
"logps/rejected": -358.0540771484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 61.127159118652344,
"margin_dpo/margin_std": 327.2840576171875,
"step": 604
},
{
"KL/chosen_KL_mean": -239.35777282714844,
"KL/mean": -294.0518798828125,
"KL/rejected_KL_mean": -348.7460632324219,
"KL/std": 237.30853271484375,
"epoch": 0.9145880574452003,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 109.38829040527344,
"fcm_dpo/q_t": 0.49998942017555237,
"grad_norm": 0.002217090455815196,
"learning_rate": 1.1274439638981532e-08,
"logits/chosen": 1.2849633693695068,
"logits/rejected": 1.2634557485580444,
"logps/chosen": -300.9706115722656,
"logps/ref_chosen": -61.61284637451172,
"logps/ref_rejected": -79.34398651123047,
"logps/rejected": -428.09002685546875,
"loss": 1.3863,
"margin_dpo/margin_mean": 109.38829040527344,
"margin_dpo/margin_std": 277.03143310546875,
"step": 605
},
{
"KL/chosen_KL_mean": -208.34193420410156,
"KL/mean": -277.22564697265625,
"KL/rejected_KL_mean": -346.10943603515625,
"KL/std": 279.93414306640625,
"epoch": 0.9160997732426304,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 137.76748657226562,
"fcm_dpo/q_t": 0.4999866485595703,
"grad_norm": 0.00246420968323946,
"learning_rate": 1.0885186502381016e-08,
"logits/chosen": 1.2400498390197754,
"logits/rejected": 1.221116542816162,
"logps/chosen": -262.80615234375,
"logps/ref_chosen": -54.46424102783203,
"logps/ref_rejected": -79.62708282470703,
"logps/rejected": -425.73651123046875,
"loss": 1.3862,
"margin_dpo/margin_mean": 137.76748657226562,
"margin_dpo/margin_std": 318.7607727050781,
"step": 606
},
{
"KL/chosen_KL_mean": -250.8245391845703,
"KL/mean": -299.26092529296875,
"KL/rejected_KL_mean": -347.69732666015625,
"KL/std": 316.63677978515625,
"epoch": 0.9176114890400605,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 96.87279510498047,
"fcm_dpo/q_t": 0.49999064207077026,
"grad_norm": 0.0016430618707090616,
"learning_rate": 1.0502621921127774e-08,
"logits/chosen": 1.2330322265625,
"logits/rejected": 1.2289059162139893,
"logps/chosen": -313.6854248046875,
"logps/ref_chosen": -62.86086654663086,
"logps/ref_rejected": -72.5501937866211,
"logps/rejected": -420.2475280761719,
"loss": 1.3863,
"margin_dpo/margin_mean": 96.87281036376953,
"margin_dpo/margin_std": 384.65234375,
"step": 607
},
{
"KL/chosen_KL_mean": -240.77908325195312,
"KL/mean": -311.6595458984375,
"KL/rejected_KL_mean": -382.5400390625,
"KL/std": 341.69573974609375,
"epoch": 0.9191232048374905,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 141.76089477539062,
"fcm_dpo/q_t": 0.49998629093170166,
"grad_norm": 0.002434444148093462,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": 1.211251974105835,
"logits/rejected": 1.174188494682312,
"logps/chosen": -303.9598083496094,
"logps/ref_chosen": -63.18071746826172,
"logps/ref_rejected": -99.15888214111328,
"logps/rejected": -481.69891357421875,
"loss": 1.3862,
"margin_dpo/margin_mean": 141.7609100341797,
"margin_dpo/margin_std": 400.3000793457031,
"step": 608
},
{
"KL/chosen_KL_mean": -180.31060791015625,
"KL/mean": -238.63388061523438,
"KL/rejected_KL_mean": -296.9571533203125,
"KL/std": 239.36196899414062,
"epoch": 0.9206349206349206,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 116.64653015136719,
"fcm_dpo/q_t": 0.49998870491981506,
"grad_norm": 0.001932095387019217,
"learning_rate": 9.757601041885694e-09,
"logits/chosen": 1.271007776260376,
"logits/rejected": 1.262525200843811,
"logps/chosen": -228.933837890625,
"logps/ref_chosen": -48.62322235107422,
"logps/ref_rejected": -68.28271484375,
"logps/rejected": -365.2398681640625,
"loss": 1.3862,
"margin_dpo/margin_mean": 116.64653015136719,
"margin_dpo/margin_std": 259.3739318847656,
"step": 609
},
{
"KL/chosen_KL_mean": -303.2109680175781,
"KL/mean": -337.4134521484375,
"KL/rejected_KL_mean": -371.615966796875,
"KL/std": 341.649658203125,
"epoch": 0.9221466364323507,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 68.40501403808594,
"fcm_dpo/q_t": 0.49999338388442993,
"grad_norm": 0.0016822778852656484,
"learning_rate": 9.395165583732379e-09,
"logits/chosen": 1.248294711112976,
"logits/rejected": 1.2520670890808105,
"logps/chosen": -375.8760986328125,
"logps/ref_chosen": -72.66513061523438,
"logps/ref_rejected": -87.15310668945312,
"logps/rejected": -458.7690734863281,
"loss": 1.3863,
"margin_dpo/margin_mean": 68.40501403808594,
"margin_dpo/margin_std": 410.22003173828125,
"step": 610
},
{
"KL/chosen_KL_mean": -202.11181640625,
"KL/mean": -256.2706604003906,
"KL/rejected_KL_mean": -310.42950439453125,
"KL/std": 272.47833251953125,
"epoch": 0.9236583522297808,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 108.31767272949219,
"fcm_dpo/q_t": 0.49998950958251953,
"grad_norm": 0.0027471468783915043,
"learning_rate": 9.03946036001449e-09,
"logits/chosen": 1.3028149604797363,
"logits/rejected": 1.2825827598571777,
"logps/chosen": -250.42039489746094,
"logps/ref_chosen": -48.30857849121094,
"logps/ref_rejected": -70.6141128540039,
"logps/rejected": -381.0436096191406,
"loss": 1.3863,
"margin_dpo/margin_mean": 108.31765747070312,
"margin_dpo/margin_std": 253.734375,
"step": 611
},
{
"KL/chosen_KL_mean": -241.52801513671875,
"KL/mean": -312.7408447265625,
"KL/rejected_KL_mean": -383.95361328125,
"KL/std": 254.8455810546875,
"epoch": 0.9251700680272109,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 142.42562866210938,
"fcm_dpo/q_t": 0.4999862313270569,
"grad_norm": 0.0026474539190530777,
"learning_rate": 8.690495320571839e-09,
"logits/chosen": 1.2259531021118164,
"logits/rejected": 1.1953879594802856,
"logps/chosen": -302.75958251953125,
"logps/ref_chosen": -61.23155975341797,
"logps/ref_rejected": -94.37979888916016,
"logps/rejected": -478.33343505859375,
"loss": 1.3862,
"margin_dpo/margin_mean": 142.42559814453125,
"margin_dpo/margin_std": 282.14910888671875,
"step": 612
},
{
"KL/chosen_KL_mean": -215.14187622070312,
"KL/mean": -248.74453735351562,
"KL/rejected_KL_mean": -282.34716796875,
"KL/std": 317.31195068359375,
"epoch": 0.926681783824641,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 67.20530700683594,
"fcm_dpo/q_t": 0.4999935030937195,
"grad_norm": 0.0015928486827760935,
"learning_rate": 8.348280226706722e-09,
"logits/chosen": 1.2564080953598022,
"logits/rejected": 1.26479971408844,
"logps/chosen": -269.125,
"logps/ref_chosen": -53.98310852050781,
"logps/ref_rejected": -58.32208251953125,
"logps/rejected": -340.66925048828125,
"loss": 1.3863,
"margin_dpo/margin_mean": 67.20530700683594,
"margin_dpo/margin_std": 282.4429626464844,
"step": 613
},
{
"KL/chosen_KL_mean": -235.35379028320312,
"KL/mean": -268.393798828125,
"KL/rejected_KL_mean": -301.4338073730469,
"KL/std": 251.66497802734375,
"epoch": 0.9281934996220711,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 66.08002471923828,
"fcm_dpo/q_t": 0.49999362230300903,
"grad_norm": 0.0015550341922789812,
"learning_rate": 8.012824650910937e-09,
"logits/chosen": 1.275475025177002,
"logits/rejected": 1.2758376598358154,
"logps/chosen": -295.5968322753906,
"logps/ref_chosen": -60.24303436279297,
"logps/ref_rejected": -72.26258850097656,
"logps/rejected": -373.6964111328125,
"loss": 1.3863,
"margin_dpo/margin_mean": 66.08003234863281,
"margin_dpo/margin_std": 330.7596435546875,
"step": 614
},
{
"KL/chosen_KL_mean": -288.22857666015625,
"KL/mean": -350.49859619140625,
"KL/rejected_KL_mean": -412.76861572265625,
"KL/std": 364.74468994140625,
"epoch": 0.9297052154195011,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 124.53999328613281,
"fcm_dpo/q_t": 0.49998795986175537,
"grad_norm": 0.0024001309648156166,
"learning_rate": 7.684137976598088e-09,
"logits/chosen": 1.2872897386550903,
"logits/rejected": 1.2527488470077515,
"logps/chosen": -360.3232727050781,
"logps/ref_chosen": -72.09467315673828,
"logps/ref_rejected": -104.02980041503906,
"logps/rejected": -516.7984008789062,
"loss": 1.3862,
"margin_dpo/margin_mean": 124.53999328613281,
"margin_dpo/margin_std": 407.09320068359375,
"step": 615
},
{
"KL/chosen_KL_mean": -214.70602416992188,
"KL/mean": -280.04010009765625,
"KL/rejected_KL_mean": -345.3741455078125,
"KL/std": 338.1444091796875,
"epoch": 0.9312169312169312,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.6681365966797,
"fcm_dpo/q_t": 0.4999873638153076,
"grad_norm": 0.002586161717772484,
"learning_rate": 7.36222939784098e-09,
"logits/chosen": 1.3120701313018799,
"logits/rejected": 1.2948602437973022,
"logps/chosen": -273.23675537109375,
"logps/ref_chosen": -58.530723571777344,
"logps/ref_rejected": -75.48025512695312,
"logps/rejected": -420.85443115234375,
"loss": 1.3862,
"margin_dpo/margin_mean": 130.6681365966797,
"margin_dpo/margin_std": 311.08026123046875,
"step": 616
},
{
"KL/chosen_KL_mean": -231.34402465820312,
"KL/mean": -275.96807861328125,
"KL/rejected_KL_mean": -320.59210205078125,
"KL/std": 250.75045776367188,
"epoch": 0.9327286470143613,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.24805450439453,
"fcm_dpo/q_t": 0.49999135732650757,
"grad_norm": 0.0019385057967156172,
"learning_rate": 7.047107919114586e-09,
"logits/chosen": 1.3058488368988037,
"logits/rejected": 1.2810614109039307,
"logps/chosen": -288.95269775390625,
"logps/ref_chosen": -57.608673095703125,
"logps/ref_rejected": -81.22109985351562,
"logps/rejected": -401.81317138671875,
"loss": 1.3863,
"margin_dpo/margin_mean": 89.24806213378906,
"margin_dpo/margin_std": 294.38568115234375,
"step": 617
},
{
"KL/chosen_KL_mean": -219.1179656982422,
"KL/mean": -298.6544189453125,
"KL/rejected_KL_mean": -378.19085693359375,
"KL/std": 302.0130310058594,
"epoch": 0.9342403628117913,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 159.0728759765625,
"fcm_dpo/q_t": 0.49998462200164795,
"grad_norm": 0.003072551218792796,
"learning_rate": 6.738782355044048e-09,
"logits/chosen": 1.2910728454589844,
"logits/rejected": 1.2586588859558105,
"logps/chosen": -275.81390380859375,
"logps/ref_chosen": -56.69594192504883,
"logps/ref_rejected": -85.92362976074219,
"logps/rejected": -464.1144714355469,
"loss": 1.3862,
"margin_dpo/margin_mean": 159.0728759765625,
"margin_dpo/margin_std": 343.8255920410156,
"step": 618
},
{
"KL/chosen_KL_mean": -202.87046813964844,
"KL/mean": -281.81170654296875,
"KL/rejected_KL_mean": -360.75299072265625,
"KL/std": 335.5523681640625,
"epoch": 0.9357520786092215,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 157.8825225830078,
"fcm_dpo/q_t": 0.4999847114086151,
"grad_norm": 0.002635043114423752,
"learning_rate": 6.437261330158206e-09,
"logits/chosen": 1.328965425491333,
"logits/rejected": 1.297224760055542,
"logps/chosen": -256.92889404296875,
"logps/ref_chosen": -54.05841827392578,
"logps/ref_rejected": -83.55493927001953,
"logps/rejected": -444.30792236328125,
"loss": 1.3862,
"margin_dpo/margin_mean": 157.8825225830078,
"margin_dpo/margin_std": 431.3692626953125,
"step": 619
},
{
"KL/chosen_KL_mean": -240.50149536132812,
"KL/mean": -257.7876281738281,
"KL/rejected_KL_mean": -275.0737609863281,
"KL/std": 267.0464782714844,
"epoch": 0.9372637944066515,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 34.57228088378906,
"fcm_dpo/q_t": 0.4999966323375702,
"grad_norm": 0.0017283577471971512,
"learning_rate": 6.142553278648238e-09,
"logits/chosen": 1.2990144491195679,
"logits/rejected": 1.3006207942962646,
"logps/chosen": -303.8712158203125,
"logps/ref_chosen": -63.36971664428711,
"logps/ref_rejected": -65.68269348144531,
"logps/rejected": -340.7564392089844,
"loss": 1.3863,
"margin_dpo/margin_mean": 34.57228088378906,
"margin_dpo/margin_std": 325.6532287597656,
"step": 620
},
{
"KL/chosen_KL_mean": -205.8194580078125,
"KL/mean": -296.57342529296875,
"KL/rejected_KL_mean": -387.327392578125,
"KL/std": 351.635498046875,
"epoch": 0.9387755102040817,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 181.50796508789062,
"fcm_dpo/q_t": 0.49998241662979126,
"grad_norm": 0.0033599112648516893,
"learning_rate": 5.854666444131934e-09,
"logits/chosen": 1.2818564176559448,
"logits/rejected": 1.236130952835083,
"logps/chosen": -258.14068603515625,
"logps/ref_chosen": -52.321224212646484,
"logps/ref_rejected": -88.09001159667969,
"logps/rejected": -475.41741943359375,
"loss": 1.3862,
"margin_dpo/margin_mean": 181.50796508789062,
"margin_dpo/margin_std": 466.3568115234375,
"step": 621
},
{
"KL/chosen_KL_mean": -252.88168334960938,
"KL/mean": -307.3717956542969,
"KL/rejected_KL_mean": -361.8619384765625,
"KL/std": 309.87310791015625,
"epoch": 0.9402872260015117,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 108.98025512695312,
"fcm_dpo/q_t": 0.49998944997787476,
"grad_norm": 0.0022065993398427963,
"learning_rate": 5.573608879422875e-09,
"logits/chosen": 1.2621958255767822,
"logits/rejected": 1.2421605587005615,
"logps/chosen": -312.74713134765625,
"logps/ref_chosen": -59.86545944213867,
"logps/ref_rejected": -81.86668395996094,
"logps/rejected": -443.7286071777344,
"loss": 1.3863,
"margin_dpo/margin_mean": 108.98025512695312,
"margin_dpo/margin_std": 352.85650634765625,
"step": 622
},
{
"KL/chosen_KL_mean": -274.98095703125,
"KL/mean": -340.2034606933594,
"KL/rejected_KL_mean": -405.4259338378906,
"KL/std": 373.55999755859375,
"epoch": 0.9417989417989417,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.4449462890625,
"fcm_dpo/q_t": 0.4999873638153076,
"grad_norm": 0.002324384171515703,
"learning_rate": 5.299388446305342e-09,
"logits/chosen": 1.2612360715866089,
"logits/rejected": 1.245605707168579,
"logps/chosen": -342.34942626953125,
"logps/ref_chosen": -67.36846160888672,
"logps/ref_rejected": -82.02733612060547,
"logps/rejected": -487.4532470703125,
"loss": 1.3862,
"margin_dpo/margin_mean": 130.4449462890625,
"margin_dpo/margin_std": 484.2095642089844,
"step": 623
},
{
"KL/chosen_KL_mean": -195.49905395507812,
"KL/mean": -272.87152099609375,
"KL/rejected_KL_mean": -350.2440490722656,
"KL/std": 312.8592834472656,
"epoch": 0.9433106575963719,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 154.7449951171875,
"fcm_dpo/q_t": 0.4999850392341614,
"grad_norm": 0.002652752446010709,
"learning_rate": 5.03201281531429e-09,
"logits/chosen": 1.2726714611053467,
"logits/rejected": 1.2336527109146118,
"logps/chosen": -246.52560424804688,
"logps/ref_chosen": -51.02655029296875,
"logps/ref_rejected": -76.49203491210938,
"logps/rejected": -426.736083984375,
"loss": 1.3862,
"margin_dpo/margin_mean": 154.7449951171875,
"margin_dpo/margin_std": 322.3939514160156,
"step": 624
},
{
"KL/chosen_KL_mean": -213.73834228515625,
"KL/mean": -271.16351318359375,
"KL/rejected_KL_mean": -328.5887145996094,
"KL/std": 293.5513000488281,
"epoch": 0.9448223733938019,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 114.85034942626953,
"fcm_dpo/q_t": 0.4999888837337494,
"grad_norm": 0.0026168376207351685,
"learning_rate": 4.7714894655209174e-09,
"logits/chosen": 1.3009436130523682,
"logits/rejected": 1.2550803422927856,
"logps/chosen": -267.9459533691406,
"logps/ref_chosen": -54.20761489868164,
"logps/ref_rejected": -84.93669128417969,
"logps/rejected": -413.525390625,
"loss": 1.3862,
"margin_dpo/margin_mean": 114.85035705566406,
"margin_dpo/margin_std": 311.7130126953125,
"step": 625
},
{
"KL/chosen_KL_mean": -169.8229522705078,
"KL/mean": -267.14849853515625,
"KL/rejected_KL_mean": -364.47406005859375,
"KL/std": 340.929931640625,
"epoch": 0.9463340891912321,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 194.651123046875,
"fcm_dpo/q_t": 0.499981164932251,
"grad_norm": 0.0032611822243779898,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": 1.2939801216125488,
"logits/rejected": 1.2345411777496338,
"logps/chosen": -214.88497924804688,
"logps/ref_chosen": -45.06201934814453,
"logps/ref_rejected": -89.66368103027344,
"logps/rejected": -454.13775634765625,
"loss": 1.3862,
"margin_dpo/margin_mean": 194.651123046875,
"margin_dpo/margin_std": 399.38238525390625,
"step": 626
},
{
"KL/chosen_KL_mean": -223.85272216796875,
"KL/mean": -312.509033203125,
"KL/rejected_KL_mean": -401.1653747558594,
"KL/std": 350.2436218261719,
"epoch": 0.9478458049886621,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 177.31263732910156,
"fcm_dpo/q_t": 0.4999828338623047,
"grad_norm": 0.0033359085209667683,
"learning_rate": 4.271028567242818e-09,
"logits/chosen": 1.2410776615142822,
"logits/rejected": 1.183640718460083,
"logps/chosen": -282.643798828125,
"logps/ref_chosen": -58.791053771972656,
"logps/ref_rejected": -94.90802001953125,
"logps/rejected": -496.0733947753906,
"loss": 1.3862,
"margin_dpo/margin_mean": 177.3126220703125,
"margin_dpo/margin_std": 358.99542236328125,
"step": 627
},
{
"KL/chosen_KL_mean": -220.1020050048828,
"KL/mean": -270.9043273925781,
"KL/rejected_KL_mean": -321.7066650390625,
"KL/std": 269.397705078125,
"epoch": 0.9493575207860923,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 101.60467529296875,
"fcm_dpo/q_t": 0.49999016523361206,
"grad_norm": 0.0020546908490359783,
"learning_rate": 4.0311050177251895e-09,
"logits/chosen": 1.292949914932251,
"logits/rejected": 1.277388572692871,
"logps/chosen": -272.90557861328125,
"logps/ref_chosen": -52.80357360839844,
"logps/ref_rejected": -76.49468994140625,
"logps/rejected": -398.20135498046875,
"loss": 1.3863,
"margin_dpo/margin_mean": 101.60467529296875,
"margin_dpo/margin_std": 336.76214599609375,
"step": 628
},
{
"KL/chosen_KL_mean": -258.51788330078125,
"KL/mean": -296.933349609375,
"KL/rejected_KL_mean": -335.34881591796875,
"KL/std": 292.3125,
"epoch": 0.9508692365835223,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 76.83094787597656,
"fcm_dpo/q_t": 0.4999925494194031,
"grad_norm": 0.0015813360223546624,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": 1.2685928344726562,
"logits/rejected": 1.2776813507080078,
"logps/chosen": -329.2353820800781,
"logps/ref_chosen": -70.71749877929688,
"logps/ref_rejected": -78.96273803710938,
"logps/rejected": -414.31158447265625,
"loss": 1.3863,
"margin_dpo/margin_mean": 76.83094787597656,
"margin_dpo/margin_std": 334.86175537109375,
"step": 629
},
{
"KL/chosen_KL_mean": -204.28152465820312,
"KL/mean": -270.9383544921875,
"KL/rejected_KL_mean": -337.5951843261719,
"KL/std": 271.621826171875,
"epoch": 0.9523809523809523,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 133.31365966796875,
"fcm_dpo/q_t": 0.49998709559440613,
"grad_norm": 0.0021148163359612226,
"learning_rate": 3.5719052736323806e-09,
"logits/chosen": 1.272836685180664,
"logits/rejected": 1.2661714553833008,
"logps/chosen": -260.4829406738281,
"logps/ref_chosen": -56.201412200927734,
"logps/ref_rejected": -74.69807434082031,
"logps/rejected": -412.29327392578125,
"loss": 1.3862,
"margin_dpo/margin_mean": 133.31365966796875,
"margin_dpo/margin_std": 350.02862548828125,
"step": 630
},
{
"KL/chosen_KL_mean": -239.1774444580078,
"KL/mean": -327.0312805175781,
"KL/rejected_KL_mean": -414.8851318359375,
"KL/std": 334.594482421875,
"epoch": 0.9538926681783825,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 175.70767211914062,
"fcm_dpo/q_t": 0.4999829828739166,
"grad_norm": 0.0031651852186769247,
"learning_rate": 3.352641923861144e-09,
"logits/chosen": 1.3142149448394775,
"logits/rejected": 1.264453411102295,
"logps/chosen": -297.998046875,
"logps/ref_chosen": -58.82059860229492,
"logps/ref_rejected": -96.51437377929688,
"logps/rejected": -511.3995056152344,
"loss": 1.3862,
"margin_dpo/margin_mean": 175.70765686035156,
"margin_dpo/margin_std": 397.9634094238281,
"step": 631
},
{
"KL/chosen_KL_mean": -214.4776153564453,
"KL/mean": -264.37371826171875,
"KL/rejected_KL_mean": -314.269775390625,
"KL/std": 248.91575622558594,
"epoch": 0.9554043839758125,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.79216003417969,
"fcm_dpo/q_t": 0.4999903440475464,
"grad_norm": 0.0018715175101533532,
"learning_rate": 3.140277830901428e-09,
"logits/chosen": 1.289548397064209,
"logits/rejected": 1.2905744314193726,
"logps/chosen": -273.263671875,
"logps/ref_chosen": -58.786048889160156,
"logps/ref_rejected": -67.21923828125,
"logps/rejected": -381.489013671875,
"loss": 1.3863,
"margin_dpo/margin_mean": 99.79216003417969,
"margin_dpo/margin_std": 307.36798095703125,
"step": 632
},
{
"KL/chosen_KL_mean": -178.64141845703125,
"KL/mean": -238.59205627441406,
"KL/rejected_KL_mean": -298.5426940917969,
"KL/std": 253.90550231933594,
"epoch": 0.9569160997732427,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 119.90129089355469,
"fcm_dpo/q_t": 0.4999883770942688,
"grad_norm": 0.002127637853845954,
"learning_rate": 2.9348189350335007e-09,
"logits/chosen": 1.2887210845947266,
"logits/rejected": 1.2725958824157715,
"logps/chosen": -230.77162170410156,
"logps/ref_chosen": -52.13019561767578,
"logps/ref_rejected": -67.23016357421875,
"logps/rejected": -365.7728576660156,
"loss": 1.3862,
"margin_dpo/margin_mean": 119.90128326416016,
"margin_dpo/margin_std": 286.3564453125,
"step": 633
},
{
"KL/chosen_KL_mean": -235.68812561035156,
"KL/mean": -247.49952697753906,
"KL/rejected_KL_mean": -259.3109436035156,
"KL/std": 264.06658935546875,
"epoch": 0.9584278155706727,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 23.622833251953125,
"fcm_dpo/q_t": 0.49999770522117615,
"grad_norm": 0.0016156296478584409,
"learning_rate": 2.736270983384276e-09,
"logits/chosen": 1.3023512363433838,
"logits/rejected": 1.3197746276855469,
"logps/chosen": -296.66790771484375,
"logps/ref_chosen": -60.97979736328125,
"logps/ref_rejected": -58.50825119018555,
"logps/rejected": -317.8191833496094,
"loss": 1.3863,
"margin_dpo/margin_mean": 23.62282943725586,
"margin_dpo/margin_std": 295.5430908203125,
"step": 634
},
{
"KL/chosen_KL_mean": -270.3648376464844,
"KL/mean": -338.20135498046875,
"KL/rejected_KL_mean": -406.0378723144531,
"KL/std": 269.56549072265625,
"epoch": 0.9599395313681028,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 135.67300415039062,
"fcm_dpo/q_t": 0.4999868869781494,
"grad_norm": 0.0024493515957146883,
"learning_rate": 2.5446395297668287e-09,
"logits/chosen": 1.2402102947235107,
"logits/rejected": 1.2276395559310913,
"logps/chosen": -336.3378601074219,
"logps/ref_chosen": -65.9730224609375,
"logps/ref_rejected": -85.61317443847656,
"logps/rejected": -491.6510314941406,
"loss": 1.3862,
"margin_dpo/margin_mean": 135.67300415039062,
"margin_dpo/margin_std": 318.53192138671875,
"step": 635
},
{
"KL/chosen_KL_mean": -190.3470001220703,
"KL/mean": -266.31427001953125,
"KL/rejected_KL_mean": -342.281494140625,
"KL/std": 254.54660034179688,
"epoch": 0.9614512471655329,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 151.93450927734375,
"fcm_dpo/q_t": 0.4999852776527405,
"grad_norm": 0.002561971079558134,
"learning_rate": 2.359929934524829e-09,
"logits/chosen": 1.2839725017547607,
"logits/rejected": 1.2400085926055908,
"logps/chosen": -239.48716735839844,
"logps/ref_chosen": -49.140167236328125,
"logps/ref_rejected": -81.26971435546875,
"logps/rejected": -423.55120849609375,
"loss": 1.3862,
"margin_dpo/margin_mean": 151.93450927734375,
"margin_dpo/margin_std": 261.3985595703125,
"step": 636
},
{
"KL/chosen_KL_mean": -266.4497985839844,
"KL/mean": -329.82757568359375,
"KL/rejected_KL_mean": -393.205322265625,
"KL/std": 320.78643798828125,
"epoch": 0.9629629629629629,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 126.7555160522461,
"fcm_dpo/q_t": 0.49998772144317627,
"grad_norm": 0.002606100868433714,
"learning_rate": 2.1821473643827137e-09,
"logits/chosen": 1.2443512678146362,
"logits/rejected": 1.230696201324463,
"logps/chosen": -340.1463928222656,
"logps/ref_chosen": -73.69658660888672,
"logps/ref_rejected": -83.01487731933594,
"logps/rejected": -476.22021484375,
"loss": 1.3862,
"margin_dpo/margin_mean": 126.75550842285156,
"margin_dpo/margin_std": 355.1866149902344,
"step": 637
},
{
"KL/chosen_KL_mean": -243.36972045898438,
"KL/mean": -302.26416015625,
"KL/rejected_KL_mean": -361.1585388183594,
"KL/std": 264.62188720703125,
"epoch": 0.9644746787603931,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 117.788818359375,
"fcm_dpo/q_t": 0.4999886155128479,
"grad_norm": 0.0024036893155425787,
"learning_rate": 2.0112967923011646e-09,
"logits/chosen": 1.290708065032959,
"logits/rejected": 1.2749075889587402,
"logps/chosen": -306.15130615234375,
"logps/ref_chosen": -62.78158187866211,
"logps/ref_rejected": -85.40478515625,
"logps/rejected": -446.5633239746094,
"loss": 1.3862,
"margin_dpo/margin_mean": 117.788818359375,
"margin_dpo/margin_std": 344.58038330078125,
"step": 638
},
{
"KL/chosen_KL_mean": -215.73281860351562,
"KL/mean": -291.6164855957031,
"KL/rejected_KL_mean": -367.5001220703125,
"KL/std": 315.470947265625,
"epoch": 0.9659863945578231,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 151.767333984375,
"fcm_dpo/q_t": 0.49998530745506287,
"grad_norm": 0.002579744439572096,
"learning_rate": 1.847382997337943e-09,
"logits/chosen": 1.2794743776321411,
"logits/rejected": 1.251741886138916,
"logps/chosen": -269.4994201660156,
"logps/ref_chosen": -53.76658630371094,
"logps/ref_rejected": -72.30009460449219,
"logps/rejected": -439.80023193359375,
"loss": 1.3862,
"margin_dpo/margin_mean": 151.767333984375,
"margin_dpo/margin_std": 374.64910888671875,
"step": 639
},
{
"KL/chosen_KL_mean": -203.71878051757812,
"KL/mean": -270.4925842285156,
"KL/rejected_KL_mean": -337.2663879394531,
"KL/std": 287.63861083984375,
"epoch": 0.9674981103552532,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 133.547607421875,
"fcm_dpo/q_t": 0.49998706579208374,
"grad_norm": 0.0021755900233983994,
"learning_rate": 1.690410564514244e-09,
"logits/chosen": 1.3102037906646729,
"logits/rejected": 1.282835602760315,
"logps/chosen": -255.13656616210938,
"logps/ref_chosen": -51.41777801513672,
"logps/ref_rejected": -77.27879333496094,
"logps/rejected": -414.545166015625,
"loss": 1.3862,
"margin_dpo/margin_mean": 133.54759216308594,
"margin_dpo/margin_std": 318.0604248046875,
"step": 640
},
{
"KL/chosen_KL_mean": -253.23318481445312,
"KL/mean": -302.85894775390625,
"KL/rejected_KL_mean": -352.48468017578125,
"KL/std": 273.1927795410156,
"epoch": 0.9690098261526833,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.2514877319336,
"fcm_dpo/q_t": 0.49999040365219116,
"grad_norm": 0.0019294384401291609,
"learning_rate": 1.5403838846864692e-09,
"logits/chosen": 1.2644659280776978,
"logits/rejected": 1.2682955265045166,
"logps/chosen": -324.287841796875,
"logps/ref_chosen": -71.0546646118164,
"logps/ref_rejected": -82.2440185546875,
"logps/rejected": -434.72869873046875,
"loss": 1.3863,
"margin_dpo/margin_mean": 99.2514877319336,
"margin_dpo/margin_std": 286.9643859863281,
"step": 641
},
{
"KL/chosen_KL_mean": -276.5420227050781,
"KL/mean": -290.9453430175781,
"KL/rejected_KL_mean": -305.3486633300781,
"KL/std": 272.9807434082031,
"epoch": 0.9705215419501134,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 28.806636810302734,
"fcm_dpo/q_t": 0.49999719858169556,
"grad_norm": 0.001907353405840695,
"learning_rate": 1.3973071544233218e-09,
"logits/chosen": 1.2331547737121582,
"logits/rejected": 1.250314474105835,
"logps/chosen": -345.4713134765625,
"logps/ref_chosen": -68.92927551269531,
"logps/ref_rejected": -70.85682678222656,
"logps/rejected": -376.20550537109375,
"loss": 1.3863,
"margin_dpo/margin_mean": 28.8066349029541,
"margin_dpo/margin_std": 318.6273193359375,
"step": 642
},
{
"KL/chosen_KL_mean": -229.24893188476562,
"KL/mean": -313.97808837890625,
"KL/rejected_KL_mean": -398.70721435546875,
"KL/std": 334.5547790527344,
"epoch": 0.9720332577475435,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 169.45823669433594,
"fcm_dpo/q_t": 0.4999835789203644,
"grad_norm": 0.003335776040330529,
"learning_rate": 1.261184375888541e-09,
"logits/chosen": 1.2515963315963745,
"logits/rejected": 1.2289414405822754,
"logps/chosen": -294.5579833984375,
"logps/ref_chosen": -65.30903625488281,
"logps/ref_rejected": -83.61613464355469,
"logps/rejected": -482.3233337402344,
"loss": 1.3862,
"margin_dpo/margin_mean": 169.458251953125,
"margin_dpo/margin_std": 398.0389099121094,
"step": 643
},
{
"KL/chosen_KL_mean": -195.06884765625,
"KL/mean": -232.06622314453125,
"KL/rejected_KL_mean": -269.0635986328125,
"KL/std": 229.22683715820312,
"epoch": 0.9735449735449735,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 73.99472045898438,
"fcm_dpo/q_t": 0.49999281764030457,
"grad_norm": 0.0016697602113708854,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": 1.319830060005188,
"logits/rejected": 1.3086210489273071,
"logps/chosen": -246.0714569091797,
"logps/ref_chosen": -51.002601623535156,
"logps/ref_rejected": -64.46372985839844,
"logps/rejected": -333.5273132324219,
"loss": 1.3863,
"margin_dpo/margin_mean": 73.99470520019531,
"margin_dpo/margin_std": 218.09716796875,
"step": 644
},
{
"KL/chosen_KL_mean": -238.21824645996094,
"KL/mean": -267.15887451171875,
"KL/rejected_KL_mean": -296.0995178222656,
"KL/std": 247.69064331054688,
"epoch": 0.9750566893424036,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 57.881282806396484,
"fcm_dpo/q_t": 0.4999943971633911,
"grad_norm": 0.001644918229430914,
"learning_rate": 1.0098157099674987e-09,
"logits/chosen": 1.2519395351409912,
"logits/rejected": 1.2480509281158447,
"logps/chosen": -299.181640625,
"logps/ref_chosen": -60.963409423828125,
"logps/ref_rejected": -69.73353576660156,
"logps/rejected": -365.83306884765625,
"loss": 1.3863,
"margin_dpo/margin_mean": 57.88128662109375,
"margin_dpo/margin_std": 300.2716064453125,
"step": 645
},
{
"KL/chosen_KL_mean": -262.1314697265625,
"KL/mean": -314.58929443359375,
"KL/rejected_KL_mean": -367.047119140625,
"KL/std": 342.90087890625,
"epoch": 0.9765684051398337,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.91563415527344,
"fcm_dpo/q_t": 0.4999898374080658,
"grad_norm": 0.0021173881832510233,
"learning_rate": 8.945768539031783e-10,
"logits/chosen": 1.2763549089431763,
"logits/rejected": 1.2538020610809326,
"logps/chosen": -324.42156982421875,
"logps/ref_chosen": -62.290069580078125,
"logps/ref_rejected": -85.54812622070312,
"logps/rejected": -452.59521484375,
"loss": 1.3863,
"margin_dpo/margin_mean": 104.91563415527344,
"margin_dpo/margin_std": 296.7239990234375,
"step": 646
},
{
"KL/chosen_KL_mean": -277.2899475097656,
"KL/mean": -347.98150634765625,
"KL/rejected_KL_mean": -418.67303466796875,
"KL/std": 369.2318115234375,
"epoch": 0.9780801209372638,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 141.3831024169922,
"fcm_dpo/q_t": 0.49998632073402405,
"grad_norm": 0.002953104441985488,
"learning_rate": 7.863060120144316e-10,
"logits/chosen": 1.2947700023651123,
"logits/rejected": 1.2425816059112549,
"logps/chosen": -344.8058166503906,
"logps/ref_chosen": -67.515869140625,
"logps/ref_rejected": -101.50871276855469,
"logps/rejected": -520.1817626953125,
"loss": 1.3862,
"margin_dpo/margin_mean": 141.3831024169922,
"margin_dpo/margin_std": 402.0182189941406,
"step": 647
},
{
"KL/chosen_KL_mean": -248.36538696289062,
"KL/mean": -310.21771240234375,
"KL/rejected_KL_mean": -372.0700378417969,
"KL/std": 275.6451416015625,
"epoch": 0.9795918367346939,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 123.70465087890625,
"fcm_dpo/q_t": 0.49998801946640015,
"grad_norm": 0.002356282202526927,
"learning_rate": 6.850062128694045e-10,
"logits/chosen": 1.2679743766784668,
"logits/rejected": 1.2473814487457275,
"logps/chosen": -312.9613037109375,
"logps/ref_chosen": -64.59593963623047,
"logps/ref_rejected": -83.384033203125,
"logps/rejected": -455.4540710449219,
"loss": 1.3862,
"margin_dpo/margin_mean": 123.70465087890625,
"margin_dpo/margin_std": 370.0848693847656,
"step": 648
},
{
"KL/chosen_KL_mean": -202.95803833007812,
"KL/mean": -254.43443298339844,
"KL/rejected_KL_mean": -305.91082763671875,
"KL/std": 240.42996215820312,
"epoch": 0.981103552532124,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.95277404785156,
"fcm_dpo/q_t": 0.4999900460243225,
"grad_norm": 0.0025812452659010887,
"learning_rate": 5.906802900412788e-10,
"logits/chosen": 1.308830738067627,
"logits/rejected": 1.2813698053359985,
"logps/chosen": -252.2677001953125,
"logps/ref_chosen": -49.30964660644531,
"logps/ref_rejected": -73.73710632324219,
"logps/rejected": -379.64794921875,
"loss": 1.3863,
"margin_dpo/margin_mean": 102.95277404785156,
"margin_dpo/margin_std": 293.2994384765625,
"step": 649
},
{
"KL/chosen_KL_mean": -175.028076171875,
"KL/mean": -253.37965393066406,
"KL/rejected_KL_mean": -331.731201171875,
"KL/std": 274.1990661621094,
"epoch": 0.982615268329554,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 156.70315551757812,
"fcm_dpo/q_t": 0.49998483061790466,
"grad_norm": 0.002816020045429468,
"learning_rate": 5.033308820289184e-10,
"logits/chosen": 1.3074793815612793,
"logits/rejected": 1.2863340377807617,
"logps/chosen": -230.09133911132812,
"logps/ref_chosen": -55.06325912475586,
"logps/ref_rejected": -77.39610290527344,
"logps/rejected": -409.1273193359375,
"loss": 1.3862,
"margin_dpo/margin_mean": 156.70314025878906,
"margin_dpo/margin_std": 366.322998046875,
"step": 650
},
{
"KL/chosen_KL_mean": -214.81884765625,
"KL/mean": -288.9599609375,
"KL/rejected_KL_mean": -363.10113525390625,
"KL/std": 292.44451904296875,
"epoch": 0.9841269841269841,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 148.2822723388672,
"fcm_dpo/q_t": 0.49998563528060913,
"grad_norm": 0.002445911755785346,
"learning_rate": 4.2296043218295606e-10,
"logits/chosen": 1.288893461227417,
"logits/rejected": 1.2606956958770752,
"logps/chosen": -268.8840026855469,
"logps/ref_chosen": -54.065162658691406,
"logps/ref_rejected": -77.79080200195312,
"logps/rejected": -440.89190673828125,
"loss": 1.3862,
"margin_dpo/margin_mean": 148.2822723388672,
"margin_dpo/margin_std": 326.09967041015625,
"step": 651
},
{
"KL/chosen_KL_mean": -275.9385986328125,
"KL/mean": -314.86370849609375,
"KL/rejected_KL_mean": -353.788818359375,
"KL/std": 326.7114562988281,
"epoch": 0.9856386999244142,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 77.85020446777344,
"fcm_dpo/q_t": 0.4999924600124359,
"grad_norm": 0.0021558792795985937,
"learning_rate": 3.4957118863768176e-10,
"logits/chosen": 1.2859222888946533,
"logits/rejected": 1.265965223312378,
"logps/chosen": -339.57891845703125,
"logps/ref_chosen": -63.64030456542969,
"logps/ref_rejected": -78.86882019042969,
"logps/rejected": -432.6576232910156,
"loss": 1.3863,
"margin_dpo/margin_mean": 77.85020446777344,
"margin_dpo/margin_std": 423.6152648925781,
"step": 652
},
{
"KL/chosen_KL_mean": -228.88973999023438,
"KL/mean": -280.29022216796875,
"KL/rejected_KL_mean": -331.6907043457031,
"KL/std": 262.5857849121094,
"epoch": 0.9871504157218443,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.80097961425781,
"fcm_dpo/q_t": 0.4999900460243225,
"grad_norm": 0.001885903999209404,
"learning_rate": 2.831652042480093e-10,
"logits/chosen": 1.2586016654968262,
"logits/rejected": 1.2437849044799805,
"logps/chosen": -290.55810546875,
"logps/ref_chosen": -61.668373107910156,
"logps/ref_rejected": -73.83012390136719,
"logps/rejected": -405.52081298828125,
"loss": 1.3863,
"margin_dpo/margin_mean": 102.80097961425781,
"margin_dpo/margin_std": 331.70751953125,
"step": 653
},
{
"KL/chosen_KL_mean": -235.8380126953125,
"KL/mean": -312.2591552734375,
"KL/rejected_KL_mean": -388.6803283691406,
"KL/std": 327.74395751953125,
"epoch": 0.9886621315192744,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 152.84230041503906,
"fcm_dpo/q_t": 0.4999852180480957,
"grad_norm": 0.002882143948227167,
"learning_rate": 2.2374433653205016e-10,
"logits/chosen": 1.2653387784957886,
"logits/rejected": 1.2258384227752686,
"logps/chosen": -293.4062805175781,
"logps/ref_chosen": -57.568267822265625,
"logps/ref_rejected": -87.74789428710938,
"logps/rejected": -476.42822265625,
"loss": 1.3862,
"margin_dpo/margin_mean": 152.84230041503906,
"margin_dpo/margin_std": 319.22900390625,
"step": 654
},
{
"KL/chosen_KL_mean": -178.378173828125,
"KL/mean": -272.81280517578125,
"KL/rejected_KL_mean": -367.2474670410156,
"KL/std": 287.7238464355469,
"epoch": 0.9901738473167044,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 188.86932373046875,
"fcm_dpo/q_t": 0.49998173117637634,
"grad_norm": 0.003305543912574649,
"learning_rate": 1.7131024761923852e-10,
"logits/chosen": 1.2574398517608643,
"logits/rejected": 1.2300739288330078,
"logps/chosen": -230.5253143310547,
"logps/ref_chosen": -52.14714813232422,
"logps/ref_rejected": -80.85014343261719,
"logps/rejected": -448.09759521484375,
"loss": 1.3862,
"margin_dpo/margin_mean": 188.86932373046875,
"margin_dpo/margin_std": 355.13861083984375,
"step": 655
},
{
"KL/chosen_KL_mean": -268.1697692871094,
"KL/mean": -328.5416564941406,
"KL/rejected_KL_mean": -388.91357421875,
"KL/std": 299.40435791015625,
"epoch": 0.9916855631141346,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 120.74380493164062,
"fcm_dpo/q_t": 0.499988317489624,
"grad_norm": 0.0024166665971279144,
"learning_rate": 1.2586440420372934e-10,
"logits/chosen": 1.2543549537658691,
"logits/rejected": 1.2443668842315674,
"logps/chosen": -341.4264831542969,
"logps/ref_chosen": -73.25672912597656,
"logps/ref_rejected": -85.35127258300781,
"logps/rejected": -474.26483154296875,
"loss": 1.3862,
"margin_dpo/margin_mean": 120.74380493164062,
"margin_dpo/margin_std": 330.2796630859375,
"step": 656
},
{
"KL/chosen_KL_mean": -185.50474548339844,
"KL/mean": -276.64654541015625,
"KL/rejected_KL_mean": -367.788330078125,
"KL/std": 325.72955322265625,
"epoch": 0.9931972789115646,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 182.28355407714844,
"fcm_dpo/q_t": 0.4999823570251465,
"grad_norm": 0.00303459819406271,
"learning_rate": 8.740807750345913e-11,
"logits/chosen": 1.3113363981246948,
"logits/rejected": 1.2885608673095703,
"logps/chosen": -235.2281494140625,
"logps/ref_chosen": -49.72339630126953,
"logps/ref_rejected": -75.1568603515625,
"logps/rejected": -442.9451904296875,
"loss": 1.3862,
"margin_dpo/margin_mean": 182.28355407714844,
"margin_dpo/margin_std": 396.7044677734375,
"step": 657
},
{
"KL/chosen_KL_mean": -248.78866577148438,
"KL/mean": -298.27093505859375,
"KL/rejected_KL_mean": -347.7532043457031,
"KL/std": 330.69586181640625,
"epoch": 0.9947089947089947,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 98.96455383300781,
"fcm_dpo/q_t": 0.49999043345451355,
"grad_norm": 0.0021189674735069275,
"learning_rate": 5.594234322453539e-11,
"logits/chosen": 1.2897260189056396,
"logits/rejected": 1.271322250366211,
"logps/chosen": -311.83502197265625,
"logps/ref_chosen": -63.04634094238281,
"logps/ref_rejected": -83.44963073730469,
"logps/rejected": -431.2028503417969,
"loss": 1.3863,
"margin_dpo/margin_mean": 98.96455383300781,
"margin_dpo/margin_std": 376.11456298828125,
"step": 658
},
{
"KL/chosen_KL_mean": -244.0128936767578,
"KL/mean": -291.29986572265625,
"KL/rejected_KL_mean": -338.5868225097656,
"KL/std": 314.4420166015625,
"epoch": 0.9962207105064248,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 94.57395935058594,
"fcm_dpo/q_t": 0.4999908208847046,
"grad_norm": 0.0024397147353738546,
"learning_rate": 3.146808153123293e-11,
"logits/chosen": 1.3078527450561523,
"logits/rejected": 1.281451940536499,
"logps/chosen": -299.09307861328125,
"logps/ref_chosen": -55.0802001953125,
"logps/ref_rejected": -71.91049194335938,
"logps/rejected": -410.497314453125,
"loss": 1.3863,
"margin_dpo/margin_mean": 94.5739517211914,
"margin_dpo/margin_std": 354.7186279296875,
"step": 659
},
{
"KL/chosen_KL_mean": -202.13861083984375,
"KL/mean": -280.438720703125,
"KL/rejected_KL_mean": -358.7388610839844,
"KL/std": 307.3833923339844,
"epoch": 0.9977324263038548,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 156.60025024414062,
"fcm_dpo/q_t": 0.49998486042022705,
"grad_norm": 0.002643043175339699,
"learning_rate": 1.3985977021235829e-11,
"logits/chosen": 1.329068660736084,
"logits/rejected": 1.3038125038146973,
"logps/chosen": -256.6645202636719,
"logps/ref_chosen": -54.525917053222656,
"logps/ref_rejected": -81.23604583740234,
"logps/rejected": -439.97491455078125,
"loss": 1.3862,
"margin_dpo/margin_mean": 156.60025024414062,
"margin_dpo/margin_std": 385.9468688964844,
"step": 660
},
{
"KL/chosen_KL_mean": -255.35760498046875,
"KL/mean": -313.8293762207031,
"KL/rejected_KL_mean": -372.3011474609375,
"KL/std": 311.5862121582031,
"epoch": 0.999244142101285,
"fcm_dpo/beta": 3.8711289107595803e-07,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 116.94351959228516,
"fcm_dpo/q_t": 0.4999886751174927,
"grad_norm": 0.0025908024981617928,
"learning_rate": 3.4965187065971735e-12,
"logits/chosen": 1.2794947624206543,
"logits/rejected": 1.2472293376922607,
"logps/chosen": -315.7302551269531,
"logps/ref_chosen": -60.37263870239258,
"logps/ref_rejected": -77.42874145507812,
"logps/rejected": -449.7298583984375,
"loss": 1.3862,
"margin_dpo/margin_mean": 116.94351959228516,
"margin_dpo/margin_std": 371.20477294921875,
"step": 661
},
{
"epoch": 0.999244142101285,
"step": 661,
"total_flos": 0.0,
"train_loss": 1.370076598991244,
"train_runtime": 1650.2951,
"train_samples_per_second": 25.654,
"train_steps_per_second": 0.401
}
],
"logging_steps": 1,
"max_steps": 661,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink