jackf857/qwen3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.45-s_star-0.4
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
cde57acec7c0becd789a1aa90d3ab66b81991715
qwen3-8b-base-new-dpo-ultra…/trainer_state.json
ModelHub XC cde57acec7 初始化项目，由ModelHub XC社区提供模型 
Model: jackf857/qwen3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.45-s_star-0.4
Source: Original Platform
2026-05-12 06:43:36 +08:00

11057 lines
412 KiB
JSON
Raw Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9989528795811519,
"eval_steps": 200,
"global_step": 477,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": -0.04536247253417969,
"KL/mean": -0.05389120429754257,
"KL/rejected_KL_mean": -0.06241607666015625,
"KL/std": 0.5513200759887695,
"epoch": 0.0020942408376963353,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01704825460910797,
"fcm_dpo/q_t": 0.499957412481308,
"grad_norm": 14.618651390075684,
"learning_rate": 0.0,
"logits/chosen": 1.9854813814163208,
"logits/rejected": 1.7737188339233398,
"logps/chosen": -267.0936584472656,
"logps/ref_chosen": -267.04827880859375,
"logps/ref_rejected": -219.76657104492188,
"logps/rejected": -219.82899475097656,
"loss": 5.5446,
"margin_dpo/margin_mean": 0.01704716682434082,
"margin_dpo/margin_std": 0.7467707991600037,
"step": 1
},
{
"KL/chosen_KL_mean": 0.07550811767578125,
"KL/mean": 0.03060000389814377,
"KL/rejected_KL_mean": -0.014301300048828125,
"KL/std": 0.6403332352638245,
"epoch": 0.004188481675392671,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08979703485965729,
"fcm_dpo/q_t": 0.4997755289077759,
"grad_norm": 15.140149116516113,
"learning_rate": 1.0416666666666666e-08,
"logits/chosen": 2.1993117332458496,
"logits/rejected": 2.1368322372436523,
"logps/chosen": -256.5101623535156,
"logps/ref_chosen": -256.585693359375,
"logps/ref_rejected": -239.92742919921875,
"logps/rejected": -239.94171142578125,
"loss": 5.5417,
"margin_dpo/margin_mean": 0.08979731798171997,
"margin_dpo/margin_std": 0.9745746850967407,
"step": 2
},
{
"KL/chosen_KL_mean": 0.00390625,
"KL/mean": -0.015328258275985718,
"KL/rejected_KL_mean": -0.03455162048339844,
"KL/std": 0.625785768032074,
"epoch": 0.0062827225130890054,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03845793008804321,
"fcm_dpo/q_t": 0.4999038279056549,
"grad_norm": 14.649983406066895,
"learning_rate": 2.083333333333333e-08,
"logits/chosen": 2.2561516761779785,
"logits/rejected": 2.2956507205963135,
"logps/chosen": -265.66497802734375,
"logps/ref_chosen": -265.66888427734375,
"logps/ref_rejected": -284.3778991699219,
"logps/rejected": -284.4124755859375,
"loss": 5.5437,
"margin_dpo/margin_mean": 0.03845754265785217,
"margin_dpo/margin_std": 0.8079719543457031,
"step": 3
},
{
"KL/chosen_KL_mean": 0.07561492919921875,
"KL/mean": 0.07756663113832474,
"KL/rejected_KL_mean": 0.07953453063964844,
"KL/std": 0.6961261630058289,
"epoch": 0.008376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.003922984004020691,
"fcm_dpo/q_t": 0.5000097751617432,
"grad_norm": 15.700592041015625,
"learning_rate": 3.125e-08,
"logits/chosen": 1.8715617656707764,
"logits/rejected": 1.8110389709472656,
"logps/chosen": -319.431884765625,
"logps/ref_chosen": -319.50750732421875,
"logps/ref_rejected": -296.22314453125,
"logps/rejected": -296.1436462402344,
"loss": 5.5455,
"margin_dpo/margin_mean": -0.003923103213310242,
"margin_dpo/margin_std": 1.0326601266860962,
"step": 4
},
{
"KL/chosen_KL_mean": -0.003139495849609375,
"KL/mean": 0.02464383840560913,
"KL/rejected_KL_mean": 0.05242919921875,
"KL/std": 0.6424128413200378,
"epoch": 0.010471204188481676,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0555671751499176,
"fcm_dpo/q_t": 0.5001389384269714,
"grad_norm": 15.680787086486816,
"learning_rate": 4.166666666666666e-08,
"logits/chosen": 1.9470841884613037,
"logits/rejected": 1.9636645317077637,
"logps/chosen": -292.8761291503906,
"logps/ref_chosen": -292.87298583984375,
"logps/ref_rejected": -270.3802795410156,
"logps/rejected": -270.327880859375,
"loss": 5.5475,
"margin_dpo/margin_mean": -0.05556736886501312,
"margin_dpo/margin_std": 0.878207802772522,
"step": 5
},
{
"KL/chosen_KL_mean": 0.00652313232421875,
"KL/mean": 0.008331656455993652,
"KL/rejected_KL_mean": 0.0101470947265625,
"KL/std": 0.5935744643211365,
"epoch": 0.012565445026178011,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0036214888095855713,
"fcm_dpo/q_t": 0.5000090599060059,
"grad_norm": 15.472807884216309,
"learning_rate": 5.208333333333333e-08,
"logits/chosen": 2.1412649154663086,
"logits/rejected": 2.078343152999878,
"logps/chosen": -280.2789306640625,
"logps/ref_chosen": -280.28546142578125,
"logps/ref_rejected": -242.7611541748047,
"logps/rejected": -242.75100708007812,
"loss": 5.5454,
"margin_dpo/margin_mean": -0.0036224573850631714,
"margin_dpo/margin_std": 0.7638767957687378,
"step": 6
},
{
"KL/chosen_KL_mean": -0.004520416259765625,
"KL/mean": 0.03132270276546478,
"KL/rejected_KL_mean": 0.06715965270996094,
"KL/std": 0.6809048652648926,
"epoch": 0.014659685863874346,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0716867744922638,
"fcm_dpo/q_t": 0.5001791715621948,
"grad_norm": 15.662071228027344,
"learning_rate": 6.25e-08,
"logits/chosen": 1.8676385879516602,
"logits/rejected": 1.7663965225219727,
"logps/chosen": -255.11024475097656,
"logps/ref_chosen": -255.10572814941406,
"logps/ref_rejected": -251.15402221679688,
"logps/rejected": -251.08685302734375,
"loss": 5.5482,
"margin_dpo/margin_mean": -0.07168655097484589,
"margin_dpo/margin_std": 0.9891082644462585,
"step": 7
},
{
"KL/chosen_KL_mean": -0.05980682373046875,
"KL/mean": -0.008247092366218567,
"KL/rejected_KL_mean": 0.043323516845703125,
"KL/std": 0.7069255709648132,
"epoch": 0.016753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.10313859581947327,
"fcm_dpo/q_t": 0.5002578496932983,
"grad_norm": 15.891398429870605,
"learning_rate": 7.291666666666667e-08,
"logits/chosen": 2.325185537338257,
"logits/rejected": 2.135742664337158,
"logps/chosen": -345.24139404296875,
"logps/ref_chosen": -345.18157958984375,
"logps/ref_rejected": -277.3522644042969,
"logps/rejected": -277.3089599609375,
"loss": 5.5494,
"margin_dpo/margin_mean": -0.10313853621482849,
"margin_dpo/margin_std": 1.0472654104232788,
"step": 8
},
{
"KL/chosen_KL_mean": -0.05593681335449219,
"KL/mean": -0.04206794500350952,
"KL/rejected_KL_mean": -0.0281982421875,
"KL/std": 0.6409730315208435,
"epoch": 0.018848167539267015,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.027742207050323486,
"fcm_dpo/q_t": 0.5000693798065186,
"grad_norm": 14.116764068603516,
"learning_rate": 8.333333333333333e-08,
"logits/chosen": 2.199899435043335,
"logits/rejected": 2.1797549724578857,
"logps/chosen": -276.7862854003906,
"logps/ref_chosen": -276.7303161621094,
"logps/ref_rejected": -266.19952392578125,
"logps/rejected": -266.2276916503906,
"loss": 5.5464,
"margin_dpo/margin_mean": -0.027742385864257812,
"margin_dpo/margin_std": 0.8435088396072388,
"step": 9
},
{
"KL/chosen_KL_mean": -0.027740478515625,
"KL/mean": -0.010075248777866364,
"KL/rejected_KL_mean": 0.00759124755859375,
"KL/std": 0.5887585282325745,
"epoch": 0.020942408376963352,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.035333022475242615,
"fcm_dpo/q_t": 0.5000883340835571,
"grad_norm": 13.785626411437988,
"learning_rate": 9.375e-08,
"logits/chosen": 1.9849461317062378,
"logits/rejected": 2.041294813156128,
"logps/chosen": -246.5900115966797,
"logps/ref_chosen": -246.562255859375,
"logps/ref_rejected": -224.70401000976562,
"logps/rejected": -224.6964111328125,
"loss": 5.5467,
"margin_dpo/margin_mean": -0.03533339500427246,
"margin_dpo/margin_std": 0.7649649381637573,
"step": 10
},
{
"KL/chosen_KL_mean": 0.059051513671875,
"KL/mean": 0.010689198970794678,
"KL/rejected_KL_mean": -0.03765869140625,
"KL/std": 0.6329272389411926,
"epoch": 0.023036649214659685,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09670479595661163,
"fcm_dpo/q_t": 0.499758243560791,
"grad_norm": 15.459443092346191,
"learning_rate": 1.0416666666666667e-07,
"logits/chosen": 1.970166802406311,
"logits/rejected": 1.9005579948425293,
"logps/chosen": -290.4420166015625,
"logps/ref_chosen": -290.5010986328125,
"logps/ref_rejected": -253.9559326171875,
"logps/rejected": -253.99354553222656,
"loss": 5.5414,
"margin_dpo/margin_mean": 0.09670477360486984,
"margin_dpo/margin_std": 0.9046503305435181,
"step": 11
},
{
"KL/chosen_KL_mean": 0.040882110595703125,
"KL/mean": 0.030767112970352173,
"KL/rejected_KL_mean": 0.020654678344726562,
"KL/std": 0.614155650138855,
"epoch": 0.025130890052356022,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02022460103034973,
"fcm_dpo/q_t": 0.4999494254589081,
"grad_norm": 14.839339256286621,
"learning_rate": 1.1458333333333332e-07,
"logits/chosen": 1.5951333045959473,
"logits/rejected": 1.579640507698059,
"logps/chosen": -278.4776611328125,
"logps/ref_chosen": -278.5185241699219,
"logps/ref_rejected": -273.56207275390625,
"logps/rejected": -273.54144287109375,
"loss": 5.5444,
"margin_dpo/margin_mean": 0.020224541425704956,
"margin_dpo/margin_std": 0.8575910925865173,
"step": 12
},
{
"KL/chosen_KL_mean": -0.10135078430175781,
"KL/mean": -0.05781416594982147,
"KL/rejected_KL_mean": -0.014280319213867188,
"KL/std": 0.5546329617500305,
"epoch": 0.027225130890052355,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.08706018328666687,
"fcm_dpo/q_t": 0.5002176761627197,
"grad_norm": 14.821869850158691,
"learning_rate": 1.25e-07,
"logits/chosen": 1.8845791816711426,
"logits/rejected": 2.0167646408081055,
"logps/chosen": -264.75848388671875,
"logps/ref_chosen": -264.6571350097656,
"logps/ref_rejected": -274.3492736816406,
"logps/rejected": -274.3635559082031,
"loss": 5.5487,
"margin_dpo/margin_mean": -0.08706018328666687,
"margin_dpo/margin_std": 0.7981866002082825,
"step": 13
},
{
"KL/chosen_KL_mean": 0.0448760986328125,
"KL/mean": -0.011957794427871704,
"KL/rejected_KL_mean": -0.0688018798828125,
"KL/std": 0.5699095129966736,
"epoch": 0.02931937172774869,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11367706954479218,
"fcm_dpo/q_t": 0.4997158348560333,
"grad_norm": 15.356264114379883,
"learning_rate": 1.3541666666666666e-07,
"logits/chosen": 2.0706424713134766,
"logits/rejected": 2.1114251613616943,
"logps/chosen": -274.0697326660156,
"logps/ref_chosen": -274.1145935058594,
"logps/ref_rejected": -261.0951843261719,
"logps/rejected": -261.1639709472656,
"loss": 5.5407,
"margin_dpo/margin_mean": 0.11367689073085785,
"margin_dpo/margin_std": 0.7376527786254883,
"step": 14
},
{
"KL/chosen_KL_mean": 0.02631378173828125,
"KL/mean": 0.055424295365810394,
"KL/rejected_KL_mean": 0.08452606201171875,
"KL/std": 0.6501664519309998,
"epoch": 0.031413612565445025,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.05820421874523163,
"fcm_dpo/q_t": 0.5001454949378967,
"grad_norm": 16.085472106933594,
"learning_rate": 1.4583333333333335e-07,
"logits/chosen": 2.1451964378356934,
"logits/rejected": 2.0267412662506104,
"logps/chosen": -288.55841064453125,
"logps/ref_chosen": -288.584716796875,
"logps/ref_rejected": -224.8094482421875,
"logps/rejected": -224.72491455078125,
"loss": 5.5476,
"margin_dpo/margin_mean": -0.05820445716381073,
"margin_dpo/margin_std": 0.8958753943443298,
"step": 15
},
{
"KL/chosen_KL_mean": 0.00731658935546875,
"KL/mean": 0.004071265459060669,
"KL/rejected_KL_mean": 0.00083160400390625,
"KL/std": 0.6132818460464478,
"epoch": 0.033507853403141365,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.006486058235168457,
"fcm_dpo/q_t": 0.4999837875366211,
"grad_norm": 16.61728858947754,
"learning_rate": 1.5624999999999999e-07,
"logits/chosen": 1.8354623317718506,
"logits/rejected": 1.6800379753112793,
"logps/chosen": -283.8358154296875,
"logps/ref_chosen": -283.8431091308594,
"logps/ref_rejected": -248.63645935058594,
"logps/rejected": -248.6356201171875,
"loss": 5.545,
"margin_dpo/margin_mean": 0.006484881043434143,
"margin_dpo/margin_std": 0.8758120536804199,
"step": 16
},
{
"KL/chosen_KL_mean": 0.07244110107421875,
"KL/mean": 0.06645964086055756,
"KL/rejected_KL_mean": 0.06048583984375,
"KL/std": 0.5597188472747803,
"epoch": 0.0356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.011956393718719482,
"fcm_dpo/q_t": 0.49997010827064514,
"grad_norm": 15.15675163269043,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": 1.8342028856277466,
"logits/rejected": 1.786647915840149,
"logps/chosen": -254.56114196777344,
"logps/ref_chosen": -254.6335906982422,
"logps/ref_rejected": -245.3981170654297,
"logps/rejected": -245.33763122558594,
"loss": 5.5448,
"margin_dpo/margin_mean": 0.011956632137298584,
"margin_dpo/margin_std": 0.7229968309402466,
"step": 17
},
{
"KL/chosen_KL_mean": -0.0007228851318359375,
"KL/mean": 0.06241019070148468,
"KL/rejected_KL_mean": 0.1255321502685547,
"KL/std": 0.6303836107254028,
"epoch": 0.03769633507853403,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.12625613808631897,
"fcm_dpo/q_t": 0.5003156065940857,
"grad_norm": 15.067169189453125,
"learning_rate": 1.7708333333333334e-07,
"logits/chosen": 1.8873498439788818,
"logits/rejected": 1.8011493682861328,
"logps/chosen": -287.750244140625,
"logps/ref_chosen": -287.7495422363281,
"logps/ref_rejected": -273.03363037109375,
"logps/rejected": -272.9080810546875,
"loss": 5.5503,
"margin_dpo/margin_mean": -0.12625686824321747,
"margin_dpo/margin_std": 0.896041214466095,
"step": 18
},
{
"KL/chosen_KL_mean": -0.023553848266601562,
"KL/mean": -0.029172401875257492,
"KL/rejected_KL_mean": -0.03479766845703125,
"KL/std": 0.59148770570755,
"epoch": 0.039790575916230364,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.011247612535953522,
"fcm_dpo/q_t": 0.4999718964099884,
"grad_norm": 14.825756072998047,
"learning_rate": 1.875e-07,
"logits/chosen": 1.9977781772613525,
"logits/rejected": 1.9584683179855347,
"logps/chosen": -273.4274597167969,
"logps/ref_chosen": -273.4039001464844,
"logps/ref_rejected": -260.09722900390625,
"logps/rejected": -260.13201904296875,
"loss": 5.5448,
"margin_dpo/margin_mean": 0.011247284710407257,
"margin_dpo/margin_std": 0.773921549320221,
"step": 19
},
{
"KL/chosen_KL_mean": 0.07031822204589844,
"KL/mean": 0.049061864614486694,
"KL/rejected_KL_mean": 0.0278167724609375,
"KL/std": 0.6591533422470093,
"epoch": 0.041884816753926704,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.042512863874435425,
"fcm_dpo/q_t": 0.4998937249183655,
"grad_norm": 15.503080368041992,
"learning_rate": 1.9791666666666664e-07,
"logits/chosen": 1.8373280763626099,
"logits/rejected": 1.7833504676818848,
"logps/chosen": -292.89178466796875,
"logps/ref_chosen": -292.96209716796875,
"logps/ref_rejected": -256.602783203125,
"logps/rejected": -256.574951171875,
"loss": 5.5436,
"margin_dpo/margin_mean": 0.042513057589530945,
"margin_dpo/margin_std": 0.8287696838378906,
"step": 20
},
{
"KL/chosen_KL_mean": -0.009965896606445312,
"KL/mean": 0.0010740011930465698,
"KL/rejected_KL_mean": 0.012109756469726562,
"KL/std": 0.6445425152778625,
"epoch": 0.04397905759162304,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.022067517042160034,
"fcm_dpo/q_t": 0.500055193901062,
"grad_norm": 15.471956253051758,
"learning_rate": 2.0833333333333333e-07,
"logits/chosen": 2.0407376289367676,
"logits/rejected": 1.9655239582061768,
"logps/chosen": -292.3371887207031,
"logps/ref_chosen": -292.3271789550781,
"logps/ref_rejected": -274.20635986328125,
"logps/rejected": -274.19427490234375,
"loss": 5.5461,
"margin_dpo/margin_mean": -0.022068947553634644,
"margin_dpo/margin_std": 0.8329710960388184,
"step": 21
},
{
"KL/chosen_KL_mean": 0.09007644653320312,
"KL/mean": 0.04034734144806862,
"KL/rejected_KL_mean": -0.009380340576171875,
"KL/std": 0.5518816709518433,
"epoch": 0.04607329842931937,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09946237504482269,
"fcm_dpo/q_t": 0.49975138902664185,
"grad_norm": 13.731009483337402,
"learning_rate": 2.1875e-07,
"logits/chosen": 2.002246856689453,
"logits/rejected": 2.0066237449645996,
"logps/chosen": -253.17613220214844,
"logps/ref_chosen": -253.2662353515625,
"logps/ref_rejected": -241.4681396484375,
"logps/rejected": -241.4775390625,
"loss": 5.5413,
"margin_dpo/margin_mean": 0.09946297109127045,
"margin_dpo/margin_std": 0.7452505230903625,
"step": 22
},
{
"KL/chosen_KL_mean": 0.020843505859375,
"KL/mean": 0.05904676765203476,
"KL/rejected_KL_mean": 0.0972442626953125,
"KL/std": 0.7336594462394714,
"epoch": 0.048167539267015703,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.07639683783054352,
"fcm_dpo/q_t": 0.5001909732818604,
"grad_norm": 15.244080543518066,
"learning_rate": 2.2916666666666663e-07,
"logits/chosen": 1.8692981004714966,
"logits/rejected": 1.8475254774093628,
"logps/chosen": -275.4007263183594,
"logps/ref_chosen": -275.4216003417969,
"logps/ref_rejected": -264.44024658203125,
"logps/rejected": -264.3429870605469,
"loss": 5.5484,
"margin_dpo/margin_mean": -0.07639613747596741,
"margin_dpo/margin_std": 1.02485191822052,
"step": 23
},
{
"KL/chosen_KL_mean": -0.03753471374511719,
"KL/mean": -0.006300576031208038,
"KL/rejected_KL_mean": 0.024938583374023438,
"KL/std": 0.6965287923812866,
"epoch": 0.050261780104712044,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06248579919338226,
"fcm_dpo/q_t": 0.5001561641693115,
"grad_norm": 16.480966567993164,
"learning_rate": 2.3958333333333335e-07,
"logits/chosen": 1.9098074436187744,
"logits/rejected": 1.9367527961730957,
"logps/chosen": -293.41510009765625,
"logps/ref_chosen": -293.3775634765625,
"logps/ref_rejected": -301.59014892578125,
"logps/rejected": -301.5652160644531,
"loss": 5.5478,
"margin_dpo/margin_mean": -0.062486037611961365,
"margin_dpo/margin_std": 0.9366015791893005,
"step": 24
},
{
"KL/chosen_KL_mean": 0.07129096984863281,
"KL/mean": 0.03325209766626358,
"KL/rejected_KL_mean": -0.004787445068359375,
"KL/std": 0.5905327200889587,
"epoch": 0.05235602094240838,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07607276737689972,
"fcm_dpo/q_t": 0.49980980157852173,
"grad_norm": 14.5157470703125,
"learning_rate": 2.5e-07,
"logits/chosen": 1.721674919128418,
"logits/rejected": 1.756920576095581,
"logps/chosen": -240.86239624023438,
"logps/ref_chosen": -240.93368530273438,
"logps/ref_rejected": -235.3230743408203,
"logps/rejected": -235.32786560058594,
"loss": 5.5422,
"margin_dpo/margin_mean": 0.07607249915599823,
"margin_dpo/margin_std": 0.806990921497345,
"step": 25
},
{
"KL/chosen_KL_mean": 0.06521224975585938,
"KL/mean": 0.023147452622652054,
"KL/rejected_KL_mean": -0.018918991088867188,
"KL/std": 0.6094150543212891,
"epoch": 0.05445026178010471,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08413281291723251,
"fcm_dpo/q_t": 0.49978965520858765,
"grad_norm": 15.577874183654785,
"learning_rate": 2.604166666666667e-07,
"logits/chosen": 1.7217767238616943,
"logits/rejected": 1.7037371397018433,
"logps/chosen": -273.53009033203125,
"logps/ref_chosen": -273.5953369140625,
"logps/ref_rejected": -234.49148559570312,
"logps/rejected": -234.51040649414062,
"loss": 5.5419,
"margin_dpo/margin_mean": 0.08413246273994446,
"margin_dpo/margin_std": 0.7173627614974976,
"step": 26
},
{
"KL/chosen_KL_mean": 0.07335662841796875,
"KL/mean": 0.025179728865623474,
"KL/rejected_KL_mean": -0.022989273071289062,
"KL/std": 0.6592622995376587,
"epoch": 0.05654450261780105,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09633858501911163,
"fcm_dpo/q_t": 0.49975913763046265,
"grad_norm": 15.121481895446777,
"learning_rate": 2.708333333333333e-07,
"logits/chosen": 2.2555246353149414,
"logits/rejected": 2.032055616378784,
"logps/chosen": -314.1905212402344,
"logps/ref_chosen": -314.26385498046875,
"logps/ref_rejected": -242.06735229492188,
"logps/rejected": -242.09031677246094,
"loss": 5.5414,
"margin_dpo/margin_mean": 0.09633757174015045,
"margin_dpo/margin_std": 0.8887685537338257,
"step": 27
},
{
"KL/chosen_KL_mean": 0.1266918182373047,
"KL/mean": 0.03975718468427658,
"KL/rejected_KL_mean": -0.047176361083984375,
"KL/std": 0.581898033618927,
"epoch": 0.05863874345549738,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1738647222518921,
"fcm_dpo/q_t": 0.49956533312797546,
"grad_norm": 15.378426551818848,
"learning_rate": 2.8125e-07,
"logits/chosen": 2.039022922515869,
"logits/rejected": 2.04673171043396,
"logps/chosen": -240.6121368408203,
"logps/ref_chosen": -240.7388153076172,
"logps/ref_rejected": -237.8331298828125,
"logps/rejected": -237.8803253173828,
"loss": 5.5383,
"margin_dpo/margin_mean": 0.1738651543855667,
"margin_dpo/margin_std": 0.8048579692840576,
"step": 28
},
{
"KL/chosen_KL_mean": 0.07528114318847656,
"KL/mean": 0.04480874538421631,
"KL/rejected_KL_mean": 0.014341354370117188,
"KL/std": 0.6803877949714661,
"epoch": 0.060732984293193716,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06094604730606079,
"fcm_dpo/q_t": 0.4998476505279541,
"grad_norm": 14.76402759552002,
"learning_rate": 2.916666666666667e-07,
"logits/chosen": 1.9146788120269775,
"logits/rejected": 1.794429898262024,
"logps/chosen": -307.754150390625,
"logps/ref_chosen": -307.82940673828125,
"logps/ref_rejected": -281.8109130859375,
"logps/rejected": -281.79656982421875,
"loss": 5.5428,
"margin_dpo/margin_mean": 0.06094469130039215,
"margin_dpo/margin_std": 0.9543280005455017,
"step": 29
},
{
"KL/chosen_KL_mean": 0.03249931335449219,
"KL/mean": 0.02733471244573593,
"KL/rejected_KL_mean": 0.022161483764648438,
"KL/std": 0.5754698514938354,
"epoch": 0.06282722513089005,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.010352656245231628,
"fcm_dpo/q_t": 0.4999741315841675,
"grad_norm": 14.39789867401123,
"learning_rate": 3.020833333333333e-07,
"logits/chosen": 1.7222105264663696,
"logits/rejected": 1.7153657674789429,
"logps/chosen": -247.9812774658203,
"logps/ref_chosen": -248.01376342773438,
"logps/ref_rejected": -259.0014953613281,
"logps/rejected": -258.9793395996094,
"loss": 5.5448,
"margin_dpo/margin_mean": 0.010352000594139099,
"margin_dpo/margin_std": 0.806208610534668,
"step": 30
},
{
"KL/chosen_KL_mean": 0.10100173950195312,
"KL/mean": 0.04322759807109833,
"KL/rejected_KL_mean": -0.014553070068359375,
"KL/std": 0.5859971046447754,
"epoch": 0.06492146596858639,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11553975939750671,
"fcm_dpo/q_t": 0.49971115589141846,
"grad_norm": 13.664345741271973,
"learning_rate": 3.1249999999999997e-07,
"logits/chosen": 1.9450936317443848,
"logits/rejected": 1.8830111026763916,
"logps/chosen": -239.96180725097656,
"logps/ref_chosen": -240.0628204345703,
"logps/ref_rejected": -201.91021728515625,
"logps/rejected": -201.9247589111328,
"loss": 5.5406,
"margin_dpo/margin_mean": 0.11553928256034851,
"margin_dpo/margin_std": 0.751106321811676,
"step": 31
},
{
"KL/chosen_KL_mean": 0.15114593505859375,
"KL/mean": 0.07302998751401901,
"KL/rejected_KL_mean": -0.005084991455078125,
"KL/std": 0.657054603099823,
"epoch": 0.06701570680628273,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1562250405550003,
"fcm_dpo/q_t": 0.49960944056510925,
"grad_norm": 15.576884269714355,
"learning_rate": 3.2291666666666666e-07,
"logits/chosen": 1.918774962425232,
"logits/rejected": 1.8419352769851685,
"logps/chosen": -293.9810791015625,
"logps/ref_chosen": -294.1322326660156,
"logps/ref_rejected": -265.8150634765625,
"logps/rejected": -265.8201599121094,
"loss": 5.539,
"margin_dpo/margin_mean": 0.15622559189796448,
"margin_dpo/margin_std": 0.8563711047172546,
"step": 32
},
{
"KL/chosen_KL_mean": 0.24457740783691406,
"KL/mean": 0.1883525848388672,
"KL/rejected_KL_mean": 0.13211822509765625,
"KL/std": 0.6640927195549011,
"epoch": 0.06910994764397906,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11245375871658325,
"fcm_dpo/q_t": 0.4997188448905945,
"grad_norm": 14.962160110473633,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": 1.9812177419662476,
"logits/rejected": 1.9455798864364624,
"logps/chosen": -275.4105224609375,
"logps/ref_chosen": -275.6551208496094,
"logps/ref_rejected": -253.5972137451172,
"logps/rejected": -253.465087890625,
"loss": 5.5408,
"margin_dpo/margin_mean": 0.11245402693748474,
"margin_dpo/margin_std": 0.9471386075019836,
"step": 33
},
{
"KL/chosen_KL_mean": 0.24161911010742188,
"KL/mean": 0.1785203069448471,
"KL/rejected_KL_mean": 0.11542320251464844,
"KL/std": 0.7460881471633911,
"epoch": 0.0712041884816754,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12618765234947205,
"fcm_dpo/q_t": 0.49968454241752625,
"grad_norm": 17.706989288330078,
"learning_rate": 3.4375e-07,
"logits/chosen": 2.1137003898620605,
"logits/rejected": 2.008443593978882,
"logps/chosen": -332.7284851074219,
"logps/ref_chosen": -332.9701232910156,
"logps/ref_rejected": -278.2213439941406,
"logps/rejected": -278.10595703125,
"loss": 5.5402,
"margin_dpo/margin_mean": 0.12618720531463623,
"margin_dpo/margin_std": 0.9816871881484985,
"step": 34
},
{
"KL/chosen_KL_mean": 0.1520099639892578,
"KL/mean": 0.13822707533836365,
"KL/rejected_KL_mean": 0.12445068359375,
"KL/std": 0.6340049505233765,
"epoch": 0.07329842931937172,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02756369858980179,
"fcm_dpo/q_t": 0.49993109703063965,
"grad_norm": 13.682244300842285,
"learning_rate": 3.541666666666667e-07,
"logits/chosen": 1.7967126369476318,
"logits/rejected": 1.8555289506912231,
"logps/chosen": -258.27685546875,
"logps/ref_chosen": -258.42889404296875,
"logps/ref_rejected": -259.5119934082031,
"logps/rejected": -259.3875427246094,
"loss": 5.5442,
"margin_dpo/margin_mean": 0.02756364643573761,
"margin_dpo/margin_std": 0.8654166460037231,
"step": 35
},
{
"KL/chosen_KL_mean": 0.4261493682861328,
"KL/mean": 0.22535449266433716,
"KL/rejected_KL_mean": 0.024557113647460938,
"KL/std": 0.79774010181427,
"epoch": 0.07539267015706806,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.40159767866134644,
"fcm_dpo/q_t": 0.4989960491657257,
"grad_norm": 17.678897857666016,
"learning_rate": 3.645833333333333e-07,
"logits/chosen": 1.9235519170761108,
"logits/rejected": 1.8702714443206787,
"logps/chosen": -290.1932067871094,
"logps/ref_chosen": -290.6193542480469,
"logps/ref_rejected": -280.2503967285156,
"logps/rejected": -280.2258605957031,
"loss": 5.5292,
"margin_dpo/margin_mean": 0.40159714221954346,
"margin_dpo/margin_std": 1.0169625282287598,
"step": 36
},
{
"KL/chosen_KL_mean": 0.29315185546875,
"KL/mean": 0.20537252724170685,
"KL/rejected_KL_mean": 0.11758995056152344,
"KL/std": 0.7490401864051819,
"epoch": 0.0774869109947644,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1755593866109848,
"fcm_dpo/q_t": 0.4995611608028412,
"grad_norm": 15.22133731842041,
"learning_rate": 3.75e-07,
"logits/chosen": 1.8605339527130127,
"logits/rejected": 1.8499176502227783,
"logps/chosen": -303.2796630859375,
"logps/ref_chosen": -303.57281494140625,
"logps/ref_rejected": -294.56878662109375,
"logps/rejected": -294.451171875,
"loss": 5.5383,
"margin_dpo/margin_mean": 0.1755588799715042,
"margin_dpo/margin_std": 1.0125519037246704,
"step": 37
},
{
"KL/chosen_KL_mean": 0.3296089172363281,
"KL/mean": 0.2202475368976593,
"KL/rejected_KL_mean": 0.1108856201171875,
"KL/std": 0.763922393321991,
"epoch": 0.07958115183246073,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.21871574223041534,
"fcm_dpo/q_t": 0.49945324659347534,
"grad_norm": 16.027008056640625,
"learning_rate": 3.8541666666666665e-07,
"logits/chosen": 1.8954304456710815,
"logits/rejected": 1.9512522220611572,
"logps/chosen": -272.223876953125,
"logps/ref_chosen": -272.5534973144531,
"logps/ref_rejected": -266.9136047363281,
"logps/rejected": -266.802734375,
"loss": 5.5365,
"margin_dpo/margin_mean": 0.21871612966060638,
"margin_dpo/margin_std": 0.9921301603317261,
"step": 38
},
{
"KL/chosen_KL_mean": 0.5544986724853516,
"KL/mean": 0.3141814172267914,
"KL/rejected_KL_mean": 0.07386398315429688,
"KL/std": 0.77060467004776,
"epoch": 0.08167539267015707,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.48063284158706665,
"fcm_dpo/q_t": 0.4987984299659729,
"grad_norm": 15.96513557434082,
"learning_rate": 3.958333333333333e-07,
"logits/chosen": 1.7163671255111694,
"logits/rejected": 1.6469464302062988,
"logps/chosen": -264.83203125,
"logps/ref_chosen": -265.38653564453125,
"logps/ref_rejected": -244.34774780273438,
"logps/rejected": -244.27392578125,
"loss": 5.5261,
"margin_dpo/margin_mean": 0.48063212633132935,
"margin_dpo/margin_std": 0.8729540109634399,
"step": 39
},
{
"KL/chosen_KL_mean": 0.4694805145263672,
"KL/mean": 0.27980729937553406,
"KL/rejected_KL_mean": 0.0901336669921875,
"KL/std": 0.7455384135246277,
"epoch": 0.08376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.37933290004730225,
"fcm_dpo/q_t": 0.4990517199039459,
"grad_norm": 14.94265365600586,
"learning_rate": 4.0625e-07,
"logits/chosen": 1.7584089040756226,
"logits/rejected": 1.744396448135376,
"logps/chosen": -274.42431640625,
"logps/ref_chosen": -274.893798828125,
"logps/ref_rejected": -244.3120880126953,
"logps/rejected": -244.22195434570312,
"loss": 5.5301,
"margin_dpo/margin_mean": 0.37933290004730225,
"margin_dpo/margin_std": 0.9794178605079651,
"step": 40
},
{
"KL/chosen_KL_mean": 0.383026123046875,
"KL/mean": 0.278035968542099,
"KL/rejected_KL_mean": 0.1730499267578125,
"KL/std": 0.7958909869194031,
"epoch": 0.08586387434554973,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20995977520942688,
"fcm_dpo/q_t": 0.4994751214981079,
"grad_norm": 15.472199440002441,
"learning_rate": 4.1666666666666667e-07,
"logits/chosen": 2.015687942504883,
"logits/rejected": 1.9452970027923584,
"logps/chosen": -284.4161071777344,
"logps/ref_chosen": -284.7991027832031,
"logps/ref_rejected": -248.8882598876953,
"logps/rejected": -248.71519470214844,
"loss": 5.5369,
"margin_dpo/margin_mean": 0.2099606841802597,
"margin_dpo/margin_std": 1.0590481758117676,
"step": 41
},
{
"KL/chosen_KL_mean": 0.49579811096191406,
"KL/mean": 0.3066490590572357,
"KL/rejected_KL_mean": 0.11749649047851562,
"KL/std": 0.7752858400344849,
"epoch": 0.08795811518324607,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3783026337623596,
"fcm_dpo/q_t": 0.4990542531013489,
"grad_norm": 14.61644172668457,
"learning_rate": 4.270833333333333e-07,
"logits/chosen": 1.8188252449035645,
"logits/rejected": 1.8732718229293823,
"logps/chosen": -245.3438720703125,
"logps/ref_chosen": -245.83966064453125,
"logps/ref_rejected": -259.2870788574219,
"logps/rejected": -259.1695861816406,
"loss": 5.5302,
"margin_dpo/margin_mean": 0.3783027231693268,
"margin_dpo/margin_std": 1.054323434829712,
"step": 42
},
{
"KL/chosen_KL_mean": 0.4672527313232422,
"KL/mean": 0.31816428899765015,
"KL/rejected_KL_mean": 0.1690826416015625,
"KL/std": 0.8266512155532837,
"epoch": 0.09005235602094241,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.29816895723342896,
"fcm_dpo/q_t": 0.49925461411476135,
"grad_norm": 15.704439163208008,
"learning_rate": 4.375e-07,
"logits/chosen": 1.7950459718704224,
"logits/rejected": 1.7692830562591553,
"logps/chosen": -307.9302978515625,
"logps/ref_chosen": -308.39752197265625,
"logps/ref_rejected": -270.3797912597656,
"logps/rejected": -270.2107238769531,
"loss": 5.5334,
"margin_dpo/margin_mean": 0.2981690466403961,
"margin_dpo/margin_std": 1.0226835012435913,
"step": 43
},
{
"KL/chosen_KL_mean": 0.7249507904052734,
"KL/mean": 0.47041821479797363,
"KL/rejected_KL_mean": 0.2158832550048828,
"KL/std": 0.8857836127281189,
"epoch": 0.09214659685863874,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5090519785881042,
"fcm_dpo/q_t": 0.49872738122940063,
"grad_norm": 15.815662384033203,
"learning_rate": 4.479166666666667e-07,
"logits/chosen": 1.9265743494033813,
"logits/rejected": 1.9330271482467651,
"logps/chosen": -274.63677978515625,
"logps/ref_chosen": -275.36175537109375,
"logps/ref_rejected": -279.7540588378906,
"logps/rejected": -279.5381774902344,
"loss": 5.525,
"margin_dpo/margin_mean": 0.5090534090995789,
"margin_dpo/margin_std": 1.1619629859924316,
"step": 44
},
{
"KL/chosen_KL_mean": 0.7300357818603516,
"KL/mean": 0.4381192922592163,
"KL/rejected_KL_mean": 0.1461963653564453,
"KL/std": 1.0366795063018799,
"epoch": 0.09424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.583842396736145,
"fcm_dpo/q_t": 0.4985404908657074,
"grad_norm": 16.86652183532715,
"learning_rate": 4.5833333333333327e-07,
"logits/chosen": 1.8563649654388428,
"logits/rejected": 1.8279879093170166,
"logps/chosen": -314.39013671875,
"logps/ref_chosen": -315.1202087402344,
"logps/ref_rejected": -290.34210205078125,
"logps/rejected": -290.1958923339844,
"loss": 5.522,
"margin_dpo/margin_mean": 0.5838421583175659,
"margin_dpo/margin_std": 1.3088961839675903,
"step": 45
},
{
"KL/chosen_KL_mean": 0.8227634429931641,
"KL/mean": 0.49031269550323486,
"KL/rejected_KL_mean": 0.15786361694335938,
"KL/std": 1.0303242206573486,
"epoch": 0.09633507853403141,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6648975014686584,
"fcm_dpo/q_t": 0.49833783507347107,
"grad_norm": 17.709739685058594,
"learning_rate": 4.6874999999999996e-07,
"logits/chosen": 1.6669872999191284,
"logits/rejected": 1.668416142463684,
"logps/chosen": -292.1178283691406,
"logps/ref_chosen": -292.94061279296875,
"logps/ref_rejected": -264.0979309082031,
"logps/rejected": -263.9400634765625,
"loss": 5.5188,
"margin_dpo/margin_mean": 0.6648980975151062,
"margin_dpo/margin_std": 1.3021197319030762,
"step": 46
},
{
"KL/chosen_KL_mean": 0.6330909729003906,
"KL/mean": 0.4338260293006897,
"KL/rejected_KL_mean": 0.23456192016601562,
"KL/std": 1.0144609212875366,
"epoch": 0.09842931937172775,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3985220491886139,
"fcm_dpo/q_t": 0.4990037679672241,
"grad_norm": 14.713279724121094,
"learning_rate": 4.791666666666667e-07,
"logits/chosen": 1.8358039855957031,
"logits/rejected": 1.9415535926818848,
"logps/chosen": -254.21958923339844,
"logps/ref_chosen": -254.85269165039062,
"logps/ref_rejected": -255.88380432128906,
"logps/rejected": -255.6492156982422,
"loss": 5.5295,
"margin_dpo/margin_mean": 0.3985212445259094,
"margin_dpo/margin_std": 1.4352999925613403,
"step": 47
},
{
"KL/chosen_KL_mean": 0.7173347473144531,
"KL/mean": 0.39810723066329956,
"KL/rejected_KL_mean": 0.07886505126953125,
"KL/std": 1.0975290536880493,
"epoch": 0.10052356020942409,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.638471245765686,
"fcm_dpo/q_t": 0.49840396642684937,
"grad_norm": 14.869832038879395,
"learning_rate": 4.895833333333333e-07,
"logits/chosen": 1.8391691446304321,
"logits/rejected": 1.82039213180542,
"logps/chosen": -261.7904052734375,
"logps/ref_chosen": -262.5077209472656,
"logps/ref_rejected": -239.25320434570312,
"logps/rejected": -239.17434692382812,
"loss": 5.5199,
"margin_dpo/margin_mean": 0.6384709477424622,
"margin_dpo/margin_std": 1.3757047653198242,
"step": 48
},
{
"KL/chosen_KL_mean": 0.9507026672363281,
"KL/mean": 0.6429872512817383,
"KL/rejected_KL_mean": 0.33528709411621094,
"KL/std": 1.1882615089416504,
"epoch": 0.10261780104712041,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.61541748046875,
"fcm_dpo/q_t": 0.49846160411834717,
"grad_norm": 15.200157165527344,
"learning_rate": 5e-07,
"logits/chosen": 1.7383028268814087,
"logits/rejected": 1.8577663898468018,
"logps/chosen": -248.57717895507812,
"logps/ref_chosen": -249.5278778076172,
"logps/ref_rejected": -254.6465301513672,
"logps/rejected": -254.311279296875,
"loss": 5.5209,
"margin_dpo/margin_mean": 0.615418553352356,
"margin_dpo/margin_std": 1.670445442199707,
"step": 49
},
{
"KL/chosen_KL_mean": 0.9584369659423828,
"KL/mean": 0.714724063873291,
"KL/rejected_KL_mean": 0.4709930419921875,
"KL/std": 1.2188169956207275,
"epoch": 0.10471204188481675,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.48745203018188477,
"fcm_dpo/q_t": 0.4987815022468567,
"grad_norm": 14.659168243408203,
"learning_rate": 4.999932966293553e-07,
"logits/chosen": 1.902108907699585,
"logits/rejected": 1.9085142612457275,
"logps/chosen": -291.54327392578125,
"logps/ref_chosen": -292.501708984375,
"logps/ref_rejected": -294.6496887207031,
"logps/rejected": -294.1787109375,
"loss": 5.526,
"margin_dpo/margin_mean": 0.48745256662368774,
"margin_dpo/margin_std": 1.5779855251312256,
"step": 50
},
{
"KL/chosen_KL_mean": 0.9927196502685547,
"KL/mean": 0.6409055590629578,
"KL/rejected_KL_mean": 0.28908538818359375,
"KL/std": 1.3663586378097534,
"epoch": 0.1068062827225131,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7036239504814148,
"fcm_dpo/q_t": 0.498241126537323,
"grad_norm": 14.71286678314209,
"learning_rate": 4.999731868769026e-07,
"logits/chosen": 1.5570727586746216,
"logits/rejected": 1.6826603412628174,
"logps/chosen": -258.869140625,
"logps/ref_chosen": -259.8619079589844,
"logps/ref_rejected": -293.1994934082031,
"logps/rejected": -292.910400390625,
"loss": 5.5175,
"margin_dpo/margin_mean": 0.703624963760376,
"margin_dpo/margin_std": 1.8807909488677979,
"step": 51
},
{
"KL/chosen_KL_mean": 1.238037109375,
"KL/mean": 0.7984499335289001,
"KL/rejected_KL_mean": 0.35886383056640625,
"KL/std": 1.439669132232666,
"epoch": 0.10890052356020942,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8791704773902893,
"fcm_dpo/q_t": 0.4978024363517761,
"grad_norm": 17.054243087768555,
"learning_rate": 4.99939671821067e-07,
"logits/chosen": 2.004744052886963,
"logits/rejected": 1.925959587097168,
"logps/chosen": -297.1872863769531,
"logps/ref_chosen": -298.42529296875,
"logps/ref_rejected": -278.5128479003906,
"logps/rejected": -278.1539611816406,
"loss": 5.5105,
"margin_dpo/margin_mean": 0.8791694045066833,
"margin_dpo/margin_std": 1.9268198013305664,
"step": 52
},
{
"KL/chosen_KL_mean": 1.1849250793457031,
"KL/mean": 0.8460350632667542,
"KL/rejected_KL_mean": 0.5071334838867188,
"KL/std": 1.4284039735794067,
"epoch": 0.11099476439790576,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6777990460395813,
"fcm_dpo/q_t": 0.49830570816993713,
"grad_norm": 15.994030952453613,
"learning_rate": 4.998927532591591e-07,
"logits/chosen": 1.5852761268615723,
"logits/rejected": 1.6465017795562744,
"logps/chosen": -274.1235656738281,
"logps/ref_chosen": -275.3085021972656,
"logps/ref_rejected": -267.97381591796875,
"logps/rejected": -267.4667053222656,
"loss": 5.5185,
"margin_dpo/margin_mean": 0.6777995824813843,
"margin_dpo/margin_std": 1.7592477798461914,
"step": 53
},
{
"KL/chosen_KL_mean": 1.2797298431396484,
"KL/mean": 0.9198053479194641,
"KL/rejected_KL_mean": 0.5598831176757812,
"KL/std": 1.4647531509399414,
"epoch": 0.1130890052356021,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.719858705997467,
"fcm_dpo/q_t": 0.4982006251811981,
"grad_norm": 14.507118225097656,
"learning_rate": 4.998324337072792e-07,
"logits/chosen": 1.733615756034851,
"logits/rejected": 1.765891432762146,
"logps/chosen": -296.8675842285156,
"logps/ref_chosen": -298.1473388671875,
"logps/ref_rejected": -269.7539367675781,
"logps/rejected": -269.19403076171875,
"loss": 5.5169,
"margin_dpo/margin_mean": 0.7198596000671387,
"margin_dpo/margin_std": 1.952254295349121,
"step": 54
},
{
"KL/chosen_KL_mean": 1.1664409637451172,
"KL/mean": 0.8456940650939941,
"KL/rejected_KL_mean": 0.5249443054199219,
"KL/std": 1.3166602849960327,
"epoch": 0.11518324607329843,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6415013074874878,
"fcm_dpo/q_t": 0.4983964264392853,
"grad_norm": 12.762544631958008,
"learning_rate": 4.997587164001815e-07,
"logits/chosen": 1.521553874015808,
"logits/rejected": 1.5612958669662476,
"logps/chosen": -215.5835418701172,
"logps/ref_chosen": -216.74998474121094,
"logps/ref_rejected": -202.7234649658203,
"logps/rejected": -202.19850158691406,
"loss": 5.5199,
"margin_dpo/margin_mean": 0.6415011882781982,
"margin_dpo/margin_std": 1.7675597667694092,
"step": 55
},
{
"KL/chosen_KL_mean": 1.2932567596435547,
"KL/mean": 0.7695432305335999,
"KL/rejected_KL_mean": 0.2458324432373047,
"KL/std": 1.81564199924469,
"epoch": 0.11727748691099477,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0474203824996948,
"fcm_dpo/q_t": 0.4973820149898529,
"grad_norm": 14.452275276184082,
"learning_rate": 4.996716052911017e-07,
"logits/chosen": 1.5290578603744507,
"logits/rejected": 1.615087866783142,
"logps/chosen": -242.39816284179688,
"logps/ref_chosen": -243.69140625,
"logps/ref_rejected": -234.970947265625,
"logps/rejected": -234.72509765625,
"loss": 5.504,
"margin_dpo/margin_mean": 1.0474200248718262,
"margin_dpo/margin_std": 2.32503604888916,
"step": 56
},
{
"KL/chosen_KL_mean": 1.7666187286376953,
"KL/mean": 1.0542978048324585,
"KL/rejected_KL_mean": 0.34197998046875,
"KL/std": 1.7339706420898438,
"epoch": 0.1193717277486911,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4246318340301514,
"fcm_dpo/q_t": 0.4964395761489868,
"grad_norm": 17.599245071411133,
"learning_rate": 4.99571105051544e-07,
"logits/chosen": 2.0374603271484375,
"logits/rejected": 1.935616135597229,
"logps/chosen": -308.9797058105469,
"logps/ref_chosen": -310.746337890625,
"logps/ref_rejected": -270.6407165527344,
"logps/rejected": -270.2987060546875,
"loss": 5.489,
"margin_dpo/margin_mean": 1.424631118774414,
"margin_dpo/margin_std": 2.1392083168029785,
"step": 57
},
{
"KL/chosen_KL_mean": 1.3137035369873047,
"KL/mean": 0.8551568388938904,
"KL/rejected_KL_mean": 0.3966217041015625,
"KL/std": 1.615386962890625,
"epoch": 0.12146596858638743,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.9170898199081421,
"fcm_dpo/q_t": 0.49770745635032654,
"grad_norm": 14.97243881225586,
"learning_rate": 4.994572210710314e-07,
"logits/chosen": 1.744799256324768,
"logits/rejected": 1.7298442125320435,
"logps/chosen": -257.4992980957031,
"logps/ref_chosen": -258.8130187988281,
"logps/ref_rejected": -268.2101745605469,
"logps/rejected": -267.8135681152344,
"loss": 5.509,
"margin_dpo/margin_mean": 0.917088508605957,
"margin_dpo/margin_std": 1.958569884300232,
"step": 58
},
{
"KL/chosen_KL_mean": 1.4245281219482422,
"KL/mean": 1.0160744190216064,
"KL/rejected_KL_mean": 0.6076259613037109,
"KL/std": 1.923659086227417,
"epoch": 0.12356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8169017434120178,
"fcm_dpo/q_t": 0.4979582726955414,
"grad_norm": 15.254434585571289,
"learning_rate": 4.993299594568162e-07,
"logits/chosen": 1.7835378646850586,
"logits/rejected": 1.7226471900939941,
"logps/chosen": -275.8169250488281,
"logps/ref_chosen": -277.2414245605469,
"logps/ref_rejected": -259.13519287109375,
"logps/rejected": -258.5275573730469,
"loss": 5.5133,
"margin_dpo/margin_mean": 0.8169001340866089,
"margin_dpo/margin_std": 2.5672593116760254,
"step": 59
},
{
"KL/chosen_KL_mean": 1.5428886413574219,
"KL/mean": 0.9007205367088318,
"KL/rejected_KL_mean": 0.2585639953613281,
"KL/std": 1.732983112335205,
"epoch": 0.1256544502617801,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2843198776245117,
"fcm_dpo/q_t": 0.496789813041687,
"grad_norm": 14.663458824157715,
"learning_rate": 4.991893270335525e-07,
"logits/chosen": 1.8974318504333496,
"logits/rejected": 1.943682312965393,
"logps/chosen": -249.80296325683594,
"logps/ref_chosen": -251.34585571289062,
"logps/ref_rejected": -251.4120635986328,
"logps/rejected": -251.1534881591797,
"loss": 5.4946,
"margin_dpo/margin_mean": 1.2843196392059326,
"margin_dpo/margin_std": 2.2694783210754395,
"step": 60
},
{
"KL/chosen_KL_mean": 1.6408882141113281,
"KL/mean": 1.103342056274414,
"KL/rejected_KL_mean": 0.5658016204833984,
"KL/std": 1.8047294616699219,
"epoch": 0.12774869109947645,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.075092077255249,
"fcm_dpo/q_t": 0.49731284379959106,
"grad_norm": 15.019508361816406,
"learning_rate": 4.990353313429303e-07,
"logits/chosen": 1.7713136672973633,
"logits/rejected": 1.725186824798584,
"logps/chosen": -267.34173583984375,
"logps/ref_chosen": -268.98260498046875,
"logps/ref_rejected": -249.34474182128906,
"logps/rejected": -248.7789306640625,
"loss": 5.503,
"margin_dpo/margin_mean": 1.0750921964645386,
"margin_dpo/margin_std": 2.428819417953491,
"step": 61
},
{
"KL/chosen_KL_mean": 2.0969371795654297,
"KL/mean": 1.4681521654129028,
"KL/rejected_KL_mean": 0.8393688201904297,
"KL/std": 2.045714855194092,
"epoch": 0.12984293193717278,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2575669288635254,
"fcm_dpo/q_t": 0.4968569278717041,
"grad_norm": 14.69105052947998,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": 1.9424972534179688,
"logits/rejected": 1.9180412292480469,
"logps/chosen": -262.1065673828125,
"logps/ref_chosen": -264.2034912109375,
"logps/ref_rejected": -260.2911376953125,
"logps/rejected": -259.4517822265625,
"loss": 5.4958,
"margin_dpo/margin_mean": 1.2575657367706299,
"margin_dpo/margin_std": 2.6261508464813232,
"step": 62
},
{
"KL/chosen_KL_mean": 1.8828144073486328,
"KL/mean": 1.2346196174621582,
"KL/rejected_KL_mean": 0.5864162445068359,
"KL/std": 2.0843284130096436,
"epoch": 0.1319371727748691,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2964017391204834,
"fcm_dpo/q_t": 0.49675989151000977,
"grad_norm": 14.768925666809082,
"learning_rate": 4.986872839090852e-07,
"logits/chosen": 1.7671276330947876,
"logits/rejected": 1.7914533615112305,
"logps/chosen": -269.5854797363281,
"logps/ref_chosen": -271.4682922363281,
"logps/ref_rejected": -273.0804748535156,
"logps/rejected": -272.4940490722656,
"loss": 5.4943,
"margin_dpo/margin_mean": 1.2964012622833252,
"margin_dpo/margin_std": 2.744537591934204,
"step": 63
},
{
"KL/chosen_KL_mean": 2.306966781616211,
"KL/mean": 1.525729775428772,
"KL/rejected_KL_mean": 0.7444896697998047,
"KL/std": 2.388784170150757,
"epoch": 0.13403141361256546,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5624661445617676,
"fcm_dpo/q_t": 0.4960950016975403,
"grad_norm": 15.698229789733887,
"learning_rate": 4.9849325083059e-07,
"logits/chosen": 1.7998967170715332,
"logits/rejected": 1.8028191328048706,
"logps/chosen": -271.29644775390625,
"logps/ref_chosen": -273.6033935546875,
"logps/ref_rejected": -259.54107666015625,
"logps/rejected": -258.79656982421875,
"loss": 5.484,
"margin_dpo/margin_mean": 1.5624663829803467,
"margin_dpo/margin_std": 3.152646064758301,
"step": 64
},
{
"KL/chosen_KL_mean": 2.3221817016601562,
"KL/mean": 1.769188404083252,
"KL/rejected_KL_mean": 1.2161998748779297,
"KL/std": 2.1231086254119873,
"epoch": 0.13612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1059739589691162,
"fcm_dpo/q_t": 0.4972355365753174,
"grad_norm": 14.570204734802246,
"learning_rate": 4.982858918131906e-07,
"logits/chosen": 1.8644115924835205,
"logits/rejected": 1.962466835975647,
"logps/chosen": -256.19549560546875,
"logps/ref_chosen": -258.5176696777344,
"logps/ref_rejected": -264.9944763183594,
"logps/rejected": -263.77825927734375,
"loss": 5.5019,
"margin_dpo/margin_mean": 1.1059731245040894,
"margin_dpo/margin_std": 2.760225534439087,
"step": 65
},
{
"KL/chosen_KL_mean": 2.2520103454589844,
"KL/mean": 1.5462473630905151,
"KL/rejected_KL_mean": 0.8404960632324219,
"KL/std": 2.6777021884918213,
"epoch": 0.1382198952879581,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.411508321762085,
"fcm_dpo/q_t": 0.49647241830825806,
"grad_norm": 14.818696022033691,
"learning_rate": 4.980652179769217e-07,
"logits/chosen": 1.9537615776062012,
"logits/rejected": 2.05753493309021,
"logps/chosen": -265.2574157714844,
"logps/ref_chosen": -267.5094299316406,
"logps/ref_rejected": -277.03228759765625,
"logps/rejected": -276.1917724609375,
"loss": 5.4903,
"margin_dpo/margin_mean": 1.4115078449249268,
"margin_dpo/margin_std": 3.628791332244873,
"step": 66
},
{
"KL/chosen_KL_mean": 2.579130172729492,
"KL/mean": 1.9668623208999634,
"KL/rejected_KL_mean": 1.354593276977539,
"KL/std": 2.6205031871795654,
"epoch": 0.14031413612565444,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.224534511566162,
"fcm_dpo/q_t": 0.49693942070007324,
"grad_norm": 14.398809432983398,
"learning_rate": 4.978312411558517e-07,
"logits/chosen": 1.6984463930130005,
"logits/rejected": 1.705766201019287,
"logps/chosen": -254.8858184814453,
"logps/ref_chosen": -257.4649353027344,
"logps/ref_rejected": -247.29161071777344,
"logps/rejected": -245.93704223632812,
"loss": 5.4975,
"margin_dpo/margin_mean": 1.2245352268218994,
"margin_dpo/margin_std": 3.272260904312134,
"step": 67
},
{
"KL/chosen_KL_mean": 2.4789600372314453,
"KL/mean": 1.6513326168060303,
"KL/rejected_KL_mean": 0.8237094879150391,
"KL/std": 2.9045519828796387,
"epoch": 0.1424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6552443504333496,
"fcm_dpo/q_t": 0.4958641231060028,
"grad_norm": 14.775374412536621,
"learning_rate": 4.975839738974473e-07,
"logits/chosen": 1.700913667678833,
"logits/rejected": 1.6477922201156616,
"logps/chosen": -288.0052185058594,
"logps/ref_chosen": -290.4841613769531,
"logps/ref_rejected": -260.8398132324219,
"logps/rejected": -260.01611328125,
"loss": 5.4809,
"margin_dpo/margin_mean": 1.6552444696426392,
"margin_dpo/margin_std": 3.8591935634613037,
"step": 68
},
{
"KL/chosen_KL_mean": 2.830690383911133,
"KL/mean": 1.7516111135482788,
"KL/rejected_KL_mean": 0.6725292205810547,
"KL/std": 2.9886691570281982,
"epoch": 0.14450261780104712,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.1581647396087646,
"fcm_dpo/q_t": 0.4946069121360779,
"grad_norm": 15.165844917297363,
"learning_rate": 4.97323429461901e-07,
"logits/chosen": 1.8554754257202148,
"logits/rejected": 1.893682599067688,
"logps/chosen": -254.806884765625,
"logps/ref_chosen": -257.6375732421875,
"logps/ref_rejected": -235.6666717529297,
"logps/rejected": -234.994140625,
"loss": 5.4608,
"margin_dpo/margin_mean": 2.1581649780273438,
"margin_dpo/margin_std": 3.7994115352630615,
"step": 69
},
{
"KL/chosen_KL_mean": 3.446706771850586,
"KL/mean": 2.358335494995117,
"KL/rejected_KL_mean": 1.2699604034423828,
"KL/std": 3.0744781494140625,
"epoch": 0.14659685863874344,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.176751136779785,
"fcm_dpo/q_t": 0.4945610463619232,
"grad_norm": 15.59477424621582,
"learning_rate": 4.970496218214204e-07,
"logits/chosen": 1.6959162950515747,
"logits/rejected": 1.6481292247772217,
"logps/chosen": -284.01849365234375,
"logps/ref_chosen": -287.4651794433594,
"logps/ref_rejected": -253.4770965576172,
"logps/rejected": -252.20713806152344,
"loss": 5.4602,
"margin_dpo/margin_mean": 2.1767518520355225,
"margin_dpo/margin_std": 3.827840566635132,
"step": 70
},
{
"KL/chosen_KL_mean": 3.4551048278808594,
"KL/mean": 2.5311903953552246,
"KL/rejected_KL_mean": 1.6072845458984375,
"KL/std": 3.245633840560913,
"epoch": 0.1486910994764398,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.847829818725586,
"fcm_dpo/q_t": 0.4953823685646057,
"grad_norm": 14.982972145080566,
"learning_rate": 4.967625656594781e-07,
"logits/chosen": 1.964093804359436,
"logits/rejected": 1.9719585180282593,
"logps/chosen": -277.4803161621094,
"logps/ref_chosen": -280.9353942871094,
"logps/ref_rejected": -275.998779296875,
"logps/rejected": -274.3915100097656,
"loss": 5.4736,
"margin_dpo/margin_mean": 1.8478305339813232,
"margin_dpo/margin_std": 4.226970195770264,
"step": 71
},
{
"KL/chosen_KL_mean": 3.068317413330078,
"KL/mean": 2.0340206623077393,
"KL/rejected_KL_mean": 0.9997177124023438,
"KL/std": 3.0637056827545166,
"epoch": 0.15078534031413612,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.0686075687408447,
"fcm_dpo/q_t": 0.4948310852050781,
"grad_norm": 15.693150520324707,
"learning_rate": 4.964622763700252e-07,
"logits/chosen": 1.7748774290084839,
"logits/rejected": 1.9190484285354614,
"logps/chosen": -227.07720947265625,
"logps/ref_chosen": -230.14553833007812,
"logps/ref_rejected": -253.37661743164062,
"logps/rejected": -252.37689208984375,
"loss": 5.4646,
"margin_dpo/margin_mean": 2.068606376647949,
"margin_dpo/margin_std": 3.9270946979522705,
"step": 72
},
{
"KL/chosen_KL_mean": 3.863943099975586,
"KL/mean": 2.885406494140625,
"KL/rejected_KL_mean": 1.9068660736083984,
"KL/std": 3.2366580963134766,
"epoch": 0.15287958115183245,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9570707082748413,
"fcm_dpo/q_t": 0.49510952830314636,
"grad_norm": 14.728974342346191,
"learning_rate": 4.961487700566646e-07,
"logits/chosen": 1.6069412231445312,
"logits/rejected": 1.621825933456421,
"logps/chosen": -261.0023498535156,
"logps/ref_chosen": -264.86627197265625,
"logps/ref_rejected": -245.21304321289062,
"logps/rejected": -243.30616760253906,
"loss": 5.4694,
"margin_dpo/margin_mean": 1.957069993019104,
"margin_dpo/margin_std": 4.404537677764893,
"step": 73
},
{
"KL/chosen_KL_mean": 3.792757034301758,
"KL/mean": 2.8318533897399902,
"KL/rejected_KL_mean": 1.8709602355957031,
"KL/std": 3.567683458328247,
"epoch": 0.1549738219895288,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.921801209449768,
"fcm_dpo/q_t": 0.4951982796192169,
"grad_norm": 15.760645866394043,
"learning_rate": 4.958220635317885e-07,
"logits/chosen": 1.634965419769287,
"logits/rejected": 1.6836990118026733,
"logps/chosen": -330.7607421875,
"logps/ref_chosen": -334.55352783203125,
"logps/ref_rejected": -325.8403015136719,
"logps/rejected": -323.9693298339844,
"loss": 5.471,
"margin_dpo/margin_mean": 1.9218016862869263,
"margin_dpo/margin_std": 4.589147567749023,
"step": 74
},
{
"KL/chosen_KL_mean": 4.494831085205078,
"KL/mean": 3.165982723236084,
"KL/rejected_KL_mean": 1.8371295928955078,
"KL/std": 3.5341873168945312,
"epoch": 0.15706806282722513,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.657700538635254,
"fcm_dpo/q_t": 0.4933589696884155,
"grad_norm": 15.213676452636719,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": 1.6492834091186523,
"logits/rejected": 1.7265129089355469,
"logps/chosen": -255.92706298828125,
"logps/ref_chosen": -260.421875,
"logps/ref_rejected": -273.0882263183594,
"logps/rejected": -271.2510986328125,
"loss": 5.4415,
"margin_dpo/margin_mean": 2.657698631286621,
"margin_dpo/margin_std": 4.120148658752441,
"step": 75
},
{
"KL/chosen_KL_mean": 4.497751235961914,
"KL/mean": 3.0013351440429688,
"KL/rejected_KL_mean": 1.5049152374267578,
"KL/std": 3.860976457595825,
"epoch": 0.15916230366492146,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.992838144302368,
"fcm_dpo/q_t": 0.4925258159637451,
"grad_norm": 16.776811599731445,
"learning_rate": 4.951291206355559e-07,
"logits/chosen": 1.6961089372634888,
"logits/rejected": 1.6757653951644897,
"logps/chosen": -267.2377014160156,
"logps/ref_chosen": -271.7354431152344,
"logps/ref_rejected": -262.64111328125,
"logps/rejected": -261.13623046875,
"loss": 5.429,
"margin_dpo/margin_mean": 2.992838144302368,
"margin_dpo/margin_std": 4.868131637573242,
"step": 76
},
{
"KL/chosen_KL_mean": 4.730262756347656,
"KL/mean": 3.5549404621124268,
"KL/rejected_KL_mean": 2.3796234130859375,
"KL/std": 4.021265029907227,
"epoch": 0.1612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.3506383895874023,
"fcm_dpo/q_t": 0.4941290318965912,
"grad_norm": 14.854421615600586,
"learning_rate": 4.947629214246236e-07,
"logits/chosen": 1.7227365970611572,
"logits/rejected": 1.5612424612045288,
"logps/chosen": -289.94036865234375,
"logps/ref_chosen": -294.6706237792969,
"logps/ref_rejected": -247.6900177001953,
"logps/rejected": -245.3104248046875,
"loss": 5.4545,
"margin_dpo/margin_mean": 2.350637674331665,
"margin_dpo/margin_std": 5.070836067199707,
"step": 77
},
{
"KL/chosen_KL_mean": 5.183525085449219,
"KL/mean": 3.701960325241089,
"KL/rejected_KL_mean": 2.2203903198242188,
"KL/std": 3.653484344482422,
"epoch": 0.16335078534031414,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.9631338119506836,
"fcm_dpo/q_t": 0.49259960651397705,
"grad_norm": 14.6358060836792,
"learning_rate": 4.943835963210323e-07,
"logits/chosen": 1.7129117250442505,
"logits/rejected": 1.6462018489837646,
"logps/chosen": -242.96875,
"logps/ref_chosen": -248.1522979736328,
"logps/ref_rejected": -209.12808227539062,
"logps/rejected": -206.9076690673828,
"loss": 5.4303,
"margin_dpo/margin_mean": 2.963132619857788,
"margin_dpo/margin_std": 5.155320167541504,
"step": 78
},
{
"KL/chosen_KL_mean": 4.984601974487305,
"KL/mean": 3.4764275550842285,
"KL/rejected_KL_mean": 1.9682502746582031,
"KL/std": 4.8389410972595215,
"epoch": 0.16544502617801046,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.016352653503418,
"fcm_dpo/q_t": 0.4924572706222534,
"grad_norm": 16.72459602355957,
"learning_rate": 4.939911656668361e-07,
"logits/chosen": 1.8387343883514404,
"logits/rejected": 1.9029288291931152,
"logps/chosen": -254.48977661132812,
"logps/ref_chosen": -259.4743957519531,
"logps/ref_rejected": -246.9627227783203,
"logps/rejected": -244.99447631835938,
"loss": 5.4301,
"margin_dpo/margin_mean": 3.016352891921997,
"margin_dpo/margin_std": 6.287435531616211,
"step": 79
},
{
"KL/chosen_KL_mean": 5.328868865966797,
"KL/mean": 4.002456188201904,
"KL/rejected_KL_mean": 2.6760520935058594,
"KL/std": 4.738064289093018,
"epoch": 0.16753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.652810573577881,
"fcm_dpo/q_t": 0.4933757185935974,
"grad_norm": 15.615460395812988,
"learning_rate": 4.935856505068998e-07,
"logits/chosen": 1.5117417573928833,
"logits/rejected": 1.4887609481811523,
"logps/chosen": -280.59765625,
"logps/ref_chosen": -285.926513671875,
"logps/ref_rejected": -253.77996826171875,
"logps/rejected": -251.1039276123047,
"loss": 5.4436,
"margin_dpo/margin_mean": 2.652811288833618,
"margin_dpo/margin_std": 5.693536758422852,
"step": 80
},
{
"KL/chosen_KL_mean": 5.9272003173828125,
"KL/mean": 4.476429462432861,
"KL/rejected_KL_mean": 3.025655746459961,
"KL/std": 4.470715522766113,
"epoch": 0.16963350785340314,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.9015376567840576,
"fcm_dpo/q_t": 0.49275198578834534,
"grad_norm": 14.92487907409668,
"learning_rate": 4.93167072587771e-07,
"logits/chosen": 1.745436429977417,
"logits/rejected": 1.6883275508880615,
"logps/chosen": -256.6669921875,
"logps/ref_chosen": -262.5942077636719,
"logps/ref_rejected": -236.687744140625,
"logps/rejected": -233.66207885742188,
"loss": 5.4343,
"margin_dpo/margin_mean": 2.9015393257141113,
"margin_dpo/margin_std": 6.303562641143799,
"step": 81
},
{
"KL/chosen_KL_mean": 6.398937225341797,
"KL/mean": 4.894276142120361,
"KL/rejected_KL_mean": 3.3896026611328125,
"KL/std": 4.335615634918213,
"epoch": 0.17172774869109947,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.009336233139038,
"fcm_dpo/q_t": 0.4924847185611725,
"grad_norm": 15.691886901855469,
"learning_rate": 4.92735454356513e-07,
"logits/chosen": 1.8371354341506958,
"logits/rejected": 1.7929863929748535,
"logps/chosen": -264.64422607421875,
"logps/ref_chosen": -271.04315185546875,
"logps/ref_rejected": -234.03359985351562,
"logps/rejected": -230.64398193359375,
"loss": 5.4292,
"margin_dpo/margin_mean": 3.009336233139038,
"margin_dpo/margin_std": 5.506655216217041,
"step": 82
},
{
"KL/chosen_KL_mean": 6.437170028686523,
"KL/mean": 4.668962001800537,
"KL/rejected_KL_mean": 2.9007644653320312,
"KL/std": 4.886711597442627,
"epoch": 0.17382198952879582,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.5364108085632324,
"fcm_dpo/q_t": 0.4911731779575348,
"grad_norm": 15.337501525878906,
"learning_rate": 4.922908189595017e-07,
"logits/chosen": 1.7183247804641724,
"logits/rejected": 1.793430209159851,
"logps/chosen": -259.8481140136719,
"logps/ref_chosen": -266.2852783203125,
"logps/ref_rejected": -274.32501220703125,
"logps/rejected": -271.42425537109375,
"loss": 5.4096,
"margin_dpo/margin_mean": 3.536411762237549,
"margin_dpo/margin_std": 6.47735071182251,
"step": 83
},
{
"KL/chosen_KL_mean": 6.663084030151367,
"KL/mean": 5.296658039093018,
"KL/rejected_KL_mean": 3.9302330017089844,
"KL/std": 5.117173671722412,
"epoch": 0.17591623036649215,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.7328550815582275,
"fcm_dpo/q_t": 0.493173211812973,
"grad_norm": 15.184418678283691,
"learning_rate": 4.918331902411841e-07,
"logits/chosen": 1.8807992935180664,
"logits/rejected": 1.8695476055145264,
"logps/chosen": -280.1175231933594,
"logps/ref_chosen": -286.7806091308594,
"logps/ref_rejected": -276.1176452636719,
"logps/rejected": -272.18743896484375,
"loss": 5.4415,
"margin_dpo/margin_mean": 2.732853651046753,
"margin_dpo/margin_std": 6.627993583679199,
"step": 84
},
{
"KL/chosen_KL_mean": 6.175373077392578,
"KL/mean": 5.1813273429870605,
"KL/rejected_KL_mean": 4.187278747558594,
"KL/std": 4.361845970153809,
"epoch": 0.17801047120418848,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9880964756011963,
"fcm_dpo/q_t": 0.4950335621833801,
"grad_norm": 13.90788459777832,
"learning_rate": 4.913625927427995e-07,
"logits/chosen": 1.8967005014419556,
"logits/rejected": 1.9972286224365234,
"logps/chosen": -228.8544921875,
"logps/ref_chosen": -235.02987670898438,
"logps/ref_rejected": -259.80999755859375,
"logps/rejected": -255.62274169921875,
"loss": 5.4693,
"margin_dpo/margin_mean": 1.9880964756011963,
"margin_dpo/margin_std": 5.34002161026001,
"step": 85
},
{
"KL/chosen_KL_mean": 7.878423690795898,
"KL/mean": 6.022519111633301,
"KL/rejected_KL_mean": 4.166624069213867,
"KL/std": 5.585781574249268,
"epoch": 0.18010471204188483,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.711801052093506,
"fcm_dpo/q_t": 0.4907352328300476,
"grad_norm": 16.465648651123047,
"learning_rate": 4.908790517010636e-07,
"logits/chosen": 1.9428482055664062,
"logits/rejected": 1.995483160018921,
"logps/chosen": -293.434326171875,
"logps/ref_chosen": -301.312744140625,
"logps/ref_rejected": -288.8368225097656,
"logps/rejected": -284.6701965332031,
"loss": 5.404,
"margin_dpo/margin_mean": 3.7118024826049805,
"margin_dpo/margin_std": 7.516912460327148,
"step": 86
},
{
"KL/chosen_KL_mean": 8.02880859375,
"KL/mean": 5.975081443786621,
"KL/rejected_KL_mean": 3.921346664428711,
"KL/std": 5.473031044006348,
"epoch": 0.18219895287958116,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.10746431350708,
"fcm_dpo/q_t": 0.48974609375,
"grad_norm": 15.32911205291748,
"learning_rate": 4.903825930468148e-07,
"logits/chosen": 1.7233219146728516,
"logits/rejected": 1.7099597454071045,
"logps/chosen": -257.6371765136719,
"logps/ref_chosen": -265.6659851074219,
"logps/ref_rejected": -242.0697479248047,
"logps/rejected": -238.14840698242188,
"loss": 5.387,
"margin_dpo/margin_mean": 4.10746431350708,
"margin_dpo/margin_std": 6.42798376083374,
"step": 87
},
{
"KL/chosen_KL_mean": 7.276971817016602,
"KL/mean": 5.6905622482299805,
"KL/rejected_KL_mean": 4.104146957397461,
"KL/std": 5.471114158630371,
"epoch": 0.18429319371727748,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.1728219985961914,
"fcm_dpo/q_t": 0.4920782744884491,
"grad_norm": 14.059208869934082,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": 1.7752293348312378,
"logits/rejected": 1.7928274869918823,
"logps/chosen": -253.44447326660156,
"logps/ref_chosen": -260.721435546875,
"logps/ref_rejected": -262.6593322753906,
"logps/rejected": -258.55517578125,
"loss": 5.4247,
"margin_dpo/margin_mean": 3.1728243827819824,
"margin_dpo/margin_std": 7.120820999145508,
"step": 88
},
{
"KL/chosen_KL_mean": 8.954307556152344,
"KL/mean": 7.050387859344482,
"KL/rejected_KL_mean": 5.14647102355957,
"KL/std": 5.224122524261475,
"epoch": 0.18638743455497384,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.8078417778015137,
"fcm_dpo/q_t": 0.4904939532279968,
"grad_norm": 15.816296577453613,
"learning_rate": 4.893510300863676e-07,
"logits/chosen": 1.7604267597198486,
"logits/rejected": 1.7203480005264282,
"logps/chosen": -248.0485076904297,
"logps/ref_chosen": -257.0028076171875,
"logps/ref_rejected": -254.66232299804688,
"logps/rejected": -249.515869140625,
"loss": 5.3988,
"margin_dpo/margin_mean": 3.807842493057251,
"margin_dpo/margin_std": 6.273983955383301,
"step": 89
},
{
"KL/chosen_KL_mean": 8.962364196777344,
"KL/mean": 7.273670196533203,
"KL/rejected_KL_mean": 5.58497428894043,
"KL/std": 5.62357759475708,
"epoch": 0.18848167539267016,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3773882389068604,
"fcm_dpo/q_t": 0.49156898260116577,
"grad_norm": 15.256770133972168,
"learning_rate": 4.8881598109976e-07,
"logits/chosen": 2.0314841270446777,
"logits/rejected": 1.976723551750183,
"logps/chosen": -289.4480285644531,
"logps/ref_chosen": -298.4103698730469,
"logps/ref_rejected": -262.8761901855469,
"logps/rejected": -257.2912292480469,
"loss": 5.4166,
"margin_dpo/margin_mean": 3.3773880004882812,
"margin_dpo/margin_std": 6.8097686767578125,
"step": 90
},
{
"KL/chosen_KL_mean": 8.793439865112305,
"KL/mean": 7.109142780303955,
"KL/rejected_KL_mean": 5.424846649169922,
"KL/std": 5.423921585083008,
"epoch": 0.1905759162303665,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.368590831756592,
"fcm_dpo/q_t": 0.4915948510169983,
"grad_norm": 14.69740104675293,
"learning_rate": 4.882681251368548e-07,
"logits/chosen": 1.6287318468093872,
"logits/rejected": 1.6709989309310913,
"logps/chosen": -218.73043823242188,
"logps/ref_chosen": -227.5238800048828,
"logps/ref_rejected": -227.11428833007812,
"logps/rejected": -221.689453125,
"loss": 5.4173,
"margin_dpo/margin_mean": 3.368590831756592,
"margin_dpo/margin_std": 7.276670455932617,
"step": 91
},
{
"KL/chosen_KL_mean": 9.003055572509766,
"KL/mean": 7.0606303215026855,
"KL/rejected_KL_mean": 5.118196487426758,
"KL/std": 6.609164714813232,
"epoch": 0.19267015706806281,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.8848636150360107,
"fcm_dpo/q_t": 0.4903002381324768,
"grad_norm": 15.576513290405273,
"learning_rate": 4.877074915775048e-07,
"logits/chosen": 1.769035816192627,
"logits/rejected": 1.6670876741409302,
"logps/chosen": -266.3791809082031,
"logps/ref_chosen": -275.3822326660156,
"logps/ref_rejected": -269.8356018066406,
"logps/rejected": -264.7174072265625,
"loss": 5.3976,
"margin_dpo/margin_mean": 3.8848628997802734,
"margin_dpo/margin_std": 7.702731132507324,
"step": 92
},
{
"KL/chosen_KL_mean": 9.199054718017578,
"KL/mean": 7.457064151763916,
"KL/rejected_KL_mean": 5.715080261230469,
"KL/std": 5.752642631530762,
"epoch": 0.19476439790575917,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.483973503112793,
"fcm_dpo/q_t": 0.4913066029548645,
"grad_norm": 14.088767051696777,
"learning_rate": 4.871341104867864e-07,
"logits/chosen": 1.7151176929473877,
"logits/rejected": 1.6953684091567993,
"logps/chosen": -218.91732788085938,
"logps/ref_chosen": -228.1163787841797,
"logps/ref_rejected": -228.26266479492188,
"logps/rejected": -222.54757690429688,
"loss": 5.4126,
"margin_dpo/margin_mean": 3.4839727878570557,
"margin_dpo/margin_std": 7.183119773864746,
"step": 93
},
{
"KL/chosen_KL_mean": 10.13239860534668,
"KL/mean": 8.115608215332031,
"KL/rejected_KL_mean": 6.098814010620117,
"KL/std": 6.654098987579346,
"epoch": 0.1968586387434555,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.033576965332031,
"fcm_dpo/q_t": 0.489940881729126,
"grad_norm": 15.566554069519043,
"learning_rate": 4.865480126133871e-07,
"logits/chosen": 1.7142863273620605,
"logits/rejected": 1.8038783073425293,
"logps/chosen": -273.89483642578125,
"logps/ref_chosen": -284.0272521972656,
"logps/ref_rejected": -282.1720275878906,
"logps/rejected": -276.0732116699219,
"loss": 5.3938,
"margin_dpo/margin_mean": 4.0335774421691895,
"margin_dpo/margin_std": 8.711073875427246,
"step": 94
},
{
"KL/chosen_KL_mean": 9.834869384765625,
"KL/mean": 7.612403869628906,
"KL/rejected_KL_mean": 5.389949798583984,
"KL/std": 6.842440128326416,
"epoch": 0.19895287958115182,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.444918632507324,
"fcm_dpo/q_t": 0.4889160394668579,
"grad_norm": 15.96788215637207,
"learning_rate": 4.859492293879573e-07,
"logits/chosen": 1.8246318101882935,
"logits/rejected": 1.7539443969726562,
"logps/chosen": -293.2149658203125,
"logps/ref_chosen": -303.0498352050781,
"logps/ref_rejected": -259.8980712890625,
"logps/rejected": -254.50814819335938,
"loss": 5.3776,
"margin_dpo/margin_mean": 4.444916725158691,
"margin_dpo/margin_std": 8.767796516418457,
"step": 95
},
{
"KL/chosen_KL_mean": 9.86693000793457,
"KL/mean": 7.7021965980529785,
"KL/rejected_KL_mean": 5.537467956542969,
"KL/std": 6.970180511474609,
"epoch": 0.20104712041884817,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.329456329345703,
"fcm_dpo/q_t": 0.4892018437385559,
"grad_norm": 15.623520851135254,
"learning_rate": 4.853377929214243e-07,
"logits/chosen": 1.7852537631988525,
"logits/rejected": 1.7468805313110352,
"logps/chosen": -260.7207336425781,
"logps/ref_chosen": -270.58770751953125,
"logps/ref_rejected": -238.90069580078125,
"logps/rejected": -233.36322021484375,
"loss": 5.3808,
"margin_dpo/margin_mean": 4.329455375671387,
"margin_dpo/margin_std": 7.833381175994873,
"step": 96
},
{
"KL/chosen_KL_mean": 9.781450271606445,
"KL/mean": 7.512989521026611,
"KL/rejected_KL_mean": 5.244525909423828,
"KL/std": 6.864675521850586,
"epoch": 0.2031413612565445,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.53691291809082,
"fcm_dpo/q_t": 0.4886978268623352,
"grad_norm": 15.31660270690918,
"learning_rate": 4.847137360032699e-07,
"logits/chosen": 1.8315813541412354,
"logits/rejected": 1.7155046463012695,
"logps/chosen": -286.40411376953125,
"logps/ref_chosen": -296.1855773925781,
"logps/ref_rejected": -260.35333251953125,
"logps/rejected": -255.10879516601562,
"loss": 5.3753,
"margin_dpo/margin_mean": 4.5369133949279785,
"margin_dpo/margin_std": 9.493367195129395,
"step": 97
},
{
"KL/chosen_KL_mean": 10.113018035888672,
"KL/mean": 7.6967668533325195,
"KL/rejected_KL_mean": 5.280508041381836,
"KL/std": 7.416313648223877,
"epoch": 0.20523560209424083,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.8325090408325195,
"fcm_dpo/q_t": 0.48795896768569946,
"grad_norm": 15.75379467010498,
"learning_rate": 4.84077092099773e-07,
"logits/chosen": 2.001626491546631,
"logits/rejected": 1.9910184144973755,
"logps/chosen": -267.6023254394531,
"logps/ref_chosen": -277.7153625488281,
"logps/ref_rejected": -272.9094543457031,
"logps/rejected": -267.6289367675781,
"loss": 5.3648,
"margin_dpo/margin_mean": 4.8325090408325195,
"margin_dpo/margin_std": 9.47249984741211,
"step": 98
},
{
"KL/chosen_KL_mean": 10.356014251708984,
"KL/mean": 7.7745161056518555,
"KL/rejected_KL_mean": 5.193023681640625,
"KL/std": 7.610881328582764,
"epoch": 0.20732984293193718,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.162997722625732,
"fcm_dpo/q_t": 0.48712167143821716,
"grad_norm": 15.03431224822998,
"learning_rate": 4.834278953522137e-07,
"logits/chosen": 1.7922710180282593,
"logits/rejected": 1.7652747631072998,
"logps/chosen": -262.1817932128906,
"logps/ref_chosen": -272.5378112792969,
"logps/ref_rejected": -244.6329345703125,
"logps/rejected": -239.43991088867188,
"loss": 5.3527,
"margin_dpo/margin_mean": 5.162997722625732,
"margin_dpo/margin_std": 10.040070533752441,
"step": 99
},
{
"KL/chosen_KL_mean": 9.370292663574219,
"KL/mean": 7.144068717956543,
"KL/rejected_KL_mean": 4.917854309082031,
"KL/std": 7.839417457580566,
"epoch": 0.2094240837696335,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.452436923980713,
"fcm_dpo/q_t": 0.4889054000377655,
"grad_norm": 15.50412368774414,
"learning_rate": 4.827661805750437e-07,
"logits/chosen": 1.7104381322860718,
"logits/rejected": 1.6327906847000122,
"logps/chosen": -280.44207763671875,
"logps/ref_chosen": -289.8123474121094,
"logps/ref_rejected": -274.29071044921875,
"logps/rejected": -269.3728332519531,
"loss": 5.3786,
"margin_dpo/margin_mean": 4.452437400817871,
"margin_dpo/margin_std": 9.089731216430664,
"step": 100
},
{
"KL/chosen_KL_mean": 9.941091537475586,
"KL/mean": 7.673428535461426,
"KL/rejected_KL_mean": 5.405767440795898,
"KL/std": 8.093482971191406,
"epoch": 0.21151832460732983,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.535319805145264,
"fcm_dpo/q_t": 0.48869484663009644,
"grad_norm": 15.68911075592041,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": 1.7467799186706543,
"logits/rejected": 1.9193204641342163,
"logps/chosen": -235.77203369140625,
"logps/ref_chosen": -245.71311950683594,
"logps/ref_rejected": -272.08935546875,
"logps/rejected": -266.68359375,
"loss": 5.3766,
"margin_dpo/margin_mean": 4.5353193283081055,
"margin_dpo/margin_std": 9.767685890197754,
"step": 101
},
{
"KL/chosen_KL_mean": 10.382354736328125,
"KL/mean": 7.193247318267822,
"KL/rejected_KL_mean": 4.004146575927734,
"KL/std": 7.537840843200684,
"epoch": 0.2136125654450262,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.378210067749023,
"fcm_dpo/q_t": 0.4840970039367676,
"grad_norm": 15.268579483032227,
"learning_rate": 4.814053395442932e-07,
"logits/chosen": 1.5934196710586548,
"logits/rejected": 1.6801108121871948,
"logps/chosen": -199.56005859375,
"logps/ref_chosen": -209.94239807128906,
"logps/ref_rejected": -230.66897583007812,
"logps/rejected": -226.66481018066406,
"loss": 5.3031,
"margin_dpo/margin_mean": 6.378209114074707,
"margin_dpo/margin_std": 8.972631454467773,
"step": 102
},
{
"KL/chosen_KL_mean": 9.419265747070312,
"KL/mean": 7.0547308921813965,
"KL/rejected_KL_mean": 4.690199851989746,
"KL/std": 8.282856941223145,
"epoch": 0.2157068062827225,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.729057312011719,
"fcm_dpo/q_t": 0.48822057247161865,
"grad_norm": 16.341489791870117,
"learning_rate": 4.807062862684873e-07,
"logits/chosen": 1.7020974159240723,
"logits/rejected": 1.7979882955551147,
"logps/chosen": -241.88037109375,
"logps/ref_chosen": -251.2996368408203,
"logps/ref_rejected": -277.4847412109375,
"logps/rejected": -272.7945251464844,
"loss": 5.3694,
"margin_dpo/margin_mean": 4.729056358337402,
"margin_dpo/margin_std": 10.193910598754883,
"step": 103
},
{
"KL/chosen_KL_mean": 8.38427734375,
"KL/mean": 6.163862705230713,
"KL/rejected_KL_mean": 3.943441390991211,
"KL/std": 8.543988227844238,
"epoch": 0.21780104712041884,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.440829753875732,
"fcm_dpo/q_t": 0.4889433681964874,
"grad_norm": 14.319323539733887,
"learning_rate": 4.799948609147061e-07,
"logits/chosen": 1.757680892944336,
"logits/rejected": 1.796667218208313,
"logps/chosen": -249.39512634277344,
"logps/ref_chosen": -257.7794189453125,
"logps/ref_rejected": -227.91546630859375,
"logps/rejected": -223.97201538085938,
"loss": 5.3825,
"margin_dpo/margin_mean": 4.440830230712891,
"margin_dpo/margin_std": 11.020442962646484,
"step": 104
},
{
"KL/chosen_KL_mean": 10.291145324707031,
"KL/mean": 6.228615760803223,
"KL/rejected_KL_mean": 2.1660938262939453,
"KL/std": 9.67591381072998,
"epoch": 0.2198952879581152,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.125062942504883,
"fcm_dpo/q_t": 0.47975796461105347,
"grad_norm": 17.875959396362305,
"learning_rate": 4.792711016345321e-07,
"logits/chosen": 1.8722009658813477,
"logits/rejected": 1.8510180711746216,
"logps/chosen": -301.8028869628906,
"logps/ref_chosen": -312.09405517578125,
"logps/ref_rejected": -277.97332763671875,
"logps/rejected": -275.8072204589844,
"loss": 5.2393,
"margin_dpo/margin_mean": 8.125061988830566,
"margin_dpo/margin_std": 10.691678047180176,
"step": 105
},
{
"KL/chosen_KL_mean": 6.734123229980469,
"KL/mean": 3.977590560913086,
"KL/rejected_KL_mean": 1.221048355102539,
"KL/std": 8.91702938079834,
"epoch": 0.22198952879581152,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.513080596923828,
"fcm_dpo/q_t": 0.48627668619155884,
"grad_norm": 16.416852951049805,
"learning_rate": 4.785350472409791e-07,
"logits/chosen": 1.6931140422821045,
"logits/rejected": 1.6858336925506592,
"logps/chosen": -278.6322326660156,
"logps/ref_chosen": -285.3663330078125,
"logps/ref_rejected": -262.2779541015625,
"logps/rejected": -261.056884765625,
"loss": 5.3415,
"margin_dpo/margin_mean": 5.513082027435303,
"margin_dpo/margin_std": 11.19965934753418,
"step": 106
},
{
"KL/chosen_KL_mean": 9.352890014648438,
"KL/mean": 5.674660682678223,
"KL/rejected_KL_mean": 1.9964275360107422,
"KL/std": 9.54844856262207,
"epoch": 0.22408376963350785,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.356465816497803,
"fcm_dpo/q_t": 0.48166128993034363,
"grad_norm": 16.4990234375,
"learning_rate": 4.777867372064105e-07,
"logits/chosen": 1.9865798950195312,
"logits/rejected": 2.064659833908081,
"logps/chosen": -286.9407043457031,
"logps/ref_chosen": -296.2935791015625,
"logps/ref_rejected": -291.5162353515625,
"logps/rejected": -289.519775390625,
"loss": 5.2719,
"margin_dpo/margin_mean": 7.356466293334961,
"margin_dpo/margin_std": 11.690131187438965,
"step": 107
},
{
"KL/chosen_KL_mean": 9.395183563232422,
"KL/mean": 5.331313610076904,
"KL/rejected_KL_mean": 1.267435073852539,
"KL/std": 8.979668617248535,
"epoch": 0.2261780104712042,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.127754211425781,
"fcm_dpo/q_t": 0.47975727915763855,
"grad_norm": 16.080812454223633,
"learning_rate": 4.770262116604223e-07,
"logits/chosen": 1.9511507749557495,
"logits/rejected": 1.8253110647201538,
"logps/chosen": -278.7291564941406,
"logps/ref_chosen": -288.12432861328125,
"logps/ref_rejected": -232.35720825195312,
"logps/rejected": -231.08978271484375,
"loss": 5.2387,
"margin_dpo/margin_mean": 8.127753257751465,
"margin_dpo/margin_std": 10.594194412231445,
"step": 108
},
{
"KL/chosen_KL_mean": 8.109546661376953,
"KL/mean": 3.8482954502105713,
"KL/rejected_KL_mean": -0.4129467010498047,
"KL/std": 10.208244323730469,
"epoch": 0.22827225130890053,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.522490501403809,
"fcm_dpo/q_t": 0.4788376986980438,
"grad_norm": 17.691587448120117,
"learning_rate": 4.7625351138769166e-07,
"logits/chosen": 1.5900017023086548,
"logits/rejected": 1.5221468210220337,
"logps/chosen": -309.0920104980469,
"logps/ref_chosen": -317.2015686035156,
"logps/ref_rejected": -278.0653381347656,
"logps/rejected": -278.478271484375,
"loss": 5.2299,
"margin_dpo/margin_mean": 8.522488594055176,
"margin_dpo/margin_std": 12.911687850952148,
"step": 109
},
{
"KL/chosen_KL_mean": 8.590599060058594,
"KL/mean": 4.59503173828125,
"KL/rejected_KL_mean": 0.5994701385498047,
"KL/std": 10.171649932861328,
"epoch": 0.23036649214659685,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.9911394119262695,
"fcm_dpo/q_t": 0.48011514544487,
"grad_norm": 15.929461479187012,
"learning_rate": 4.75468677825789e-07,
"logits/chosen": 1.877347469329834,
"logits/rejected": 1.881067156791687,
"logps/chosen": -254.34884643554688,
"logps/ref_chosen": -262.939453125,
"logps/ref_rejected": -257.82275390625,
"logps/rejected": -257.2232971191406,
"loss": 5.2504,
"margin_dpo/margin_mean": 7.9911394119262695,
"margin_dpo/margin_std": 13.278604507446289,
"step": 110
},
{
"KL/chosen_KL_mean": 9.09178352355957,
"KL/mean": 4.779868125915527,
"KL/rejected_KL_mean": 0.4679603576660156,
"KL/std": 11.26836109161377,
"epoch": 0.2324607329842932,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.623827934265137,
"fcm_dpo/q_t": 0.4785763621330261,
"grad_norm": 18.236873626708984,
"learning_rate": 4.7467175306295647e-07,
"logits/chosen": 1.8864248991012573,
"logits/rejected": 1.7907862663269043,
"logps/chosen": -310.06201171875,
"logps/ref_chosen": -319.15380859375,
"logps/ref_rejected": -281.7733154296875,
"logps/rejected": -281.30535888671875,
"loss": 5.2262,
"margin_dpo/margin_mean": 8.62382698059082,
"margin_dpo/margin_std": 12.825776100158691,
"step": 111
},
{
"KL/chosen_KL_mean": 5.521121978759766,
"KL/mean": 3.3389346599578857,
"KL/rejected_KL_mean": 1.156747817993164,
"KL/std": 10.577245712280273,
"epoch": 0.23455497382198953,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.364378452301025,
"fcm_dpo/q_t": 0.48913276195526123,
"grad_norm": 14.586549758911133,
"learning_rate": 4.7386277983585053e-07,
"logits/chosen": 1.85859215259552,
"logits/rejected": 1.967081904411316,
"logps/chosen": -244.97930908203125,
"logps/ref_chosen": -250.50042724609375,
"logps/ref_rejected": -268.32635498046875,
"logps/rejected": -267.16961669921875,
"loss": 5.3963,
"margin_dpo/margin_mean": 4.364377975463867,
"margin_dpo/margin_std": 14.259346008300781,
"step": 112
},
{
"KL/chosen_KL_mean": 6.750080108642578,
"KL/mean": 1.6503314971923828,
"KL/rejected_KL_mean": -3.449411392211914,
"KL/std": 11.677305221557617,
"epoch": 0.23664921465968586,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.199491500854492,
"fcm_dpo/q_t": 0.47468462586402893,
"grad_norm": 16.903900146484375,
"learning_rate": 4.7304180152725024e-07,
"logits/chosen": 1.6014283895492554,
"logits/rejected": 1.636258602142334,
"logps/chosen": -272.6515197753906,
"logps/ref_chosen": -279.4015808105469,
"logps/ref_rejected": -266.6392822265625,
"logps/rejected": -270.0887145996094,
"loss": 5.17,
"margin_dpo/margin_mean": 10.199490547180176,
"margin_dpo/margin_std": 14.30715560913086,
"step": 113
},
{
"KL/chosen_KL_mean": 5.200408935546875,
"KL/mean": 2.613903522491455,
"KL/rejected_KL_mean": 0.02741241455078125,
"KL/std": 10.772842407226562,
"epoch": 0.2387434554973822,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.172990322113037,
"fcm_dpo/q_t": 0.4871406853199005,
"grad_norm": 15.496525764465332,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": 1.6856958866119385,
"logits/rejected": 1.6697858572006226,
"logps/chosen": -249.06007385253906,
"logps/ref_chosen": -254.26048278808594,
"logps/ref_rejected": -241.8769989013672,
"logps/rejected": -241.84957885742188,
"loss": 5.3599,
"margin_dpo/margin_mean": 5.172990322113037,
"margin_dpo/margin_std": 13.062189102172852,
"step": 114
},
{
"KL/chosen_KL_mean": 5.646036148071289,
"KL/mean": 2.675328493118286,
"KL/rejected_KL_mean": -0.2953834533691406,
"KL/std": 11.957541465759277,
"epoch": 0.24083769633507854,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.941422939300537,
"fcm_dpo/q_t": 0.48527175188064575,
"grad_norm": 15.836464881896973,
"learning_rate": 4.7136400641330245e-07,
"logits/chosen": 1.7326550483703613,
"logits/rejected": 1.7668747901916504,
"logps/chosen": -283.1584777832031,
"logps/ref_chosen": -288.80450439453125,
"logps/ref_rejected": -266.8573913574219,
"logps/rejected": -267.15277099609375,
"loss": 5.3344,
"margin_dpo/margin_mean": 5.941421985626221,
"margin_dpo/margin_std": 14.47954273223877,
"step": 115
},
{
"KL/chosen_KL_mean": 6.954935073852539,
"KL/mean": 2.911851644515991,
"KL/rejected_KL_mean": -1.1312294006347656,
"KL/std": 11.175189971923828,
"epoch": 0.24293193717277486,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.086163520812988,
"fcm_dpo/q_t": 0.47990795969963074,
"grad_norm": 16.428619384765625,
"learning_rate": 4.70507279583015e-07,
"logits/chosen": 1.6866003274917603,
"logits/rejected": 1.6639567613601685,
"logps/chosen": -265.23504638671875,
"logps/ref_chosen": -272.18994140625,
"logps/ref_rejected": -253.14334106445312,
"logps/rejected": -254.27456665039062,
"loss": 5.2496,
"margin_dpo/margin_mean": 8.086163520812988,
"margin_dpo/margin_std": 14.127702713012695,
"step": 116
},
{
"KL/chosen_KL_mean": 5.766324996948242,
"KL/mean": 1.491805076599121,
"KL/rejected_KL_mean": -2.782712936401367,
"KL/std": 11.109378814697266,
"epoch": 0.2450261780104712,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.549039840698242,
"fcm_dpo/q_t": 0.4787558615207672,
"grad_norm": 15.702656745910645,
"learning_rate": 4.6963872761652834e-07,
"logits/chosen": 1.5461552143096924,
"logits/rejected": 1.5219987630844116,
"logps/chosen": -244.6363067626953,
"logps/ref_chosen": -250.4026336669922,
"logps/ref_rejected": -229.63853454589844,
"logps/rejected": -232.4212646484375,
"loss": 5.2298,
"margin_dpo/margin_mean": 8.549040794372559,
"margin_dpo/margin_std": 12.602401733398438,
"step": 117
},
{
"KL/chosen_KL_mean": 5.609523773193359,
"KL/mean": 1.0851911306381226,
"KL/rejected_KL_mean": -3.439149856567383,
"KL/std": 12.804605484008789,
"epoch": 0.24712041884816754,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.048680305480957,
"fcm_dpo/q_t": 0.4775315523147583,
"grad_norm": 20.030120849609375,
"learning_rate": 4.687583970916486e-07,
"logits/chosen": 1.7872297763824463,
"logits/rejected": 1.8385579586029053,
"logps/chosen": -252.786376953125,
"logps/ref_chosen": -258.3959045410156,
"logps/ref_rejected": -268.84228515625,
"logps/rejected": -272.2814636230469,
"loss": 5.2143,
"margin_dpo/margin_mean": 9.048680305480957,
"margin_dpo/margin_std": 14.543923377990723,
"step": 118
},
{
"KL/chosen_KL_mean": 6.068717956542969,
"KL/mean": 2.253279685974121,
"KL/rejected_KL_mean": -1.5621623992919922,
"KL/std": 14.264659881591797,
"epoch": 0.24921465968586387,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.630876541137695,
"fcm_dpo/q_t": 0.48115378618240356,
"grad_norm": 15.853008270263672,
"learning_rate": 4.6786633521783005e-07,
"logits/chosen": 1.8772937059402466,
"logits/rejected": 1.8533079624176025,
"logps/chosen": -311.23382568359375,
"logps/ref_chosen": -317.30255126953125,
"logps/ref_rejected": -300.47552490234375,
"logps/rejected": -302.0376892089844,
"loss": 5.2793,
"margin_dpo/margin_mean": 7.630877494812012,
"margin_dpo/margin_std": 16.963003158569336,
"step": 119
},
{
"KL/chosen_KL_mean": 6.961023330688477,
"KL/mean": 2.4848945140838623,
"KL/rejected_KL_mean": -1.9912357330322266,
"KL/std": 13.374311447143555,
"epoch": 0.2513089005235602,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.952258110046387,
"fcm_dpo/q_t": 0.4778059124946594,
"grad_norm": 16.386856079101562,
"learning_rate": 4.669625898336438e-07,
"logits/chosen": 2.0444533824920654,
"logits/rejected": 2.03185772895813,
"logps/chosen": -298.29339599609375,
"logps/ref_chosen": -305.2544250488281,
"logps/ref_rejected": -259.40716552734375,
"logps/rejected": -261.39837646484375,
"loss": 5.2241,
"margin_dpo/margin_mean": 8.952259063720703,
"margin_dpo/margin_std": 16.330432891845703,
"step": 120
},
{
"KL/chosen_KL_mean": 3.5496482849121094,
"KL/mean": 0.8425604104995728,
"KL/rejected_KL_mean": -1.8645248413085938,
"KL/std": 11.928267478942871,
"epoch": 0.2534031413612565,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.414175987243652,
"fcm_dpo/q_t": 0.48656564950942993,
"grad_norm": 15.524222373962402,
"learning_rate": 4.6604720940421207e-07,
"logits/chosen": 1.5826594829559326,
"logits/rejected": 1.6027311086654663,
"logps/chosen": -213.091064453125,
"logps/ref_chosen": -216.64073181152344,
"logps/ref_rejected": -224.36422729492188,
"logps/rejected": -226.22874450683594,
"loss": 5.3549,
"margin_dpo/margin_mean": 5.414175987243652,
"margin_dpo/margin_std": 14.486465454101562,
"step": 121
},
{
"KL/chosen_KL_mean": 5.696184158325195,
"KL/mean": 1.9863008260726929,
"KL/rejected_KL_mean": -1.7235946655273438,
"KL/std": 14.075178146362305,
"epoch": 0.2554973821989529,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.419771671295166,
"fcm_dpo/q_t": 0.4816259443759918,
"grad_norm": 15.780983924865723,
"learning_rate": 4.651202430186092e-07,
"logits/chosen": 1.6831539869308472,
"logits/rejected": 1.6807143688201904,
"logps/chosen": -263.12261962890625,
"logps/ref_chosen": -268.8188171386719,
"logps/ref_rejected": -272.55853271484375,
"logps/rejected": -274.2821044921875,
"loss": 5.2877,
"margin_dpo/margin_mean": 7.419772624969482,
"margin_dpo/margin_std": 17.622047424316406,
"step": 122
},
{
"KL/chosen_KL_mean": 8.446487426757812,
"KL/mean": 3.178858757019043,
"KL/rejected_KL_mean": -2.088764190673828,
"KL/std": 15.126729011535645,
"epoch": 0.25759162303664923,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.535255432128906,
"fcm_dpo/q_t": 0.4739333391189575,
"grad_norm": 18.45299530029297,
"learning_rate": 4.6418174038722924e-07,
"logits/chosen": 1.8908565044403076,
"logits/rejected": 1.8965411186218262,
"logps/chosen": -313.2452697753906,
"logps/ref_chosen": -321.6917419433594,
"logps/ref_rejected": -279.1517333984375,
"logps/rejected": -281.240478515625,
"loss": 5.1704,
"margin_dpo/margin_mean": 10.535255432128906,
"margin_dpo/margin_std": 18.469093322753906,
"step": 123
},
{
"KL/chosen_KL_mean": 6.057577133178711,
"KL/mean": 1.2710437774658203,
"KL/rejected_KL_mean": -3.515501022338867,
"KL/std": 14.487335205078125,
"epoch": 0.25968586387434556,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.573090553283691,
"fcm_dpo/q_t": 0.47625818848609924,
"grad_norm": 17.11802864074707,
"learning_rate": 4.6323175183912023e-07,
"logits/chosen": 1.8385354280471802,
"logits/rejected": 1.7378027439117432,
"logps/chosen": -265.61761474609375,
"logps/ref_chosen": -271.6752014160156,
"logps/ref_rejected": -221.85577392578125,
"logps/rejected": -225.3712921142578,
"loss": 5.2017,
"margin_dpo/margin_mean": 9.573091506958008,
"margin_dpo/margin_std": 16.087444305419922,
"step": 124
},
{
"KL/chosen_KL_mean": 5.609090805053711,
"KL/mean": 0.8871803283691406,
"KL/rejected_KL_mean": -3.8347320556640625,
"KL/std": 14.195379257202148,
"epoch": 0.2617801047120419,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.443817138671875,
"fcm_dpo/q_t": 0.4766680598258972,
"grad_norm": 16.373830795288086,
"learning_rate": 4.6227032831928483e-07,
"logits/chosen": 1.627115249633789,
"logits/rejected": 1.6331775188446045,
"logps/chosen": -264.7899475097656,
"logps/ref_chosen": -270.3990478515625,
"logps/ref_rejected": -264.16925048828125,
"logps/rejected": -268.00396728515625,
"loss": 5.2127,
"margin_dpo/margin_mean": 9.443815231323242,
"margin_dpo/margin_std": 18.61271095275879,
"step": 125
},
{
"KL/chosen_KL_mean": 6.839256286621094,
"KL/mean": 1.4984056949615479,
"KL/rejected_KL_mean": -3.8424301147460938,
"KL/std": 15.581066131591797,
"epoch": 0.2638743455497382,
"fcm_dpo/beta": 0.010156591422855854,
"fcm_dpo/delta": 0.022172488272190094,
"fcm_dpo/margin": 10.681697845458984,
"fcm_dpo/q_t": 0.47326067090034485,
"grad_norm": 16.927207946777344,
"learning_rate": 4.612975213859487e-07,
"logits/chosen": 1.724934458732605,
"logits/rejected": 1.699325442314148,
"logps/chosen": -306.41204833984375,
"logps/ref_chosen": -313.25128173828125,
"logps/ref_rejected": -283.0433349609375,
"logps/rejected": -286.8857421875,
"loss": 5.1609,
"margin_dpo/margin_mean": 10.681696891784668,
"margin_dpo/margin_std": 18.312498092651367,
"step": 126
},
{
"KL/chosen_KL_mean": 8.03831672668457,
"KL/mean": 1.868345022201538,
"KL/rejected_KL_mean": -4.301609039306641,
"KL/std": 16.05044937133789,
"epoch": 0.26596858638743454,
"fcm_dpo/beta": 0.010299422778189182,
"fcm_dpo/delta": 0.04627181217074394,
"fcm_dpo/margin": 12.339926719665527,
"fcm_dpo/q_t": 0.4686408042907715,
"grad_norm": 17.30465316772461,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": 1.6262192726135254,
"logits/rejected": 1.548923373222351,
"logps/chosen": -290.5256652832031,
"logps/ref_chosen": -298.5639953613281,
"logps/ref_rejected": -274.021728515625,
"logps/rejected": -278.3233337402344,
"loss": 5.0892,
"margin_dpo/margin_mean": 12.339926719665527,
"margin_dpo/margin_std": 18.16909408569336,
"step": 127
},
{
"KL/chosen_KL_mean": 8.606454849243164,
"KL/mean": 1.7851275205612183,
"KL/rejected_KL_mean": -5.036199569702148,
"KL/std": 17.000141143798828,
"epoch": 0.2680628272251309,
"fcm_dpo/beta": 0.010715160518884659,
"fcm_dpo/delta": 0.023808792233467102,
"fcm_dpo/margin": 13.642653465270996,
"fcm_dpo/q_t": 0.46404626965522766,
"grad_norm": 17.175697326660156,
"learning_rate": 4.5931796656116837e-07,
"logits/chosen": 1.6542246341705322,
"logits/rejected": 1.690248966217041,
"logps/chosen": -248.5244140625,
"logps/ref_chosen": -257.130859375,
"logps/ref_rejected": -247.71636962890625,
"logps/rejected": -252.75254821777344,
"loss": 5.0312,
"margin_dpo/margin_mean": 13.642655372619629,
"margin_dpo/margin_std": 20.29867935180664,
"step": 128
},
{
"KL/chosen_KL_mean": 8.863290786743164,
"KL/mean": 2.632221221923828,
"KL/rejected_KL_mean": -3.5988426208496094,
"KL/std": 17.10430908203125,
"epoch": 0.27015706806282724,
"fcm_dpo/beta": 0.010969205759465694,
"fcm_dpo/delta": 0.04741775244474411,
"fcm_dpo/margin": 12.46213150024414,
"fcm_dpo/q_t": 0.4665260910987854,
"grad_norm": 17.762378692626953,
"learning_rate": 4.5831132482724193e-07,
"logits/chosen": 1.6477328538894653,
"logits/rejected": 1.6887083053588867,
"logps/chosen": -286.0972595214844,
"logps/ref_chosen": -294.9605712890625,
"logps/ref_rejected": -272.6249694824219,
"logps/rejected": -276.22381591796875,
"loss": 5.0714,
"margin_dpo/margin_mean": 12.46213150024414,
"margin_dpo/margin_std": 20.405351638793945,
"step": 129
},
{
"KL/chosen_KL_mean": 7.510087966918945,
"KL/mean": 1.6374802589416504,
"KL/rejected_KL_mean": -4.235134124755859,
"KL/std": 15.58251667022705,
"epoch": 0.27225130890052357,
"fcm_dpo/beta": 0.011182182468473911,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 11.745227813720703,
"fcm_dpo/q_t": 0.4675550162792206,
"grad_norm": 22.702251434326172,
"learning_rate": 4.5729351198915705e-07,
"logits/chosen": 1.811161756515503,
"logits/rejected": 1.8970874547958374,
"logps/chosen": -268.93212890625,
"logps/ref_chosen": -276.4422302246094,
"logps/ref_rejected": -290.241455078125,
"logps/rejected": -294.4765930175781,
"loss": 5.085,
"margin_dpo/margin_mean": 11.745227813720703,
"margin_dpo/margin_std": 19.256053924560547,
"step": 130
},
{
"KL/chosen_KL_mean": 5.034919738769531,
"KL/mean": 0.19638243317604065,
"KL/rejected_KL_mean": -4.64215087890625,
"KL/std": 16.58043098449707,
"epoch": 0.2743455497382199,
"fcm_dpo/beta": 0.011311189271509647,
"fcm_dpo/delta": 0.02286328561604023,
"fcm_dpo/margin": 9.677083015441895,
"fcm_dpo/q_t": 0.4732462167739868,
"grad_norm": 19.667495727539062,
"learning_rate": 4.5626458262912735e-07,
"logits/chosen": 1.5953093767166138,
"logits/rejected": 1.5892738103866577,
"logps/chosen": -299.789794921875,
"logps/ref_chosen": -304.82470703125,
"logps/ref_rejected": -289.9726867675781,
"logps/rejected": -294.61480712890625,
"loss": 5.1813,
"margin_dpo/margin_mean": 9.677083015441895,
"margin_dpo/margin_std": 21.382164001464844,
"step": 131
},
{
"KL/chosen_KL_mean": 7.070009231567383,
"KL/mean": 0.573898434638977,
"KL/rejected_KL_mean": -5.922212600708008,
"KL/std": 18.59527587890625,
"epoch": 0.2764397905759162,
"fcm_dpo/beta": 0.011468597687780857,
"fcm_dpo/delta": 0.032270729541778564,
"fcm_dpo/margin": 12.99222183227539,
"fcm_dpo/q_t": 0.4638496935367584,
"grad_norm": 20.325056076049805,
"learning_rate": 4.5522459192551166e-07,
"logits/chosen": 1.4823801517486572,
"logits/rejected": 1.5927780866622925,
"logps/chosen": -253.24472045898438,
"logps/ref_chosen": -260.31475830078125,
"logps/ref_rejected": -280.93756103515625,
"logps/rejected": -286.8597717285156,
"loss": 5.0492,
"margin_dpo/margin_mean": 12.992222785949707,
"margin_dpo/margin_std": 23.399463653564453,
"step": 132
},
{
"KL/chosen_KL_mean": 7.743677139282227,
"KL/mean": 1.0414981842041016,
"KL/rejected_KL_mean": -5.660671234130859,
"KL/std": 17.203453063964844,
"epoch": 0.27853403141361255,
"fcm_dpo/beta": 0.011720841750502586,
"fcm_dpo/delta": 0.014634376391768456,
"fcm_dpo/margin": 13.40435791015625,
"fcm_dpo/q_t": 0.4613617956638336,
"grad_norm": 18.77317237854004,
"learning_rate": 4.541735956498554e-07,
"logits/chosen": 1.5894581079483032,
"logits/rejected": 1.5555914640426636,
"logps/chosen": -271.37689208984375,
"logps/ref_chosen": -279.12060546875,
"logps/ref_rejected": -247.22982788085938,
"logps/rejected": -252.8905029296875,
"loss": 5.0094,
"margin_dpo/margin_mean": 13.40435791015625,
"margin_dpo/margin_std": 21.21170997619629,
"step": 133
},
{
"KL/chosen_KL_mean": 1.6970939636230469,
"KL/mean": -3.0371992588043213,
"KL/rejected_KL_mean": -7.77149772644043,
"KL/std": 17.570058822631836,
"epoch": 0.2806282722513089,
"fcm_dpo/beta": 0.011929137632250786,
"fcm_dpo/delta": 0.022021804004907608,
"fcm_dpo/margin": 9.46860408782959,
"fcm_dpo/q_t": 0.4723929166793823,
"grad_norm": 24.406070709228516,
"learning_rate": 4.5311165016389914e-07,
"logits/chosen": 1.8009541034698486,
"logits/rejected": 1.8369526863098145,
"logps/chosen": -313.9070129394531,
"logps/ref_chosen": -315.6040954589844,
"logps/ref_rejected": -291.45635986328125,
"logps/rejected": -299.22784423828125,
"loss": 5.169,
"margin_dpo/margin_mean": 9.46860408782959,
"margin_dpo/margin_std": 20.294403076171875,
"step": 134
},
{
"KL/chosen_KL_mean": 5.026142120361328,
"KL/mean": -1.5437160730361938,
"KL/rejected_KL_mean": -8.113567352294922,
"KL/std": 18.06447410583496,
"epoch": 0.28272251308900526,
"fcm_dpo/beta": 0.012354599311947823,
"fcm_dpo/delta": 0.05824194476008415,
"fcm_dpo/margin": 13.139720916748047,
"fcm_dpo/q_t": 0.4605092704296112,
"grad_norm": 20.61244773864746,
"learning_rate": 4.520388124165564e-07,
"logits/chosen": 1.43618643283844,
"logits/rejected": 1.419420838356018,
"logps/chosen": -277.8468322753906,
"logps/ref_chosen": -282.87298583984375,
"logps/ref_rejected": -265.0682373046875,
"logps/rejected": -273.1817932128906,
"loss": 4.9883,
"margin_dpo/margin_mean": 13.13971996307373,
"margin_dpo/margin_std": 19.358867645263672,
"step": 135
},
{
"KL/chosen_KL_mean": 4.645238876342773,
"KL/mean": -1.692807674407959,
"KL/rejected_KL_mean": -8.030860900878906,
"KL/std": 17.679540634155273,
"epoch": 0.2848167539267016,
"fcm_dpo/beta": 0.012806904502213001,
"fcm_dpo/delta": 0.03608924150466919,
"fcm_dpo/margin": 12.676102638244629,
"fcm_dpo/q_t": 0.4605832099914551,
"grad_norm": 23.0947322845459,
"learning_rate": 4.5095513994085974e-07,
"logits/chosen": 1.490594744682312,
"logits/rejected": 1.6552977561950684,
"logps/chosen": -258.7966613769531,
"logps/ref_chosen": -263.4419250488281,
"logps/ref_rejected": -251.43150329589844,
"logps/rejected": -259.46234130859375,
"loss": 5.0083,
"margin_dpo/margin_mean": 12.676101684570312,
"margin_dpo/margin_std": 21.883665084838867,
"step": 136
},
{
"KL/chosen_KL_mean": 2.6373767852783203,
"KL/mean": -3.0409834384918213,
"KL/rejected_KL_mean": -8.719337463378906,
"KL/std": 19.185440063476562,
"epoch": 0.2869109947643979,
"fcm_dpo/beta": 0.01320512406527996,
"fcm_dpo/delta": 0.06166880577802658,
"fcm_dpo/margin": 11.356720924377441,
"fcm_dpo/q_t": 0.46354466676712036,
"grad_norm": 23.587217330932617,
"learning_rate": 4.498606908508753e-07,
"logits/chosen": 1.5931267738342285,
"logits/rejected": 1.5913976430892944,
"logps/chosen": -292.63336181640625,
"logps/ref_chosen": -295.2707214355469,
"logps/ref_rejected": -283.9851989746094,
"logps/rejected": -292.70458984375,
"loss": 5.0482,
"margin_dpo/margin_mean": 11.356720924377441,
"margin_dpo/margin_std": 20.716747283935547,
"step": 137
},
{
"KL/chosen_KL_mean": 1.1577720642089844,
"KL/mean": -4.458102703094482,
"KL/rejected_KL_mean": -10.073986053466797,
"KL/std": 18.46007537841797,
"epoch": 0.28900523560209423,
"fcm_dpo/beta": 0.01372732874006033,
"fcm_dpo/delta": 0.00854338239878416,
"fcm_dpo/margin": 11.231751441955566,
"fcm_dpo/q_t": 0.4628608822822571,
"grad_norm": 22.117218017578125,
"learning_rate": 4.487555238385862e-07,
"logits/chosen": 1.4696087837219238,
"logits/rejected": 1.460563063621521,
"logps/chosen": -281.4001159667969,
"logps/ref_chosen": -282.5578918457031,
"logps/ref_rejected": -262.4772644042969,
"logps/rejected": -272.55126953125,
"loss": 5.0642,
"margin_dpo/margin_mean": 11.231752395629883,
"margin_dpo/margin_std": 23.57673454284668,
"step": 138
},
{
"KL/chosen_KL_mean": 2.1462669372558594,
"KL/mean": -1.9316972494125366,
"KL/rejected_KL_mean": -6.009664535522461,
"KL/std": 19.203439712524414,
"epoch": 0.29109947643979056,
"fcm_dpo/beta": 0.01388666033744812,
"fcm_dpo/delta": 0.0230011697858572,
"fcm_dpo/margin": 8.15593147277832,
"fcm_dpo/q_t": 0.4726797342300415,
"grad_norm": 22.397058486938477,
"learning_rate": 4.476396981707453e-07,
"logits/chosen": 1.6937816143035889,
"logits/rejected": 1.7701420783996582,
"logps/chosen": -261.5045471191406,
"logps/ref_chosen": -263.65081787109375,
"logps/ref_rejected": -266.2420959472656,
"logps/rejected": -272.25177001953125,
"loss": 5.217,
"margin_dpo/margin_mean": 8.155933380126953,
"margin_dpo/margin_std": 23.77894401550293,
"step": 139
},
{
"KL/chosen_KL_mean": 5.735929489135742,
"KL/mean": -1.7937304973602295,
"KL/rejected_KL_mean": -9.323375701904297,
"KL/std": 19.06015396118164,
"epoch": 0.2931937172774869,
"fcm_dpo/beta": 0.014193753711879253,
"fcm_dpo/delta": 0.055229805409908295,
"fcm_dpo/margin": 15.059309005737305,
"fcm_dpo/q_t": 0.4484260678291321,
"grad_norm": 21.740379333496094,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": 1.7132786512374878,
"logits/rejected": 1.6778305768966675,
"logps/chosen": -266.6465148925781,
"logps/ref_chosen": -272.3824462890625,
"logps/ref_rejected": -248.96444702148438,
"logps/rejected": -258.287841796875,
"loss": 4.8335,
"margin_dpo/margin_mean": 15.059309005737305,
"margin_dpo/margin_std": 21.372209548950195,
"step": 140
},
{
"KL/chosen_KL_mean": 3.1934146881103516,
"KL/mean": -3.918694257736206,
"KL/rejected_KL_mean": -11.030794143676758,
"KL/std": 20.240835189819336,
"epoch": 0.29528795811518327,
"fcm_dpo/beta": 0.014990235678851604,
"fcm_dpo/delta": 0.04028759524226189,
"fcm_dpo/margin": 14.224204063415527,
"fcm_dpo/q_t": 0.4487554430961609,
"grad_norm": 33.108272552490234,
"learning_rate": 4.453763107901675e-07,
"logits/chosen": 1.7549117803573608,
"logits/rejected": 1.8129609823226929,
"logps/chosen": -284.87762451171875,
"logps/ref_chosen": -288.071044921875,
"logps/ref_rejected": -274.83477783203125,
"logps/rejected": -285.86553955078125,
"loss": 4.8777,
"margin_dpo/margin_mean": 14.224204063415527,
"margin_dpo/margin_std": 24.404754638671875,
"step": 141
},
{
"KL/chosen_KL_mean": 5.986888885498047,
"KL/mean": -0.9714588522911072,
"KL/rejected_KL_mean": -7.929801940917969,
"KL/std": 19.997905731201172,
"epoch": 0.2973821989528796,
"fcm_dpo/beta": 0.015509484335780144,
"fcm_dpo/delta": 0.04578714445233345,
"fcm_dpo/margin": 13.916696548461914,
"fcm_dpo/q_t": 0.44861719012260437,
"grad_norm": 25.78396224975586,
"learning_rate": 4.4422887045602674e-07,
"logits/chosen": 1.5206505060195923,
"logits/rejected": 1.406781792640686,
"logps/chosen": -283.00494384765625,
"logps/ref_chosen": -288.9918518066406,
"logps/ref_rejected": -228.26071166992188,
"logps/rejected": -236.1905059814453,
"loss": 4.8791,
"margin_dpo/margin_mean": 13.916695594787598,
"margin_dpo/margin_std": 24.268972396850586,
"step": 142
},
{
"KL/chosen_KL_mean": 5.79694938659668,
"KL/mean": -2.3009872436523438,
"KL/rejected_KL_mean": -10.398920059204102,
"KL/std": 21.152034759521484,
"epoch": 0.2994764397905759,
"fcm_dpo/beta": 0.016140975058078766,
"fcm_dpo/delta": 0.07417097687721252,
"fcm_dpo/margin": 16.19586181640625,
"fcm_dpo/q_t": 0.438321053981781,
"grad_norm": 29.62942123413086,
"learning_rate": 4.4307101421701755e-07,
"logits/chosen": 1.4169150590896606,
"logits/rejected": 1.4356873035430908,
"logps/chosen": -283.6775207519531,
"logps/ref_chosen": -289.4744873046875,
"logps/ref_rejected": -265.130615234375,
"logps/rejected": -275.5295715332031,
"loss": 4.7569,
"margin_dpo/margin_mean": 16.195859909057617,
"margin_dpo/margin_std": 26.71329689025879,
"step": 143
},
{
"KL/chosen_KL_mean": 7.127115249633789,
"KL/mean": 0.3925158381462097,
"KL/rejected_KL_mean": -6.342063903808594,
"KL/std": 20.461181640625,
"epoch": 0.30157068062827225,
"fcm_dpo/beta": 0.01703416183590889,
"fcm_dpo/delta": 0.06011633202433586,
"fcm_dpo/margin": 13.469175338745117,
"fcm_dpo/q_t": 0.4462902843952179,
"grad_norm": 28.699085235595703,
"learning_rate": 4.419028041654559e-07,
"logits/chosen": 1.70430588722229,
"logits/rejected": 1.688096046447754,
"logps/chosen": -283.4592590332031,
"logps/ref_chosen": -290.5863342285156,
"logps/ref_rejected": -263.41046142578125,
"logps/rejected": -269.7525329589844,
"loss": 4.8609,
"margin_dpo/margin_mean": 13.4691743850708,
"margin_dpo/margin_std": 24.161060333251953,
"step": 144
},
{
"KL/chosen_KL_mean": 7.376377105712891,
"KL/mean": -1.2903668880462646,
"KL/rejected_KL_mean": -9.957107543945312,
"KL/std": 19.41018295288086,
"epoch": 0.3036649214659686,
"fcm_dpo/beta": 0.01746426150202751,
"fcm_dpo/delta": -0.00025831814855337143,
"fcm_dpo/margin": 17.333486557006836,
"fcm_dpo/q_t": 0.4287872910499573,
"grad_norm": 27.49614715576172,
"learning_rate": 4.4072430294890166e-07,
"logits/chosen": 1.5508968830108643,
"logits/rejected": 1.4520549774169922,
"logps/chosen": -257.16387939453125,
"logps/ref_chosen": -264.5402526855469,
"logps/ref_rejected": -214.7958221435547,
"logps/rejected": -224.7529296875,
"loss": 4.6247,
"margin_dpo/margin_mean": 17.333484649658203,
"margin_dpo/margin_std": 24.21767234802246,
"step": 145
},
{
"KL/chosen_KL_mean": 3.2908153533935547,
"KL/mean": -3.671146869659424,
"KL/rejected_KL_mean": -10.633108139038086,
"KL/std": 19.569717407226562,
"epoch": 0.3057591623036649,
"fcm_dpo/beta": 0.017695514485239983,
"fcm_dpo/delta": 0.050446767359972,
"fcm_dpo/margin": 13.923921585083008,
"fcm_dpo/q_t": 0.44180774688720703,
"grad_norm": 46.415531158447266,
"learning_rate": 4.395355737667985e-07,
"logits/chosen": 1.5119810104370117,
"logits/rejected": 1.626979947090149,
"logps/chosen": -266.8889465332031,
"logps/ref_chosen": -270.17974853515625,
"logps/ref_rejected": -261.2323913574219,
"logps/rejected": -271.865478515625,
"loss": 4.8104,
"margin_dpo/margin_mean": 13.92392349243164,
"margin_dpo/margin_std": 23.901309967041016,
"step": 146
},
{
"KL/chosen_KL_mean": 1.9331626892089844,
"KL/mean": -3.9541797637939453,
"KL/rejected_KL_mean": -9.841531753540039,
"KL/std": 21.822551727294922,
"epoch": 0.3078534031413613,
"fcm_dpo/beta": 0.018427424132823944,
"fcm_dpo/delta": 0.02790326438844204,
"fcm_dpo/margin": 11.774698257446289,
"fcm_dpo/q_t": 0.4492885172367096,
"grad_norm": 29.35231590270996,
"learning_rate": 4.3833668036708483e-07,
"logits/chosen": 1.5314257144927979,
"logits/rejected": 1.5078436136245728,
"logps/chosen": -280.8474426269531,
"logps/ref_chosen": -282.7806091308594,
"logps/ref_rejected": -249.23399353027344,
"logps/rejected": -259.0755310058594,
"loss": 4.9819,
"margin_dpo/margin_mean": 11.774696350097656,
"margin_dpo/margin_std": 26.265121459960938,
"step": 147
},
{
"KL/chosen_KL_mean": 2.4545116424560547,
"KL/mean": -3.7157320976257324,
"KL/rejected_KL_mean": -9.885980606079102,
"KL/std": 20.505939483642578,
"epoch": 0.3099476439790576,
"fcm_dpo/beta": 0.01864941045641899,
"fcm_dpo/delta": 0.018809720873832703,
"fcm_dpo/margin": 12.340494155883789,
"fcm_dpo/q_t": 0.4464648365974426,
"grad_norm": 30.132938385009766,
"learning_rate": 4.3712768704277524e-07,
"logits/chosen": 1.5904818773269653,
"logits/rejected": 1.6076817512512207,
"logps/chosen": -226.46279907226562,
"logps/ref_chosen": -228.9173126220703,
"logps/ref_rejected": -226.81222534179688,
"logps/rejected": -236.69822692871094,
"loss": 4.914,
"margin_dpo/margin_mean": 12.340494155883789,
"margin_dpo/margin_std": 25.5549373626709,
"step": 148
},
{
"KL/chosen_KL_mean": 4.127834320068359,
"KL/mean": -3.567676544189453,
"KL/rejected_KL_mean": -11.263181686401367,
"KL/std": 21.19050407409668,
"epoch": 0.31204188481675393,
"fcm_dpo/beta": 0.019410088658332825,
"fcm_dpo/delta": 0.05117492750287056,
"fcm_dpo/margin": 15.391016006469727,
"fcm_dpo/q_t": 0.4308072626590729,
"grad_norm": 32.52208709716797,
"learning_rate": 4.3590865862851263e-07,
"logits/chosen": 1.9011356830596924,
"logits/rejected": 1.8911197185516357,
"logps/chosen": -306.95806884765625,
"logps/ref_chosen": -311.0859375,
"logps/ref_rejected": -303.225341796875,
"logps/rejected": -314.4885559082031,
"loss": 4.6801,
"margin_dpo/margin_mean": 15.39101791381836,
"margin_dpo/margin_std": 25.0798282623291,
"step": 149
},
{
"KL/chosen_KL_mean": 1.6150665283203125,
"KL/mean": -5.609472751617432,
"KL/rejected_KL_mean": -12.834024429321289,
"KL/std": 20.40045166015625,
"epoch": 0.31413612565445026,
"fcm_dpo/beta": 0.020297512412071228,
"fcm_dpo/delta": 0.07883753627538681,
"fcm_dpo/margin": 14.449088096618652,
"fcm_dpo/q_t": 0.4320872724056244,
"grad_norm": 32.55631637573242,
"learning_rate": 4.346796604970912e-07,
"logits/chosen": 1.540358066558838,
"logits/rejected": 1.5534651279449463,
"logps/chosen": -268.86834716796875,
"logps/ref_chosen": -270.48345947265625,
"logps/ref_rejected": -269.0491638183594,
"logps/rejected": -281.8831481933594,
"loss": 4.7022,
"margin_dpo/margin_mean": 14.449089050292969,
"margin_dpo/margin_std": 24.21239471435547,
"step": 150
},
{
"KL/chosen_KL_mean": 5.48326301574707,
"KL/mean": -5.53632926940918,
"KL/rejected_KL_mean": -16.55592155456543,
"KL/std": 22.382789611816406,
"epoch": 0.3162303664921466,
"fcm_dpo/beta": 0.020343992859125137,
"fcm_dpo/delta": -0.05105812847614288,
"fcm_dpo/margin": 22.03917121887207,
"fcm_dpo/q_t": 0.3968949019908905,
"grad_norm": 35.151790618896484,
"learning_rate": 4.3344075855595097e-07,
"logits/chosen": 1.4414728879928589,
"logits/rejected": 1.447618007659912,
"logps/chosen": -282.68243408203125,
"logps/ref_chosen": -288.16571044921875,
"logps/ref_rejected": -264.7412414550781,
"logps/rejected": -281.29718017578125,
"loss": 4.2681,
"margin_dpo/margin_mean": 22.039173126220703,
"margin_dpo/margin_std": 27.89799690246582,
"step": 151
},
{
"KL/chosen_KL_mean": 3.7835044860839844,
"KL/mean": -4.452916145324707,
"KL/rejected_KL_mean": -12.689334869384766,
"KL/std": 22.493972778320312,
"epoch": 0.3183246073298429,
"fcm_dpo/beta": 0.020089194178581238,
"fcm_dpo/delta": 0.01719110645353794,
"fcm_dpo/margin": 16.472835540771484,
"fcm_dpo/q_t": 0.4237143099308014,
"grad_norm": 39.374393463134766,
"learning_rate": 4.3219201924364323e-07,
"logits/chosen": 1.5648020505905151,
"logits/rejected": 1.6551036834716797,
"logps/chosen": -256.1549987792969,
"logps/ref_chosen": -259.9385070800781,
"logps/ref_rejected": -266.2453308105469,
"logps/rejected": -278.9346923828125,
"loss": 4.6358,
"margin_dpo/margin_mean": 16.47283363342285,
"margin_dpo/margin_std": 26.881832122802734,
"step": 152
},
{
"KL/chosen_KL_mean": 2.586366653442383,
"KL/mean": -9.49244213104248,
"KL/rejected_KL_mean": -21.571243286132812,
"KL/std": 23.96262550354004,
"epoch": 0.3204188481675393,
"fcm_dpo/beta": 0.019452663138508797,
"fcm_dpo/delta": -0.07453102618455887,
"fcm_dpo/margin": 24.157608032226562,
"fcm_dpo/q_t": 0.3905426859855652,
"grad_norm": 40.339393615722656,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": 1.6833831071853638,
"logits/rejected": 1.7517095804214478,
"logps/chosen": -262.9906311035156,
"logps/ref_chosen": -265.5769958496094,
"logps/ref_rejected": -277.6246032714844,
"logps/rejected": -299.19586181640625,
"loss": 4.1366,
"margin_dpo/margin_mean": 24.157608032226562,
"margin_dpo/margin_std": 26.23731803894043,
"step": 153
},
{
"KL/chosen_KL_mean": 0.8299999237060547,
"KL/mean": -7.864226818084717,
"KL/rejected_KL_mean": -16.558452606201172,
"KL/std": 24.469377517700195,
"epoch": 0.3225130890052356,
"fcm_dpo/beta": 0.019323289394378662,
"fcm_dpo/delta": 0.002240743488073349,
"fcm_dpo/margin": 17.38846206665039,
"fcm_dpo/q_t": 0.42226171493530273,
"grad_norm": 36.89560317993164,
"learning_rate": 4.2966529689388064e-07,
"logits/chosen": 1.360990047454834,
"logits/rejected": 1.327060341835022,
"logps/chosen": -281.44342041015625,
"logps/ref_chosen": -282.2734069824219,
"logps/ref_rejected": -261.5135498046875,
"logps/rejected": -278.072021484375,
"loss": 4.6639,
"margin_dpo/margin_mean": 17.38846206665039,
"margin_dpo/margin_std": 30.103727340698242,
"step": 154
},
{
"KL/chosen_KL_mean": -0.40489959716796875,
"KL/mean": -6.861968994140625,
"KL/rejected_KL_mean": -13.319042205810547,
"KL/std": 22.191448211669922,
"epoch": 0.32460732984293195,
"fcm_dpo/beta": 0.019459176808595657,
"fcm_dpo/delta": 0.032827116549015045,
"fcm_dpo/margin": 12.914140701293945,
"fcm_dpo/q_t": 0.44211363792419434,
"grad_norm": 35.24588394165039,
"learning_rate": 4.2838744935687716e-07,
"logits/chosen": 1.6091328859329224,
"logits/rejected": 1.7022383213043213,
"logps/chosen": -279.116455078125,
"logps/ref_chosen": -278.7115783691406,
"logps/ref_rejected": -298.0945739746094,
"logps/rejected": -311.41363525390625,
"loss": 4.8997,
"margin_dpo/margin_mean": 12.914140701293945,
"margin_dpo/margin_std": 27.220930099487305,
"step": 155
},
{
"KL/chosen_KL_mean": 1.2807083129882812,
"KL/mean": -9.533502578735352,
"KL/rejected_KL_mean": -20.34770965576172,
"KL/std": 24.442045211791992,
"epoch": 0.3267015706806283,
"fcm_dpo/beta": 0.01958465948700905,
"fcm_dpo/delta": -0.024999193847179413,
"fcm_dpo/margin": 21.628427505493164,
"fcm_dpo/q_t": 0.4033975303173065,
"grad_norm": 31.40673828125,
"learning_rate": 4.271000354423425e-07,
"logits/chosen": 1.6639182567596436,
"logits/rejected": 1.694714903831482,
"logps/chosen": -267.0704345703125,
"logps/ref_chosen": -268.35113525390625,
"logps/ref_rejected": -241.9660186767578,
"logps/rejected": -262.313720703125,
"loss": 4.4131,
"margin_dpo/margin_mean": 21.628427505493164,
"margin_dpo/margin_std": 32.28349304199219,
"step": 156
},
{
"KL/chosen_KL_mean": -0.10972976684570312,
"KL/mean": -7.63818883895874,
"KL/rejected_KL_mean": -15.166645050048828,
"KL/std": 23.81902503967285,
"epoch": 0.3287958115183246,
"fcm_dpo/beta": 0.019391320645809174,
"fcm_dpo/delta": 0.02052324265241623,
"fcm_dpo/margin": 15.056900024414062,
"fcm_dpo/q_t": 0.43288618326187134,
"grad_norm": 46.412200927734375,
"learning_rate": 4.258031241903777e-07,
"logits/chosen": 1.4068256616592407,
"logits/rejected": 1.4659873247146606,
"logps/chosen": -265.51336669921875,
"logps/ref_chosen": -265.40362548828125,
"logps/ref_rejected": -257.82440185546875,
"logps/rejected": -272.99102783203125,
"loss": 4.7695,
"margin_dpo/margin_mean": 15.056900024414062,
"margin_dpo/margin_std": 27.764768600463867,
"step": 157
},
{
"KL/chosen_KL_mean": 1.0236034393310547,
"KL/mean": -7.27507209777832,
"KL/rejected_KL_mean": -15.573741912841797,
"KL/std": 23.505292892456055,
"epoch": 0.3308900523560209,
"fcm_dpo/beta": 0.020102355629205704,
"fcm_dpo/delta": 0.02125120908021927,
"fcm_dpo/margin": 16.59735107421875,
"fcm_dpo/q_t": 0.42306697368621826,
"grad_norm": 35.24538803100586,
"learning_rate": 4.2449678515039743e-07,
"logits/chosen": 1.2856690883636475,
"logits/rejected": 1.317069411277771,
"logps/chosen": -280.1387023925781,
"logps/ref_chosen": -281.1622619628906,
"logps/ref_rejected": -268.2899475097656,
"logps/rejected": -283.8636779785156,
"loss": 4.6476,
"margin_dpo/margin_mean": 16.597352981567383,
"margin_dpo/margin_std": 27.941625595092773,
"step": 158
},
{
"KL/chosen_KL_mean": 0.037982940673828125,
"KL/mean": -6.434523582458496,
"KL/rejected_KL_mean": -12.907020568847656,
"KL/std": 24.362865447998047,
"epoch": 0.33298429319371725,
"fcm_dpo/beta": 0.019898485392332077,
"fcm_dpo/delta": -0.001014402136206627,
"fcm_dpo/margin": 12.94499397277832,
"fcm_dpo/q_t": 0.44250720739364624,
"grad_norm": 39.46014404296875,
"learning_rate": 4.2318108837739986e-07,
"logits/chosen": 1.5753355026245117,
"logits/rejected": 1.46786367893219,
"logps/chosen": -311.28704833984375,
"logps/ref_chosen": -311.32501220703125,
"logps/ref_rejected": -248.85350036621094,
"logps/rejected": -261.760498046875,
"loss": 5.012,
"margin_dpo/margin_mean": 12.94499397277832,
"margin_dpo/margin_std": 32.133689880371094,
"step": 159
},
{
"KL/chosen_KL_mean": 3.813638687133789,
"KL/mean": -6.125799655914307,
"KL/rejected_KL_mean": -16.06523895263672,
"KL/std": 24.1287899017334,
"epoch": 0.33507853403141363,
"fcm_dpo/beta": 0.01965140737593174,
"fcm_dpo/delta": -0.023824896663427353,
"fcm_dpo/margin": 19.878883361816406,
"fcm_dpo/q_t": 0.41106274724006653,
"grad_norm": 50.02199172973633,
"learning_rate": 4.218561044282098e-07,
"logits/chosen": 1.4759557247161865,
"logits/rejected": 1.4768081903457642,
"logps/chosen": -263.29534912109375,
"logps/ref_chosen": -267.10894775390625,
"logps/ref_rejected": -258.66162109375,
"logps/rejected": -274.72686767578125,
"loss": 4.4502,
"margin_dpo/margin_mean": 19.878883361816406,
"margin_dpo/margin_std": 27.98908805847168,
"step": 160
},
{
"KL/chosen_KL_mean": 3.346097946166992,
"KL/mean": -6.985053539276123,
"KL/rejected_KL_mean": -17.31619644165039,
"KL/std": 25.046649932861328,
"epoch": 0.33717277486910996,
"fcm_dpo/beta": 0.019791923463344574,
"fcm_dpo/delta": -0.009587782435119152,
"fcm_dpo/margin": 20.662309646606445,
"fcm_dpo/q_t": 0.40747061371803284,
"grad_norm": 42.32925033569336,
"learning_rate": 4.2052190435769554e-07,
"logits/chosen": 1.7133197784423828,
"logits/rejected": 1.6081936359405518,
"logps/chosen": -298.5462646484375,
"logps/ref_chosen": -301.89239501953125,
"logps/ref_rejected": -247.645263671875,
"logps/rejected": -264.9614562988281,
"loss": 4.4483,
"margin_dpo/margin_mean": 20.662309646606445,
"margin_dpo/margin_std": 31.163915634155273,
"step": 161
},
{
"KL/chosen_KL_mean": 4.65040397644043,
"KL/mean": -4.603542327880859,
"KL/rejected_KL_mean": -13.857484817504883,
"KL/std": 24.12982940673828,
"epoch": 0.3392670157068063,
"fcm_dpo/beta": 0.01887052319943905,
"fcm_dpo/delta": -0.011755827814340591,
"fcm_dpo/margin": 18.507890701293945,
"fcm_dpo/q_t": 0.4189423620700836,
"grad_norm": 36.829593658447266,
"learning_rate": 4.1917855971495763e-07,
"logits/chosen": 1.603858232498169,
"logits/rejected": 1.5859763622283936,
"logps/chosen": -283.5826416015625,
"logps/ref_chosen": -288.2330627441406,
"logps/ref_rejected": -252.5148162841797,
"logps/rejected": -266.3722839355469,
"loss": 4.5906,
"margin_dpo/margin_mean": 18.507892608642578,
"margin_dpo/margin_std": 29.093059539794922,
"step": 162
},
{
"KL/chosen_KL_mean": 3.3226470947265625,
"KL/mean": -7.70544958114624,
"KL/rejected_KL_mean": -18.73355484008789,
"KL/std": 23.491344451904297,
"epoch": 0.3413612565445026,
"fcm_dpo/beta": 0.019159583374857903,
"fcm_dpo/delta": -0.024082660675048828,
"fcm_dpo/margin": 22.05620002746582,
"fcm_dpo/q_t": 0.40153104066848755,
"grad_norm": 37.20431137084961,
"learning_rate": 4.1782614253949255e-07,
"logits/chosen": 1.6068838834762573,
"logits/rejected": 1.5826996564865112,
"logps/chosen": -283.4411926269531,
"logps/ref_chosen": -286.7638244628906,
"logps/ref_rejected": -262.8302917480469,
"logps/rejected": -281.5638122558594,
"loss": 4.2868,
"margin_dpo/margin_mean": 22.056196212768555,
"margin_dpo/margin_std": 26.158645629882812,
"step": 163
},
{
"KL/chosen_KL_mean": -3.850292205810547,
"KL/mean": -13.29897403717041,
"KL/rejected_KL_mean": -22.747655868530273,
"KL/std": 25.1888370513916,
"epoch": 0.34345549738219894,
"fcm_dpo/beta": 0.018868062645196915,
"fcm_dpo/delta": -0.02091181091964245,
"fcm_dpo/margin": 18.897357940673828,
"fcm_dpo/q_t": 0.4178180396556854,
"grad_norm": 32.7020378112793,
"learning_rate": 4.164647253573289e-07,
"logits/chosen": 1.732231616973877,
"logits/rejected": 1.7001898288726807,
"logps/chosen": -264.1287841796875,
"logps/ref_chosen": -260.2784729003906,
"logps/ref_rejected": -217.10556030273438,
"logps/rejected": -239.8532257080078,
"loss": 4.5442,
"margin_dpo/margin_mean": 18.897357940673828,
"margin_dpo/margin_std": 28.706422805786133,
"step": 164
},
{
"KL/chosen_KL_mean": 0.1812763214111328,
"KL/mean": -7.839303970336914,
"KL/rejected_KL_mean": -15.859880447387695,
"KL/std": 23.385269165039062,
"epoch": 0.34554973821989526,
"fcm_dpo/beta": 0.01889457367360592,
"fcm_dpo/delta": 0.024164600297808647,
"fcm_dpo/margin": 16.041149139404297,
"fcm_dpo/q_t": 0.4299403131008148,
"grad_norm": 36.616493225097656,
"learning_rate": 4.1509438117713863e-07,
"logits/chosen": 1.7898118495941162,
"logits/rejected": 1.8426463603973389,
"logps/chosen": -270.373046875,
"logps/ref_chosen": -270.5543212890625,
"logps/ref_rejected": -259.80218505859375,
"logps/rejected": -275.66204833984375,
"loss": 4.6946,
"margin_dpo/margin_mean": 16.041149139404297,
"margin_dpo/margin_std": 26.979326248168945,
"step": 165
},
{
"KL/chosen_KL_mean": -2.423433303833008,
"KL/mean": -9.657724380493164,
"KL/rejected_KL_mean": -16.892017364501953,
"KL/std": 25.662090301513672,
"epoch": 0.34764397905759165,
"fcm_dpo/beta": 0.018830828368663788,
"fcm_dpo/delta": 0.014171226881444454,
"fcm_dpo/margin": 14.468596458435059,
"fcm_dpo/q_t": 0.4390319287776947,
"grad_norm": 32.48873519897461,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": 1.5254061222076416,
"logits/rejected": 1.5039020776748657,
"logps/chosen": -272.7178649902344,
"logps/ref_chosen": -270.29443359375,
"logps/ref_rejected": -230.40151977539062,
"logps/rejected": -247.29351806640625,
"loss": 4.9205,
"margin_dpo/margin_mean": 14.468595504760742,
"margin_dpo/margin_std": 33.09213638305664,
"step": 166
},
{
"KL/chosen_KL_mean": -2.779369354248047,
"KL/mean": -11.354179382324219,
"KL/rejected_KL_mean": -19.92898941040039,
"KL/std": 25.046934127807617,
"epoch": 0.34973821989528797,
"fcm_dpo/beta": 0.01971183903515339,
"fcm_dpo/delta": 0.028924930840730667,
"fcm_dpo/margin": 17.14962387084961,
"fcm_dpo/q_t": 0.42319583892822266,
"grad_norm": 34.71977233886719,
"learning_rate": 4.123272062470633e-07,
"logits/chosen": 1.7265127897262573,
"logits/rejected": 1.6851749420166016,
"logps/chosen": -276.2399597167969,
"logps/ref_chosen": -273.4605712890625,
"logps/ref_rejected": -248.86997985839844,
"logps/rejected": -268.7989501953125,
"loss": 4.6612,
"margin_dpo/margin_mean": 17.14962387084961,
"margin_dpo/margin_std": 30.061779022216797,
"step": 167
},
{
"KL/chosen_KL_mean": 0.3898735046386719,
"KL/mean": -10.855436325073242,
"KL/rejected_KL_mean": -22.100749969482422,
"KL/std": 27.144193649291992,
"epoch": 0.3518324607329843,
"fcm_dpo/beta": 0.01884337142109871,
"fcm_dpo/delta": -0.07109567523002625,
"fcm_dpo/margin": 22.490629196166992,
"fcm_dpo/q_t": 0.4015496075153351,
"grad_norm": 35.232513427734375,
"learning_rate": 4.1093052389237174e-07,
"logits/chosen": 1.7364689111709595,
"logits/rejected": 1.7493667602539062,
"logps/chosen": -305.3348388671875,
"logps/ref_chosen": -305.7247009277344,
"logps/ref_rejected": -290.16650390625,
"logps/rejected": -312.26727294921875,
"loss": 4.3533,
"margin_dpo/margin_mean": 22.490631103515625,
"margin_dpo/margin_std": 29.51881217956543,
"step": 168
},
{
"KL/chosen_KL_mean": -0.20895767211914062,
"KL/mean": -13.058430671691895,
"KL/rejected_KL_mean": -25.90790557861328,
"KL/std": 26.704998016357422,
"epoch": 0.3539267015706806,
"fcm_dpo/beta": 0.017862707376480103,
"fcm_dpo/delta": -0.09691692888736725,
"fcm_dpo/margin": 25.698944091796875,
"fcm_dpo/q_t": 0.3947722017765045,
"grad_norm": 29.24917221069336,
"learning_rate": 4.0952521132208267e-07,
"logits/chosen": 1.2933053970336914,
"logits/rejected": 1.3155572414398193,
"logps/chosen": -252.3792266845703,
"logps/ref_chosen": -252.1702880859375,
"logps/ref_rejected": -233.34750366210938,
"logps/rejected": -259.2554016113281,
"loss": 4.2295,
"margin_dpo/margin_mean": 25.698944091796875,
"margin_dpo/margin_std": 29.882102966308594,
"step": 169
},
{
"KL/chosen_KL_mean": 0.6535549163818359,
"KL/mean": -10.402743339538574,
"KL/rejected_KL_mean": -21.459043502807617,
"KL/std": 25.29845428466797,
"epoch": 0.35602094240837695,
"fcm_dpo/beta": 0.017152002081274986,
"fcm_dpo/delta": 0.02156105265021324,
"fcm_dpo/margin": 22.11260223388672,
"fcm_dpo/q_t": 0.41253462433815,
"grad_norm": 37.989990234375,
"learning_rate": 4.081113438988443e-07,
"logits/chosen": 1.4381834268569946,
"logits/rejected": 1.4736790657043457,
"logps/chosen": -301.8248596191406,
"logps/ref_chosen": -302.4783935546875,
"logps/ref_rejected": -259.40777587890625,
"logps/rejected": -280.8668212890625,
"loss": 4.4294,
"margin_dpo/margin_mean": 22.112600326538086,
"margin_dpo/margin_std": 30.194551467895508,
"step": 170
},
{
"KL/chosen_KL_mean": 0.401123046875,
"KL/mean": -10.984270095825195,
"KL/rejected_KL_mean": -22.36966896057129,
"KL/std": 25.969249725341797,
"epoch": 0.3581151832460733,
"fcm_dpo/beta": 0.016981348395347595,
"fcm_dpo/delta": -0.0327591747045517,
"fcm_dpo/margin": 22.770790100097656,
"fcm_dpo/q_t": 0.41038233041763306,
"grad_norm": 33.292850494384766,
"learning_rate": 4.0668899744407567e-07,
"logits/chosen": 1.3699983358383179,
"logits/rejected": 1.4796205759048462,
"logps/chosen": -250.24560546875,
"logps/ref_chosen": -250.646728515625,
"logps/ref_rejected": -259.1659851074219,
"logps/rejected": -281.5356750488281,
"loss": 4.409,
"margin_dpo/margin_mean": 22.770788192749023,
"margin_dpo/margin_std": 29.17012596130371,
"step": 171
},
{
"KL/chosen_KL_mean": 1.918487548828125,
"KL/mean": -5.691974639892578,
"KL/rejected_KL_mean": -13.302434921264648,
"KL/std": 26.688800811767578,
"epoch": 0.36020942408376966,
"fcm_dpo/beta": 0.017365295439958572,
"fcm_dpo/delta": 0.04691143333911896,
"fcm_dpo/margin": 15.220909118652344,
"fcm_dpo/q_t": 0.4379444122314453,
"grad_norm": 31.966800689697266,
"learning_rate": 4.0525824823390043e-07,
"logits/chosen": 1.688840627670288,
"logits/rejected": 1.6983768939971924,
"logps/chosen": -327.1064453125,
"logps/ref_chosen": -329.02490234375,
"logps/ref_rejected": -293.0215759277344,
"logps/rejected": -306.3240051269531,
"loss": 4.8557,
"margin_dpo/margin_mean": 15.220909118652344,
"margin_dpo/margin_std": 30.862401962280273,
"step": 172
},
{
"KL/chosen_KL_mean": 1.177724838256836,
"KL/mean": -8.230151176452637,
"KL/rejected_KL_mean": -17.638029098510742,
"KL/std": 26.464139938354492,
"epoch": 0.362303664921466,
"fcm_dpo/beta": 0.017997927963733673,
"fcm_dpo/delta": 0.020180756226181984,
"fcm_dpo/margin": 18.815750122070312,
"fcm_dpo/q_t": 0.4219967722892761,
"grad_norm": 31.154638290405273,
"learning_rate": 4.0381917299505686e-07,
"logits/chosen": 1.525386095046997,
"logits/rejected": 1.496148705482483,
"logps/chosen": -288.962890625,
"logps/ref_chosen": -290.140625,
"logps/ref_rejected": -271.474853515625,
"logps/rejected": -289.11285400390625,
"loss": 4.5925,
"margin_dpo/margin_mean": 18.815750122070312,
"margin_dpo/margin_std": 29.17994499206543,
"step": 173
},
{
"KL/chosen_KL_mean": 2.210672378540039,
"KL/mean": -9.635655403137207,
"KL/rejected_KL_mean": -21.481983184814453,
"KL/std": 25.624143600463867,
"epoch": 0.3643979057591623,
"fcm_dpo/beta": 0.017743976786732674,
"fcm_dpo/delta": -0.022125719115138054,
"fcm_dpo/margin": 23.69266128540039,
"fcm_dpo/q_t": 0.40354806184768677,
"grad_norm": 42.779747009277344,
"learning_rate": 4.0237184890078243e-07,
"logits/chosen": 1.5114091634750366,
"logits/rejected": 1.4241846799850464,
"logps/chosen": -327.0965881347656,
"logps/ref_chosen": -329.3072509765625,
"logps/ref_rejected": -256.6839599609375,
"logps/rejected": -278.16595458984375,
"loss": 4.3127,
"margin_dpo/margin_mean": 23.692659378051758,
"margin_dpo/margin_std": 29.146427154541016,
"step": 174
},
{
"KL/chosen_KL_mean": -0.10210800170898438,
"KL/mean": -9.668460845947266,
"KL/rejected_KL_mean": -19.23480987548828,
"KL/std": 26.648395538330078,
"epoch": 0.36649214659685864,
"fcm_dpo/beta": 0.017704615369439125,
"fcm_dpo/delta": 0.0033662477508187294,
"fcm_dpo/margin": 19.132694244384766,
"fcm_dpo/q_t": 0.42134740948677063,
"grad_norm": 32.94378662109375,
"learning_rate": 4.00916353566676e-07,
"logits/chosen": 1.6281156539916992,
"logits/rejected": 1.6203536987304688,
"logps/chosen": -276.0043640136719,
"logps/ref_chosen": -275.9023132324219,
"logps/ref_rejected": -281.4163513183594,
"logps/rejected": -300.65118408203125,
"loss": 4.6253,
"margin_dpo/margin_mean": 19.1326961517334,
"margin_dpo/margin_std": 31.65574836730957,
"step": 175
},
{
"KL/chosen_KL_mean": -1.8588676452636719,
"KL/mean": -8.88139533996582,
"KL/rejected_KL_mean": -15.903926849365234,
"KL/std": 26.206148147583008,
"epoch": 0.36858638743455496,
"fcm_dpo/beta": 0.017541371285915375,
"fcm_dpo/delta": -0.027952371165156364,
"fcm_dpo/margin": 14.045061111450195,
"fcm_dpo/q_t": 0.4421733021736145,
"grad_norm": 33.93702697753906,
"learning_rate": 3.994527650465352e-07,
"logits/chosen": 1.3971831798553467,
"logits/rejected": 1.4549391269683838,
"logps/chosen": -245.423095703125,
"logps/ref_chosen": -243.56423950195312,
"logps/ref_rejected": -239.12905883789062,
"logps/rejected": -255.03297424316406,
"loss": 4.9662,
"margin_dpo/margin_mean": 14.045061111450195,
"margin_dpo/margin_std": 31.961017608642578,
"step": 176
},
{
"KL/chosen_KL_mean": -2.177288055419922,
"KL/mean": -9.260589599609375,
"KL/rejected_KL_mean": -16.343902587890625,
"KL/std": 26.89461326599121,
"epoch": 0.3706806282722513,
"fcm_dpo/beta": 0.017628297209739685,
"fcm_dpo/delta": 0.03455434739589691,
"fcm_dpo/margin": 14.166617393493652,
"fcm_dpo/q_t": 0.4423142969608307,
"grad_norm": 62.61311340332031,
"learning_rate": 3.979811618281705e-07,
"logits/chosen": 1.5508711338043213,
"logits/rejected": 1.4726340770721436,
"logps/chosen": -295.48095703125,
"logps/ref_chosen": -293.30364990234375,
"logps/ref_rejected": -292.293701171875,
"logps/rejected": -308.6376037597656,
"loss": 4.9989,
"margin_dpo/margin_mean": 14.166617393493652,
"margin_dpo/margin_std": 34.50107192993164,
"step": 177
},
{
"KL/chosen_KL_mean": 2.100444793701172,
"KL/mean": -9.132177352905273,
"KL/rejected_KL_mean": -20.36479949951172,
"KL/std": 28.20733070373535,
"epoch": 0.37277486910994767,
"fcm_dpo/beta": 0.017436780035495758,
"fcm_dpo/delta": -0.02598407492041588,
"fcm_dpo/margin": 22.465251922607422,
"fcm_dpo/q_t": 0.4109533429145813,
"grad_norm": 36.81523132324219,
"learning_rate": 3.9650162282919654e-07,
"logits/chosen": 1.6348121166229248,
"logits/rejected": 1.6586459875106812,
"logps/chosen": -274.0257873535156,
"logps/ref_chosen": -276.126220703125,
"logps/ref_rejected": -240.31179809570312,
"logps/rejected": -260.6766052246094,
"loss": 4.4651,
"margin_dpo/margin_mean": 22.46525001525879,
"margin_dpo/margin_std": 33.34449768066406,
"step": 178
},
{
"KL/chosen_KL_mean": 1.2245655059814453,
"KL/mean": -6.5566229820251465,
"KL/rejected_KL_mean": -14.337806701660156,
"KL/std": 27.405574798583984,
"epoch": 0.374869109947644,
"fcm_dpo/beta": 0.01756238378584385,
"fcm_dpo/delta": 0.03585870563983917,
"fcm_dpo/margin": 15.56238079071045,
"fcm_dpo/q_t": 0.4376697540283203,
"grad_norm": 37.79298782348633,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": 1.316421389579773,
"logits/rejected": 1.4284144639968872,
"logps/chosen": -248.6378631591797,
"logps/ref_chosen": -249.8624267578125,
"logps/ref_rejected": -271.18603515625,
"logps/rejected": -285.52386474609375,
"loss": 4.8619,
"margin_dpo/margin_mean": 15.562381744384766,
"margin_dpo/margin_std": 31.931869506835938,
"step": 179
},
{
"KL/chosen_KL_mean": 2.530637741088867,
"KL/mean": -7.931081295013428,
"KL/rejected_KL_mean": -18.39280128479004,
"KL/std": 26.43503189086914,
"epoch": 0.3769633507853403,
"fcm_dpo/beta": 0.017923174425959587,
"fcm_dpo/delta": -0.00823952816426754,
"fcm_dpo/margin": 20.92344856262207,
"fcm_dpo/q_t": 0.4146185517311096,
"grad_norm": 44.96994400024414,
"learning_rate": 3.935190552834828e-07,
"logits/chosen": 1.5395066738128662,
"logits/rejected": 1.5768020153045654,
"logps/chosen": -258.13848876953125,
"logps/ref_chosen": -260.66912841796875,
"logps/ref_rejected": -229.25491333007812,
"logps/rejected": -247.64773559570312,
"loss": 4.5056,
"margin_dpo/margin_mean": 20.92344856262207,
"margin_dpo/margin_std": 31.313364028930664,
"step": 180
},
{
"KL/chosen_KL_mean": 4.5495452880859375,
"KL/mean": -5.502880096435547,
"KL/rejected_KL_mean": -15.555295944213867,
"KL/std": 26.449636459350586,
"epoch": 0.37905759162303665,
"fcm_dpo/beta": 0.017637405544519424,
"fcm_dpo/delta": -0.005057952366769314,
"fcm_dpo/margin": 20.104846954345703,
"fcm_dpo/q_t": 0.4186038374900818,
"grad_norm": 36.245113372802734,
"learning_rate": 3.920161866827889e-07,
"logits/chosen": 1.5702896118164062,
"logits/rejected": 1.54368257522583,
"logps/chosen": -286.3390197753906,
"logps/ref_chosen": -290.8885803222656,
"logps/ref_rejected": -275.1922302246094,
"logps/rejected": -290.7475280761719,
"loss": 4.5959,
"margin_dpo/margin_mean": 20.104846954345703,
"margin_dpo/margin_std": 32.911582946777344,
"step": 181
},
{
"KL/chosen_KL_mean": 2.2509727478027344,
"KL/mean": -10.04803466796875,
"KL/rejected_KL_mean": -22.347047805786133,
"KL/std": 26.909276962280273,
"epoch": 0.381151832460733,
"fcm_dpo/beta": 0.017666857689619064,
"fcm_dpo/delta": -0.036299802362918854,
"fcm_dpo/margin": 24.598037719726562,
"fcm_dpo/q_t": 0.3986150324344635,
"grad_norm": 47.16189193725586,
"learning_rate": 3.90505702185e-07,
"logits/chosen": 1.46827232837677,
"logits/rejected": 1.4841911792755127,
"logps/chosen": -268.31585693359375,
"logps/ref_chosen": -270.56683349609375,
"logps/ref_rejected": -260.3321533203125,
"logps/rejected": -282.6791687011719,
"loss": 4.2547,
"margin_dpo/margin_mean": 24.598037719726562,
"margin_dpo/margin_std": 28.5032958984375,
"step": 182
},
{
"KL/chosen_KL_mean": 2.6596717834472656,
"KL/mean": -7.440838813781738,
"KL/rejected_KL_mean": -17.541349411010742,
"KL/std": 26.812868118286133,
"epoch": 0.3832460732984293,
"fcm_dpo/beta": 0.01747625507414341,
"fcm_dpo/delta": 0.048673342913389206,
"fcm_dpo/margin": 20.20101547241211,
"fcm_dpo/q_t": 0.4192884564399719,
"grad_norm": 35.38688659667969,
"learning_rate": 3.889876827928156e-07,
"logits/chosen": 1.5745729207992554,
"logits/rejected": 1.6223291158676147,
"logps/chosen": -258.3683166503906,
"logps/ref_chosen": -261.0279846191406,
"logps/ref_rejected": -240.72361755371094,
"logps/rejected": -258.2649841308594,
"loss": 4.5729,
"margin_dpo/margin_mean": 20.20101547241211,
"margin_dpo/margin_std": 32.505943298339844,
"step": 183
},
{
"KL/chosen_KL_mean": 6.955654144287109,
"KL/mean": -7.538846492767334,
"KL/rejected_KL_mean": -22.033349990844727,
"KL/std": 29.865995407104492,
"epoch": 0.38534031413612563,
"fcm_dpo/beta": 0.016830746084451675,
"fcm_dpo/delta": -0.09457923471927643,
"fcm_dpo/margin": 28.989009857177734,
"fcm_dpo/q_t": 0.3883908689022064,
"grad_norm": 36.70336151123047,
"learning_rate": 3.874622099130087e-07,
"logits/chosen": 1.4726858139038086,
"logits/rejected": 1.4355816841125488,
"logps/chosen": -302.1282043457031,
"logps/ref_chosen": -309.0838317871094,
"logps/ref_rejected": -263.6435241699219,
"logps/rejected": -285.6768798828125,
"loss": 4.1719,
"margin_dpo/margin_mean": 28.989009857177734,
"margin_dpo/margin_std": 35.74262237548828,
"step": 184
},
{
"KL/chosen_KL_mean": 1.0868282318115234,
"KL/mean": -9.744384765625,
"KL/rejected_KL_mean": -20.57560157775879,
"KL/std": 28.781902313232422,
"epoch": 0.387434554973822,
"fcm_dpo/beta": 0.0165347158908844,
"fcm_dpo/delta": 0.006427157670259476,
"fcm_dpo/margin": 21.66242218017578,
"fcm_dpo/q_t": 0.41854044795036316,
"grad_norm": 39.357120513916016,
"learning_rate": 3.859293653520604e-07,
"logits/chosen": 1.453484058380127,
"logits/rejected": 1.5226836204528809,
"logps/chosen": -264.6148681640625,
"logps/ref_chosen": -265.70172119140625,
"logps/ref_rejected": -257.5074157714844,
"logps/rejected": -278.0830078125,
"loss": 4.5754,
"margin_dpo/margin_mean": 21.662424087524414,
"margin_dpo/margin_std": 34.37306594848633,
"step": 185
},
{
"KL/chosen_KL_mean": -0.38506507873535156,
"KL/mean": -10.96863842010498,
"KL/rejected_KL_mean": -21.55221176147461,
"KL/std": 27.273874282836914,
"epoch": 0.38952879581151834,
"fcm_dpo/beta": 0.016737408936023712,
"fcm_dpo/delta": 0.014673182740807533,
"fcm_dpo/margin": 21.167146682739258,
"fcm_dpo/q_t": 0.4183846414089203,
"grad_norm": 47.73256301879883,
"learning_rate": 3.8438923131177237e-07,
"logits/chosen": 1.437438726425171,
"logits/rejected": 1.4502127170562744,
"logps/chosen": -292.11004638671875,
"logps/ref_chosen": -291.7249755859375,
"logps/ref_rejected": -273.54010009765625,
"logps/rejected": -295.09234619140625,
"loss": 4.5356,
"margin_dpo/margin_mean": 21.16714859008789,
"margin_dpo/margin_std": 31.127275466918945,
"step": 186
},
{
"KL/chosen_KL_mean": 0.13139915466308594,
"KL/mean": -10.805803298950195,
"KL/rejected_KL_mean": -21.74300193786621,
"KL/std": 29.678203582763672,
"epoch": 0.39162303664921466,
"fcm_dpo/beta": 0.016892198473215103,
"fcm_dpo/delta": 0.031411200761795044,
"fcm_dpo/margin": 21.874393463134766,
"fcm_dpo/q_t": 0.4164515733718872,
"grad_norm": 37.13667678833008,
"learning_rate": 3.828418903848593e-07,
"logits/chosen": 1.5591447353363037,
"logits/rejected": 1.5824084281921387,
"logps/chosen": -321.5704650878906,
"logps/ref_chosen": -321.70184326171875,
"logps/ref_rejected": -261.8386535644531,
"logps/rejected": -283.5816650390625,
"loss": 4.5952,
"margin_dpo/margin_mean": 21.874393463134766,
"margin_dpo/margin_std": 37.063011169433594,
"step": 187
},
{
"KL/chosen_KL_mean": -1.804117202758789,
"KL/mean": -12.928169250488281,
"KL/rejected_KL_mean": -24.052230834960938,
"KL/std": 29.605234146118164,
"epoch": 0.393717277486911,
"fcm_dpo/beta": 0.01713397167623043,
"fcm_dpo/delta": -0.009998919442296028,
"fcm_dpo/margin": 22.248119354248047,
"fcm_dpo/q_t": 0.41325312852859497,
"grad_norm": 33.81171798706055,
"learning_rate": 3.812874255505191e-07,
"logits/chosen": 1.3632234334945679,
"logits/rejected": 1.333018183708191,
"logps/chosen": -257.71954345703125,
"logps/ref_chosen": -255.91542053222656,
"logps/ref_rejected": -245.1119842529297,
"logps/rejected": -269.1642150878906,
"loss": 4.5532,
"margin_dpo/margin_mean": 22.248119354248047,
"margin_dpo/margin_std": 36.244773864746094,
"step": 188
},
{
"KL/chosen_KL_mean": 3.7180252075195312,
"KL/mean": -9.177162170410156,
"KL/rejected_KL_mean": -22.07235336303711,
"KL/std": 29.054580688476562,
"epoch": 0.3958115183246073,
"fcm_dpo/beta": 0.016553619876503944,
"fcm_dpo/delta": -0.0703524798154831,
"fcm_dpo/margin": 25.790372848510742,
"fcm_dpo/q_t": 0.40066149830818176,
"grad_norm": 28.8972225189209,
"learning_rate": 3.797259201699833e-07,
"logits/chosen": 1.5498346090316772,
"logits/rejected": 1.5829347372055054,
"logps/chosen": -262.300537109375,
"logps/ref_chosen": -266.0185546875,
"logps/ref_rejected": -272.862548828125,
"logps/rejected": -294.9349060058594,
"loss": 4.2716,
"margin_dpo/margin_mean": 25.790372848510742,
"margin_dpo/margin_std": 29.882930755615234,
"step": 189
},
{
"KL/chosen_KL_mean": 2.1718368530273438,
"KL/mean": -10.560308456420898,
"KL/rejected_KL_mean": -23.292451858520508,
"KL/std": 28.673686981201172,
"epoch": 0.39790575916230364,
"fcm_dpo/beta": 0.016140490770339966,
"fcm_dpo/delta": -0.011915532872080803,
"fcm_dpo/margin": 25.464290618896484,
"fcm_dpo/q_t": 0.40527617931365967,
"grad_norm": 30.04618263244629,
"learning_rate": 3.781574579820464e-07,
"logits/chosen": 1.4085410833358765,
"logits/rejected": 1.3426133394241333,
"logps/chosen": -247.9791259765625,
"logps/ref_chosen": -250.15098571777344,
"logps/ref_rejected": -223.34393310546875,
"logps/rejected": -246.63636779785156,
"loss": 4.3943,
"margin_dpo/margin_mean": 25.464290618896484,
"margin_dpo/margin_std": 35.534278869628906,
"step": 190
},
{
"KL/chosen_KL_mean": 1.7022075653076172,
"KL/mean": -8.630979537963867,
"KL/rejected_KL_mean": -18.964176177978516,
"KL/std": 29.75131607055664,
"epoch": 0.4,
"fcm_dpo/beta": 0.01615564338862896,
"fcm_dpo/delta": 0.02544761821627617,
"fcm_dpo/margin": 20.6663818359375,
"fcm_dpo/q_t": 0.4240559935569763,
"grad_norm": 41.732566833496094,
"learning_rate": 3.765821230985757e-07,
"logits/chosen": 1.316603660583496,
"logits/rejected": 1.376888632774353,
"logps/chosen": -236.68960571289062,
"logps/ref_chosen": -238.3917999267578,
"logps/ref_rejected": -243.70272827148438,
"logps/rejected": -262.6669006347656,
"loss": 4.6813,
"margin_dpo/margin_mean": 20.6663818359375,
"margin_dpo/margin_std": 37.0047607421875,
"step": 191
},
{
"KL/chosen_KL_mean": -2.1696548461914062,
"KL/mean": -11.991387367248535,
"KL/rejected_KL_mean": -21.8131046295166,
"KL/std": 29.120777130126953,
"epoch": 0.40209424083769635,
"fcm_dpo/beta": 0.016227317973971367,
"fcm_dpo/delta": -0.005627312697470188,
"fcm_dpo/margin": 19.643455505371094,
"fcm_dpo/q_t": 0.42661046981811523,
"grad_norm": 39.09224319458008,
"learning_rate": 3.75e-07,
"logits/chosen": 1.5734920501708984,
"logits/rejected": 1.653855562210083,
"logps/chosen": -258.806640625,
"logps/ref_chosen": -256.6369934082031,
"logps/ref_rejected": -256.8894348144531,
"logps/rejected": -278.7025146484375,
"loss": 4.7535,
"margin_dpo/margin_mean": 19.64345359802246,
"margin_dpo/margin_std": 36.65842819213867,
"step": 192
},
{
"KL/chosen_KL_mean": -5.266847610473633,
"KL/mean": -15.703073501586914,
"KL/rejected_KL_mean": -26.139293670654297,
"KL/std": 31.600704193115234,
"epoch": 0.4041884816753927,
"fcm_dpo/beta": 0.016515541821718216,
"fcm_dpo/delta": 0.025459565222263336,
"fcm_dpo/margin": 20.872455596923828,
"fcm_dpo/q_t": 0.42145636677742004,
"grad_norm": 37.17631149291992,
"learning_rate": 3.734111735307796e-07,
"logits/chosen": 1.4087841510772705,
"logits/rejected": 1.5019336938858032,
"logps/chosen": -278.0664978027344,
"logps/ref_chosen": -272.7996826171875,
"logps/ref_rejected": -289.2391052246094,
"logps/rejected": -315.3783874511719,
"loss": 4.6456,
"margin_dpo/margin_mean": 20.87245750427246,
"margin_dpo/margin_std": 36.63458251953125,
"step": 193
},
{
"KL/chosen_KL_mean": -4.638954162597656,
"KL/mean": -12.322515487670898,
"KL/rejected_KL_mean": -20.006086349487305,
"KL/std": 31.54027557373047,
"epoch": 0.406282722513089,
"fcm_dpo/beta": 0.016818445175886154,
"fcm_dpo/delta": 0.04274335503578186,
"fcm_dpo/margin": 15.367134094238281,
"fcm_dpo/q_t": 0.4427103102207184,
"grad_norm": 36.010684967041016,
"learning_rate": 3.7181572889485623e-07,
"logits/chosen": 1.3490233421325684,
"logits/rejected": 1.3899866342544556,
"logps/chosen": -285.947998046875,
"logps/ref_chosen": -281.30902099609375,
"logps/ref_rejected": -246.3243408203125,
"logps/rejected": -266.3304443359375,
"loss": 4.9282,
"margin_dpo/margin_mean": 15.367136001586914,
"margin_dpo/margin_std": 34.865966796875,
"step": 194
},
{
"KL/chosen_KL_mean": -4.831840515136719,
"KL/mean": -12.867500305175781,
"KL/rejected_KL_mean": -20.903165817260742,
"KL/std": 30.93072509765625,
"epoch": 0.4083769633507853,
"fcm_dpo/beta": 0.01715153455734253,
"fcm_dpo/delta": -0.0032990314066410065,
"fcm_dpo/margin": 16.071332931518555,
"fcm_dpo/q_t": 0.4376472234725952,
"grad_norm": 35.62577438354492,
"learning_rate": 3.7021375165108377e-07,
"logits/chosen": 1.548952579498291,
"logits/rejected": 1.6486254930496216,
"logps/chosen": -271.68994140625,
"logps/ref_chosen": -266.8581237792969,
"logps/ref_rejected": -276.0855712890625,
"logps/rejected": -296.9887390136719,
"loss": 4.8675,
"margin_dpo/margin_mean": 16.071334838867188,
"margin_dpo/margin_std": 34.527015686035156,
"step": 195
},
{
"KL/chosen_KL_mean": 0.8048229217529297,
"KL/mean": -11.171314239501953,
"KL/rejected_KL_mean": -23.14745330810547,
"KL/std": 29.260290145874023,
"epoch": 0.41047120418848165,
"fcm_dpo/beta": 0.0168392863124609,
"fcm_dpo/delta": -0.062426064163446426,
"fcm_dpo/margin": 23.952272415161133,
"fcm_dpo/q_t": 0.4076106548309326,
"grad_norm": 47.47340393066406,
"learning_rate": 3.6860532770864005e-07,
"logits/chosen": 1.3038650751113892,
"logits/rejected": 1.4030379056930542,
"logps/chosen": -266.19012451171875,
"logps/ref_chosen": -266.99493408203125,
"logps/ref_rejected": -248.60513305664062,
"logps/rejected": -271.7525939941406,
"loss": 4.4078,
"margin_dpo/margin_mean": 23.952272415161133,
"margin_dpo/margin_std": 32.61539077758789,
"step": 196
},
{
"KL/chosen_KL_mean": 3.457935333251953,
"KL/mean": -11.847572326660156,
"KL/rejected_KL_mean": -27.153076171875,
"KL/std": 32.25026321411133,
"epoch": 0.41256544502617803,
"fcm_dpo/beta": 0.015746818855404854,
"fcm_dpo/delta": -0.11807969212532043,
"fcm_dpo/margin": 30.611000061035156,
"fcm_dpo/q_t": 0.38884419202804565,
"grad_norm": 45.38121795654297,
"learning_rate": 3.6699054332241985e-07,
"logits/chosen": 1.2113600969314575,
"logits/rejected": 1.1563668251037598,
"logps/chosen": -299.5318603515625,
"logps/ref_chosen": -302.98980712890625,
"logps/ref_rejected": -262.8002624511719,
"logps/rejected": -289.95330810546875,
"loss": 4.1683,
"margin_dpo/margin_mean": 30.611000061035156,
"margin_dpo/margin_std": 34.92479705810547,
"step": 197
},
{
"KL/chosen_KL_mean": 0.10808563232421875,
"KL/mean": -11.575591087341309,
"KL/rejected_KL_mean": -23.25926399230957,
"KL/std": 32.306583404541016,
"epoch": 0.41465968586387436,
"fcm_dpo/beta": 0.014680261723697186,
"fcm_dpo/delta": 0.001502858940511942,
"fcm_dpo/margin": 23.36733627319336,
"fcm_dpo/q_t": 0.42125099897384644,
"grad_norm": 32.45808029174805,
"learning_rate": 3.653694850884091e-07,
"logits/chosen": 1.5946077108383179,
"logits/rejected": 1.6851319074630737,
"logps/chosen": -296.10369873046875,
"logps/ref_chosen": -296.2117614746094,
"logps/ref_rejected": -289.8357238769531,
"logps/rejected": -313.0950012207031,
"loss": 4.6621,
"margin_dpo/margin_mean": 23.36733627319336,
"margin_dpo/margin_std": 41.27313995361328,
"step": 198
},
{
"KL/chosen_KL_mean": -1.3021049499511719,
"KL/mean": -13.162481307983398,
"KL/rejected_KL_mean": -25.02284812927246,
"KL/std": 30.66471290588379,
"epoch": 0.4167539267015707,
"fcm_dpo/beta": 0.015121981501579285,
"fcm_dpo/delta": 0.013484282419085503,
"fcm_dpo/margin": 23.72073745727539,
"fcm_dpo/q_t": 0.4176020324230194,
"grad_norm": 30.408153533935547,
"learning_rate": 3.6374223993904124e-07,
"logits/chosen": 1.2783989906311035,
"logits/rejected": 1.2025794982910156,
"logps/chosen": -256.5947265625,
"logps/ref_chosen": -255.29263305664062,
"logps/ref_rejected": -212.69761657714844,
"logps/rejected": -237.720458984375,
"loss": 4.5483,
"margin_dpo/margin_mean": 23.720739364624023,
"margin_dpo/margin_std": 37.188148498535156,
"step": 199
},
{
"KL/chosen_KL_mean": -0.3975563049316406,
"KL/mean": -10.419899940490723,
"KL/rejected_KL_mean": -20.442241668701172,
"KL/std": 32.0081787109375,
"epoch": 0.418848167539267,
"fcm_dpo/beta": 0.015231948345899582,
"fcm_dpo/delta": 0.03270227462053299,
"fcm_dpo/margin": 20.044692993164062,
"fcm_dpo/q_t": 0.43182697892189026,
"grad_norm": 41.86520767211914,
"learning_rate": 3.621088951385353e-07,
"logits/chosen": 1.7400442361831665,
"logits/rejected": 1.739106297492981,
"logps/chosen": -313.9339599609375,
"logps/ref_chosen": -313.5364074707031,
"logps/ref_rejected": -274.349609375,
"logps/rejected": -294.79180908203125,
"loss": 4.8731,
"margin_dpo/margin_mean": 20.044694900512695,
"margin_dpo/margin_std": 44.38846969604492,
"step": 200
},
{
"epoch": 0.418848167539267,
"eval_KL/chosen_KL_mean": -4.467589855194092,
"eval_KL/mean": -16.012392044067383,
"eval_KL/rejected_KL_mean": -27.557188034057617,
"eval_KL/std": 31.657482147216797,
"eval_fcm_dpo/beta": 0.015443911775946617,
"eval_logits/chosen": 1.6197923421859741,
"eval_logits/rejected": 1.661108136177063,
"eval_logps/chosen": -284.88427734375,
"eval_logps/ref_chosen": -280.41668701171875,
"eval_logps/ref_rejected": -264.6455383300781,
"eval_logps/rejected": -292.2027282714844,
"eval_loss": 0.5750734806060791,
"eval_margin_dpo/margin_mean": 23.089601516723633,
"eval_margin_dpo/margin_std": 36.13518524169922,
"eval_runtime": 89.255,
"eval_samples_per_second": 22.408,
"eval_steps_per_second": 2.801,
"step": 200
},
{
"KL/chosen_KL_mean": -2.7801513671875,
"KL/mean": -15.753643035888672,
"KL/rejected_KL_mean": -28.727128982543945,
"KL/std": 32.468223571777344,
"epoch": 0.42094240837696334,
"fcm_dpo/beta": 0.015363471582531929,
"fcm_dpo/delta": -0.03582913428544998,
"fcm_dpo/margin": 25.946979522705078,
"fcm_dpo/q_t": 0.40801653265953064,
"grad_norm": 39.52208709716797,
"learning_rate": 3.604695382782159e-07,
"logits/chosen": 1.3580992221832275,
"logits/rejected": 1.3939217329025269,
"logps/chosen": -265.4873352050781,
"logps/ref_chosen": -262.7071838378906,
"logps/ref_rejected": -243.85879516601562,
"logps/rejected": -272.5859375,
"loss": 4.422,
"margin_dpo/margin_mean": 25.946975708007812,
"margin_dpo/margin_std": 35.81454849243164,
"step": 201
},
{
"KL/chosen_KL_mean": -5.743253707885742,
"KL/mean": -17.584720611572266,
"KL/rejected_KL_mean": -29.426183700561523,
"KL/std": 32.779457092285156,
"epoch": 0.42303664921465967,
"fcm_dpo/beta": 0.014960610307753086,
"fcm_dpo/delta": -0.013768346980214119,
"fcm_dpo/margin": 23.68292808532715,
"fcm_dpo/q_t": 0.4190349876880646,
"grad_norm": 34.390716552734375,
"learning_rate": 3.588242572718162e-07,
"logits/chosen": 1.5304558277130127,
"logits/rejected": 1.508581280708313,
"logps/chosen": -270.0299987792969,
"logps/ref_chosen": -264.2867431640625,
"logps/ref_rejected": -231.6894989013672,
"logps/rejected": -261.1156921386719,
"loss": 4.6531,
"margin_dpo/margin_mean": 23.68292808532715,
"margin_dpo/margin_std": 41.36975860595703,
"step": 202
},
{
"KL/chosen_KL_mean": -4.744565963745117,
"KL/mean": -14.712820053100586,
"KL/rejected_KL_mean": -24.681068420410156,
"KL/std": 31.517353057861328,
"epoch": 0.42513089005235605,
"fcm_dpo/beta": 0.015301337465643883,
"fcm_dpo/delta": 0.06527502834796906,
"fcm_dpo/margin": 19.936498641967773,
"fcm_dpo/q_t": 0.4290742576122284,
"grad_norm": 43.57932662963867,
"learning_rate": 3.571731403507635e-07,
"logits/chosen": 1.5578314065933228,
"logits/rejected": 1.5329176187515259,
"logps/chosen": -276.0287780761719,
"logps/ref_chosen": -271.2842102050781,
"logps/ref_rejected": -247.95462036132812,
"logps/rejected": -272.63568115234375,
"loss": 4.6656,
"margin_dpo/margin_mean": 19.936500549316406,
"margin_dpo/margin_std": 32.206573486328125,
"step": 203
},
{
"KL/chosen_KL_mean": -2.7664566040039062,
"KL/mean": -16.75481605529785,
"KL/rejected_KL_mean": -30.743175506591797,
"KL/std": 32.51542663574219,
"epoch": 0.4272251308900524,
"fcm_dpo/beta": 0.015280654653906822,
"fcm_dpo/delta": -0.029274964705109596,
"fcm_dpo/margin": 27.976722717285156,
"fcm_dpo/q_t": 0.40244799852371216,
"grad_norm": 32.696537017822266,
"learning_rate": 3.5551627605944746e-07,
"logits/chosen": 1.5064336061477661,
"logits/rejected": 1.4716084003448486,
"logps/chosen": -312.89691162109375,
"logps/ref_chosen": -310.13043212890625,
"logps/ref_rejected": -267.3905334472656,
"logps/rejected": -298.1336669921875,
"loss": 4.3423,
"margin_dpo/margin_mean": 27.97671890258789,
"margin_dpo/margin_std": 38.091148376464844,
"step": 204
},
{
"KL/chosen_KL_mean": -1.1357479095458984,
"KL/mean": -14.039722442626953,
"KL/rejected_KL_mean": -26.94369125366211,
"KL/std": 31.764497756958008,
"epoch": 0.4293193717277487,
"fcm_dpo/beta": 0.01524931751191616,
"fcm_dpo/delta": -0.023136526346206665,
"fcm_dpo/margin": 25.80794906616211,
"fcm_dpo/q_t": 0.4099942147731781,
"grad_norm": 34.16581344604492,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": 1.4873193502426147,
"logits/rejected": 1.484212875366211,
"logps/chosen": -278.2964172363281,
"logps/ref_chosen": -277.16064453125,
"logps/ref_rejected": -292.91815185546875,
"logps/rejected": -319.8618469238281,
"loss": 4.4749,
"margin_dpo/margin_mean": 25.807947158813477,
"margin_dpo/margin_std": 37.94681167602539,
"step": 205
},
{
"KL/chosen_KL_mean": -1.1707172393798828,
"KL/mean": -12.986665725708008,
"KL/rejected_KL_mean": -24.802610397338867,
"KL/std": 32.0893440246582,
"epoch": 0.431413612565445,
"fcm_dpo/beta": 0.014862453565001488,
"fcm_dpo/delta": 0.019112059846520424,
"fcm_dpo/margin": 23.63189125061035,
"fcm_dpo/q_t": 0.41891059279441833,
"grad_norm": 34.6976318359375,
"learning_rate": 3.5218566107988867e-07,
"logits/chosen": 1.7092478275299072,
"logits/rejected": 1.8270341157913208,
"logps/chosen": -287.95330810546875,
"logps/ref_chosen": -286.7825927734375,
"logps/ref_rejected": -289.08270263671875,
"logps/rejected": -313.8852844238281,
"loss": 4.5861,
"margin_dpo/margin_mean": 23.63189125061035,
"margin_dpo/margin_std": 37.21580123901367,
"step": 206
},
{
"KL/chosen_KL_mean": -4.060152053833008,
"KL/mean": -14.630226135253906,
"KL/rejected_KL_mean": -25.20030975341797,
"KL/std": 33.9632568359375,
"epoch": 0.43350785340314135,
"fcm_dpo/beta": 0.01515961904078722,
"fcm_dpo/delta": 0.015925094485282898,
"fcm_dpo/margin": 21.140159606933594,
"fcm_dpo/q_t": 0.42776355147361755,
"grad_norm": 44.10590362548828,
"learning_rate": 3.505120890024195e-07,
"logits/chosen": 1.4632537364959717,
"logits/rejected": 1.5669593811035156,
"logps/chosen": -270.5924072265625,
"logps/ref_chosen": -266.5322570800781,
"logps/ref_rejected": -275.20672607421875,
"logps/rejected": -300.40704345703125,
"loss": 4.8052,
"margin_dpo/margin_mean": 21.14015769958496,
"margin_dpo/margin_std": 43.635372161865234,
"step": 207
},
{
"KL/chosen_KL_mean": -3.625772476196289,
"KL/mean": -16.000215530395508,
"KL/rejected_KL_mean": -28.374656677246094,
"KL/std": 33.34564208984375,
"epoch": 0.4356020942408377,
"fcm_dpo/beta": 0.01525797974318266,
"fcm_dpo/delta": -0.04257369041442871,
"fcm_dpo/margin": 24.74889373779297,
"fcm_dpo/q_t": 0.41384872794151306,
"grad_norm": 31.783222198486328,
"learning_rate": 3.4883312676665534e-07,
"logits/chosen": 1.755733847618103,
"logits/rejected": 1.76398766040802,
"logps/chosen": -276.4271240234375,
"logps/ref_chosen": -272.80133056640625,
"logps/ref_rejected": -288.4167785644531,
"logps/rejected": -316.7914123535156,
"loss": 4.539,
"margin_dpo/margin_mean": 24.748897552490234,
"margin_dpo/margin_std": 39.13401412963867,
"step": 208
},
{
"KL/chosen_KL_mean": -7.097347259521484,
"KL/mean": -17.537128448486328,
"KL/rejected_KL_mean": -27.976924896240234,
"KL/std": 33.336891174316406,
"epoch": 0.437696335078534,
"fcm_dpo/beta": 0.015181426890194416,
"fcm_dpo/delta": 0.0385538712143898,
"fcm_dpo/margin": 20.87958335876465,
"fcm_dpo/q_t": 0.4282674193382263,
"grad_norm": 31.456403732299805,
"learning_rate": 3.4714886441024573e-07,
"logits/chosen": 1.5062553882598877,
"logits/rejected": 1.359464406967163,
"logps/chosen": -317.7767639160156,
"logps/ref_chosen": -310.679443359375,
"logps/ref_rejected": -259.3792419433594,
"logps/rejected": -287.3561706542969,
"loss": 4.7444,
"margin_dpo/margin_mean": 20.879581451416016,
"margin_dpo/margin_std": 38.3335075378418,
"step": 209
},
{
"KL/chosen_KL_mean": -2.737762451171875,
"KL/mean": -15.045697212219238,
"KL/rejected_KL_mean": -27.353635787963867,
"KL/std": 33.89357376098633,
"epoch": 0.4397905759162304,
"fcm_dpo/beta": 0.015006419271230698,
"fcm_dpo/delta": -0.031001610681414604,
"fcm_dpo/margin": 24.615867614746094,
"fcm_dpo/q_t": 0.4150693118572235,
"grad_norm": 37.6225471496582,
"learning_rate": 3.454593922550693e-07,
"logits/chosen": 1.5236557722091675,
"logits/rejected": 1.564916729927063,
"logps/chosen": -277.7059020996094,
"logps/ref_chosen": -274.9681701660156,
"logps/ref_rejected": -283.346923828125,
"logps/rejected": -310.7005615234375,
"loss": 4.5562,
"margin_dpo/margin_mean": 24.61586570739746,
"margin_dpo/margin_std": 39.228519439697266,
"step": 210
},
{
"KL/chosen_KL_mean": -0.10589599609375,
"KL/mean": -16.420366287231445,
"KL/rejected_KL_mean": -32.73482131958008,
"KL/std": 33.75396728515625,
"epoch": 0.4418848167539267,
"fcm_dpo/beta": 0.014337250031530857,
"fcm_dpo/delta": -0.10005457699298859,
"fcm_dpo/margin": 32.6289176940918,
"fcm_dpo/q_t": 0.39139655232429504,
"grad_norm": 36.14860534667969,
"learning_rate": 3.4376480090239047e-07,
"logits/chosen": 1.4577884674072266,
"logits/rejected": 1.4486171007156372,
"logps/chosen": -266.385498046875,
"logps/ref_chosen": -266.2795715332031,
"logps/ref_rejected": -249.7120819091797,
"logps/rejected": -282.4468994140625,
"loss": 4.1621,
"margin_dpo/margin_mean": 32.6289176940918,
"margin_dpo/margin_std": 35.4855842590332,
"step": 211
},
{
"KL/chosen_KL_mean": -3.3957462310791016,
"KL/mean": -15.396642684936523,
"KL/rejected_KL_mean": -27.397533416748047,
"KL/std": 33.133121490478516,
"epoch": 0.44397905759162304,
"fcm_dpo/beta": 0.013785447925329208,
"fcm_dpo/delta": 0.005094792228192091,
"fcm_dpo/margin": 24.001785278320312,
"fcm_dpo/q_t": 0.42262667417526245,
"grad_norm": 38.02641677856445,
"learning_rate": 3.4206518122800055e-07,
"logits/chosen": 1.4611730575561523,
"logits/rejected": 1.5041024684906006,
"logps/chosen": -268.1485595703125,
"logps/ref_chosen": -264.7528076171875,
"logps/ref_rejected": -241.02842712402344,
"logps/rejected": -268.42596435546875,
"loss": 4.6317,
"margin_dpo/margin_mean": 24.001785278320312,
"margin_dpo/margin_std": 39.277198791503906,
"step": 212
},
{
"KL/chosen_KL_mean": -7.741588592529297,
"KL/mean": -18.942157745361328,
"KL/rejected_KL_mean": -30.142728805541992,
"KL/std": 33.720516204833984,
"epoch": 0.44607329842931936,
"fcm_dpo/beta": 0.014078613370656967,
"fcm_dpo/delta": 0.019538499414920807,
"fcm_dpo/margin": 22.401147842407227,
"fcm_dpo/q_t": 0.4286879301071167,
"grad_norm": 31.262754440307617,
"learning_rate": 3.403606243773448e-07,
"logits/chosen": 1.4571382999420166,
"logits/rejected": 1.5783281326293945,
"logps/chosen": -301.88018798828125,
"logps/ref_chosen": -294.13861083984375,
"logps/ref_rejected": -303.11669921875,
"logps/rejected": -333.2593994140625,
"loss": 4.697,
"margin_dpo/margin_mean": 22.40114974975586,
"margin_dpo/margin_std": 39.48596954345703,
"step": 213
},
{
"KL/chosen_KL_mean": -17.921554565429688,
"KL/mean": -28.23214340209961,
"KL/rejected_KL_mean": -38.54274368286133,
"KL/std": 32.626007080078125,
"epoch": 0.4481675392670157,
"fcm_dpo/beta": 0.014258643612265587,
"fcm_dpo/delta": 0.014283804222941399,
"fcm_dpo/margin": 20.62118148803711,
"fcm_dpo/q_t": 0.4328727722167969,
"grad_norm": 50.17946243286133,
"learning_rate": 3.3865122176063385e-07,
"logits/chosen": 1.553182601928711,
"logits/rejected": 1.7039114236831665,
"logps/chosen": -284.4983215332031,
"logps/ref_chosen": -266.5767517089844,
"logps/ref_rejected": -291.2264709472656,
"logps/rejected": -329.76922607421875,
"loss": 4.7629,
"margin_dpo/margin_mean": 20.62118148803711,
"margin_dpo/margin_std": 38.329471588134766,
"step": 214
},
{
"KL/chosen_KL_mean": -11.867902755737305,
"KL/mean": -23.135061264038086,
"KL/rejected_KL_mean": -34.4022216796875,
"KL/std": 36.483604431152344,
"epoch": 0.450261780104712,
"fcm_dpo/beta": 0.014291242696344852,
"fcm_dpo/delta": 0.011042824946343899,
"fcm_dpo/margin": 22.534318923950195,
"fcm_dpo/q_t": 0.4279981255531311,
"grad_norm": 32.39078903198242,
"learning_rate": 3.3693706504794243e-07,
"logits/chosen": 1.4903463125228882,
"logits/rejected": 1.4890713691711426,
"logps/chosen": -296.9086608886719,
"logps/ref_chosen": -285.040771484375,
"logps/ref_rejected": -258.0254821777344,
"logps/rejected": -292.42767333984375,
"loss": 4.7469,
"margin_dpo/margin_mean": 22.534318923950195,
"margin_dpo/margin_std": 43.56327819824219,
"step": 215
},
{
"KL/chosen_KL_mean": -11.920280456542969,
"KL/mean": -24.491779327392578,
"KL/rejected_KL_mean": -37.06327438354492,
"KL/std": 37.01415252685547,
"epoch": 0.4523560209424084,
"fcm_dpo/beta": 0.01426931843161583,
"fcm_dpo/delta": -0.02282012067735195,
"fcm_dpo/margin": 25.143007278442383,
"fcm_dpo/q_t": 0.4173295199871063,
"grad_norm": 62.473167419433594,
"learning_rate": 3.3521824616429284e-07,
"logits/chosen": 1.5284839868545532,
"logits/rejected": 1.5369490385055542,
"logps/chosen": -344.6716613769531,
"logps/ref_chosen": -332.7513732910156,
"logps/ref_rejected": -302.9167785644531,
"logps/rejected": -339.98004150390625,
"loss": 4.6568,
"margin_dpo/margin_mean": 25.143007278442383,
"margin_dpo/margin_std": 44.69744873046875,
"step": 216
},
{
"KL/chosen_KL_mean": -12.519142150878906,
"KL/mean": -29.5145263671875,
"KL/rejected_KL_mean": -46.50989532470703,
"KL/std": 36.492759704589844,
"epoch": 0.4544502617801047,
"fcm_dpo/beta": 0.013744605705142021,
"fcm_dpo/delta": -0.07050801068544388,
"fcm_dpo/margin": 33.990753173828125,
"fcm_dpo/q_t": 0.3944663405418396,
"grad_norm": 28.30463981628418,
"learning_rate": 3.334948572847253e-07,
"logits/chosen": 1.7275793552398682,
"logits/rejected": 1.7330214977264404,
"logps/chosen": -298.9870910644531,
"logps/ref_chosen": -286.46795654296875,
"logps/ref_rejected": -253.99298095703125,
"logps/rejected": -300.50286865234375,
"loss": 4.258,
"margin_dpo/margin_mean": 33.990753173828125,
"margin_dpo/margin_std": 44.993526458740234,
"step": 217
},
{
"KL/chosen_KL_mean": -9.264612197875977,
"KL/mean": -22.980953216552734,
"KL/rejected_KL_mean": -36.697288513183594,
"KL/std": 36.396488189697266,
"epoch": 0.45654450261780105,
"fcm_dpo/beta": 0.013674240559339523,
"fcm_dpo/delta": 0.025067811831831932,
"fcm_dpo/margin": 27.43267059326172,
"fcm_dpo/q_t": 0.4136398434638977,
"grad_norm": 41.536502838134766,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": 1.6624709367752075,
"logits/rejected": 1.7311898469924927,
"logps/chosen": -320.64312744140625,
"logps/ref_chosen": -311.3785400390625,
"logps/ref_rejected": -290.3092956542969,
"logps/rejected": -327.006591796875,
"loss": 4.4824,
"margin_dpo/margin_mean": 27.432668685913086,
"margin_dpo/margin_std": 40.15641784667969,
"step": 218
},
{
"KL/chosen_KL_mean": -9.294855117797852,
"KL/mean": -23.00322723388672,
"KL/rejected_KL_mean": -36.71159362792969,
"KL/std": 34.37981414794922,
"epoch": 0.4586387434554974,
"fcm_dpo/beta": 0.013447243720293045,
"fcm_dpo/delta": -0.02139398269355297,
"fcm_dpo/margin": 27.41674041748047,
"fcm_dpo/q_t": 0.41440269351005554,
"grad_norm": 38.7296257019043,
"learning_rate": 3.300347394584172e-07,
"logits/chosen": 1.2187390327453613,
"logits/rejected": 1.2701640129089355,
"logps/chosen": -268.6277770996094,
"logps/ref_chosen": -259.33294677734375,
"logps/ref_rejected": -261.7661437988281,
"logps/rejected": -298.47772216796875,
"loss": 4.5068,
"margin_dpo/margin_mean": 27.4167423248291,
"margin_dpo/margin_std": 40.53258514404297,
"step": 219
},
{
"KL/chosen_KL_mean": -10.534852981567383,
"KL/mean": -24.904674530029297,
"KL/rejected_KL_mean": -39.274497985839844,
"KL/std": 36.653446197509766,
"epoch": 0.4607329842931937,
"fcm_dpo/beta": 0.013439134694635868,
"fcm_dpo/delta": 0.014226877130568027,
"fcm_dpo/margin": 28.73965835571289,
"fcm_dpo/q_t": 0.41149696707725525,
"grad_norm": 49.266822814941406,
"learning_rate": 3.2829819606729477e-07,
"logits/chosen": 1.702681303024292,
"logits/rejected": 1.652176022529602,
"logps/chosen": -316.7392578125,
"logps/ref_chosen": -306.2043762207031,
"logps/ref_rejected": -256.7926330566406,
"logps/rejected": -296.067138671875,
"loss": 4.488,
"margin_dpo/margin_mean": 28.73965835571289,
"margin_dpo/margin_std": 44.04676818847656,
"step": 220
},
{
"KL/chosen_KL_mean": -11.413686752319336,
"KL/mean": -22.628131866455078,
"KL/rejected_KL_mean": -33.842586517333984,
"KL/std": 36.744773864746094,
"epoch": 0.46282722513089003,
"fcm_dpo/beta": 0.01356799341738224,
"fcm_dpo/delta": -0.017365112900733948,
"fcm_dpo/margin": 22.428897857666016,
"fcm_dpo/q_t": 0.4291653335094452,
"grad_norm": 30.916912078857422,
"learning_rate": 3.265574537815398e-07,
"logits/chosen": 1.1171040534973145,
"logits/rejected": 1.2264795303344727,
"logps/chosen": -304.7456359863281,
"logps/ref_chosen": -293.3319396972656,
"logps/ref_rejected": -282.7599182128906,
"logps/rejected": -316.60247802734375,
"loss": 4.7429,
"margin_dpo/margin_mean": 22.428895950317383,
"margin_dpo/margin_std": 41.00771713256836,
"step": 221
},
{
"KL/chosen_KL_mean": -7.823530197143555,
"KL/mean": -22.639671325683594,
"KL/rejected_KL_mean": -37.455810546875,
"KL/std": 35.31937026977539,
"epoch": 0.4649214659685864,
"fcm_dpo/beta": 0.01314732525497675,
"fcm_dpo/delta": -0.012120349332690239,
"fcm_dpo/margin": 29.632274627685547,
"fcm_dpo/q_t": 0.4094116687774658,
"grad_norm": 40.30009078979492,
"learning_rate": 3.248126059518784e-07,
"logits/chosen": 1.564622402191162,
"logits/rejected": 1.480457067489624,
"logps/chosen": -294.8296813964844,
"logps/ref_chosen": -287.0061340332031,
"logps/ref_rejected": -233.23130798339844,
"logps/rejected": -270.6871337890625,
"loss": 4.4124,
"margin_dpo/margin_mean": 29.63227653503418,
"margin_dpo/margin_std": 39.13923645019531,
"step": 222
},
{
"KL/chosen_KL_mean": -8.294769287109375,
"KL/mean": -24.715295791625977,
"KL/rejected_KL_mean": -41.13582992553711,
"KL/std": 38.33906936645508,
"epoch": 0.46701570680628274,
"fcm_dpo/beta": 0.013019093312323093,
"fcm_dpo/delta": -0.029901059344410896,
"fcm_dpo/margin": 32.841064453125,
"fcm_dpo/q_t": 0.40152767300605774,
"grad_norm": 29.832014083862305,
"learning_rate": 3.230637461492043e-07,
"logits/chosen": 1.3996307849884033,
"logits/rejected": 1.2953250408172607,
"logps/chosen": -284.1100769042969,
"logps/ref_chosen": -275.8153076171875,
"logps/ref_rejected": -238.54286193847656,
"logps/rejected": -279.6786804199219,
"loss": 4.3452,
"margin_dpo/margin_mean": 32.841064453125,
"margin_dpo/margin_std": 43.7483024597168,
"step": 223
},
{
"KL/chosen_KL_mean": -9.125528335571289,
"KL/mean": -25.0452823638916,
"KL/rejected_KL_mean": -40.96503448486328,
"KL/std": 37.11698532104492,
"epoch": 0.46910994764397906,
"fcm_dpo/beta": 0.012906144373118877,
"fcm_dpo/delta": -0.042829155921936035,
"fcm_dpo/margin": 31.839508056640625,
"fcm_dpo/q_t": 0.404636412858963,
"grad_norm": 44.4716682434082,
"learning_rate": 3.213109681595612e-07,
"logits/chosen": 1.397386908531189,
"logits/rejected": 1.4601026773452759,
"logps/chosen": -248.22592163085938,
"logps/ref_chosen": -239.1003875732422,
"logps/ref_rejected": -230.31900024414062,
"logps/rejected": -271.2840270996094,
"loss": 4.36,
"margin_dpo/margin_mean": 31.839508056640625,
"margin_dpo/margin_std": 40.82142639160156,
"step": 224
},
{
"KL/chosen_KL_mean": -15.982580184936523,
"KL/mean": -27.515350341796875,
"KL/rejected_KL_mean": -39.04811477661133,
"KL/std": 38.05552673339844,
"epoch": 0.4712041884816754,
"fcm_dpo/beta": 0.012642517685890198,
"fcm_dpo/delta": 0.04203230142593384,
"fcm_dpo/margin": 23.065526962280273,
"fcm_dpo/q_t": 0.4347875416278839,
"grad_norm": 31.074045181274414,
"learning_rate": 3.1955436597911315e-07,
"logits/chosen": 1.4961891174316406,
"logits/rejected": 1.5179147720336914,
"logps/chosen": -316.31195068359375,
"logps/ref_chosen": -300.3293762207031,
"logps/ref_rejected": -332.132568359375,
"logps/rejected": -371.1806640625,
"loss": 4.8525,
"margin_dpo/margin_mean": 23.065526962280273,
"margin_dpo/margin_std": 49.160301208496094,
"step": 225
},
{
"KL/chosen_KL_mean": -11.568771362304688,
"KL/mean": -22.770845413208008,
"KL/rejected_KL_mean": -33.97290802001953,
"KL/std": 36.24320602416992,
"epoch": 0.4732984293193717,
"fcm_dpo/beta": 0.013132845051586628,
"fcm_dpo/delta": 0.046728260815143585,
"fcm_dpo/margin": 22.404129028320312,
"fcm_dpo/q_t": 0.43278542160987854,
"grad_norm": 34.16708755493164,
"learning_rate": 3.1779403380910425e-07,
"logits/chosen": 1.3392904996871948,
"logits/rejected": 1.3999265432357788,
"logps/chosen": -256.98095703125,
"logps/ref_chosen": -245.41213989257812,
"logps/ref_rejected": -251.72999572753906,
"logps/rejected": -285.7029113769531,
"loss": 4.78,
"margin_dpo/margin_mean": 22.404129028320312,
"margin_dpo/margin_std": 43.30206298828125,
"step": 226
},
{
"KL/chosen_KL_mean": -10.773397445678711,
"KL/mean": -26.87171173095703,
"KL/rejected_KL_mean": -42.97003173828125,
"KL/std": 38.47583770751953,
"epoch": 0.47539267015706804,
"fcm_dpo/beta": 0.013252967968583107,
"fcm_dpo/delta": -0.028668414801359177,
"fcm_dpo/margin": 32.19664001464844,
"fcm_dpo/q_t": 0.40182405710220337,
"grad_norm": 31.617626190185547,
"learning_rate": 3.160300660508064e-07,
"logits/chosen": 1.4435663223266602,
"logits/rejected": 1.4892001152038574,
"logps/chosen": -287.6492004394531,
"logps/ref_chosen": -276.87579345703125,
"logps/ref_rejected": -259.0455017089844,
"logps/rejected": -302.0155334472656,
"loss": 4.369,
"margin_dpo/margin_mean": 32.19664001464844,
"margin_dpo/margin_std": 44.42658615112305,
"step": 227
},
{
"KL/chosen_KL_mean": -8.808744430541992,
"KL/mean": -25.228464126586914,
"KL/rejected_KL_mean": -41.648189544677734,
"KL/std": 37.78360366821289,
"epoch": 0.4774869109947644,
"fcm_dpo/beta": 0.013077094219624996,
"fcm_dpo/delta": -0.03130736202001572,
"fcm_dpo/margin": 32.83942794799805,
"fcm_dpo/q_t": 0.4017367660999298,
"grad_norm": 32.4455680847168,
"learning_rate": 3.1426255730045695e-07,
"logits/chosen": 1.5388902425765991,
"logits/rejected": 1.5339179039001465,
"logps/chosen": -312.77978515625,
"logps/ref_chosen": -303.9710693359375,
"logps/ref_rejected": -255.62033081054688,
"logps/rejected": -297.26849365234375,
"loss": 4.3349,
"margin_dpo/margin_mean": 32.83942794799805,
"margin_dpo/margin_std": 43.40449523925781,
"step": 228
},
{
"KL/chosen_KL_mean": -8.927270889282227,
"KL/mean": -29.90787696838379,
"KL/rejected_KL_mean": -50.888465881347656,
"KL/std": 39.940574645996094,
"epoch": 0.47958115183246075,
"fcm_dpo/beta": 0.012126508168876171,
"fcm_dpo/delta": -0.11578944325447083,
"fcm_dpo/margin": 41.961185455322266,
"fcm_dpo/q_t": 0.38210585713386536,
"grad_norm": 28.49382209777832,
"learning_rate": 3.1249160234418644e-07,
"logits/chosen": 1.2846533060073853,
"logits/rejected": 1.2547979354858398,
"logps/chosen": -290.6231994628906,
"logps/ref_chosen": -281.6959533691406,
"logps/ref_rejected": -258.93353271484375,
"logps/rejected": -309.822021484375,
"loss": 4.0764,
"margin_dpo/margin_mean": 41.96118927001953,
"margin_dpo/margin_std": 45.12513732910156,
"step": 229
},
{
"KL/chosen_KL_mean": -9.807109832763672,
"KL/mean": -26.030256271362305,
"KL/rejected_KL_mean": -42.253395080566406,
"KL/std": 38.47943878173828,
"epoch": 0.4816753926701571,
"fcm_dpo/beta": 0.011823546141386032,
"fcm_dpo/delta": 0.016186799854040146,
"fcm_dpo/margin": 32.4462776184082,
"fcm_dpo/q_t": 0.4109889566898346,
"grad_norm": 45.17549133300781,
"learning_rate": 3.1071729615293424e-07,
"logits/chosen": 1.3623743057250977,
"logits/rejected": 1.4585875272750854,
"logps/chosen": -233.76364135742188,
"logps/ref_chosen": -223.95655822753906,
"logps/ref_rejected": -238.95187377929688,
"logps/rejected": -281.20526123046875,
"loss": 4.4637,
"margin_dpo/margin_mean": 32.44627380371094,
"margin_dpo/margin_std": 46.24778747558594,
"step": 230
},
{
"KL/chosen_KL_mean": -15.893941879272461,
"KL/mean": -28.720094680786133,
"KL/rejected_KL_mean": -41.54625701904297,
"KL/std": 38.19879150390625,
"epoch": 0.4837696335078534,
"fcm_dpo/beta": 0.011899704113602638,
"fcm_dpo/delta": 0.015257255174219608,
"fcm_dpo/margin": 25.652320861816406,
"fcm_dpo/q_t": 0.4285806715488434,
"grad_norm": 38.40047073364258,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": 1.301490306854248,
"logits/rejected": 1.358456015586853,
"logps/chosen": -330.81854248046875,
"logps/ref_chosen": -314.92462158203125,
"logps/ref_rejected": -279.6347351074219,
"logps/rejected": -321.1809997558594,
"loss": 4.6728,
"margin_dpo/margin_mean": 25.652320861816406,
"margin_dpo/margin_std": 42.17586898803711,
"step": 231
},
{
"KL/chosen_KL_mean": -8.095703125,
"KL/mean": -23.29351806640625,
"KL/rejected_KL_mean": -38.491336822509766,
"KL/std": 40.04689025878906,
"epoch": 0.48586387434554973,
"fcm_dpo/beta": 0.01189206913113594,
"fcm_dpo/delta": -0.0057932473719120026,
"fcm_dpo/margin": 30.395639419555664,
"fcm_dpo/q_t": 0.41665101051330566,
"grad_norm": 28.25246810913086,
"learning_rate": 3.071590108427243e-07,
"logits/chosen": 1.3400001525878906,
"logits/rejected": 1.3999887704849243,
"logps/chosen": -271.4153137207031,
"logps/ref_chosen": -263.3196105957031,
"logps/ref_rejected": -247.13612365722656,
"logps/rejected": -285.62744140625,
"loss": 4.5218,
"margin_dpo/margin_mean": 30.395639419555664,
"margin_dpo/margin_std": 44.760887145996094,
"step": 232
},
{
"KL/chosen_KL_mean": -7.75877571105957,
"KL/mean": -23.7706356048584,
"KL/rejected_KL_mean": -39.78249740600586,
"KL/std": 39.010414123535156,
"epoch": 0.48795811518324606,
"fcm_dpo/beta": 0.011896791867911816,
"fcm_dpo/delta": -0.004583900794386864,
"fcm_dpo/margin": 32.023719787597656,
"fcm_dpo/q_t": 0.41239315271377563,
"grad_norm": 34.30588150024414,
"learning_rate": 3.05375222543809e-07,
"logits/chosen": 1.3542277812957764,
"logits/rejected": 1.4231150150299072,
"logps/chosen": -287.1209716796875,
"logps/ref_chosen": -279.36224365234375,
"logps/ref_rejected": -264.46112060546875,
"logps/rejected": -304.24365234375,
"loss": 4.4553,
"margin_dpo/margin_mean": 32.023719787597656,
"margin_dpo/margin_std": 45.20970153808594,
"step": 233
},
{
"KL/chosen_KL_mean": -7.031229019165039,
"KL/mean": -23.30236053466797,
"KL/rejected_KL_mean": -39.5734977722168,
"KL/std": 41.138160705566406,
"epoch": 0.4900523560209424,
"fcm_dpo/beta": 0.011804355308413506,
"fcm_dpo/delta": -0.015027280896902084,
"fcm_dpo/margin": 32.54227066040039,
"fcm_dpo/q_t": 0.41214871406555176,
"grad_norm": 32.93385314941406,
"learning_rate": 3.035884646397637e-07,
"logits/chosen": 1.3371347188949585,
"logits/rejected": 1.4137632846832275,
"logps/chosen": -293.4803466796875,
"logps/ref_chosen": -286.4491271972656,
"logps/ref_rejected": -294.5469665527344,
"logps/rejected": -334.1204833984375,
"loss": 4.5281,
"margin_dpo/margin_mean": 32.54227066040039,
"margin_dpo/margin_std": 51.168392181396484,
"step": 234
},
{
"KL/chosen_KL_mean": -8.793224334716797,
"KL/mean": -23.923728942871094,
"KL/rejected_KL_mean": -39.054237365722656,
"KL/std": 42.044288635253906,
"epoch": 0.49214659685863876,
"fcm_dpo/beta": 0.011966248974204063,
"fcm_dpo/delta": 0.005010334774851799,
"fcm_dpo/margin": 30.261001586914062,
"fcm_dpo/q_t": 0.4166721701622009,
"grad_norm": 55.811973571777344,
"learning_rate": 3.017988329489923e-07,
"logits/chosen": 1.4549720287322998,
"logits/rejected": 1.4591089487075806,
"logps/chosen": -291.1532897949219,
"logps/ref_chosen": -282.36004638671875,
"logps/ref_rejected": -243.79052734375,
"logps/rejected": -282.8447570800781,
"loss": 4.5759,
"margin_dpo/margin_mean": 30.261001586914062,
"margin_dpo/margin_std": 49.17083740234375,
"step": 235
},
{
"KL/chosen_KL_mean": -2.9586658477783203,
"KL/mean": -20.14302635192871,
"KL/rejected_KL_mean": -37.327392578125,
"KL/std": 39.80377197265625,
"epoch": 0.4942408376963351,
"fcm_dpo/beta": 0.011891270987689495,
"fcm_dpo/delta": -0.009616704657673836,
"fcm_dpo/margin": 34.368717193603516,
"fcm_dpo/q_t": 0.40520286560058594,
"grad_norm": 30.121191024780273,
"learning_rate": 3.000064234440111e-07,
"logits/chosen": 1.392760992050171,
"logits/rejected": 1.4052550792694092,
"logps/chosen": -282.09808349609375,
"logps/ref_chosen": -279.139404296875,
"logps/ref_rejected": -240.5850067138672,
"logps/rejected": -277.9123840332031,
"loss": 4.3816,
"margin_dpo/margin_mean": 34.368717193603516,
"margin_dpo/margin_std": 46.1241340637207,
"step": 236
},
{
"KL/chosen_KL_mean": -5.928140640258789,
"KL/mean": -20.729507446289062,
"KL/rejected_KL_mean": -35.530879974365234,
"KL/std": 36.886966705322266,
"epoch": 0.4963350785340314,
"fcm_dpo/beta": 0.011840139515697956,
"fcm_dpo/delta": 0.009517230093479156,
"fcm_dpo/margin": 29.60274314880371,
"fcm_dpo/q_t": 0.41943880915641785,
"grad_norm": 30.60716438293457,
"learning_rate": 2.9821133224630223e-07,
"logits/chosen": 1.3946075439453125,
"logits/rejected": 1.4704986810684204,
"logps/chosen": -264.7396545410156,
"logps/ref_chosen": -258.8114929199219,
"logps/ref_rejected": -266.3898010253906,
"logps/rejected": -301.920654296875,
"loss": 4.5759,
"margin_dpo/margin_mean": 29.60274314880371,
"margin_dpo/margin_std": 46.73874282836914,
"step": 237
},
{
"KL/chosen_KL_mean": -6.701227188110352,
"KL/mean": -23.007665634155273,
"KL/rejected_KL_mean": -39.31410598754883,
"KL/std": 41.285335540771484,
"epoch": 0.49842931937172774,
"fcm_dpo/beta": 0.011498241685330868,
"fcm_dpo/delta": -0.030815942212939262,
"fcm_dpo/margin": 32.61288070678711,
"fcm_dpo/q_t": 0.4142690896987915,
"grad_norm": 40.07395935058594,
"learning_rate": 2.964136556211588e-07,
"logits/chosen": 1.3910374641418457,
"logits/rejected": 1.3155585527420044,
"logps/chosen": -308.76739501953125,
"logps/ref_chosen": -302.0661926269531,
"logps/ref_rejected": -265.5518798828125,
"logps/rejected": -304.865966796875,
"loss": 4.495,
"margin_dpo/margin_mean": 32.61288070678711,
"margin_dpo/margin_std": 47.61798095703125,
"step": 238
},
{
"KL/chosen_KL_mean": -3.102130889892578,
"KL/mean": -17.093172073364258,
"KL/rejected_KL_mean": -31.084213256835938,
"KL/std": 40.21778869628906,
"epoch": 0.5005235602094241,
"fcm_dpo/beta": 0.011761010624468327,
"fcm_dpo/delta": 0.07378143817186356,
"fcm_dpo/margin": 27.982086181640625,
"fcm_dpo/q_t": 0.42560938000679016,
"grad_norm": 30.5093994140625,
"learning_rate": 2.946134899725226e-07,
"logits/chosen": 1.6279447078704834,
"logits/rejected": 1.7356630563735962,
"logps/chosen": -262.1004943847656,
"logps/ref_chosen": -258.99835205078125,
"logps/ref_rejected": -273.7834777832031,
"logps/rejected": -304.86767578125,
"loss": 4.6858,
"margin_dpo/margin_mean": 27.982086181640625,
"margin_dpo/margin_std": 50.87156295776367,
"step": 239
},
{
"KL/chosen_KL_mean": -0.7970199584960938,
"KL/mean": -18.575305938720703,
"KL/rejected_KL_mean": -36.353599548339844,
"KL/std": 39.7580680847168,
"epoch": 0.5026178010471204,
"fcm_dpo/beta": 0.011806718073785305,
"fcm_dpo/delta": -0.05412173643708229,
"fcm_dpo/margin": 35.556583404541016,
"fcm_dpo/q_t": 0.40297773480415344,
"grad_norm": 44.02696990966797,
"learning_rate": 2.9281093183781403e-07,
"logits/chosen": 1.1961954832077026,
"logits/rejected": 1.2669832706451416,
"logps/chosen": -276.320068359375,
"logps/ref_chosen": -275.5230407714844,
"logps/ref_rejected": -255.50784301757812,
"logps/rejected": -291.8614196777344,
"loss": 4.3285,
"margin_dpo/margin_mean": 35.556583404541016,
"margin_dpo/margin_std": 45.55504608154297,
"step": 240
},
{
"KL/chosen_KL_mean": -6.538116455078125,
"KL/mean": -19.914806365966797,
"KL/rejected_KL_mean": -33.29149627685547,
"KL/std": 40.47013854980469,
"epoch": 0.5047120418848168,
"fcm_dpo/beta": 0.011729340068995953,
"fcm_dpo/delta": 0.025532079860568047,
"fcm_dpo/margin": 26.753385543823242,
"fcm_dpo/q_t": 0.42871057987213135,
"grad_norm": 31.722482681274414,
"learning_rate": 2.910060778827554e-07,
"logits/chosen": 1.733362078666687,
"logits/rejected": 1.7891788482666016,
"logps/chosen": -259.901611328125,
"logps/ref_chosen": -253.36349487304688,
"logps/ref_rejected": -247.1916961669922,
"logps/rejected": -280.4831848144531,
"loss": 4.7253,
"margin_dpo/margin_mean": 26.753385543823242,
"margin_dpo/margin_std": 49.15824890136719,
"step": 241
},
{
"KL/chosen_KL_mean": -3.924562454223633,
"KL/mean": -20.041032791137695,
"KL/rejected_KL_mean": -36.15748977661133,
"KL/std": 39.7971305847168,
"epoch": 0.506806282722513,
"fcm_dpo/beta": 0.011834909208118916,
"fcm_dpo/delta": -0.010150094516575336,
"fcm_dpo/margin": 32.23292922973633,
"fcm_dpo/q_t": 0.4113154411315918,
"grad_norm": 33.408729553222656,
"learning_rate": 2.891990248961871e-07,
"logits/chosen": 1.470890760421753,
"logits/rejected": 1.4131429195404053,
"logps/chosen": -268.9126281738281,
"logps/ref_chosen": -264.9880676269531,
"logps/ref_rejected": -243.07838439941406,
"logps/rejected": -279.2358703613281,
"loss": 4.4475,
"margin_dpo/margin_mean": 32.23292541503906,
"margin_dpo/margin_std": 44.627681732177734,
"step": 242
},
{
"KL/chosen_KL_mean": 0.30600738525390625,
"KL/mean": -18.52072525024414,
"KL/rejected_KL_mean": -37.34746551513672,
"KL/std": 39.642032623291016,
"epoch": 0.5089005235602094,
"fcm_dpo/beta": 0.011598140932619572,
"fcm_dpo/delta": -0.0700819194316864,
"fcm_dpo/margin": 37.65347671508789,
"fcm_dpo/q_t": 0.3989962637424469,
"grad_norm": 53.24457931518555,
"learning_rate": 2.873898697848762e-07,
"logits/chosen": 1.5711965560913086,
"logits/rejected": 1.6385666131973267,
"logps/chosen": -315.5646667480469,
"logps/ref_chosen": -315.8706970214844,
"logps/ref_rejected": -304.3729248046875,
"logps/rejected": -341.72039794921875,
"loss": 4.2608,
"margin_dpo/margin_mean": 37.65347671508789,
"margin_dpo/margin_std": 42.59384536743164,
"step": 243
},
{
"KL/chosen_KL_mean": -3.296337127685547,
"KL/mean": -22.78225326538086,
"KL/rejected_KL_mean": -42.26817321777344,
"KL/std": 42.10606384277344,
"epoch": 0.5109947643979058,
"fcm_dpo/beta": 0.01082875207066536,
"fcm_dpo/delta": -0.05993076041340828,
"fcm_dpo/margin": 38.971832275390625,
"fcm_dpo/q_t": 0.401917040348053,
"grad_norm": 27.999408721923828,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": 1.5478038787841797,
"logits/rejected": 1.5114996433258057,
"logps/chosen": -314.39349365234375,
"logps/ref_chosen": -311.09716796875,
"logps/ref_rejected": -266.01959228515625,
"logps/rejected": -308.2877502441406,
"loss": 4.318,
"margin_dpo/margin_mean": 38.97182846069336,
"margin_dpo/margin_std": 48.343570709228516,
"step": 244
},
{
"KL/chosen_KL_mean": -3.196439743041992,
"KL/mean": -22.358665466308594,
"KL/rejected_KL_mean": -41.520896911621094,
"KL/std": 41.44492721557617,
"epoch": 0.5130890052356021,
"fcm_dpo/beta": 0.010494336485862732,
"fcm_dpo/delta": -0.050034236162900925,
"fcm_dpo/margin": 38.3244743347168,
"fcm_dpo/q_t": 0.4060443937778473,
"grad_norm": 41.397159576416016,
"learning_rate": 2.837656413735479e-07,
"logits/chosen": 1.4871952533721924,
"logits/rejected": 1.3748258352279663,
"logps/chosen": -287.9369201660156,
"logps/ref_chosen": -284.740478515625,
"logps/ref_rejected": -237.64117431640625,
"logps/rejected": -279.1620788574219,
"loss": 4.3706,
"margin_dpo/margin_mean": 38.3244743347168,
"margin_dpo/margin_std": 47.9886474609375,
"step": 245
},
{
"KL/chosen_KL_mean": -12.974367141723633,
"KL/mean": -25.873023986816406,
"KL/rejected_KL_mean": -38.77168273925781,
"KL/std": 40.77077865600586,
"epoch": 0.5151832460732985,
"fcm_dpo/beta": 0.01014913059771061,
"fcm_dpo/delta": -0.009277934208512306,
"fcm_dpo/margin": 25.79730987548828,
"fcm_dpo/q_t": 0.43964654207229614,
"grad_norm": 43.22949981689453,
"learning_rate": 2.8195076242990116e-07,
"logits/chosen": 1.5551692247390747,
"logits/rejected": 1.5850450992584229,
"logps/chosen": -285.5872497558594,
"logps/ref_chosen": -272.6128845214844,
"logps/ref_rejected": -249.99725341796875,
"logps/rejected": -288.7689514160156,
"loss": 4.845,
"margin_dpo/margin_mean": 25.79730987548828,
"margin_dpo/margin_std": 50.12140655517578,
"step": 246
},
{
"KL/chosen_KL_mean": -5.976814270019531,
"KL/mean": -22.756752014160156,
"KL/rejected_KL_mean": -39.53668975830078,
"KL/std": 41.12067413330078,
"epoch": 0.5172774869109947,
"fcm_dpo/beta": 0.01021606009453535,
"fcm_dpo/delta": 0.005496193654835224,
"fcm_dpo/margin": 33.559879302978516,
"fcm_dpo/q_t": 0.4197867214679718,
"grad_norm": 30.316707611083984,
"learning_rate": 2.801341700638307e-07,
"logits/chosen": 1.4950445890426636,
"logits/rejected": 1.507032871246338,
"logps/chosen": -280.4126281738281,
"logps/ref_chosen": -274.435791015625,
"logps/ref_rejected": -257.8125305175781,
"logps/rejected": -297.3492126464844,
"loss": 4.5279,
"margin_dpo/margin_mean": 33.55988311767578,
"margin_dpo/margin_std": 47.255096435546875,
"step": 247
},
{
"KL/chosen_KL_mean": -10.627542495727539,
"KL/mean": -24.254194259643555,
"KL/rejected_KL_mean": -37.88084411621094,
"KL/std": 40.86634826660156,
"epoch": 0.5193717277486911,
"fcm_dpo/beta": 0.010291523300111294,
"fcm_dpo/delta": 0.038031596690416336,
"fcm_dpo/margin": 27.25330352783203,
"fcm_dpo/q_t": 0.4346015453338623,
"grad_norm": 38.1819953918457,
"learning_rate": 2.7831596169367227e-07,
"logits/chosen": 1.2884039878845215,
"logits/rejected": 1.3203765153884888,
"logps/chosen": -271.9388732910156,
"logps/ref_chosen": -261.3113098144531,
"logps/ref_rejected": -231.152587890625,
"logps/rejected": -269.033447265625,
"loss": 4.7694,
"margin_dpo/margin_mean": 27.25330352783203,
"margin_dpo/margin_std": 49.18940353393555,
"step": 248
},
{
"KL/chosen_KL_mean": -12.927343368530273,
"KL/mean": -29.005685806274414,
"KL/rejected_KL_mean": -45.08403015136719,
"KL/std": 43.595123291015625,
"epoch": 0.5214659685863874,
"fcm_dpo/beta": 0.010600361973047256,
"fcm_dpo/delta": 0.020274803042411804,
"fcm_dpo/margin": 32.15668869018555,
"fcm_dpo/q_t": 0.4210623502731323,
"grad_norm": 40.379180908203125,
"learning_rate": 2.7649623482442274e-07,
"logits/chosen": 1.4619112014770508,
"logits/rejected": 1.4200809001922607,
"logps/chosen": -281.39459228515625,
"logps/ref_chosen": -268.46722412109375,
"logps/ref_rejected": -240.27597045898438,
"logps/rejected": -285.3599853515625,
"loss": 4.5757,
"margin_dpo/margin_mean": 32.15668487548828,
"margin_dpo/margin_std": 49.13870620727539,
"step": 249
},
{
"KL/chosen_KL_mean": -13.616073608398438,
"KL/mean": -29.948217391967773,
"KL/rejected_KL_mean": -46.280357360839844,
"KL/std": 44.8901481628418,
"epoch": 0.5235602094240838,
"fcm_dpo/beta": 0.010583215393126011,
"fcm_dpo/delta": -0.0072011156007647514,
"fcm_dpo/margin": 32.6642951965332,
"fcm_dpo/q_t": 0.4200817942619324,
"grad_norm": 29.176288604736328,
"learning_rate": 2.7467508704251135e-07,
"logits/chosen": 1.3896605968475342,
"logits/rejected": 1.4943935871124268,
"logps/chosen": -298.7212829589844,
"logps/ref_chosen": -285.1051940917969,
"logps/ref_rejected": -281.2331848144531,
"logps/rejected": -327.5135498046875,
"loss": 4.6045,
"margin_dpo/margin_mean": 32.6642951965332,
"margin_dpo/margin_std": 52.9317512512207,
"step": 250
},
{
"KL/chosen_KL_mean": -10.912569046020508,
"KL/mean": -27.458873748779297,
"KL/rejected_KL_mean": -44.00518035888672,
"KL/std": 42.60402297973633,
"epoch": 0.5256544502617801,
"fcm_dpo/beta": 0.010662700049579144,
"fcm_dpo/delta": -0.015465203672647476,
"fcm_dpo/margin": 33.09260559082031,
"fcm_dpo/q_t": 0.4187396764755249,
"grad_norm": 41.07495880126953,
"learning_rate": 2.7285261601056697e-07,
"logits/chosen": 1.4050275087356567,
"logits/rejected": 1.3634721040725708,
"logps/chosen": -285.02520751953125,
"logps/ref_chosen": -274.1126403808594,
"logps/ref_rejected": -256.7730407714844,
"logps/rejected": -300.7782287597656,
"loss": 4.5358,
"margin_dpo/margin_mean": 33.09260940551758,
"margin_dpo/margin_std": 48.139583587646484,
"step": 251
},
{
"KL/chosen_KL_mean": -17.89266586303711,
"KL/mean": -35.21458053588867,
"KL/rejected_KL_mean": -52.53649139404297,
"KL/std": 44.71033477783203,
"epoch": 0.5277486910994764,
"fcm_dpo/beta": 0.010413442738354206,
"fcm_dpo/delta": 0.01656465046107769,
"fcm_dpo/margin": 34.643829345703125,
"fcm_dpo/q_t": 0.4168463945388794,
"grad_norm": 30.107173919677734,
"learning_rate": 2.7102891946217994e-07,
"logits/chosen": 1.5433881282806396,
"logits/rejected": 1.514957070350647,
"logps/chosen": -307.4583740234375,
"logps/ref_chosen": -289.56573486328125,
"logps/ref_rejected": -277.9891662597656,
"logps/rejected": -330.525634765625,
"loss": 4.5739,
"margin_dpo/margin_mean": 34.643829345703125,
"margin_dpo/margin_std": 54.72654342651367,
"step": 252
},
{
"KL/chosen_KL_mean": -16.55995750427246,
"KL/mean": -30.391183853149414,
"KL/rejected_KL_mean": -44.222415924072266,
"KL/std": 44.476783752441406,
"epoch": 0.5298429319371728,
"fcm_dpo/beta": 0.010737106204032898,
"fcm_dpo/delta": 0.025508765131235123,
"fcm_dpo/margin": 27.662458419799805,
"fcm_dpo/q_t": 0.43178489804267883,
"grad_norm": 37.88166427612305,
"learning_rate": 2.692040951966617e-07,
"logits/chosen": 1.434678554534912,
"logits/rejected": 1.423270583152771,
"logps/chosen": -286.743896484375,
"logps/ref_chosen": -270.1839294433594,
"logps/ref_rejected": -245.86949157714844,
"logps/rejected": -290.0918884277344,
"loss": 4.7617,
"margin_dpo/margin_mean": 27.662458419799805,
"margin_dpo/margin_std": 51.88008499145508,
"step": 253
},
{
"KL/chosen_KL_mean": -12.813804626464844,
"KL/mean": -30.503990173339844,
"KL/rejected_KL_mean": -48.194175720214844,
"KL/std": 44.81682586669922,
"epoch": 0.5319371727748691,
"fcm_dpo/beta": 0.011003278195858002,
"fcm_dpo/delta": -0.01815183088183403,
"fcm_dpo/margin": 35.380367279052734,
"fcm_dpo/q_t": 0.4096793532371521,
"grad_norm": 29.679153442382812,
"learning_rate": 2.6737824107379947e-07,
"logits/chosen": 1.3948644399642944,
"logits/rejected": 1.383762001991272,
"logps/chosen": -273.500244140625,
"logps/ref_chosen": -260.6864318847656,
"logps/ref_rejected": -245.99508666992188,
"logps/rejected": -294.1892395019531,
"loss": 4.4201,
"margin_dpo/margin_mean": 35.38037109375,
"margin_dpo/margin_std": 45.590354919433594,
"step": 254
},
{
"KL/chosen_KL_mean": -11.986688613891602,
"KL/mean": -31.540464401245117,
"KL/rejected_KL_mean": -51.09423828125,
"KL/std": 45.224788665771484,
"epoch": 0.5340314136125655,
"fcm_dpo/beta": 0.010793380439281464,
"fcm_dpo/delta": -0.023381978273391724,
"fcm_dpo/margin": 39.1075553894043,
"fcm_dpo/q_t": 0.40392202138900757,
"grad_norm": 34.99132537841797,
"learning_rate": 2.655514550086086e-07,
"logits/chosen": 1.6440770626068115,
"logits/rejected": 1.5638293027877808,
"logps/chosen": -309.9005126953125,
"logps/ref_chosen": -297.9138488769531,
"logps/ref_rejected": -253.39952087402344,
"logps/rejected": -304.4937744140625,
"loss": 4.4037,
"margin_dpo/margin_mean": 39.10755157470703,
"margin_dpo/margin_std": 56.7020378112793,
"step": 255
},
{
"KL/chosen_KL_mean": -10.440006256103516,
"KL/mean": -30.1831111907959,
"KL/rejected_KL_mean": -49.92621612548828,
"KL/std": 46.450164794921875,
"epoch": 0.5361256544502618,
"fcm_dpo/beta": 0.010454785078763962,
"fcm_dpo/delta": -0.03796786814928055,
"fcm_dpo/margin": 39.486209869384766,
"fcm_dpo/q_t": 0.40512558817863464,
"grad_norm": 36.39244842529297,
"learning_rate": 2.6372383496608186e-07,
"logits/chosen": 1.5219961404800415,
"logits/rejected": 1.4941329956054688,
"logps/chosen": -320.36639404296875,
"logps/ref_chosen": -309.9263916015625,
"logps/ref_rejected": -253.452392578125,
"logps/rejected": -303.3786315917969,
"loss": 4.4543,
"margin_dpo/margin_mean": 39.486209869384766,
"margin_dpo/margin_std": 58.31575393676758,
"step": 256
},
{
"KL/chosen_KL_mean": -14.06297492980957,
"KL/mean": -31.715511322021484,
"KL/rejected_KL_mean": -49.3680534362793,
"KL/std": 46.08407974243164,
"epoch": 0.5382198952879581,
"fcm_dpo/beta": 0.010138208046555519,
"fcm_dpo/delta": 0.018022824078798294,
"fcm_dpo/margin": 35.30508804321289,
"fcm_dpo/q_t": 0.41767239570617676,
"grad_norm": 50.86438751220703,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": 1.1277216672897339,
"logits/rejected": 1.1563029289245605,
"logps/chosen": -273.8766174316406,
"logps/ref_chosen": -259.8136291503906,
"logps/ref_rejected": -245.84947204589844,
"logps/rejected": -295.217529296875,
"loss": 4.5203,
"margin_dpo/margin_mean": 35.30508804321289,
"margin_dpo/margin_std": 51.38655471801758,
"step": 257
},
{
"KL/chosen_KL_mean": -12.599967956542969,
"KL/mean": -32.08304977416992,
"KL/rejected_KL_mean": -51.566131591796875,
"KL/std": 46.551483154296875,
"epoch": 0.5403141361256545,
"fcm_dpo/beta": 0.01018352247774601,
"fcm_dpo/delta": 0.0029114307835698128,
"fcm_dpo/margin": 38.966163635253906,
"fcm_dpo/q_t": 0.40727880597114563,
"grad_norm": 31.537353515625,
"learning_rate": 2.600664850273538e-07,
"logits/chosen": 1.556702733039856,
"logits/rejected": 1.6439810991287231,
"logps/chosen": -277.95361328125,
"logps/ref_chosen": -265.35369873046875,
"logps/ref_rejected": -247.9174346923828,
"logps/rejected": -299.4835510253906,
"loss": 4.3671,
"margin_dpo/margin_mean": 38.966163635253906,
"margin_dpo/margin_std": 49.094520568847656,
"step": 258
},
{
"KL/chosen_KL_mean": -12.489116668701172,
"KL/mean": -30.218482971191406,
"KL/rejected_KL_mean": -47.94784164428711,
"KL/std": 46.43803787231445,
"epoch": 0.5424083769633508,
"fcm_dpo/beta": 0.010553291067481041,
"fcm_dpo/delta": 0.02626843750476837,
"fcm_dpo/margin": 35.45872497558594,
"fcm_dpo/q_t": 0.41366690397262573,
"grad_norm": 40.70254135131836,
"learning_rate": 2.582369512637302e-07,
"logits/chosen": 1.5837359428405762,
"logits/rejected": 1.5908241271972656,
"logps/chosen": -300.71234130859375,
"logps/ref_chosen": -288.2232360839844,
"logps/ref_rejected": -270.7300109863281,
"logps/rejected": -318.6778564453125,
"loss": 4.5045,
"margin_dpo/margin_mean": 35.45872497558594,
"margin_dpo/margin_std": 54.217803955078125,
"step": 259
},
{
"KL/chosen_KL_mean": -16.213422775268555,
"KL/mean": -24.49829864501953,
"KL/rejected_KL_mean": -32.783172607421875,
"KL/std": 44.51939392089844,
"epoch": 0.5445026178010471,
"fcm_dpo/beta": 0.010747314430773258,
"fcm_dpo/delta": 0.023694217205047607,
"fcm_dpo/margin": 16.569740295410156,
"fcm_dpo/q_t": 0.45913174748420715,
"grad_norm": 36.17821502685547,
"learning_rate": 2.5640697577740815e-07,
"logits/chosen": 1.4569004774093628,
"logits/rejected": 1.4987921714782715,
"logps/chosen": -297.8331298828125,
"logps/ref_chosen": -281.61968994140625,
"logps/ref_rejected": -271.03973388671875,
"logps/rejected": -303.8229064941406,
"loss": 5.1896,
"margin_dpo/margin_mean": 16.569740295410156,
"margin_dpo/margin_std": 52.53853988647461,
"step": 260
},
{
"KL/chosen_KL_mean": -20.783327102661133,
"KL/mean": -36.140594482421875,
"KL/rejected_KL_mean": -51.497886657714844,
"KL/std": 45.89060974121094,
"epoch": 0.5465968586387434,
"fcm_dpo/beta": 0.010736201889812946,
"fcm_dpo/delta": 0.01374741643667221,
"fcm_dpo/margin": 30.714555740356445,
"fcm_dpo/q_t": 0.4246878921985626,
"grad_norm": 40.811885833740234,
"learning_rate": 2.5457665670441937e-07,
"logits/chosen": 1.2206922769546509,
"logits/rejected": 1.370550274848938,
"logps/chosen": -263.2464904785156,
"logps/ref_chosen": -242.4631805419922,
"logps/ref_rejected": -244.0020751953125,
"logps/rejected": -295.4999694824219,
"loss": 4.7007,
"margin_dpo/margin_mean": 30.714557647705078,
"margin_dpo/margin_std": 56.56787109375,
"step": 261
},
{
"KL/chosen_KL_mean": -13.065511703491211,
"KL/mean": -32.089962005615234,
"KL/rejected_KL_mean": -51.114410400390625,
"KL/std": 47.31406021118164,
"epoch": 0.5486910994764398,
"fcm_dpo/beta": 0.01091125700622797,
"fcm_dpo/delta": -0.01627585105597973,
"fcm_dpo/margin": 38.04890441894531,
"fcm_dpo/q_t": 0.4048381447792053,
"grad_norm": 55.11283874511719,
"learning_rate": 2.527460921992209e-07,
"logits/chosen": 1.606964111328125,
"logits/rejected": 1.6369879245758057,
"logps/chosen": -305.0541076660156,
"logps/ref_chosen": -291.98858642578125,
"logps/ref_rejected": -273.35955810546875,
"logps/rejected": -324.4739685058594,
"loss": 4.3713,
"margin_dpo/margin_mean": 38.04890441894531,
"margin_dpo/margin_std": 51.87273406982422,
"step": 262
},
{
"KL/chosen_KL_mean": -20.500993728637695,
"KL/mean": -36.50056457519531,
"KL/rejected_KL_mean": -52.50013732910156,
"KL/std": 47.64826965332031,
"epoch": 0.5507853403141362,
"fcm_dpo/beta": 0.010693997144699097,
"fcm_dpo/delta": -0.013701440766453743,
"fcm_dpo/margin": 31.9991455078125,
"fcm_dpo/q_t": 0.42212849855422974,
"grad_norm": 45.294193267822266,
"learning_rate": 2.509153804294318e-07,
"logits/chosen": 1.2619881629943848,
"logits/rejected": 1.2392959594726562,
"logps/chosen": -295.4894714355469,
"logps/ref_chosen": -274.98846435546875,
"logps/ref_rejected": -254.75782775878906,
"logps/rejected": -307.25799560546875,
"loss": 4.6377,
"margin_dpo/margin_mean": 31.9991455078125,
"margin_dpo/margin_std": 54.831886291503906,
"step": 263
},
{
"KL/chosen_KL_mean": -11.085941314697266,
"KL/mean": -31.268726348876953,
"KL/rejected_KL_mean": -51.45151138305664,
"KL/std": 47.041046142578125,
"epoch": 0.5528795811518324,
"fcm_dpo/beta": 0.010340879671275616,
"fcm_dpo/delta": -0.05381663143634796,
"fcm_dpo/margin": 40.36557388305664,
"fcm_dpo/q_t": 0.4039081037044525,
"grad_norm": 27.04505729675293,
"learning_rate": 2.4908461957056825e-07,
"logits/chosen": 1.2077322006225586,
"logits/rejected": 1.276128888130188,
"logps/chosen": -262.7121276855469,
"logps/ref_chosen": -251.62615966796875,
"logps/ref_rejected": -251.9549560546875,
"logps/rejected": -303.406494140625,
"loss": 4.3325,
"margin_dpo/margin_mean": 40.36557388305664,
"margin_dpo/margin_std": 51.57044982910156,
"step": 264
},
{
"KL/chosen_KL_mean": -17.214717864990234,
"KL/mean": -36.08303451538086,
"KL/rejected_KL_mean": -54.95136260986328,
"KL/std": 49.44145584106445,
"epoch": 0.5549738219895288,
"fcm_dpo/beta": 0.009950151666998863,
"fcm_dpo/delta": -0.030162349343299866,
"fcm_dpo/margin": 37.73666000366211,
"fcm_dpo/q_t": 0.41336631774902344,
"grad_norm": 37.25914764404297,
"learning_rate": 2.4725390780077905e-07,
"logits/chosen": 1.388809323310852,
"logits/rejected": 1.4408124685287476,
"logps/chosen": -295.15130615234375,
"logps/ref_chosen": -277.9366149902344,
"logps/ref_rejected": -262.689453125,
"logps/rejected": -317.6408386230469,
"loss": 4.4984,
"margin_dpo/margin_mean": 37.73666000366211,
"margin_dpo/margin_std": 55.681739807128906,
"step": 265
},
{
"KL/chosen_KL_mean": -7.3596954345703125,
"KL/mean": -26.975372314453125,
"KL/rejected_KL_mean": -46.5910530090332,
"KL/std": 47.52385330200195,
"epoch": 0.5570680628272251,
"fcm_dpo/beta": 0.00993617344647646,
"fcm_dpo/delta": -0.021311882883310318,
"fcm_dpo/margin": 39.231361389160156,
"fcm_dpo/q_t": 0.40893828868865967,
"grad_norm": 37.004058837890625,
"learning_rate": 2.454233432955807e-07,
"logits/chosen": 1.655153512954712,
"logits/rejected": 1.652661681175232,
"logps/chosen": -277.80462646484375,
"logps/ref_chosen": -270.4449157714844,
"logps/ref_rejected": -259.5942077636719,
"logps/rejected": -306.1852722167969,
"loss": 4.4086,
"margin_dpo/margin_mean": 39.231361389160156,
"margin_dpo/margin_std": 50.556400299072266,
"step": 266
},
{
"KL/chosen_KL_mean": -15.378364562988281,
"KL/mean": -31.202293395996094,
"KL/rejected_KL_mean": -47.02621841430664,
"KL/std": 48.46784973144531,
"epoch": 0.5591623036649215,
"fcm_dpo/beta": 0.009753599762916565,
"fcm_dpo/delta": 0.008063675835728645,
"fcm_dpo/margin": 31.647851943969727,
"fcm_dpo/q_t": 0.42852890491485596,
"grad_norm": 35.43317413330078,
"learning_rate": 2.435930242225919e-07,
"logits/chosen": 1.227034330368042,
"logits/rejected": 1.2889069318771362,
"logps/chosen": -285.21826171875,
"logps/ref_chosen": -269.83990478515625,
"logps/ref_rejected": -246.33746337890625,
"logps/rejected": -293.36370849609375,
"loss": 4.7415,
"margin_dpo/margin_mean": 31.64785385131836,
"margin_dpo/margin_std": 59.13050079345703,
"step": 267
},
{
"KL/chosen_KL_mean": -9.642990112304688,
"KL/mean": -28.5567626953125,
"KL/rejected_KL_mean": -47.47053527832031,
"KL/std": 47.91365432739258,
"epoch": 0.5612565445026177,
"fcm_dpo/beta": 0.00979677215218544,
"fcm_dpo/delta": -0.005948986858129501,
"fcm_dpo/margin": 37.827545166015625,
"fcm_dpo/q_t": 0.41443774104118347,
"grad_norm": 34.83292770385742,
"learning_rate": 2.4176304873626984e-07,
"logits/chosen": 1.2267543077468872,
"logits/rejected": 1.1969324350357056,
"logps/chosen": -307.17694091796875,
"logps/ref_chosen": -297.533935546875,
"logps/ref_rejected": -270.3407897949219,
"logps/rejected": -317.8113098144531,
"loss": 4.4696,
"margin_dpo/margin_mean": 37.827545166015625,
"margin_dpo/margin_std": 52.79344177246094,
"step": 268
},
{
"KL/chosen_KL_mean": -13.821935653686523,
"KL/mean": -30.641128540039062,
"KL/rejected_KL_mean": -47.4603271484375,
"KL/std": 48.29499053955078,
"epoch": 0.5633507853403141,
"fcm_dpo/beta": 0.010049771517515182,
"fcm_dpo/delta": 0.039144884794950485,
"fcm_dpo/margin": 33.638389587402344,
"fcm_dpo/q_t": 0.42163753509521484,
"grad_norm": 34.716251373291016,
"learning_rate": 2.399335149726463e-07,
"logits/chosen": 1.0486133098602295,
"logits/rejected": 1.081622838973999,
"logps/chosen": -254.16078186035156,
"logps/ref_chosen": -240.33883666992188,
"logps/ref_rejected": -220.71717834472656,
"logps/rejected": -268.177490234375,
"loss": 4.6492,
"margin_dpo/margin_mean": 33.63838577270508,
"margin_dpo/margin_std": 58.49000930786133,
"step": 269
},
{
"KL/chosen_KL_mean": -9.618762969970703,
"KL/mean": -30.344589233398438,
"KL/rejected_KL_mean": -51.070404052734375,
"KL/std": 50.43669128417969,
"epoch": 0.5654450261780105,
"fcm_dpo/beta": 0.009877322241663933,
"fcm_dpo/delta": -0.04414429888129234,
"fcm_dpo/margin": 41.451622009277344,
"fcm_dpo/q_t": 0.4053528308868408,
"grad_norm": 51.390342712402344,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": 1.7349985837936401,
"logits/rejected": 1.7574037313461304,
"logps/chosen": -318.5118408203125,
"logps/ref_chosen": -308.8930969238281,
"logps/ref_rejected": -278.47869873046875,
"logps/rejected": -329.5491027832031,
"loss": 4.4327,
"margin_dpo/margin_mean": 41.451622009277344,
"margin_dpo/margin_std": 60.15433883666992,
"step": 270
},
{
"KL/chosen_KL_mean": -10.520090103149414,
"KL/mean": -29.58258819580078,
"KL/rejected_KL_mean": -48.645076751708984,
"KL/std": 47.74767303466797,
"epoch": 0.5675392670157068,
"fcm_dpo/beta": 0.010011949576437473,
"fcm_dpo/delta": 0.017741503193974495,
"fcm_dpo/margin": 38.12498474121094,
"fcm_dpo/q_t": 0.4125012457370758,
"grad_norm": 32.7165641784668,
"learning_rate": 2.3627616503391812e-07,
"logits/chosen": 1.1843674182891846,
"logits/rejected": 1.1827080249786377,
"logps/chosen": -285.94830322265625,
"logps/ref_chosen": -275.42822265625,
"logps/ref_rejected": -251.50906372070312,
"logps/rejected": -300.1541442871094,
"loss": 4.447,
"margin_dpo/margin_mean": 38.12498474121094,
"margin_dpo/margin_std": 52.87849044799805,
"step": 271
},
{
"KL/chosen_KL_mean": -8.865665435791016,
"KL/mean": -26.257308959960938,
"KL/rejected_KL_mean": -43.648956298828125,
"KL/std": 47.73676681518555,
"epoch": 0.5696335078534032,
"fcm_dpo/beta": 0.009815176017582417,
"fcm_dpo/delta": -0.016911154612898827,
"fcm_dpo/margin": 34.78328323364258,
"fcm_dpo/q_t": 0.4211246073246002,
"grad_norm": 58.63067626953125,
"learning_rate": 2.344485449913914e-07,
"logits/chosen": 1.4511405229568481,
"logits/rejected": 1.4676002264022827,
"logps/chosen": -298.2484130859375,
"logps/ref_chosen": -289.38275146484375,
"logps/ref_rejected": -274.895751953125,
"logps/rejected": -318.5447082519531,
"loss": 4.5905,
"margin_dpo/margin_mean": 34.78328323364258,
"margin_dpo/margin_std": 54.094459533691406,
"step": 272
},
{
"KL/chosen_KL_mean": -8.876047134399414,
"KL/mean": -28.207664489746094,
"KL/rejected_KL_mean": -47.53927993774414,
"KL/std": 48.09898376464844,
"epoch": 0.5717277486910994,
"fcm_dpo/beta": 0.009568422101438046,
"fcm_dpo/delta": -0.0034137628972530365,
"fcm_dpo/margin": 38.663230895996094,
"fcm_dpo/q_t": 0.414844810962677,
"grad_norm": 34.78086853027344,
"learning_rate": 2.3262175892620062e-07,
"logits/chosen": 1.498274803161621,
"logits/rejected": 1.618941307067871,
"logps/chosen": -293.8645324707031,
"logps/ref_chosen": -284.98846435546875,
"logps/ref_rejected": -273.5077819824219,
"logps/rejected": -321.04705810546875,
"loss": 4.5119,
"margin_dpo/margin_mean": 38.663230895996094,
"margin_dpo/margin_std": 57.82013702392578,
"step": 273
},
{
"KL/chosen_KL_mean": -9.224098205566406,
"KL/mean": -30.08123016357422,
"KL/rejected_KL_mean": -50.938358306884766,
"KL/std": 50.3647346496582,
"epoch": 0.5738219895287958,
"fcm_dpo/beta": 0.00954630970954895,
"fcm_dpo/delta": -0.04250919446349144,
"fcm_dpo/margin": 41.71424865722656,
"fcm_dpo/q_t": 0.4074460566043854,
"grad_norm": 29.297210693359375,
"learning_rate": 2.3079590480333827e-07,
"logits/chosen": 1.4107565879821777,
"logits/rejected": 1.4762794971466064,
"logps/chosen": -271.9036865234375,
"logps/ref_chosen": -262.6795959472656,
"logps/ref_rejected": -236.13792419433594,
"logps/rejected": -287.0762939453125,
"loss": 4.4076,
"margin_dpo/margin_mean": 41.71424865722656,
"margin_dpo/margin_std": 56.30377960205078,
"step": 274
},
{
"KL/chosen_KL_mean": -5.721000671386719,
"KL/mean": -30.27288246154785,
"KL/rejected_KL_mean": -54.82476043701172,
"KL/std": 48.94614791870117,
"epoch": 0.5759162303664922,
"fcm_dpo/beta": 0.009182119742035866,
"fcm_dpo/delta": -0.053633034229278564,
"fcm_dpo/margin": 49.103763580322266,
"fcm_dpo/q_t": 0.3949816823005676,
"grad_norm": 27.509553909301758,
"learning_rate": 2.2897108053782e-07,
"logits/chosen": 1.151371955871582,
"logits/rejected": 1.2531877756118774,
"logps/chosen": -252.1204833984375,
"logps/ref_chosen": -246.39950561523438,
"logps/ref_rejected": -247.4836883544922,
"logps/rejected": -302.3084716796875,
"loss": 4.2006,
"margin_dpo/margin_mean": 49.10376739501953,
"margin_dpo/margin_std": 55.38058853149414,
"step": 275
},
{
"KL/chosen_KL_mean": -2.3746280670166016,
"KL/mean": -24.551910400390625,
"KL/rejected_KL_mean": -46.72920227050781,
"KL/std": 48.10188674926758,
"epoch": 0.5780104712041885,
"fcm_dpo/beta": 0.008951807394623756,
"fcm_dpo/delta": 0.0025863340124487877,
"fcm_dpo/margin": 44.354576110839844,
"fcm_dpo/q_t": 0.407490611076355,
"grad_norm": 35.141151428222656,
"learning_rate": 2.2714738398943308e-07,
"logits/chosen": 1.5304458141326904,
"logits/rejected": 1.6467180252075195,
"logps/chosen": -291.2947082519531,
"logps/ref_chosen": -288.9200439453125,
"logps/ref_rejected": -291.69305419921875,
"logps/rejected": -338.4222412109375,
"loss": 4.3461,
"margin_dpo/margin_mean": 44.354576110839844,
"margin_dpo/margin_std": 53.57950210571289,
"step": 276
},
{
"KL/chosen_KL_mean": -12.331165313720703,
"KL/mean": -28.60528564453125,
"KL/rejected_KL_mean": -44.8794059753418,
"KL/std": 48.65975570678711,
"epoch": 0.5801047120418849,
"fcm_dpo/beta": 0.009269548580050468,
"fcm_dpo/delta": 0.03773127496242523,
"fcm_dpo/margin": 32.548255920410156,
"fcm_dpo/q_t": 0.429889976978302,
"grad_norm": 36.12635040283203,
"learning_rate": 2.2532491295748865e-07,
"logits/chosen": 1.2161855697631836,
"logits/rejected": 1.2608983516693115,
"logps/chosen": -269.4160461425781,
"logps/ref_chosen": -257.08489990234375,
"logps/ref_rejected": -251.61532592773438,
"logps/rejected": -296.4947509765625,
"loss": 4.7665,
"margin_dpo/margin_mean": 32.548255920410156,
"margin_dpo/margin_std": 61.996192932128906,
"step": 277
},
{
"KL/chosen_KL_mean": -17.341049194335938,
"KL/mean": -31.862226486206055,
"KL/rejected_KL_mean": -46.38340759277344,
"KL/std": 50.42668914794922,
"epoch": 0.5821989528795811,
"fcm_dpo/beta": 0.009464469738304615,
"fcm_dpo/delta": 0.029068736359477043,
"fcm_dpo/margin": 29.042362213134766,
"fcm_dpo/q_t": 0.43703651428222656,
"grad_norm": 35.64056396484375,
"learning_rate": 2.2350376517557726e-07,
"logits/chosen": 1.3662313222885132,
"logits/rejected": 1.3283146619796753,
"logps/chosen": -278.0823974609375,
"logps/ref_chosen": -260.7413635253906,
"logps/ref_rejected": -227.4325408935547,
"logps/rejected": -273.81597900390625,
"loss": 4.8913,
"margin_dpo/margin_mean": 29.042362213134766,
"margin_dpo/margin_std": 62.812740325927734,
"step": 278
},
{
"KL/chosen_KL_mean": -9.475412368774414,
"KL/mean": -32.11738586425781,
"KL/rejected_KL_mean": -54.75935363769531,
"KL/std": 52.19489288330078,
"epoch": 0.5842931937172775,
"fcm_dpo/beta": 0.009224911220371723,
"fcm_dpo/delta": -0.04645578935742378,
"fcm_dpo/margin": 45.283931732177734,
"fcm_dpo/q_t": 0.4031781256198883,
"grad_norm": 27.29572296142578,
"learning_rate": 2.2168403830632769e-07,
"logits/chosen": 1.2522985935211182,
"logits/rejected": 1.3427064418792725,
"logps/chosen": -312.4229736328125,
"logps/ref_chosen": -302.947509765625,
"logps/ref_rejected": -294.1771545410156,
"logps/rejected": -348.9365234375,
"loss": 4.3233,
"margin_dpo/margin_mean": 45.283931732177734,
"margin_dpo/margin_std": 56.65925216674805,
"step": 279
},
{
"KL/chosen_KL_mean": -9.627082824707031,
"KL/mean": -30.115880966186523,
"KL/rejected_KL_mean": -50.60468673706055,
"KL/std": 48.38449478149414,
"epoch": 0.5863874345549738,
"fcm_dpo/beta": 0.009083016775548458,
"fcm_dpo/delta": -0.004786573350429535,
"fcm_dpo/margin": 40.97761917114258,
"fcm_dpo/q_t": 0.41387462615966797,
"grad_norm": 31.47639274597168,
"learning_rate": 2.1986582993616925e-07,
"logits/chosen": 1.4386847019195557,
"logits/rejected": 1.5247678756713867,
"logps/chosen": -267.5251770019531,
"logps/ref_chosen": -257.8981018066406,
"logps/ref_rejected": -244.9084014892578,
"logps/rejected": -295.5130920410156,
"loss": 4.4853,
"margin_dpo/margin_mean": 40.97761917114258,
"margin_dpo/margin_std": 59.07532501220703,
"step": 280
},
{
"KL/chosen_KL_mean": -19.928136825561523,
"KL/mean": -39.731937408447266,
"KL/rejected_KL_mean": -59.53572463989258,
"KL/std": 52.304969787597656,
"epoch": 0.5884816753926702,
"fcm_dpo/beta": 0.009135991334915161,
"fcm_dpo/delta": 0.00806823093444109,
"fcm_dpo/margin": 39.607601165771484,
"fcm_dpo/q_t": 0.41674911975860596,
"grad_norm": 35.33727264404297,
"learning_rate": 2.1804923757009882e-07,
"logits/chosen": 1.4383091926574707,
"logits/rejected": 1.5045735836029053,
"logps/chosen": -310.122802734375,
"logps/ref_chosen": -290.1946716308594,
"logps/ref_rejected": -290.98883056640625,
"logps/rejected": -350.52459716796875,
"loss": 4.4937,
"margin_dpo/margin_mean": 39.607601165771484,
"margin_dpo/margin_std": 55.200523376464844,
"step": 281
},
{
"KL/chosen_KL_mean": -17.756895065307617,
"KL/mean": -35.99581527709961,
"KL/rejected_KL_mean": -54.2347412109375,
"KL/std": 52.38850402832031,
"epoch": 0.5905759162303665,
"fcm_dpo/beta": 0.009173966012895107,
"fcm_dpo/delta": 0.009089872241020203,
"fcm_dpo/margin": 36.47785949707031,
"fcm_dpo/q_t": 0.4234859347343445,
"grad_norm": 35.32107162475586,
"learning_rate": 2.1623435862645205e-07,
"logits/chosen": 1.3886326551437378,
"logits/rejected": 1.4161839485168457,
"logps/chosen": -326.6743469238281,
"logps/ref_chosen": -308.91741943359375,
"logps/ref_rejected": -272.2177734375,
"logps/rejected": -326.4525146484375,
"loss": 4.6586,
"margin_dpo/margin_mean": 36.47785568237305,
"margin_dpo/margin_std": 63.51108932495117,
"step": 282
},
{
"KL/chosen_KL_mean": -20.84634017944336,
"KL/mean": -40.495811462402344,
"KL/rejected_KL_mean": -60.14530944824219,
"KL/std": 49.9395751953125,
"epoch": 0.5926701570680628,
"fcm_dpo/beta": 0.00926823541522026,
"fcm_dpo/delta": -0.016706613823771477,
"fcm_dpo/margin": 39.29896926879883,
"fcm_dpo/q_t": 0.41592517495155334,
"grad_norm": 27.957590103149414,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": 1.6090337038040161,
"logits/rejected": 1.678878903388977,
"logps/chosen": -268.86834716796875,
"logps/ref_chosen": -248.02200317382812,
"logps/ref_rejected": -234.78541564941406,
"logps/rejected": -294.93072509765625,
"loss": 4.5319,
"margin_dpo/margin_mean": 39.29896926879883,
"margin_dpo/margin_std": 58.933406829833984,
"step": 283
},
{
"KL/chosen_KL_mean": -22.433128356933594,
"KL/mean": -41.53519058227539,
"KL/rejected_KL_mean": -60.63725662231445,
"KL/std": 53.89200973510742,
"epoch": 0.5947643979057592,
"fcm_dpo/beta": 0.009125684387981892,
"fcm_dpo/delta": 0.02019267901778221,
"fcm_dpo/margin": 38.204124450683594,
"fcm_dpo/q_t": 0.4207829535007477,
"grad_norm": 50.85140609741211,
"learning_rate": 2.1261013021512378e-07,
"logits/chosen": 1.3252496719360352,
"logits/rejected": 1.3272849321365356,
"logps/chosen": -286.92236328125,
"logps/ref_chosen": -264.4892578125,
"logps/ref_rejected": -255.43092346191406,
"logps/rejected": -316.0682067871094,
"loss": 4.6678,
"margin_dpo/margin_mean": 38.20412063598633,
"margin_dpo/margin_std": 68.52159881591797,
"step": 284
},
{
"KL/chosen_KL_mean": -20.986644744873047,
"KL/mean": -36.80175018310547,
"KL/rejected_KL_mean": -52.61685562133789,
"KL/std": 50.02957534790039,
"epoch": 0.5968586387434555,
"fcm_dpo/beta": 0.009410517290234566,
"fcm_dpo/delta": 0.0025642849504947662,
"fcm_dpo/margin": 31.630218505859375,
"fcm_dpo/q_t": 0.43106475472450256,
"grad_norm": 50.25341033935547,
"learning_rate": 2.1080097510381294e-07,
"logits/chosen": 1.4484590291976929,
"logits/rejected": 1.4547747373580933,
"logps/chosen": -322.455322265625,
"logps/ref_chosen": -301.46868896484375,
"logps/ref_rejected": -274.2380676269531,
"logps/rejected": -326.85491943359375,
"loss": 4.7508,
"margin_dpo/margin_mean": 31.630218505859375,
"margin_dpo/margin_std": 56.27755355834961,
"step": 285
},
{
"KL/chosen_KL_mean": -25.049720764160156,
"KL/mean": -41.187744140625,
"KL/rejected_KL_mean": -57.32576370239258,
"KL/std": 51.383689880371094,
"epoch": 0.5989528795811518,
"fcm_dpo/beta": 0.009583317674696445,
"fcm_dpo/delta": 0.06181109696626663,
"fcm_dpo/margin": 32.276039123535156,
"fcm_dpo/q_t": 0.4300505518913269,
"grad_norm": 32.433380126953125,
"learning_rate": 2.089939221172446e-07,
"logits/chosen": 1.3072019815444946,
"logits/rejected": 1.3753267526626587,
"logps/chosen": -291.6087951660156,
"logps/ref_chosen": -266.5590515136719,
"logps/ref_rejected": -280.12066650390625,
"logps/rejected": -337.4464416503906,
"loss": 4.7853,
"margin_dpo/margin_mean": 32.276039123535156,
"margin_dpo/margin_std": 64.37699127197266,
"step": 286
},
{
"KL/chosen_KL_mean": -22.22498321533203,
"KL/mean": -40.70927810668945,
"KL/rejected_KL_mean": -59.19358825683594,
"KL/std": 48.768131256103516,
"epoch": 0.6010471204188481,
"fcm_dpo/beta": 0.009829613380134106,
"fcm_dpo/delta": 0.00442691333591938,
"fcm_dpo/margin": 36.968597412109375,
"fcm_dpo/q_t": 0.4159740209579468,
"grad_norm": 43.9468879699707,
"learning_rate": 2.0718906816218595e-07,
"logits/chosen": 1.1281629800796509,
"logits/rejected": 1.1733816862106323,
"logps/chosen": -292.16131591796875,
"logps/ref_chosen": -269.93634033203125,
"logps/ref_rejected": -232.32330322265625,
"logps/rejected": -291.5168762207031,
"loss": 4.5792,
"margin_dpo/margin_mean": 36.968597412109375,
"margin_dpo/margin_std": 60.603633880615234,
"step": 287
},
{
"KL/chosen_KL_mean": -23.372922897338867,
"KL/mean": -41.4660758972168,
"KL/rejected_KL_mean": -59.5592155456543,
"KL/std": 51.44970703125,
"epoch": 0.6031413612565445,
"fcm_dpo/beta": 0.010073695331811905,
"fcm_dpo/delta": 0.008239217102527618,
"fcm_dpo/margin": 36.18629837036133,
"fcm_dpo/q_t": 0.4177555441856384,
"grad_norm": 47.44202423095703,
"learning_rate": 2.053865100274774e-07,
"logits/chosen": 1.233814001083374,
"logits/rejected": 1.1790906190872192,
"logps/chosen": -307.41839599609375,
"logps/ref_chosen": -284.04547119140625,
"logps/ref_rejected": -261.19378662109375,
"logps/rejected": -320.75299072265625,
"loss": 4.5936,
"margin_dpo/margin_mean": 36.18629837036133,
"margin_dpo/margin_std": 60.974952697753906,
"step": 288
},
{
"KL/chosen_KL_mean": -30.569286346435547,
"KL/mean": -43.77549362182617,
"KL/rejected_KL_mean": -56.981693267822266,
"KL/std": 53.21607971191406,
"epoch": 0.6052356020942409,
"fcm_dpo/beta": 0.010000809095799923,
"fcm_dpo/delta": 0.03143392875790596,
"fcm_dpo/margin": 26.41241455078125,
"fcm_dpo/q_t": 0.43992340564727783,
"grad_norm": 31.09710693359375,
"learning_rate": 2.035863443788411e-07,
"logits/chosen": 1.3066461086273193,
"logits/rejected": 1.2719676494598389,
"logps/chosen": -353.85052490234375,
"logps/ref_chosen": -323.28125,
"logps/ref_rejected": -274.2093200683594,
"logps/rejected": -331.1910400390625,
"loss": 4.9493,
"margin_dpo/margin_mean": 26.412412643432617,
"margin_dpo/margin_std": 62.61479187011719,
"step": 289
},
{
"KL/chosen_KL_mean": -23.792905807495117,
"KL/mean": -39.8603401184082,
"KL/rejected_KL_mean": -55.92776107788086,
"KL/std": 51.54084396362305,
"epoch": 0.6073298429319371,
"fcm_dpo/beta": 0.010087518952786922,
"fcm_dpo/delta": 0.01475964393466711,
"fcm_dpo/margin": 32.134864807128906,
"fcm_dpo/q_t": 0.4261983335018158,
"grad_norm": 38.690677642822266,
"learning_rate": 2.0178866775369774e-07,
"logits/chosen": 1.2787078619003296,
"logits/rejected": 1.2383533716201782,
"logps/chosen": -341.2283020019531,
"logps/ref_chosen": -317.4353942871094,
"logps/ref_rejected": -309.32403564453125,
"logps/rejected": -365.2518005371094,
"loss": 4.7017,
"margin_dpo/margin_mean": 32.13486862182617,
"margin_dpo/margin_std": 57.67361831665039,
"step": 290
},
{
"KL/chosen_KL_mean": -21.22112464904785,
"KL/mean": -44.63714599609375,
"KL/rejected_KL_mean": -68.05317687988281,
"KL/std": 55.67799377441406,
"epoch": 0.6094240837696335,
"fcm_dpo/beta": 0.009846209548413754,
"fcm_dpo/delta": -0.09073906391859055,
"fcm_dpo/margin": 46.8320426940918,
"fcm_dpo/q_t": 0.39452454447746277,
"grad_norm": 32.533935546875,
"learning_rate": 1.9999357655598891e-07,
"logits/chosen": 1.2153387069702148,
"logits/rejected": 1.377948522567749,
"logps/chosen": -286.7293395996094,
"logps/ref_chosen": -265.50823974609375,
"logps/ref_rejected": -275.1380310058594,
"logps/rejected": -343.191162109375,
"loss": 4.2942,
"margin_dpo/margin_mean": 46.83203887939453,
"margin_dpo/margin_std": 62.62254333496094,
"step": 291
},
{
"KL/chosen_KL_mean": -18.6783504486084,
"KL/mean": -37.69143295288086,
"KL/rejected_KL_mean": -56.70452117919922,
"KL/std": 52.398887634277344,
"epoch": 0.6115183246073298,
"fcm_dpo/beta": 0.00950497854501009,
"fcm_dpo/delta": 0.006776423193514347,
"fcm_dpo/margin": 38.02617645263672,
"fcm_dpo/q_t": 0.4163859188556671,
"grad_norm": 45.77560806274414,
"learning_rate": 1.9820116705100775e-07,
"logits/chosen": 1.3917005062103271,
"logits/rejected": 1.514042854309082,
"logps/chosen": -268.57965087890625,
"logps/ref_chosen": -249.9013214111328,
"logps/ref_rejected": -274.58245849609375,
"logps/rejected": -331.2869873046875,
"loss": 4.5507,
"margin_dpo/margin_mean": 38.026180267333984,
"margin_dpo/margin_std": 58.06243133544922,
"step": 292
},
{
"KL/chosen_KL_mean": -26.171293258666992,
"KL/mean": -46.48382568359375,
"KL/rejected_KL_mean": -66.79635620117188,
"KL/std": 55.30024719238281,
"epoch": 0.6136125654450262,
"fcm_dpo/beta": 0.009691167622804642,
"fcm_dpo/delta": 0.005665135104209185,
"fcm_dpo/margin": 40.625057220458984,
"fcm_dpo/q_t": 0.4102424681186676,
"grad_norm": 60.529232025146484,
"learning_rate": 1.9641153536023642e-07,
"logits/chosen": 1.3179248571395874,
"logits/rejected": 1.2775920629501343,
"logps/chosen": -322.47723388671875,
"logps/ref_chosen": -296.30596923828125,
"logps/ref_rejected": -268.0640869140625,
"logps/rejected": -334.8604431152344,
"loss": 4.4681,
"margin_dpo/margin_mean": 40.625057220458984,
"margin_dpo/margin_std": 60.73282241821289,
"step": 293
},
{
"KL/chosen_KL_mean": -22.822816848754883,
"KL/mean": -39.891197204589844,
"KL/rejected_KL_mean": -56.959564208984375,
"KL/std": 52.99144744873047,
"epoch": 0.6157068062827226,
"fcm_dpo/beta": 0.009667360223829746,
"fcm_dpo/delta": 0.016613639891147614,
"fcm_dpo/margin": 34.136741638183594,
"fcm_dpo/q_t": 0.4245774447917938,
"grad_norm": 58.12077713012695,
"learning_rate": 1.9462477745619106e-07,
"logits/chosen": 1.2750738859176636,
"logits/rejected": 1.3111969232559204,
"logps/chosen": -256.705810546875,
"logps/ref_chosen": -233.8829803466797,
"logps/ref_rejected": -224.14334106445312,
"logps/rejected": -281.1029052734375,
"loss": 4.6867,
"margin_dpo/margin_mean": 34.136741638183594,
"margin_dpo/margin_std": 60.988094329833984,
"step": 294
},
{
"KL/chosen_KL_mean": -24.004697799682617,
"KL/mean": -43.55730438232422,
"KL/rejected_KL_mean": -63.10991668701172,
"KL/std": 53.0479850769043,
"epoch": 0.6178010471204188,
"fcm_dpo/beta": 0.009808500297367573,
"fcm_dpo/delta": 0.016834016889333725,
"fcm_dpo/margin": 39.105220794677734,
"fcm_dpo/q_t": 0.41373884677886963,
"grad_norm": 38.24657440185547,
"learning_rate": 1.928409891572757e-07,
"logits/chosen": 1.404849886894226,
"logits/rejected": 1.4178699254989624,
"logps/chosen": -268.11602783203125,
"logps/ref_chosen": -244.111328125,
"logps/ref_rejected": -240.18373107910156,
"logps/rejected": -303.29364013671875,
"loss": 4.5514,
"margin_dpo/margin_mean": 39.105220794677734,
"margin_dpo/margin_std": 64.37168884277344,
"step": 295
},
{
"KL/chosen_KL_mean": -18.98125648498535,
"KL/mean": -46.19329833984375,
"KL/rejected_KL_mean": -73.40534210205078,
"KL/std": 55.29228973388672,
"epoch": 0.6198952879581152,
"fcm_dpo/beta": 0.009237255901098251,
"fcm_dpo/delta": -0.1366957724094391,
"fcm_dpo/margin": 54.424095153808594,
"fcm_dpo/q_t": 0.3833378553390503,
"grad_norm": 30.917573928833008,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": 1.2253468036651611,
"logits/rejected": 1.3825082778930664,
"logps/chosen": -303.6618347167969,
"logps/ref_chosen": -284.6805725097656,
"logps/ref_rejected": -282.1923828125,
"logps/rejected": -355.5976867675781,
"loss": 4.1243,
"margin_dpo/margin_mean": 54.424095153808594,
"margin_dpo/margin_std": 63.56785583496094,
"step": 296
},
{
"KL/chosen_KL_mean": -21.348806381225586,
"KL/mean": -42.41645431518555,
"KL/rejected_KL_mean": -63.48410415649414,
"KL/std": 54.160221099853516,
"epoch": 0.6219895287958115,
"fcm_dpo/beta": 0.008797341957688332,
"fcm_dpo/delta": 0.007480676285922527,
"fcm_dpo/margin": 42.135292053222656,
"fcm_dpo/q_t": 0.4149421155452728,
"grad_norm": 26.944902420043945,
"learning_rate": 1.8928270384706582e-07,
"logits/chosen": 1.695941686630249,
"logits/rejected": 1.707463026046753,
"logps/chosen": -301.57745361328125,
"logps/ref_chosen": -280.2286376953125,
"logps/ref_rejected": -280.8172912597656,
"logps/rejected": -344.3014221191406,
"loss": 4.5039,
"margin_dpo/margin_mean": 42.135292053222656,
"margin_dpo/margin_std": 61.98017883300781,
"step": 297
},
{
"KL/chosen_KL_mean": -15.223262786865234,
"KL/mean": -38.87238311767578,
"KL/rejected_KL_mean": -62.52149963378906,
"KL/std": 56.35419845581055,
"epoch": 0.6240837696335079,
"fcm_dpo/beta": 0.008612115867435932,
"fcm_dpo/delta": -0.07574725151062012,
"fcm_dpo/margin": 47.29825210571289,
"fcm_dpo/q_t": 0.40708616375923157,
"grad_norm": 34.92477798461914,
"learning_rate": 1.875083976558136e-07,
"logits/chosen": 1.3770982027053833,
"logits/rejected": 1.3817055225372314,
"logps/chosen": -312.22357177734375,
"logps/ref_chosen": -297.0003356933594,
"logps/ref_rejected": -279.1561279296875,
"logps/rejected": -341.6776123046875,
"loss": 4.4116,
"margin_dpo/margin_mean": 47.29825210571289,
"margin_dpo/margin_std": 66.13207244873047,
"step": 298
},
{
"KL/chosen_KL_mean": -24.688032150268555,
"KL/mean": -43.32847213745117,
"KL/rejected_KL_mean": -61.96891403198242,
"KL/std": 54.59784698486328,
"epoch": 0.6261780104712041,
"fcm_dpo/beta": 0.008580111898481846,
"fcm_dpo/delta": 0.032745346426963806,
"fcm_dpo/margin": 37.28089141845703,
"fcm_dpo/q_t": 0.42528051137924194,
"grad_norm": 30.84089469909668,
"learning_rate": 1.8573744269954297e-07,
"logits/chosen": 1.281267523765564,
"logits/rejected": 1.3654969930648804,
"logps/chosen": -296.15948486328125,
"logps/ref_chosen": -271.471435546875,
"logps/ref_rejected": -268.0870056152344,
"logps/rejected": -330.0559387207031,
"loss": 4.6057,
"margin_dpo/margin_mean": 37.28089141845703,
"margin_dpo/margin_std": 54.77768325805664,
"step": 299
},
{
"KL/chosen_KL_mean": -25.011089324951172,
"KL/mean": -44.76543426513672,
"KL/rejected_KL_mean": -64.519775390625,
"KL/std": 54.050819396972656,
"epoch": 0.6282722513089005,
"fcm_dpo/beta": 0.008571874350309372,
"fcm_dpo/delta": 0.0033765954431146383,
"fcm_dpo/margin": 39.508697509765625,
"fcm_dpo/q_t": 0.42205578088760376,
"grad_norm": 38.74055480957031,
"learning_rate": 1.839699339491937e-07,
"logits/chosen": 1.3771365880966187,
"logits/rejected": 1.364328145980835,
"logps/chosen": -331.36895751953125,
"logps/ref_chosen": -306.35784912109375,
"logps/ref_rejected": -268.64776611328125,
"logps/rejected": -333.16754150390625,
"loss": 4.6241,
"margin_dpo/margin_mean": 39.508697509765625,
"margin_dpo/margin_std": 66.10736846923828,
"step": 300
},
{
"KL/chosen_KL_mean": -23.9780216217041,
"KL/mean": -43.05587387084961,
"KL/rejected_KL_mean": -62.13372802734375,
"KL/std": 54.78178787231445,
"epoch": 0.6303664921465969,
"fcm_dpo/beta": 0.008896047249436378,
"fcm_dpo/delta": 0.06255801022052765,
"fcm_dpo/margin": 38.155696868896484,
"fcm_dpo/q_t": 0.4224284887313843,
"grad_norm": 37.444976806640625,
"learning_rate": 1.8220596619089573e-07,
"logits/chosen": 1.1716094017028809,
"logits/rejected": 1.2080966234207153,
"logps/chosen": -293.97052001953125,
"logps/ref_chosen": -269.9924621582031,
"logps/ref_rejected": -271.3135681152344,
"logps/rejected": -333.44732666015625,
"loss": 4.6205,
"margin_dpo/margin_mean": 38.15570068359375,
"margin_dpo/margin_std": 63.96094512939453,
"step": 301
},
{
"KL/chosen_KL_mean": -22.293458938598633,
"KL/mean": -44.63550567626953,
"KL/rejected_KL_mean": -66.9775390625,
"KL/std": 53.870208740234375,
"epoch": 0.6324607329842932,
"fcm_dpo/beta": 0.00888870283961296,
"fcm_dpo/delta": -0.025246508419513702,
"fcm_dpo/margin": 44.684078216552734,
"fcm_dpo/q_t": 0.4077514708042145,
"grad_norm": 31.096065521240234,
"learning_rate": 1.8044563402088682e-07,
"logits/chosen": 1.2770832777023315,
"logits/rejected": 1.3494573831558228,
"logps/chosen": -282.7541809082031,
"logps/ref_chosen": -260.460693359375,
"logps/ref_rejected": -254.2005615234375,
"logps/rejected": -321.1781005859375,
"loss": 4.4448,
"margin_dpo/margin_mean": 44.684078216552734,
"margin_dpo/margin_std": 64.02107238769531,
"step": 302
},
{
"KL/chosen_KL_mean": -20.06652069091797,
"KL/mean": -41.20815658569336,
"KL/rejected_KL_mean": -62.349796295166016,
"KL/std": 54.60296630859375,
"epoch": 0.6345549738219896,
"fcm_dpo/beta": 0.008872736245393753,
"fcm_dpo/delta": -0.021030310541391373,
"fcm_dpo/margin": 42.28326416015625,
"fcm_dpo/q_t": 0.4127102792263031,
"grad_norm": 37.00385284423828,
"learning_rate": 1.7868903184043885e-07,
"logits/chosen": 1.320084571838379,
"logits/rejected": 1.3712267875671387,
"logps/chosen": -316.16278076171875,
"logps/ref_chosen": -296.09625244140625,
"logps/ref_rejected": -264.7806701660156,
"logps/rejected": -327.13043212890625,
"loss": 4.5436,
"margin_dpo/margin_mean": 42.28326416015625,
"margin_dpo/margin_std": 66.04558563232422,
"step": 303
},
{
"KL/chosen_KL_mean": -30.992998123168945,
"KL/mean": -49.216251373291016,
"KL/rejected_KL_mean": -67.43950653076172,
"KL/std": 58.00931167602539,
"epoch": 0.6366492146596858,
"fcm_dpo/beta": 0.009011980146169662,
"fcm_dpo/delta": 0.0739751011133194,
"fcm_dpo/margin": 36.44649887084961,
"fcm_dpo/q_t": 0.42482835054397583,
"grad_norm": 35.08708572387695,
"learning_rate": 1.7693625385079574e-07,
"logits/chosen": 1.2776747941970825,
"logits/rejected": 1.2732635736465454,
"logps/chosen": -312.84246826171875,
"logps/ref_chosen": -281.8494873046875,
"logps/ref_rejected": -234.3673858642578,
"logps/rejected": -301.806884765625,
"loss": 4.6685,
"margin_dpo/margin_mean": 36.44649887084961,
"margin_dpo/margin_std": 63.821224212646484,
"step": 304
},
{
"KL/chosen_KL_mean": -15.93623161315918,
"KL/mean": -42.71954345703125,
"KL/rejected_KL_mean": -69.50286865234375,
"KL/std": 57.875244140625,
"epoch": 0.6387434554973822,
"fcm_dpo/beta": 0.008949695155024529,
"fcm_dpo/delta": -0.08474086225032806,
"fcm_dpo/margin": 53.56663513183594,
"fcm_dpo/q_t": 0.38978996872901917,
"grad_norm": 42.071353912353516,
"learning_rate": 1.7518739404812155e-07,
"logits/chosen": 1.227604627609253,
"logits/rejected": 1.3309612274169922,
"logps/chosen": -263.9789733886719,
"logps/ref_chosen": -248.0427703857422,
"logps/ref_rejected": -263.464599609375,
"logps/rejected": -332.96746826171875,
"loss": 4.2059,
"margin_dpo/margin_mean": 53.56663513183594,
"margin_dpo/margin_std": 65.27417755126953,
"step": 305
},
{
"KL/chosen_KL_mean": -14.433404922485352,
"KL/mean": -37.7215576171875,
"KL/rejected_KL_mean": -61.00971603393555,
"KL/std": 54.80039596557617,
"epoch": 0.6408376963350786,
"fcm_dpo/beta": 0.008622164838016033,
"fcm_dpo/delta": -0.028978414833545685,
"fcm_dpo/margin": 46.576324462890625,
"fcm_dpo/q_t": 0.4068741798400879,
"grad_norm": 40.38009262084961,
"learning_rate": 1.7344254621846017e-07,
"logits/chosen": 1.4350454807281494,
"logits/rejected": 1.3580363988876343,
"logps/chosen": -325.7193298339844,
"logps/ref_chosen": -311.2859191894531,
"logps/ref_rejected": -268.4949645996094,
"logps/rejected": -329.5046691894531,
"loss": 4.4003,
"margin_dpo/margin_mean": 46.576324462890625,
"margin_dpo/margin_std": 62.669273376464844,
"step": 306
},
{
"KL/chosen_KL_mean": -20.74403190612793,
"KL/mean": -46.11383056640625,
"KL/rejected_KL_mean": -71.48363494873047,
"KL/std": 58.11615753173828,
"epoch": 0.6429319371727749,
"fcm_dpo/beta": 0.008324475027620792,
"fcm_dpo/delta": -0.06760591268539429,
"fcm_dpo/margin": 50.73961639404297,
"fcm_dpo/q_t": 0.4010232090950012,
"grad_norm": 22.734024047851562,
"learning_rate": 1.717018039327053e-07,
"logits/chosen": 1.370535135269165,
"logits/rejected": 1.3642425537109375,
"logps/chosen": -290.95648193359375,
"logps/ref_chosen": -270.21246337890625,
"logps/ref_rejected": -239.1932830810547,
"logps/rejected": -310.6769104003906,
"loss": 4.3113,
"margin_dpo/margin_mean": 50.73961639404297,
"margin_dpo/margin_std": 61.37603759765625,
"step": 307
},
{
"KL/chosen_KL_mean": -27.225780487060547,
"KL/mean": -44.34769821166992,
"KL/rejected_KL_mean": -61.46961212158203,
"KL/std": 55.797428131103516,
"epoch": 0.6450261780104712,
"fcm_dpo/beta": 0.008255185559391975,
"fcm_dpo/delta": 0.05357668548822403,
"fcm_dpo/margin": 34.24382781982422,
"fcm_dpo/q_t": 0.4344278872013092,
"grad_norm": 30.90519142150879,
"learning_rate": 1.699652605415828e-07,
"logits/chosen": 1.472908854484558,
"logits/rejected": 1.48015296459198,
"logps/chosen": -315.74859619140625,
"logps/ref_chosen": -288.5228271484375,
"logps/ref_rejected": -253.90597534179688,
"logps/rejected": -315.3755798339844,
"loss": 4.779,
"margin_dpo/margin_mean": 34.24382781982422,
"margin_dpo/margin_std": 63.58550262451172,
"step": 308
},
{
"KL/chosen_KL_mean": -25.030376434326172,
"KL/mean": -50.54033660888672,
"KL/rejected_KL_mean": -76.05027770996094,
"KL/std": 59.059818267822266,
"epoch": 0.6471204188481675,
"fcm_dpo/beta": 0.008353800512850285,
"fcm_dpo/delta": -0.02783842757344246,
"fcm_dpo/margin": 51.019893646240234,
"fcm_dpo/q_t": 0.4025571048259735,
"grad_norm": 28.824357986450195,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": 1.3302932977676392,
"logits/rejected": 1.4493520259857178,
"logps/chosen": -296.92510986328125,
"logps/ref_chosen": -271.8947448730469,
"logps/ref_rejected": -260.5709228515625,
"logps/rejected": -336.62115478515625,
"loss": 4.3499,
"margin_dpo/margin_mean": 51.019893646240234,
"margin_dpo/margin_std": 68.8276138305664,
"step": 309
},
{
"KL/chosen_KL_mean": -30.154001235961914,
"KL/mean": -50.38459396362305,
"KL/rejected_KL_mean": -70.61518859863281,
"KL/std": 54.72597885131836,
"epoch": 0.6492146596858639,
"fcm_dpo/beta": 0.008251826278865337,
"fcm_dpo/delta": 0.011738432571291924,
"fcm_dpo/margin": 40.46120071411133,
"fcm_dpo/q_t": 0.42262619733810425,
"grad_norm": 37.7224006652832,
"learning_rate": 1.6650514271527465e-07,
"logits/chosen": 1.2993278503417969,
"logits/rejected": 1.412429690361023,
"logps/chosen": -300.3294677734375,
"logps/ref_chosen": -270.17547607421875,
"logps/ref_rejected": -257.6781311035156,
"logps/rejected": -328.2933349609375,
"loss": 4.5676,
"margin_dpo/margin_mean": 40.46120071411133,
"margin_dpo/margin_std": 59.413360595703125,
"step": 310
},
{
"KL/chosen_KL_mean": -28.951461791992188,
"KL/mean": -51.364383697509766,
"KL/rejected_KL_mean": -73.77729797363281,
"KL/std": 58.013572692871094,
"epoch": 0.6513089005235602,
"fcm_dpo/beta": 0.008316735737025738,
"fcm_dpo/delta": -0.003663080744445324,
"fcm_dpo/margin": 44.82583236694336,
"fcm_dpo/q_t": 0.413865327835083,
"grad_norm": 39.35993957519531,
"learning_rate": 1.647817538357072e-07,
"logits/chosen": 1.110593557357788,
"logits/rejected": 1.2548357248306274,
"logps/chosen": -292.7137145996094,
"logps/ref_chosen": -263.76226806640625,
"logps/ref_rejected": -263.8758850097656,
"logps/rejected": -337.6531677246094,
"loss": 4.5594,
"margin_dpo/margin_mean": 44.82583236694336,
"margin_dpo/margin_std": 71.53900909423828,
"step": 311
},
{
"KL/chosen_KL_mean": -26.776235580444336,
"KL/mean": -48.77787780761719,
"KL/rejected_KL_mean": -70.77953338623047,
"KL/std": 55.27427291870117,
"epoch": 0.6534031413612565,
"fcm_dpo/beta": 0.00823201984167099,
"fcm_dpo/delta": 0.00931546464562416,
"fcm_dpo/margin": 44.003299713134766,
"fcm_dpo/q_t": 0.4157472848892212,
"grad_norm": 34.14425277709961,
"learning_rate": 1.6306293495205755e-07,
"logits/chosen": 1.1987491846084595,
"logits/rejected": 1.1471325159072876,
"logps/chosen": -301.4486389160156,
"logps/ref_chosen": -274.67242431640625,
"logps/ref_rejected": -242.1553955078125,
"logps/rejected": -312.9349365234375,
"loss": 4.5322,
"margin_dpo/margin_mean": 44.00330352783203,
"margin_dpo/margin_std": 65.34341430664062,
"step": 312
},
{
"KL/chosen_KL_mean": -29.452743530273438,
"KL/mean": -50.3410530090332,
"KL/rejected_KL_mean": -71.2293701171875,
"KL/std": 59.430816650390625,
"epoch": 0.6554973821989529,
"fcm_dpo/beta": 0.008578747510910034,
"fcm_dpo/delta": 0.008986297994852066,
"fcm_dpo/margin": 41.77661895751953,
"fcm_dpo/q_t": 0.41794195771217346,
"grad_norm": 33.07001876831055,
"learning_rate": 1.6134877823936607e-07,
"logits/chosen": 1.4117703437805176,
"logits/rejected": 1.4809421300888062,
"logps/chosen": -324.6405944824219,
"logps/ref_chosen": -295.1878662109375,
"logps/ref_rejected": -271.3431091308594,
"logps/rejected": -342.5724792480469,
"loss": 4.613,
"margin_dpo/margin_mean": 41.77661895751953,
"margin_dpo/margin_std": 68.8209457397461,
"step": 313
},
{
"KL/chosen_KL_mean": -27.462417602539062,
"KL/mean": -49.15557098388672,
"KL/rejected_KL_mean": -70.84872436523438,
"KL/std": 57.39149475097656,
"epoch": 0.6575916230366492,
"fcm_dpo/beta": 0.008496898226439953,
"fcm_dpo/delta": 0.0017744582146406174,
"fcm_dpo/margin": 43.38631820678711,
"fcm_dpo/q_t": 0.41470271348953247,
"grad_norm": 45.454280853271484,
"learning_rate": 1.5963937562265522e-07,
"logits/chosen": 1.3141942024230957,
"logits/rejected": 1.3588166236877441,
"logps/chosen": -317.23028564453125,
"logps/ref_chosen": -289.767822265625,
"logps/ref_rejected": -267.0824279785156,
"logps/rejected": -337.93115234375,
"loss": 4.5466,
"margin_dpo/margin_mean": 43.386322021484375,
"margin_dpo/margin_std": 67.73545837402344,
"step": 314
},
{
"KL/chosen_KL_mean": -24.688133239746094,
"KL/mean": -48.83763885498047,
"KL/rejected_KL_mean": -72.98715209960938,
"KL/std": 58.236576080322266,
"epoch": 0.6596858638743456,
"fcm_dpo/beta": 0.008282303810119629,
"fcm_dpo/delta": -0.027523932978510857,
"fcm_dpo/margin": 48.29901885986328,
"fcm_dpo/q_t": 0.4080940783023834,
"grad_norm": 32.36628723144531,
"learning_rate": 1.5793481877199943e-07,
"logits/chosen": 1.6165395975112915,
"logits/rejected": 1.6356583833694458,
"logps/chosen": -323.107177734375,
"logps/ref_chosen": -298.4190673828125,
"logps/ref_rejected": -266.64447021484375,
"logps/rejected": -339.6316223144531,
"loss": 4.4189,
"margin_dpo/margin_mean": 48.29901885986328,
"margin_dpo/margin_std": 68.33442687988281,
"step": 315
},
{
"KL/chosen_KL_mean": -29.04701042175293,
"KL/mean": -54.41446304321289,
"KL/rejected_KL_mean": -79.78190612792969,
"KL/std": 59.92026138305664,
"epoch": 0.6617801047120419,
"fcm_dpo/beta": 0.008162661455571651,
"fcm_dpo/delta": -0.015288690105080605,
"fcm_dpo/margin": 50.734893798828125,
"fcm_dpo/q_t": 0.40513941645622253,
"grad_norm": 32.51222610473633,
"learning_rate": 1.562351990976095e-07,
"logits/chosen": 1.397526741027832,
"logits/rejected": 1.3919577598571777,
"logps/chosen": -331.2768859863281,
"logps/ref_chosen": -302.2298583984375,
"logps/ref_rejected": -272.7162170410156,
"logps/rejected": -352.49810791015625,
"loss": 4.372,
"margin_dpo/margin_mean": 50.734893798828125,
"margin_dpo/margin_std": 69.03914642333984,
"step": 316
},
{
"KL/chosen_KL_mean": -22.076305389404297,
"KL/mean": -44.926109313964844,
"KL/rejected_KL_mean": -67.77590942382812,
"KL/std": 56.510860443115234,
"epoch": 0.6638743455497382,
"fcm_dpo/beta": 0.008222212083637714,
"fcm_dpo/delta": 0.02461521327495575,
"fcm_dpo/margin": 45.69961166381836,
"fcm_dpo/q_t": 0.4130440354347229,
"grad_norm": 63.09319305419922,
"learning_rate": 1.5454060774493065e-07,
"logits/chosen": 1.3081531524658203,
"logits/rejected": 1.2788610458374023,
"logps/chosen": -268.92974853515625,
"logps/ref_chosen": -246.85342407226562,
"logps/ref_rejected": -216.5264892578125,
"logps/rejected": -284.3023681640625,
"loss": 4.4627,
"margin_dpo/margin_mean": 45.69961166381836,
"margin_dpo/margin_std": 64.08443450927734,
"step": 317
},
{
"KL/chosen_KL_mean": -25.16353416442871,
"KL/mean": -48.39657974243164,
"KL/rejected_KL_mean": -71.6296157836914,
"KL/std": 58.039100646972656,
"epoch": 0.6659685863874345,
"fcm_dpo/beta": 0.008192427456378937,
"fcm_dpo/delta": -0.007180863991379738,
"fcm_dpo/margin": 46.466087341308594,
"fcm_dpo/q_t": 0.41059932112693787,
"grad_norm": 40.38105392456055,
"learning_rate": 1.5285113558975427e-07,
"logits/chosen": 1.3573622703552246,
"logits/rejected": 1.4426628351211548,
"logps/chosen": -287.8131408691406,
"logps/ref_chosen": -262.649658203125,
"logps/ref_rejected": -251.24945068359375,
"logps/rejected": -322.8790588378906,
"loss": 4.4393,
"margin_dpo/margin_mean": 46.466087341308594,
"margin_dpo/margin_std": 62.907737731933594,
"step": 318
},
{
"KL/chosen_KL_mean": -21.974462509155273,
"KL/mean": -47.03016662597656,
"KL/rejected_KL_mean": -72.08587646484375,
"KL/std": 57.22808074951172,
"epoch": 0.6680628272251309,
"fcm_dpo/beta": 0.008105763234198093,
"fcm_dpo/delta": -0.0071238186210393906,
"fcm_dpo/margin": 50.111412048339844,
"fcm_dpo/q_t": 0.4045659303665161,
"grad_norm": 28.36729621887207,
"learning_rate": 1.5116687323334464e-07,
"logits/chosen": 1.3648066520690918,
"logits/rejected": 1.3974896669387817,
"logps/chosen": -315.36077880859375,
"logps/ref_chosen": -293.3863220214844,
"logps/ref_rejected": -238.86026000976562,
"logps/rejected": -310.9461364746094,
"loss": 4.3064,
"margin_dpo/margin_mean": 50.11140823364258,
"margin_dpo/margin_std": 57.791046142578125,
"step": 319
},
{
"KL/chosen_KL_mean": -20.64314079284668,
"KL/mean": -44.01919937133789,
"KL/rejected_KL_mean": -67.39525604248047,
"KL/std": 57.318016052246094,
"epoch": 0.6701570680628273,
"fcm_dpo/beta": 0.008033184334635735,
"fcm_dpo/delta": -0.04559609666466713,
"fcm_dpo/margin": 46.752113342285156,
"fcm_dpo/q_t": 0.4125458002090454,
"grad_norm": 42.626216888427734,
"learning_rate": 1.4948791099758052e-07,
"logits/chosen": 1.6326038837432861,
"logits/rejected": 1.6842557191848755,
"logps/chosen": -316.14697265625,
"logps/ref_chosen": -295.5038146972656,
"logps/ref_rejected": -278.524169921875,
"logps/rejected": -345.91943359375,
"loss": 4.5235,
"margin_dpo/margin_mean": 46.752113342285156,
"margin_dpo/margin_std": 71.97604370117188,
"step": 320
},
{
"KL/chosen_KL_mean": -23.548904418945312,
"KL/mean": -43.685184478759766,
"KL/rejected_KL_mean": -63.82146453857422,
"KL/std": 57.974605560302734,
"epoch": 0.6722513089005235,
"fcm_dpo/beta": 0.00815974734723568,
"fcm_dpo/delta": 0.04450944438576698,
"fcm_dpo/margin": 40.27255630493164,
"fcm_dpo/q_t": 0.4240015149116516,
"grad_norm": 26.314739227294922,
"learning_rate": 1.478143389201113e-07,
"logits/chosen": 1.7631984949111938,
"logits/rejected": 1.7144174575805664,
"logps/chosen": -304.3042297363281,
"logps/ref_chosen": -280.7553405761719,
"logps/ref_rejected": -237.47286987304688,
"logps/rejected": -301.2943420410156,
"loss": 4.6297,
"margin_dpo/margin_mean": 40.272560119628906,
"margin_dpo/margin_std": 66.70619201660156,
"step": 321
},
{
"KL/chosen_KL_mean": -22.824190139770508,
"KL/mean": -47.798336029052734,
"KL/rejected_KL_mean": -72.77247619628906,
"KL/std": 59.09068298339844,
"epoch": 0.6743455497382199,
"fcm_dpo/beta": 0.008056925609707832,
"fcm_dpo/delta": -0.038321636617183685,
"fcm_dpo/margin": 49.94829177856445,
"fcm_dpo/q_t": 0.4062731862068176,
"grad_norm": 44.12809371948242,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": 1.397947072982788,
"logits/rejected": 1.4019372463226318,
"logps/chosen": -321.9154968261719,
"logps/ref_chosen": -299.09130859375,
"logps/ref_rejected": -268.9637756347656,
"logps/rejected": -341.7362365722656,
"loss": 4.3814,
"margin_dpo/margin_mean": 49.94829177856445,
"margin_dpo/margin_std": 66.34510040283203,
"step": 322
},
{
"KL/chosen_KL_mean": -28.13748550415039,
"KL/mean": -45.56864547729492,
"KL/rejected_KL_mean": -62.999813079833984,
"KL/std": 56.300010681152344,
"epoch": 0.6764397905759162,
"fcm_dpo/beta": 0.007967677898705006,
"fcm_dpo/delta": 0.022964956238865852,
"fcm_dpo/margin": 34.86231994628906,
"fcm_dpo/q_t": 0.4358903765678406,
"grad_norm": 35.1325798034668,
"learning_rate": 1.4448372394055246e-07,
"logits/chosen": 1.5975879430770874,
"logits/rejected": 1.5509735345840454,
"logps/chosen": -304.8335266113281,
"logps/ref_chosen": -276.696044921875,
"logps/ref_rejected": -225.78790283203125,
"logps/rejected": -288.7877197265625,
"loss": 4.8489,
"margin_dpo/margin_mean": 34.86231994628906,
"margin_dpo/margin_std": 71.83058166503906,
"step": 323
},
{
"KL/chosen_KL_mean": -16.53660011291504,
"KL/mean": -44.17595291137695,
"KL/rejected_KL_mean": -71.8153076171875,
"KL/std": 57.79463577270508,
"epoch": 0.6785340314136126,
"fcm_dpo/beta": 0.007844395935535431,
"fcm_dpo/delta": -0.03678688779473305,
"fcm_dpo/margin": 55.27870178222656,
"fcm_dpo/q_t": 0.39868438243865967,
"grad_norm": 50.061527252197266,
"learning_rate": 1.428268596492364e-07,
"logits/chosen": 1.36971914768219,
"logits/rejected": 1.4502313137054443,
"logps/chosen": -247.12161254882812,
"logps/ref_chosen": -230.58499145507812,
"logps/ref_rejected": -229.8074493408203,
"logps/rejected": -301.6227722167969,
"loss": 4.2292,
"margin_dpo/margin_mean": 55.27870178222656,
"margin_dpo/margin_std": 59.602325439453125,
"step": 324
},
{
"KL/chosen_KL_mean": -20.096792221069336,
"KL/mean": -42.533958435058594,
"KL/rejected_KL_mean": -64.97111511230469,
"KL/std": 61.21430206298828,
"epoch": 0.680628272251309,
"fcm_dpo/beta": 0.007813180796802044,
"fcm_dpo/delta": 0.014284295961260796,
"fcm_dpo/margin": 44.87432861328125,
"fcm_dpo/q_t": 0.4205002188682556,
"grad_norm": 40.60066604614258,
"learning_rate": 1.4117574272818386e-07,
"logits/chosen": 1.3730337619781494,
"logits/rejected": 1.4071730375289917,
"logps/chosen": -290.6528015136719,
"logps/ref_chosen": -270.5559997558594,
"logps/ref_rejected": -266.328369140625,
"logps/rejected": -331.2994689941406,
"loss": 4.6294,
"margin_dpo/margin_mean": 44.87432861328125,
"margin_dpo/margin_std": 76.99175262451172,
"step": 325
},
{
"KL/chosen_KL_mean": -17.595491409301758,
"KL/mean": -40.99762725830078,
"KL/rejected_KL_mean": -64.3997573852539,
"KL/std": 55.9461669921875,
"epoch": 0.6827225130890052,
"fcm_dpo/beta": 0.008045146241784096,
"fcm_dpo/delta": 0.023972200229763985,
"fcm_dpo/margin": 46.80427932739258,
"fcm_dpo/q_t": 0.4126897156238556,
"grad_norm": 40.11039733886719,
"learning_rate": 1.3953046172178413e-07,
"logits/chosen": 1.284023642539978,
"logits/rejected": 1.3685574531555176,
"logps/chosen": -250.83407592773438,
"logps/ref_chosen": -233.2386016845703,
"logps/ref_rejected": -258.154541015625,
"logps/rejected": -322.5542907714844,
"loss": 4.5018,
"margin_dpo/margin_mean": 46.80427932739258,
"margin_dpo/margin_std": 70.84291076660156,
"step": 326
},
{
"KL/chosen_KL_mean": -8.933675765991211,
"KL/mean": -34.981788635253906,
"KL/rejected_KL_mean": -61.029911041259766,
"KL/std": 56.793663024902344,
"epoch": 0.6848167539267016,
"fcm_dpo/beta": 0.008016958832740784,
"fcm_dpo/delta": -0.019539086148142815,
"fcm_dpo/margin": 52.096229553222656,
"fcm_dpo/q_t": 0.4032677114009857,
"grad_norm": 28.974864959716797,
"learning_rate": 1.3789110486146468e-07,
"logits/chosen": 1.364997148513794,
"logits/rejected": 1.3957648277282715,
"logps/chosen": -277.4798583984375,
"logps/ref_chosen": -268.5461730957031,
"logps/ref_rejected": -266.5316467285156,
"logps/rejected": -327.5615234375,
"loss": 4.3732,
"margin_dpo/margin_mean": 52.096229553222656,
"margin_dpo/margin_std": 69.97504425048828,
"step": 327
},
{
"KL/chosen_KL_mean": -21.03065299987793,
"KL/mean": -43.62373352050781,
"KL/rejected_KL_mean": -66.21681213378906,
"KL/std": 57.95714569091797,
"epoch": 0.6869109947643979,
"fcm_dpo/beta": 0.008020462468266487,
"fcm_dpo/delta": 0.03876934573054314,
"fcm_dpo/margin": 45.1861572265625,
"fcm_dpo/q_t": 0.41611042618751526,
"grad_norm": 53.43826675415039,
"learning_rate": 1.362577600609588e-07,
"logits/chosen": 1.451033353805542,
"logits/rejected": 1.4784150123596191,
"logps/chosen": -315.90740966796875,
"logps/ref_chosen": -294.8768005371094,
"logps/ref_rejected": -281.09771728515625,
"logps/rejected": -347.3145446777344,
"loss": 4.5141,
"margin_dpo/margin_mean": 45.1861572265625,
"margin_dpo/margin_std": 66.3680648803711,
"step": 328
},
{
"KL/chosen_KL_mean": -28.838703155517578,
"KL/mean": -48.4996337890625,
"KL/rejected_KL_mean": -68.16056823730469,
"KL/std": 63.01567840576172,
"epoch": 0.6890052356020943,
"fcm_dpo/beta": 0.008267353288829327,
"fcm_dpo/delta": 0.016651635989546776,
"fcm_dpo/margin": 39.321868896484375,
"fcm_dpo/q_t": 0.42634207010269165,
"grad_norm": 32.08344268798828,
"learning_rate": 1.3463051491159093e-07,
"logits/chosen": 1.3955703973770142,
"logits/rejected": 1.4034247398376465,
"logps/chosen": -341.136474609375,
"logps/ref_chosen": -312.2977600097656,
"logps/ref_rejected": -304.6536560058594,
"logps/rejected": -372.814208984375,
"loss": 4.7474,
"margin_dpo/margin_mean": 39.321868896484375,
"margin_dpo/margin_std": 76.04838562011719,
"step": 329
},
{
"KL/chosen_KL_mean": -25.502676010131836,
"KL/mean": -42.34815216064453,
"KL/rejected_KL_mean": -59.193626403808594,
"KL/std": 58.84164810180664,
"epoch": 0.6910994764397905,
"fcm_dpo/beta": 0.008485514670610428,
"fcm_dpo/delta": 0.038778625428676605,
"fcm_dpo/margin": 33.690956115722656,
"fcm_dpo/q_t": 0.43506842851638794,
"grad_norm": 48.36201095581055,
"learning_rate": 1.3300945667758012e-07,
"logits/chosen": 1.2198834419250488,
"logits/rejected": 1.2783613204956055,
"logps/chosen": -319.7068786621094,
"logps/ref_chosen": -294.2042236328125,
"logps/ref_rejected": -296.2102355957031,
"logps/rejected": -355.4039001464844,
"loss": 4.8353,
"margin_dpo/margin_mean": 33.69095230102539,
"margin_dpo/margin_std": 71.08140563964844,
"step": 330
},
{
"KL/chosen_KL_mean": -18.539098739624023,
"KL/mean": -39.45647430419922,
"KL/rejected_KL_mean": -60.37385559082031,
"KL/std": 58.204124450683594,
"epoch": 0.6931937172774869,
"fcm_dpo/beta": 0.008582251146435738,
"fcm_dpo/delta": 0.0032618592958897352,
"fcm_dpo/margin": 41.83475112915039,
"fcm_dpo/q_t": 0.4177251160144806,
"grad_norm": 35.6547737121582,
"learning_rate": 1.3139467229135998e-07,
"logits/chosen": 1.4187893867492676,
"logits/rejected": 1.4320850372314453,
"logps/chosen": -286.85174560546875,
"logps/ref_chosen": -268.3126525878906,
"logps/ref_rejected": -255.5543212890625,
"logps/rejected": -315.92816162109375,
"loss": 4.5625,
"margin_dpo/margin_mean": 41.834747314453125,
"margin_dpo/margin_std": 66.47994995117188,
"step": 331
},
{
"KL/chosen_KL_mean": -19.064308166503906,
"KL/mean": -45.1978645324707,
"KL/rejected_KL_mean": -71.33141326904297,
"KL/std": 59.10099411010742,
"epoch": 0.6952879581151833,
"fcm_dpo/beta": 0.00829013716429472,
"fcm_dpo/delta": -0.03619233891367912,
"fcm_dpo/margin": 52.2671012878418,
"fcm_dpo/q_t": 0.4004819691181183,
"grad_norm": 35.77095031738281,
"learning_rate": 1.2978624834891626e-07,
"logits/chosen": 1.3465325832366943,
"logits/rejected": 1.3774622678756714,
"logps/chosen": -282.8147277832031,
"logps/ref_chosen": -263.75042724609375,
"logps/ref_rejected": -234.5144500732422,
"logps/rejected": -305.8458251953125,
"loss": 4.3568,
"margin_dpo/margin_mean": 52.2671012878418,
"margin_dpo/margin_std": 72.7847900390625,
"step": 332
},
{
"KL/chosen_KL_mean": -18.448471069335938,
"KL/mean": -38.11326599121094,
"KL/rejected_KL_mean": -57.778076171875,
"KL/std": 56.81516647338867,
"epoch": 0.6973821989528796,
"fcm_dpo/beta": 0.008496787399053574,
"fcm_dpo/delta": 0.03937835618853569,
"fcm_dpo/margin": 39.329593658447266,
"fcm_dpo/q_t": 0.42261365056037903,
"grad_norm": 51.38772964477539,
"learning_rate": 1.281842711051438e-07,
"logits/chosen": 1.4903069734573364,
"logits/rejected": 1.4716383218765259,
"logps/chosen": -305.264892578125,
"logps/ref_chosen": -286.81646728515625,
"logps/ref_rejected": -260.4169616699219,
"logps/rejected": -318.1950378417969,
"loss": 4.6185,
"margin_dpo/margin_mean": 39.3296012878418,
"margin_dpo/margin_std": 64.0001449584961,
"step": 333
},
{
"KL/chosen_KL_mean": -16.50014305114746,
"KL/mean": -40.04962158203125,
"KL/rejected_KL_mean": -63.59909439086914,
"KL/std": 59.867618560791016,
"epoch": 0.6994764397905759,
"fcm_dpo/beta": 0.008586030453443527,
"fcm_dpo/delta": -0.03872523456811905,
"fcm_dpo/margin": 47.09894561767578,
"fcm_dpo/q_t": 0.40821734070777893,
"grad_norm": 45.0473747253418,
"learning_rate": 1.2658882646922033e-07,
"logits/chosen": 1.2563035488128662,
"logits/rejected": 1.2555757761001587,
"logps/chosen": -309.8110656738281,
"logps/ref_chosen": -293.3109130859375,
"logps/ref_rejected": -264.1449279785156,
"logps/rejected": -327.7440185546875,
"loss": 4.525,
"margin_dpo/margin_mean": 47.09895324707031,
"margin_dpo/margin_std": 74.84980773925781,
"step": 334
},
{
"KL/chosen_KL_mean": -29.5706729888916,
"KL/mean": -50.683982849121094,
"KL/rejected_KL_mean": -71.79728698730469,
"KL/std": 58.2430305480957,
"epoch": 0.7015706806282722,
"fcm_dpo/beta": 0.008247178047895432,
"fcm_dpo/delta": -0.010653029195964336,
"fcm_dpo/margin": 42.226619720458984,
"fcm_dpo/q_t": 0.4203599989414215,
"grad_norm": 31.517383575439453,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": 1.1738187074661255,
"logits/rejected": 1.2567086219787598,
"logps/chosen": -299.3270263671875,
"logps/ref_chosen": -269.75634765625,
"logps/ref_rejected": -241.08438110351562,
"logps/rejected": -312.88165283203125,
"loss": 4.6694,
"margin_dpo/margin_mean": 42.226619720458984,
"margin_dpo/margin_std": 74.86090850830078,
"step": 335
},
{
"KL/chosen_KL_mean": -22.192955017089844,
"KL/mean": -42.18242645263672,
"KL/rejected_KL_mean": -62.171897888183594,
"KL/std": 58.28977584838867,
"epoch": 0.7036649214659686,
"fcm_dpo/beta": 0.00838224682956934,
"fcm_dpo/delta": 0.029553044587373734,
"fcm_dpo/margin": 39.978946685791016,
"fcm_dpo/q_t": 0.4237605035305023,
"grad_norm": 39.035865783691406,
"learning_rate": 1.2341787690142435e-07,
"logits/chosen": 1.3819284439086914,
"logits/rejected": 1.4114774465560913,
"logps/chosen": -259.56475830078125,
"logps/ref_chosen": -237.37179565429688,
"logps/ref_rejected": -253.5463409423828,
"logps/rejected": -315.71826171875,
"loss": 4.6372,
"margin_dpo/margin_mean": 39.978946685791016,
"margin_dpo/margin_std": 67.77619171142578,
"step": 336
},
{
"KL/chosen_KL_mean": -26.687501907348633,
"KL/mean": -49.499855041503906,
"KL/rejected_KL_mean": -72.31220245361328,
"KL/std": 57.879478454589844,
"epoch": 0.7057591623036649,
"fcm_dpo/beta": 0.008477190509438515,
"fcm_dpo/delta": -0.020977266132831573,
"fcm_dpo/margin": 45.62470626831055,
"fcm_dpo/q_t": 0.4121153652667999,
"grad_norm": 46.522640228271484,
"learning_rate": 1.2184254201795363e-07,
"logits/chosen": 1.2859256267547607,
"logits/rejected": 1.3341453075408936,
"logps/chosen": -287.9951477050781,
"logps/ref_chosen": -261.3076477050781,
"logps/ref_rejected": -252.3170166015625,
"logps/rejected": -324.6291809082031,
"loss": 4.5192,
"margin_dpo/margin_mean": 45.62470626831055,
"margin_dpo/margin_std": 72.3380126953125,
"step": 337
},
{
"KL/chosen_KL_mean": -11.782232284545898,
"KL/mean": -39.000064849853516,
"KL/rejected_KL_mean": -66.21788787841797,
"KL/std": 60.04515838623047,
"epoch": 0.7078534031413612,
"fcm_dpo/beta": 0.008186925202608109,
"fcm_dpo/delta": -0.04778764396905899,
"fcm_dpo/margin": 54.43565368652344,
"fcm_dpo/q_t": 0.39773881435394287,
"grad_norm": 27.990068435668945,
"learning_rate": 1.202740798300168e-07,
"logits/chosen": 1.5228660106658936,
"logits/rejected": 1.5129181146621704,
"logps/chosen": -280.553955078125,
"logps/ref_chosen": -268.7717590332031,
"logps/ref_rejected": -232.3352813720703,
"logps/rejected": -298.55316162109375,
"loss": 4.2827,
"margin_dpo/margin_mean": 54.43565368652344,
"margin_dpo/margin_std": 70.96080780029297,
"step": 338
},
{
"KL/chosen_KL_mean": -20.091079711914062,
"KL/mean": -47.560150146484375,
"KL/rejected_KL_mean": -75.02922058105469,
"KL/std": 61.880332946777344,
"epoch": 0.7099476439790576,
"fcm_dpo/beta": 0.007898853160440922,
"fcm_dpo/delta": -0.035964664071798325,
"fcm_dpo/margin": 54.938148498535156,
"fcm_dpo/q_t": 0.3994678258895874,
"grad_norm": 42.823509216308594,
"learning_rate": 1.1871257444948096e-07,
"logits/chosen": 1.256564736366272,
"logits/rejected": 1.224652647972107,
"logps/chosen": -313.7042236328125,
"logps/ref_chosen": -293.6131286621094,
"logps/ref_rejected": -281.87738037109375,
"logps/rejected": -356.9066162109375,
"loss": 4.3348,
"margin_dpo/margin_mean": 54.938148498535156,
"margin_dpo/margin_std": 74.72753143310547,
"step": 339
},
{
"KL/chosen_KL_mean": -25.699125289916992,
"KL/mean": -48.140480041503906,
"KL/rejected_KL_mean": -70.58184814453125,
"KL/std": 61.5643424987793,
"epoch": 0.7120418848167539,
"fcm_dpo/beta": 0.00794089213013649,
"fcm_dpo/delta": 0.04462888091802597,
"fcm_dpo/margin": 44.882720947265625,
"fcm_dpo/q_t": 0.41869547963142395,
"grad_norm": 25.24543571472168,
"learning_rate": 1.1715810961514072e-07,
"logits/chosen": 0.9644868969917297,
"logits/rejected": 1.0749053955078125,
"logps/chosen": -277.4561767578125,
"logps/ref_chosen": -251.75704956054688,
"logps/ref_rejected": -255.76441955566406,
"logps/rejected": -326.34625244140625,
"loss": 4.5771,
"margin_dpo/margin_mean": 44.88272476196289,
"margin_dpo/margin_std": 71.99385070800781,
"step": 340
},
{
"KL/chosen_KL_mean": -38.51675033569336,
"KL/mean": -56.909446716308594,
"KL/rejected_KL_mean": -75.30213928222656,
"KL/std": 64.48269653320312,
"epoch": 0.7141361256544503,
"fcm_dpo/beta": 0.008220081217586994,
"fcm_dpo/delta": 0.029346494004130363,
"fcm_dpo/margin": 36.785396575927734,
"fcm_dpo/q_t": 0.43140214681625366,
"grad_norm": 49.63078689575195,
"learning_rate": 1.1561076868822755e-07,
"logits/chosen": 1.3204576969146729,
"logits/rejected": 1.4248378276824951,
"logps/chosen": -345.1643981933594,
"logps/ref_chosen": -306.64764404296875,
"logps/ref_rejected": -301.4248352050781,
"logps/rejected": -376.72698974609375,
"loss": 4.8246,
"margin_dpo/margin_mean": 36.78539276123047,
"margin_dpo/margin_std": 76.30755615234375,
"step": 341
},
{
"KL/chosen_KL_mean": -22.643159866333008,
"KL/mean": -45.68241882324219,
"KL/rejected_KL_mean": -68.7216796875,
"KL/std": 61.156394958496094,
"epoch": 0.7162303664921466,
"fcm_dpo/beta": 0.00822380930185318,
"fcm_dpo/delta": -0.035644009709358215,
"fcm_dpo/margin": 46.07852554321289,
"fcm_dpo/q_t": 0.41292059421539307,
"grad_norm": 28.56557846069336,
"learning_rate": 1.1407063464793965e-07,
"logits/chosen": 1.4937210083007812,
"logits/rejected": 1.5514423847198486,
"logps/chosen": -285.31207275390625,
"logps/ref_chosen": -262.6689147949219,
"logps/ref_rejected": -260.8409118652344,
"logps/rejected": -329.5625915527344,
"loss": 4.4882,
"margin_dpo/margin_mean": 46.078521728515625,
"margin_dpo/margin_std": 67.86953735351562,
"step": 342
},
{
"KL/chosen_KL_mean": -23.73531723022461,
"KL/mean": -44.780399322509766,
"KL/rejected_KL_mean": -65.82548522949219,
"KL/std": 60.82894515991211,
"epoch": 0.7183246073298429,
"fcm_dpo/beta": 0.008099392056465149,
"fcm_dpo/delta": 0.030330460518598557,
"fcm_dpo/margin": 42.09016799926758,
"fcm_dpo/q_t": 0.4220288395881653,
"grad_norm": 30.66365623474121,
"learning_rate": 1.125377900869913e-07,
"logits/chosen": 1.606204867362976,
"logits/rejected": 1.6237492561340332,
"logps/chosen": -323.36126708984375,
"logps/ref_chosen": -299.6259765625,
"logps/ref_rejected": -261.2701416015625,
"logps/rejected": -327.09564208984375,
"loss": 4.6368,
"margin_dpo/margin_mean": 42.09016418457031,
"margin_dpo/margin_std": 72.11952209472656,
"step": 343
},
{
"KL/chosen_KL_mean": -24.897951126098633,
"KL/mean": -49.33934783935547,
"KL/rejected_KL_mean": -73.7807388305664,
"KL/std": 62.17938232421875,
"epoch": 0.7204188481675393,
"fcm_dpo/beta": 0.00840503815561533,
"fcm_dpo/delta": -0.0125031853094697,
"fcm_dpo/margin": 48.882781982421875,
"fcm_dpo/q_t": 0.4062635004520416,
"grad_norm": 35.609527587890625,
"learning_rate": 1.110123172071844e-07,
"logits/chosen": 1.2275300025939941,
"logits/rejected": 1.2526569366455078,
"logps/chosen": -315.8483581542969,
"logps/ref_chosen": -290.950439453125,
"logps/ref_rejected": -255.15374755859375,
"logps/rejected": -328.93450927734375,
"loss": 4.4319,
"margin_dpo/margin_mean": 48.882781982421875,
"margin_dpo/margin_std": 70.47264099121094,
"step": 344
},
{
"KL/chosen_KL_mean": -21.042957305908203,
"KL/mean": -41.27716827392578,
"KL/rejected_KL_mean": -61.51138687133789,
"KL/std": 59.13972091674805,
"epoch": 0.7225130890052356,
"fcm_dpo/beta": 0.00830150581896305,
"fcm_dpo/delta": 0.030441733077168465,
"fcm_dpo/margin": 40.46842956542969,
"fcm_dpo/q_t": 0.42245370149612427,
"grad_norm": 37.91876220703125,
"learning_rate": 1.09494297815e-07,
"logits/chosen": 1.3884456157684326,
"logits/rejected": 1.4642207622528076,
"logps/chosen": -294.308837890625,
"logps/ref_chosen": -273.2658996582031,
"logps/ref_rejected": -263.9664306640625,
"logps/rejected": -325.4778137207031,
"loss": 4.5937,
"margin_dpo/margin_mean": 40.46843338012695,
"margin_dpo/margin_std": 62.32899856567383,
"step": 345
},
{
"KL/chosen_KL_mean": -33.96916198730469,
"KL/mean": -57.84877014160156,
"KL/rejected_KL_mean": -81.7283706665039,
"KL/std": 61.43087387084961,
"epoch": 0.724607329842932,
"fcm_dpo/beta": 0.007993143051862717,
"fcm_dpo/delta": -0.08858863264322281,
"fcm_dpo/margin": 47.75920104980469,
"fcm_dpo/q_t": 0.41108354926109314,
"grad_norm": 34.72679138183594,
"learning_rate": 1.0798381331721107e-07,
"logits/chosen": 1.302663803100586,
"logits/rejected": 1.3537399768829346,
"logps/chosen": -296.65130615234375,
"logps/ref_chosen": -262.6821594238281,
"logps/ref_rejected": -226.46859741210938,
"logps/rejected": -308.19696044921875,
"loss": 4.4971,
"margin_dpo/margin_mean": 47.75920867919922,
"margin_dpo/margin_std": 67.78038024902344,
"step": 346
},
{
"KL/chosen_KL_mean": -30.05909538269043,
"KL/mean": -51.16781234741211,
"KL/rejected_KL_mean": -72.27653503417969,
"KL/std": 61.078372955322266,
"epoch": 0.7267015706806282,
"fcm_dpo/beta": 0.007945908233523369,
"fcm_dpo/delta": 0.0373678132891655,
"fcm_dpo/margin": 42.21742630004883,
"fcm_dpo/q_t": 0.4221636950969696,
"grad_norm": 30.520584106445312,
"learning_rate": 1.0648094471651722e-07,
"logits/chosen": 1.356184720993042,
"logits/rejected": 1.3794606924057007,
"logps/chosen": -293.8836669921875,
"logps/ref_chosen": -263.8246154785156,
"logps/ref_rejected": -237.9121551513672,
"logps/rejected": -310.188720703125,
"loss": 4.6194,
"margin_dpo/margin_mean": 42.217430114746094,
"margin_dpo/margin_std": 69.29716491699219,
"step": 347
},
{
"KL/chosen_KL_mean": -23.193214416503906,
"KL/mean": -41.58753204345703,
"KL/rejected_KL_mean": -59.98184585571289,
"KL/std": 60.363502502441406,
"epoch": 0.7287958115183246,
"fcm_dpo/beta": 0.008062899112701416,
"fcm_dpo/delta": 0.030613910406827927,
"fcm_dpo/margin": 36.78863525390625,
"fcm_dpo/q_t": 0.43188756704330444,
"grad_norm": 31.7449951171875,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": 1.1868278980255127,
"logits/rejected": 1.2123560905456543,
"logps/chosen": -301.3465881347656,
"logps/ref_chosen": -278.1533203125,
"logps/ref_rejected": -263.39691162109375,
"logps/rejected": -323.3787536621094,
"loss": 4.7326,
"margin_dpo/margin_mean": 36.788631439208984,
"margin_dpo/margin_std": 64.37882995605469,
"step": 348
},
{
"KL/chosen_KL_mean": -26.564422607421875,
"KL/mean": -52.15089416503906,
"KL/rejected_KL_mean": -77.73736572265625,
"KL/std": 64.47464752197266,
"epoch": 0.7308900523560209,
"fcm_dpo/beta": 0.00800681859254837,
"fcm_dpo/delta": -0.05323091894388199,
"fcm_dpo/margin": 51.17293930053711,
"fcm_dpo/q_t": 0.40616288781166077,
"grad_norm": 27.40471649169922,
"learning_rate": 1.0349837717080347e-07,
"logits/chosen": 1.6410548686981201,
"logits/rejected": 1.6932549476623535,
"logps/chosen": -345.01953125,
"logps/ref_chosen": -318.455078125,
"logps/ref_rejected": -288.5691833496094,
"logps/rejected": -366.3065185546875,
"loss": 4.4206,
"margin_dpo/margin_mean": 51.172943115234375,
"margin_dpo/margin_std": 72.1238784790039,
"step": 349
},
{
"KL/chosen_KL_mean": -29.941728591918945,
"KL/mean": -52.43879318237305,
"KL/rejected_KL_mean": -74.93585205078125,
"KL/std": 63.54440689086914,
"epoch": 0.7329842931937173,
"fcm_dpo/beta": 0.00789142120629549,
"fcm_dpo/delta": -0.034058570861816406,
"fcm_dpo/margin": 44.99413299560547,
"fcm_dpo/q_t": 0.41845959424972534,
"grad_norm": 35.786888122558594,
"learning_rate": 1.0201883817182949e-07,
"logits/chosen": 1.4704442024230957,
"logits/rejected": 1.4845540523529053,
"logps/chosen": -314.13092041015625,
"logps/ref_chosen": -284.1891784667969,
"logps/ref_rejected": -247.94589233398438,
"logps/rejected": -322.8817443847656,
"loss": 4.616,
"margin_dpo/margin_mean": 44.99413299560547,
"margin_dpo/margin_std": 74.6503677368164,
"step": 350
},
{
"KL/chosen_KL_mean": -27.57063102722168,
"KL/mean": -43.758853912353516,
"KL/rejected_KL_mean": -59.947086334228516,
"KL/std": 60.97875213623047,
"epoch": 0.7350785340314137,
"fcm_dpo/beta": 0.0077229635789990425,
"fcm_dpo/delta": 0.0042969174683094025,
"fcm_dpo/margin": 32.376461029052734,
"fcm_dpo/q_t": 0.4431162476539612,
"grad_norm": 37.373355865478516,
"learning_rate": 1.0054723495346482e-07,
"logits/chosen": 1.3978153467178345,
"logits/rejected": 1.3665990829467773,
"logps/chosen": -330.87872314453125,
"logps/ref_chosen": -303.30810546875,
"logps/ref_rejected": -260.1866455078125,
"logps/rejected": -320.13372802734375,
"loss": 4.9431,
"margin_dpo/margin_mean": 32.376461029052734,
"margin_dpo/margin_std": 73.97547149658203,
"step": 351
},
{
"KL/chosen_KL_mean": -24.06132698059082,
"KL/mean": -49.409461975097656,
"KL/rejected_KL_mean": -74.75761413574219,
"KL/std": 62.60258865356445,
"epoch": 0.7371727748691099,
"fcm_dpo/beta": 0.007590837776660919,
"fcm_dpo/delta": -0.04276890680193901,
"fcm_dpo/margin": 50.69627380371094,
"fcm_dpo/q_t": 0.4107002317905426,
"grad_norm": 30.83437156677246,
"learning_rate": 9.908364643332398e-08,
"logits/chosen": 1.3391189575195312,
"logits/rejected": 1.5173217058181763,
"logps/chosen": -270.8684387207031,
"logps/ref_chosen": -246.80709838867188,
"logps/ref_rejected": -253.13223266601562,
"logps/rejected": -327.8898620605469,
"loss": 4.4599,
"margin_dpo/margin_mean": 50.69627380371094,
"margin_dpo/margin_std": 70.11902618408203,
"step": 352
},
{
"KL/chosen_KL_mean": -26.750572204589844,
"KL/mean": -52.218353271484375,
"KL/rejected_KL_mean": -77.6861343383789,
"KL/std": 61.98667526245117,
"epoch": 0.7392670157068063,
"fcm_dpo/beta": 0.007327005732804537,
"fcm_dpo/delta": -0.033968254923820496,
"fcm_dpo/margin": 50.9355583190918,
"fcm_dpo/q_t": 0.41317909955978394,
"grad_norm": 31.25110626220703,
"learning_rate": 9.76281510992176e-08,
"logits/chosen": 1.1865570545196533,
"logits/rejected": 1.2616384029388428,
"logps/chosen": -288.3520202636719,
"logps/ref_chosen": -261.6014709472656,
"logps/ref_rejected": -262.74761962890625,
"logps/rejected": -340.4337463378906,
"loss": 4.4683,
"margin_dpo/margin_mean": 50.9355583190918,
"margin_dpo/margin_std": 70.31192016601562,
"step": 353
},
{
"KL/chosen_KL_mean": -27.61041259765625,
"KL/mean": -42.82272720336914,
"KL/rejected_KL_mean": -58.0350456237793,
"KL/std": 61.23716354370117,
"epoch": 0.7413612565445026,
"fcm_dpo/beta": 0.007398069370537996,
"fcm_dpo/delta": 0.042706623673439026,
"fcm_dpo/margin": 30.42462158203125,
"fcm_dpo/q_t": 0.4486212134361267,
"grad_norm": 37.50423049926758,
"learning_rate": 9.618082700494318e-08,
"logits/chosen": 1.2743968963623047,
"logits/rejected": 1.3830703496932983,
"logps/chosen": -279.75885009765625,
"logps/ref_chosen": -252.14840698242188,
"logps/ref_rejected": -244.1017608642578,
"logps/rejected": -302.13677978515625,
"loss": 5.0227,
"margin_dpo/margin_mean": 30.42462158203125,
"margin_dpo/margin_std": 76.54853820800781,
"step": 354
},
{
"KL/chosen_KL_mean": -28.34946060180664,
"KL/mean": -53.75284194946289,
"KL/rejected_KL_mean": -79.15621948242188,
"KL/std": 63.44832992553711,
"epoch": 0.743455497382199,
"fcm_dpo/beta": 0.007384137716144323,
"fcm_dpo/delta": -0.009487075731158257,
"fcm_dpo/margin": 50.806766510009766,
"fcm_dpo/q_t": 0.41258612275123596,
"grad_norm": 30.895038604736328,
"learning_rate": 9.474175176609956e-08,
"logits/chosen": 1.4611732959747314,
"logits/rejected": 1.5832594633102417,
"logps/chosen": -314.5119934082031,
"logps/ref_chosen": -286.16253662109375,
"logps/ref_rejected": -272.3275146484375,
"logps/rejected": -351.4837646484375,
"loss": 4.481,
"margin_dpo/margin_mean": 50.806766510009766,
"margin_dpo/margin_std": 72.48587036132812,
"step": 355
},
{
"KL/chosen_KL_mean": -21.091541290283203,
"KL/mean": -41.81569290161133,
"KL/rejected_KL_mean": -62.53986358642578,
"KL/std": 58.343475341796875,
"epoch": 0.7455497382198953,
"fcm_dpo/beta": 0.007514188997447491,
"fcm_dpo/delta": 0.029240809381008148,
"fcm_dpo/margin": 41.44832229614258,
"fcm_dpo/q_t": 0.42722052335739136,
"grad_norm": 34.9584846496582,
"learning_rate": 9.331100255592436e-08,
"logits/chosen": 1.315157175064087,
"logits/rejected": 1.3648688793182373,
"logps/chosen": -219.2443084716797,
"logps/ref_chosen": -198.1527862548828,
"logps/ref_rejected": -209.80569458007812,
"logps/rejected": -272.3455505371094,
"loss": 4.6401,
"margin_dpo/margin_mean": 41.44831848144531,
"margin_dpo/margin_std": 64.64385986328125,
"step": 356
},
{
"KL/chosen_KL_mean": -31.06587028503418,
"KL/mean": -53.74104309082031,
"KL/rejected_KL_mean": -76.41620635986328,
"KL/std": 59.96897888183594,
"epoch": 0.7476439790575916,
"fcm_dpo/beta": 0.007646047510206699,
"fcm_dpo/delta": 0.012101225554943085,
"fcm_dpo/margin": 45.350341796875,
"fcm_dpo/q_t": 0.4210367798805237,
"grad_norm": 38.74076461791992,
"learning_rate": 9.18886561011557e-08,
"logits/chosen": 1.3575685024261475,
"logits/rejected": 1.412861943244934,
"logps/chosen": -287.5730285644531,
"logps/ref_chosen": -256.50714111328125,
"logps/ref_rejected": -237.40769958496094,
"logps/rejected": -313.82391357421875,
"loss": 4.6022,
"margin_dpo/margin_mean": 45.350341796875,
"margin_dpo/margin_std": 73.01123809814453,
"step": 357
},
{
"KL/chosen_KL_mean": -20.08008575439453,
"KL/mean": -45.896324157714844,
"KL/rejected_KL_mean": -71.71257019042969,
"KL/std": 62.05603790283203,
"epoch": 0.749738219895288,
"fcm_dpo/beta": 0.007647480815649033,
"fcm_dpo/delta": -0.023323874920606613,
"fcm_dpo/margin": 51.63248825073242,
"fcm_dpo/q_t": 0.40921440720558167,
"grad_norm": 26.73249626159668,
"learning_rate": 9.047478867791731e-08,
"logits/chosen": 1.5963877439498901,
"logits/rejected": 1.5494128465652466,
"logps/chosen": -312.5273132324219,
"logps/ref_chosen": -292.447265625,
"logps/ref_rejected": -255.47872924804688,
"logps/rejected": -327.1913146972656,
"loss": 4.4678,
"margin_dpo/margin_mean": 51.63249588012695,
"margin_dpo/margin_std": 76.36697387695312,
"step": 358
},
{
"KL/chosen_KL_mean": -20.328577041625977,
"KL/mean": -44.36174774169922,
"KL/rejected_KL_mean": -68.39491271972656,
"KL/std": 62.28682327270508,
"epoch": 0.7518324607329843,
"fcm_dpo/beta": 0.007573855575174093,
"fcm_dpo/delta": 0.012442296370863914,
"fcm_dpo/margin": 48.06633377075195,
"fcm_dpo/q_t": 0.4150635302066803,
"grad_norm": 29.436304092407227,
"learning_rate": 8.906947610762825e-08,
"logits/chosen": 1.5220872163772583,
"logits/rejected": 1.492361068725586,
"logps/chosen": -314.08856201171875,
"logps/ref_chosen": -293.7599792480469,
"logps/ref_rejected": -259.4142761230469,
"logps/rejected": -327.8092041015625,
"loss": 4.4952,
"margin_dpo/margin_mean": 48.06633377075195,
"margin_dpo/margin_std": 68.04971313476562,
"step": 359
},
{
"KL/chosen_KL_mean": -29.602924346923828,
"KL/mean": -50.32772445678711,
"KL/rejected_KL_mean": -71.05253601074219,
"KL/std": 62.25556564331055,
"epoch": 0.7539267015706806,
"fcm_dpo/beta": 0.00775428581982851,
"fcm_dpo/delta": 0.03547399491071701,
"fcm_dpo/margin": 41.44960021972656,
"fcm_dpo/q_t": 0.4252384901046753,
"grad_norm": 25.99398422241211,
"learning_rate": 8.76727937529367e-08,
"logits/chosen": 1.428528904914856,
"logits/rejected": 1.3796058893203735,
"logps/chosen": -330.51666259765625,
"logps/ref_chosen": -300.91375732421875,
"logps/ref_rejected": -256.4080810546875,
"logps/rejected": -327.4606018066406,
"loss": 4.6291,
"margin_dpo/margin_mean": 41.44960021972656,
"margin_dpo/margin_std": 65.40933227539062,
"step": 360
},
{
"KL/chosen_KL_mean": -17.207592010498047,
"KL/mean": -45.10061264038086,
"KL/rejected_KL_mean": -72.99363708496094,
"KL/std": 63.61145782470703,
"epoch": 0.7560209424083769,
"fcm_dpo/beta": 0.007767850067466497,
"fcm_dpo/delta": -0.0348568931221962,
"fcm_dpo/margin": 55.78605270385742,
"fcm_dpo/q_t": 0.40066930651664734,
"grad_norm": 28.439056396484375,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": 0.9440096616744995,
"logits/rejected": 1.0340206623077393,
"logps/chosen": -273.76788330078125,
"logps/ref_chosen": -256.5603332519531,
"logps/ref_rejected": -271.043701171875,
"logps/rejected": -344.03729248046875,
"loss": 4.3285,
"margin_dpo/margin_mean": 55.78605270385742,
"margin_dpo/margin_std": 75.44465637207031,
"step": 361
},
{
"KL/chosen_KL_mean": -23.494083404541016,
"KL/mean": -46.04978942871094,
"KL/rejected_KL_mean": -68.60549926757812,
"KL/std": 58.13932800292969,
"epoch": 0.7581151832460733,
"fcm_dpo/beta": 0.007708454504609108,
"fcm_dpo/delta": 0.01965925097465515,
"fcm_dpo/margin": 45.111412048339844,
"fcm_dpo/q_t": 0.4195477068424225,
"grad_norm": 69.2497787475586,
"learning_rate": 8.490561882286135e-08,
"logits/chosen": 1.4437376260757446,
"logits/rejected": 1.3562790155410767,
"logps/chosen": -318.194580078125,
"logps/ref_chosen": -294.700439453125,
"logps/ref_rejected": -239.67922973632812,
"logps/rejected": -308.28472900390625,
"loss": 4.5829,
"margin_dpo/margin_mean": 45.11141586303711,
"margin_dpo/margin_std": 71.23416900634766,
"step": 362
},
{
"KL/chosen_KL_mean": -22.342344284057617,
"KL/mean": -49.643497467041016,
"KL/rejected_KL_mean": -76.94465637207031,
"KL/std": 64.40416717529297,
"epoch": 0.7602094240837697,
"fcm_dpo/beta": 0.007676102686673403,
"fcm_dpo/delta": -0.020748523995280266,
"fcm_dpo/margin": 54.60232162475586,
"fcm_dpo/q_t": 0.40338873863220215,
"grad_norm": 36.314090728759766,
"learning_rate": 8.353527464267104e-08,
"logits/chosen": 1.3692359924316406,
"logits/rejected": 1.3743070363998413,
"logps/chosen": -316.0182189941406,
"logps/ref_chosen": -293.67584228515625,
"logps/ref_rejected": -260.85675048828125,
"logps/rejected": -337.8014221191406,
"loss": 4.3686,
"margin_dpo/margin_mean": 54.60232162475586,
"margin_dpo/margin_std": 74.3442611694336,
"step": 363
},
{
"KL/chosen_KL_mean": -39.35921096801758,
"KL/mean": -59.600502014160156,
"KL/rejected_KL_mean": -79.84181213378906,
"KL/std": 62.217586517333984,
"epoch": 0.762303664921466,
"fcm_dpo/beta": 0.00767554622143507,
"fcm_dpo/delta": 0.027282362803816795,
"fcm_dpo/margin": 40.48260498046875,
"fcm_dpo/q_t": 0.4288344383239746,
"grad_norm": 29.953201293945312,
"learning_rate": 8.217385746050742e-08,
"logits/chosen": 1.0847784280776978,
"logits/rejected": 1.1279144287109375,
"logps/chosen": -318.2546081542969,
"logps/ref_chosen": -278.8953857421875,
"logps/ref_rejected": -283.8898620605469,
"logps/rejected": -363.73162841796875,
"loss": 4.7261,
"margin_dpo/margin_mean": 40.48260498046875,
"margin_dpo/margin_std": 72.49263000488281,
"step": 364
},
{
"KL/chosen_KL_mean": -23.04935073852539,
"KL/mean": -48.64548873901367,
"KL/rejected_KL_mean": -74.24163818359375,
"KL/std": 63.86309051513672,
"epoch": 0.7643979057591623,
"fcm_dpo/beta": 0.007841559126973152,
"fcm_dpo/delta": -0.0018529277294874191,
"fcm_dpo/margin": 51.192283630371094,
"fcm_dpo/q_t": 0.4081072509288788,
"grad_norm": 41.78466796875,
"learning_rate": 8.082144028504231e-08,
"logits/chosen": 1.339031457901001,
"logits/rejected": 1.3882323503494263,
"logps/chosen": -288.6932678222656,
"logps/ref_chosen": -265.6439208984375,
"logps/ref_rejected": -255.40138244628906,
"logps/rejected": -329.64300537109375,
"loss": 4.4171,
"margin_dpo/margin_mean": 51.192283630371094,
"margin_dpo/margin_std": 72.82061004638672,
"step": 365
},
{
"KL/chosen_KL_mean": -27.14842987060547,
"KL/mean": -52.876365661621094,
"KL/rejected_KL_mean": -78.60430908203125,
"KL/std": 63.30706024169922,
"epoch": 0.7664921465968586,
"fcm_dpo/beta": 0.007694230880588293,
"fcm_dpo/delta": 0.003679636400192976,
"fcm_dpo/margin": 51.45587158203125,
"fcm_dpo/q_t": 0.40837374329566956,
"grad_norm": 37.82375717163086,
"learning_rate": 7.947809564230445e-08,
"logits/chosen": 1.4413959980010986,
"logits/rejected": 1.4565752744674683,
"logps/chosen": -307.3729248046875,
"logps/ref_chosen": -280.2244873046875,
"logps/ref_rejected": -248.72366333007812,
"logps/rejected": -327.3279724121094,
"loss": 4.4214,
"margin_dpo/margin_mean": 51.45586395263672,
"margin_dpo/margin_std": 70.79524230957031,
"step": 366
},
{
"KL/chosen_KL_mean": -23.835987091064453,
"KL/mean": -50.60634231567383,
"KL/rejected_KL_mean": -77.37670135498047,
"KL/std": 62.218441009521484,
"epoch": 0.768586387434555,
"fcm_dpo/beta": 0.007677263580262661,
"fcm_dpo/delta": -0.01209146250039339,
"fcm_dpo/margin": 53.54071807861328,
"fcm_dpo/q_t": 0.40493521094322205,
"grad_norm": 41.89437484741211,
"learning_rate": 7.814389557179016e-08,
"logits/chosen": 1.3644987344741821,
"logits/rejected": 1.3721001148223877,
"logps/chosen": -316.77264404296875,
"logps/ref_chosen": -292.9366760253906,
"logps/ref_rejected": -261.88763427734375,
"logps/rejected": -339.2643127441406,
"loss": 4.3442,
"margin_dpo/margin_mean": 53.54071807861328,
"margin_dpo/margin_std": 68.99994659423828,
"step": 367
},
{
"KL/chosen_KL_mean": -18.622913360595703,
"KL/mean": -48.8709602355957,
"KL/rejected_KL_mean": -79.11900329589844,
"KL/std": 64.39937591552734,
"epoch": 0.7706806282722513,
"fcm_dpo/beta": 0.0074094366282224655,
"fcm_dpo/delta": -0.08462084829807281,
"fcm_dpo/margin": 60.49608612060547,
"fcm_dpo/q_t": 0.3952578604221344,
"grad_norm": 39.50826644897461,
"learning_rate": 7.681891162260015e-08,
"logits/chosen": 1.1243423223495483,
"logits/rejected": 1.1874594688415527,
"logps/chosen": -303.5286865234375,
"logps/ref_chosen": -284.90576171875,
"logps/ref_rejected": -256.59197998046875,
"logps/rejected": -335.71099853515625,
"loss": 4.2209,
"margin_dpo/margin_mean": 60.49608612060547,
"margin_dpo/margin_std": 68.90132141113281,
"step": 368
},
{
"KL/chosen_KL_mean": -23.85970687866211,
"KL/mean": -45.43791198730469,
"KL/rejected_KL_mean": -67.01612091064453,
"KL/std": 61.029144287109375,
"epoch": 0.7727748691099476,
"fcm_dpo/beta": 0.007211884018033743,
"fcm_dpo/delta": 0.0024794889613986015,
"fcm_dpo/margin": 43.15642166137695,
"fcm_dpo/q_t": 0.4275315999984741,
"grad_norm": 32.8515625,
"learning_rate": 7.550321484960251e-08,
"logits/chosen": 1.3259495496749878,
"logits/rejected": 1.3368303775787354,
"logps/chosen": -299.775634765625,
"logps/ref_chosen": -275.91595458984375,
"logps/ref_rejected": -264.79736328125,
"logps/rejected": -331.8134765625,
"loss": 4.6791,
"margin_dpo/margin_mean": 43.15641403198242,
"margin_dpo/margin_std": 72.02017974853516,
"step": 369
},
{
"KL/chosen_KL_mean": -24.69597816467285,
"KL/mean": -51.460182189941406,
"KL/rejected_KL_mean": -78.22439575195312,
"KL/std": 63.278751373291016,
"epoch": 0.774869109947644,
"fcm_dpo/beta": 0.007086962461471558,
"fcm_dpo/delta": -0.035770583897829056,
"fcm_dpo/margin": 53.528411865234375,
"fcm_dpo/q_t": 0.41154617071151733,
"grad_norm": 23.188228607177734,
"learning_rate": 7.419687580962222e-08,
"logits/chosen": 1.1783413887023926,
"logits/rejected": 1.2379190921783447,
"logps/chosen": -269.63800048828125,
"logps/ref_chosen": -244.9420166015625,
"logps/ref_rejected": -235.61978149414062,
"logps/rejected": -313.8441467285156,
"loss": 4.4625,
"margin_dpo/margin_mean": 53.528411865234375,
"margin_dpo/margin_std": 76.80668640136719,
"step": 370
},
{
"KL/chosen_KL_mean": -28.699094772338867,
"KL/mean": -49.59718704223633,
"KL/rejected_KL_mean": -70.49527740478516,
"KL/std": 62.270164489746094,
"epoch": 0.7769633507853403,
"fcm_dpo/beta": 0.00724080391228199,
"fcm_dpo/delta": 0.04666786640882492,
"fcm_dpo/margin": 41.79617691040039,
"fcm_dpo/q_t": 0.42907005548477173,
"grad_norm": 46.07411193847656,
"learning_rate": 7.289996455765748e-08,
"logits/chosen": 1.2584155797958374,
"logits/rejected": 1.245367169380188,
"logps/chosen": -318.0489501953125,
"logps/ref_chosen": -289.3498229980469,
"logps/ref_rejected": -243.70205688476562,
"logps/rejected": -314.19732666015625,
"loss": 4.688,
"margin_dpo/margin_mean": 41.79617691040039,
"margin_dpo/margin_std": 69.91704559326172,
"step": 371
},
{
"KL/chosen_KL_mean": -24.16677474975586,
"KL/mean": -54.58926010131836,
"KL/rejected_KL_mean": -85.01174926757812,
"KL/std": 65.71511840820312,
"epoch": 0.7790575916230367,
"fcm_dpo/beta": 0.00708200316876173,
"fcm_dpo/delta": -0.033582255244255066,
"fcm_dpo/margin": 60.84496307373047,
"fcm_dpo/q_t": 0.3992455005645752,
"grad_norm": 35.841552734375,
"learning_rate": 7.161255064312283e-08,
"logits/chosen": 1.5282446146011353,
"logits/rejected": 1.519301414489746,
"logps/chosen": -345.8202819824219,
"logps/ref_chosen": -321.65350341796875,
"logps/ref_rejected": -282.73773193359375,
"logps/rejected": -367.74945068359375,
"loss": 4.284,
"margin_dpo/margin_mean": 60.84495544433594,
"margin_dpo/margin_std": 72.75962829589844,
"step": 372
},
{
"KL/chosen_KL_mean": -25.180253982543945,
"KL/mean": -52.30124282836914,
"KL/rejected_KL_mean": -79.42222595214844,
"KL/std": 66.4149398803711,
"epoch": 0.7811518324607329,
"fcm_dpo/beta": 0.0070937094278633595,
"fcm_dpo/delta": 0.015240887179970741,
"fcm_dpo/margin": 54.241973876953125,
"fcm_dpo/q_t": 0.4105536639690399,
"grad_norm": 41.76082229614258,
"learning_rate": 7.033470310611945e-08,
"logits/chosen": 1.276640772819519,
"logits/rejected": 1.2320808172225952,
"logps/chosen": -334.9307556152344,
"logps/ref_chosen": -309.75048828125,
"logps/ref_rejected": -267.41925048828125,
"logps/rejected": -346.8414611816406,
"loss": 4.4296,
"margin_dpo/margin_mean": 54.24197769165039,
"margin_dpo/margin_std": 72.85505676269531,
"step": 373
},
{
"KL/chosen_KL_mean": -26.792285919189453,
"KL/mean": -46.979881286621094,
"KL/rejected_KL_mean": -67.16746520996094,
"KL/std": 63.515899658203125,
"epoch": 0.7832460732984293,
"fcm_dpo/beta": 0.0071641867980360985,
"fcm_dpo/delta": 0.017971232533454895,
"fcm_dpo/margin": 40.37517547607422,
"fcm_dpo/q_t": 0.43322664499282837,
"grad_norm": 45.671104431152344,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": 1.430773138999939,
"logits/rejected": 1.4034602642059326,
"logps/chosen": -337.7864074707031,
"logps/ref_chosen": -310.9941101074219,
"logps/ref_rejected": -279.94915771484375,
"logps/rejected": -347.11663818359375,
"loss": 4.7503,
"margin_dpo/margin_mean": 40.37517547607422,
"margin_dpo/margin_std": 72.25169372558594,
"step": 374
},
{
"KL/chosen_KL_mean": -26.015838623046875,
"KL/mean": -50.00862503051758,
"KL/rejected_KL_mean": -74.00141906738281,
"KL/std": 61.2200927734375,
"epoch": 0.7853403141361257,
"fcm_dpo/beta": 0.007448050193488598,
"fcm_dpo/delta": 0.01717124693095684,
"fcm_dpo/margin": 47.985595703125,
"fcm_dpo/q_t": 0.4177404046058655,
"grad_norm": 30.2922306060791,
"learning_rate": 6.780798075635675e-08,
"logits/chosen": 1.535935640335083,
"logits/rejected": 1.4619958400726318,
"logps/chosen": -335.19927978515625,
"logps/ref_chosen": -309.18341064453125,
"logps/ref_rejected": -256.9963073730469,
"logps/rejected": -330.9977111816406,
"loss": 4.5554,
"margin_dpo/margin_mean": 47.985599517822266,
"margin_dpo/margin_std": 75.4139175415039,
"step": 375
},
{
"KL/chosen_KL_mean": -31.192398071289062,
"KL/mean": -53.65875244140625,
"KL/rejected_KL_mean": -76.12509155273438,
"KL/std": 64.35493469238281,
"epoch": 0.787434554973822,
"fcm_dpo/beta": 0.007259708363562822,
"fcm_dpo/delta": -0.045814137905836105,
"fcm_dpo/margin": 44.93268585205078,
"fcm_dpo/q_t": 0.4243854880332947,
"grad_norm": 31.516435623168945,
"learning_rate": 6.655924144404906e-08,
"logits/chosen": 1.553802728652954,
"logits/rejected": 1.6345727443695068,
"logps/chosen": -313.46759033203125,
"logps/ref_chosen": -282.275146484375,
"logps/ref_rejected": -255.17941284179688,
"logps/rejected": -331.30450439453125,
"loss": 4.6516,
"margin_dpo/margin_mean": 44.93268966674805,
"margin_dpo/margin_std": 73.13008117675781,
"step": 376
},
{
"KL/chosen_KL_mean": -28.07622528076172,
"KL/mean": -49.12818908691406,
"KL/rejected_KL_mean": -70.18014526367188,
"KL/std": 61.01256561279297,
"epoch": 0.7895287958115184,
"fcm_dpo/beta": 0.0072228261269629,
"fcm_dpo/delta": 0.04093307629227638,
"fcm_dpo/margin": 42.103912353515625,
"fcm_dpo/q_t": 0.4301294684410095,
"grad_norm": 46.87041473388672,
"learning_rate": 6.532033950290885e-08,
"logits/chosen": 1.4122331142425537,
"logits/rejected": 1.4553221464157104,
"logps/chosen": -320.9534606933594,
"logps/ref_chosen": -292.8772277832031,
"logps/ref_rejected": -266.78509521484375,
"logps/rejected": -336.9652099609375,
"loss": 4.7059,
"margin_dpo/margin_mean": 42.103912353515625,
"margin_dpo/margin_std": 73.83411407470703,
"step": 377
},
{
"KL/chosen_KL_mean": -28.152355194091797,
"KL/mean": -49.32393264770508,
"KL/rejected_KL_mean": -70.49549865722656,
"KL/std": 63.85035705566406,
"epoch": 0.7916230366492146,
"fcm_dpo/beta": 0.007573708426207304,
"fcm_dpo/delta": 0.04489870369434357,
"fcm_dpo/margin": 42.34314727783203,
"fcm_dpo/q_t": 0.42605486512184143,
"grad_norm": 34.113529205322266,
"learning_rate": 6.409134137148736e-08,
"logits/chosen": 1.2811505794525146,
"logits/rejected": 1.3435945510864258,
"logps/chosen": -301.0460510253906,
"logps/ref_chosen": -272.8937072753906,
"logps/ref_rejected": -293.058349609375,
"logps/rejected": -363.5538635253906,
"loss": 4.6475,
"margin_dpo/margin_mean": 42.34314727783203,
"margin_dpo/margin_std": 69.5819091796875,
"step": 378
},
{
"KL/chosen_KL_mean": -26.611143112182617,
"KL/mean": -48.94253158569336,
"KL/rejected_KL_mean": -71.27391815185547,
"KL/std": 60.81068420410156,
"epoch": 0.793717277486911,
"fcm_dpo/beta": 0.007608911953866482,
"fcm_dpo/delta": 0.010132553055882454,
"fcm_dpo/margin": 44.66277313232422,
"fcm_dpo/q_t": 0.42162272334098816,
"grad_norm": 48.30427551269531,
"learning_rate": 6.28723129572247e-08,
"logits/chosen": 1.328374981880188,
"logits/rejected": 1.3346821069717407,
"logps/chosen": -285.75274658203125,
"logps/ref_chosen": -259.1416015625,
"logps/ref_rejected": -229.64996337890625,
"logps/rejected": -300.9239196777344,
"loss": 4.6286,
"margin_dpo/margin_mean": 44.662776947021484,
"margin_dpo/margin_std": 75.78076171875,
"step": 379
},
{
"KL/chosen_KL_mean": -32.59101104736328,
"KL/mean": -55.79750442504883,
"KL/rejected_KL_mean": -79.00399780273438,
"KL/std": 64.7029037475586,
"epoch": 0.7958115183246073,
"fcm_dpo/beta": 0.007695009466260672,
"fcm_dpo/delta": -0.0028763366863131523,
"fcm_dpo/margin": 46.41297912597656,
"fcm_dpo/q_t": 0.4187002182006836,
"grad_norm": 36.6245231628418,
"learning_rate": 6.166331963291519e-08,
"logits/chosen": 1.2380932569503784,
"logits/rejected": 1.159091830253601,
"logps/chosen": -328.9289855957031,
"logps/ref_chosen": -296.33795166015625,
"logps/ref_rejected": -283.7359924316406,
"logps/rejected": -362.739990234375,
"loss": 4.5558,
"margin_dpo/margin_mean": 46.41297912597656,
"margin_dpo/margin_std": 72.37862396240234,
"step": 380
},
{
"KL/chosen_KL_mean": -31.32663345336914,
"KL/mean": -56.64055252075195,
"KL/rejected_KL_mean": -81.9544677734375,
"KL/std": 64.01226806640625,
"epoch": 0.7979057591623037,
"fcm_dpo/beta": 0.007697650697082281,
"fcm_dpo/delta": 0.010398047976195812,
"fcm_dpo/margin": 50.62784194946289,
"fcm_dpo/q_t": 0.41058602929115295,
"grad_norm": 37.08723449707031,
"learning_rate": 6.046442623320145e-08,
"logits/chosen": 1.106405258178711,
"logits/rejected": 1.1405856609344482,
"logps/chosen": -280.2998046875,
"logps/ref_chosen": -248.97320556640625,
"logps/ref_rejected": -259.5140075683594,
"logps/rejected": -341.4684753417969,
"loss": 4.4377,
"margin_dpo/margin_mean": 50.62784194946289,
"margin_dpo/margin_std": 72.49524688720703,
"step": 381
},
{
"KL/chosen_KL_mean": -19.252281188964844,
"KL/mean": -50.923675537109375,
"KL/rejected_KL_mean": -82.59505462646484,
"KL/std": 64.12224578857422,
"epoch": 0.8,
"fcm_dpo/beta": 0.007423033006489277,
"fcm_dpo/delta": -0.10267797112464905,
"fcm_dpo/margin": 63.3427734375,
"fcm_dpo/q_t": 0.3905338943004608,
"grad_norm": 35.46590805053711,
"learning_rate": 5.9275697051098275e-08,
"logits/chosen": 1.2661614418029785,
"logits/rejected": 1.4135240316390991,
"logps/chosen": -299.9245910644531,
"logps/ref_chosen": -280.67230224609375,
"logps/ref_rejected": -272.4359130859375,
"logps/rejected": -355.03094482421875,
"loss": 4.1828,
"margin_dpo/margin_mean": 63.3427734375,
"margin_dpo/margin_std": 72.86822509765625,
"step": 382
},
{
"KL/chosen_KL_mean": -27.320846557617188,
"KL/mean": -50.53596878051758,
"KL/rejected_KL_mean": -73.75108337402344,
"KL/std": 61.96195602416992,
"epoch": 0.8020942408376963,
"fcm_dpo/beta": 0.0072087873704731464,
"fcm_dpo/delta": 0.022169094532728195,
"fcm_dpo/margin": 46.43023681640625,
"fcm_dpo/q_t": 0.42245474457740784,
"grad_norm": 53.77853775024414,
"learning_rate": 5.809719583454414e-08,
"logits/chosen": 0.8659422397613525,
"logits/rejected": 0.9049512147903442,
"logps/chosen": -294.97930908203125,
"logps/ref_chosen": -267.658447265625,
"logps/ref_rejected": -262.9810791015625,
"logps/rejected": -336.732177734375,
"loss": 4.5948,
"margin_dpo/margin_mean": 46.43023681640625,
"margin_dpo/margin_std": 71.89701080322266,
"step": 383
},
{
"KL/chosen_KL_mean": -32.981590270996094,
"KL/mean": -50.9521369934082,
"KL/rejected_KL_mean": -68.92266845703125,
"KL/std": 61.502723693847656,
"epoch": 0.8041884816753927,
"fcm_dpo/beta": 0.0074386936612427235,
"fcm_dpo/delta": 0.04118882864713669,
"fcm_dpo/margin": 35.94107437133789,
"fcm_dpo/q_t": 0.43864670395851135,
"grad_norm": 63.32637405395508,
"learning_rate": 5.6928985782982524e-08,
"logits/chosen": 1.4148743152618408,
"logits/rejected": 1.5957931280136108,
"logps/chosen": -299.2281494140625,
"logps/ref_chosen": -266.2465515136719,
"logps/ref_rejected": -281.1450500488281,
"logps/rejected": -350.0677490234375,
"loss": 4.8763,
"margin_dpo/margin_mean": 35.94107437133789,
"margin_dpo/margin_std": 77.21729278564453,
"step": 384
},
{
"KL/chosen_KL_mean": -29.206470489501953,
"KL/mean": -52.04616165161133,
"KL/rejected_KL_mean": -74.8858413696289,
"KL/std": 62.6688232421875,
"epoch": 0.806282722513089,
"fcm_dpo/beta": 0.007602612022310495,
"fcm_dpo/delta": 0.018680794164538383,
"fcm_dpo/margin": 45.679378509521484,
"fcm_dpo/q_t": 0.4202457666397095,
"grad_norm": 30.920242309570312,
"learning_rate": 5.57711295439732e-08,
"logits/chosen": 1.284320592880249,
"logits/rejected": 1.2636933326721191,
"logps/chosen": -303.5123291015625,
"logps/ref_chosen": -274.30584716796875,
"logps/ref_rejected": -242.94247436523438,
"logps/rejected": -317.82830810546875,
"loss": 4.5784,
"margin_dpo/margin_mean": 45.679378509521484,
"margin_dpo/margin_std": 72.00262451171875,
"step": 385
},
{
"KL/chosen_KL_mean": -22.99639129638672,
"KL/mean": -51.93415069580078,
"KL/rejected_KL_mean": -80.87190246582031,
"KL/std": 67.61793518066406,
"epoch": 0.8083769633507853,
"fcm_dpo/beta": 0.0072924308478832245,
"fcm_dpo/delta": -0.063094861805439,
"fcm_dpo/margin": 57.87550735473633,
"fcm_dpo/q_t": 0.40138837695121765,
"grad_norm": 26.728208541870117,
"learning_rate": 5.4623689209832484e-08,
"logits/chosen": 1.487105369567871,
"logits/rejected": 1.5577136278152466,
"logps/chosen": -332.2628173828125,
"logps/ref_chosen": -309.2664489746094,
"logps/ref_rejected": -298.4182434082031,
"logps/rejected": -379.2901306152344,
"loss": 4.3501,
"margin_dpo/margin_mean": 57.87551498413086,
"margin_dpo/margin_std": 74.2111587524414,
"step": 386
},
{
"KL/chosen_KL_mean": -24.85624122619629,
"KL/mean": -51.09400939941406,
"KL/rejected_KL_mean": -77.33177947998047,
"KL/std": 64.32537078857422,
"epoch": 0.8104712041884817,
"fcm_dpo/beta": 0.007194596342742443,
"fcm_dpo/delta": -0.014300383627414703,
"fcm_dpo/margin": 52.47553253173828,
"fcm_dpo/q_t": 0.4129951596260071,
"grad_norm": 38.05221939086914,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": 1.3143092393875122,
"logits/rejected": 1.3520739078521729,
"logps/chosen": -288.87713623046875,
"logps/ref_chosen": -264.0209045410156,
"logps/ref_rejected": -264.3447570800781,
"logps/rejected": -341.676513671875,
"loss": 4.4934,
"margin_dpo/margin_mean": 52.47553253173828,
"margin_dpo/margin_std": 76.32706451416016,
"step": 387
},
{
"KL/chosen_KL_mean": -36.69129180908203,
"KL/mean": -56.626888275146484,
"KL/rejected_KL_mean": -76.56248474121094,
"KL/std": 62.020477294921875,
"epoch": 0.812565445026178,
"fcm_dpo/beta": 0.007130505982786417,
"fcm_dpo/delta": 0.0102801863104105,
"fcm_dpo/margin": 39.87118148803711,
"fcm_dpo/q_t": 0.43438929319381714,
"grad_norm": 73.09330749511719,
"learning_rate": 5.2360301829254745e-08,
"logits/chosen": 1.2919379472732544,
"logits/rejected": 1.3333224058151245,
"logps/chosen": -299.9101257324219,
"logps/ref_chosen": -263.2188415527344,
"logps/ref_rejected": -236.08546447753906,
"logps/rejected": -312.6479187011719,
"loss": 4.8043,
"margin_dpo/margin_mean": 39.87118148803711,
"margin_dpo/margin_std": 77.06546783447266,
"step": 388
},
{
"KL/chosen_KL_mean": -27.810565948486328,
"KL/mean": -54.52623748779297,
"KL/rejected_KL_mean": -81.2419204711914,
"KL/std": 61.5937614440918,
"epoch": 0.8146596858638744,
"fcm_dpo/beta": 0.007093273103237152,
"fcm_dpo/delta": -0.0129660964012146,
"fcm_dpo/margin": 53.43135452270508,
"fcm_dpo/q_t": 0.411451131105423,
"grad_norm": 34.63319778442383,
"learning_rate": 5.1244476161413806e-08,
"logits/chosen": 1.5643380880355835,
"logits/rejected": 1.5534164905548096,
"logps/chosen": -301.6084899902344,
"logps/ref_chosen": -273.79791259765625,
"logps/ref_rejected": -242.4090118408203,
"logps/rejected": -323.6509094238281,
"loss": 4.4611,
"margin_dpo/margin_mean": 53.43135070800781,
"margin_dpo/margin_std": 74.87136840820312,
"step": 389
},
{
"KL/chosen_KL_mean": -28.37144660949707,
"KL/mean": -54.154396057128906,
"KL/rejected_KL_mean": -79.9373550415039,
"KL/std": 61.98411178588867,
"epoch": 0.8167539267015707,
"fcm_dpo/beta": 0.007195638492703438,
"fcm_dpo/delta": -0.010860946960747242,
"fcm_dpo/margin": 51.56591033935547,
"fcm_dpo/q_t": 0.4138355553150177,
"grad_norm": 31.387096405029297,
"learning_rate": 5.013930914912476e-08,
"logits/chosen": 1.2626863718032837,
"logits/rejected": 1.3200889825820923,
"logps/chosen": -303.6866149902344,
"logps/ref_chosen": -275.3151550292969,
"logps/ref_rejected": -278.1514587402344,
"logps/rejected": -358.0888366699219,
"loss": 4.5442,
"margin_dpo/margin_mean": 51.56591033935547,
"margin_dpo/margin_std": 80.60596466064453,
"step": 390
},
{
"KL/chosen_KL_mean": -34.32966613769531,
"KL/mean": -58.90526580810547,
"KL/rejected_KL_mean": -83.48086547851562,
"KL/std": 65.26823425292969,
"epoch": 0.818848167539267,
"fcm_dpo/beta": 0.007021555211395025,
"fcm_dpo/delta": -0.014854553155601025,
"fcm_dpo/margin": 49.15119171142578,
"fcm_dpo/q_t": 0.4196704626083374,
"grad_norm": 25.932903289794922,
"learning_rate": 4.904486005914027e-08,
"logits/chosen": 1.3541665077209473,
"logits/rejected": 1.3399139642715454,
"logps/chosen": -309.52899169921875,
"logps/ref_chosen": -275.1993408203125,
"logps/ref_rejected": -261.32122802734375,
"logps/rejected": -344.80206298828125,
"loss": 4.5185,
"margin_dpo/margin_mean": 49.15119934082031,
"margin_dpo/margin_std": 67.3121337890625,
"step": 391
},
{
"KL/chosen_KL_mean": -27.980928421020508,
"KL/mean": -59.93144989013672,
"KL/rejected_KL_mean": -91.88199615478516,
"KL/std": 65.07289123535156,
"epoch": 0.8209424083769633,
"fcm_dpo/beta": 0.006673329975455999,
"fcm_dpo/delta": -0.07043564319610596,
"fcm_dpo/margin": 63.901065826416016,
"fcm_dpo/q_t": 0.3997809886932373,
"grad_norm": 30.282054901123047,
"learning_rate": 4.796118758344353e-08,
"logits/chosen": 1.322252869606018,
"logits/rejected": 1.3225462436676025,
"logps/chosen": -327.8086853027344,
"logps/ref_chosen": -299.8277587890625,
"logps/ref_rejected": -251.24798583984375,
"logps/rejected": -343.1300048828125,
"loss": 4.2429,
"margin_dpo/margin_mean": 63.901065826416016,
"margin_dpo/margin_std": 69.360107421875,
"step": 392
},
{
"KL/chosen_KL_mean": -27.79674530029297,
"KL/mean": -50.54197311401367,
"KL/rejected_KL_mean": -73.28719329833984,
"KL/std": 62.20020294189453,
"epoch": 0.8230366492146597,
"fcm_dpo/beta": 0.006766752805560827,
"fcm_dpo/delta": 0.036494992673397064,
"fcm_dpo/margin": 45.490455627441406,
"fcm_dpo/q_t": 0.4280647039413452,
"grad_norm": 31.35349464416504,
"learning_rate": 4.688834983610082e-08,
"logits/chosen": 1.3172001838684082,
"logits/rejected": 1.2955164909362793,
"logps/chosen": -308.2578430175781,
"logps/ref_chosen": -280.4610900878906,
"logps/ref_rejected": -228.34823608398438,
"logps/rejected": -301.63543701171875,
"loss": 4.6621,
"margin_dpo/margin_mean": 45.49045181274414,
"margin_dpo/margin_std": 74.2087631225586,
"step": 393
},
{
"KL/chosen_KL_mean": -23.677614212036133,
"KL/mean": -46.4250602722168,
"KL/rejected_KL_mean": -69.17251586914062,
"KL/std": 62.090328216552734,
"epoch": 0.8251308900523561,
"fcm_dpo/beta": 0.00686721783131361,
"fcm_dpo/delta": 0.022358842194080353,
"fcm_dpo/margin": 45.494895935058594,
"fcm_dpo/q_t": 0.4278073012828827,
"grad_norm": 32.50846862792969,
"learning_rate": 4.582640435014459e-08,
"logits/chosen": 1.4290770292282104,
"logits/rejected": 1.5399553775787354,
"logps/chosen": -343.2777099609375,
"logps/ref_chosen": -319.60009765625,
"logps/ref_rejected": -316.42822265625,
"logps/rejected": -385.6007385253906,
"loss": 4.6475,
"margin_dpo/margin_mean": 45.494895935058594,
"margin_dpo/margin_std": 71.92081451416016,
"step": 394
},
{
"KL/chosen_KL_mean": -32.48970031738281,
"KL/mean": -56.57379913330078,
"KL/rejected_KL_mean": -80.65789794921875,
"KL/std": 61.502525329589844,
"epoch": 0.8272251308900523,
"fcm_dpo/beta": 0.006931242998689413,
"fcm_dpo/delta": 0.008774535730481148,
"fcm_dpo/margin": 48.16818618774414,
"fcm_dpo/q_t": 0.42228472232818604,
"grad_norm": 29.18105697631836,
"learning_rate": 4.477540807448832e-08,
"logits/chosen": 1.1601319313049316,
"logits/rejected": 1.2798205614089966,
"logps/chosen": -292.9075622558594,
"logps/ref_chosen": -260.4178466796875,
"logps/ref_rejected": -270.04498291015625,
"logps/rejected": -350.702880859375,
"loss": 4.5643,
"margin_dpo/margin_mean": 48.16818618774414,
"margin_dpo/margin_std": 69.345458984375,
"step": 395
},
{
"KL/chosen_KL_mean": -32.005760192871094,
"KL/mean": -56.341041564941406,
"KL/rejected_KL_mean": -80.67633819580078,
"KL/std": 63.819515228271484,
"epoch": 0.8293193717277487,
"fcm_dpo/beta": 0.0071111805737018585,
"fcm_dpo/delta": 0.021238427609205246,
"fcm_dpo/margin": 48.670570373535156,
"fcm_dpo/q_t": 0.4193718731403351,
"grad_norm": 32.6487922668457,
"learning_rate": 4.373541737087263e-08,
"logits/chosen": 1.2952619791030884,
"logits/rejected": 1.279961109161377,
"logps/chosen": -312.7087707519531,
"logps/ref_chosen": -280.7030029296875,
"logps/ref_rejected": -250.76385498046875,
"logps/rejected": -331.4402160644531,
"loss": 4.5757,
"margin_dpo/margin_mean": 48.670570373535156,
"margin_dpo/margin_std": 76.09983825683594,
"step": 396
},
{
"KL/chosen_KL_mean": -27.94131088256836,
"KL/mean": -51.001068115234375,
"KL/rejected_KL_mean": -74.06082916259766,
"KL/std": 64.63763427734375,
"epoch": 0.831413612565445,
"fcm_dpo/beta": 0.007109126076102257,
"fcm_dpo/delta": -0.01439717784523964,
"fcm_dpo/margin": 46.119537353515625,
"fcm_dpo/q_t": 0.4233459234237671,
"grad_norm": 28.73206329345703,
"learning_rate": 4.270648801084295e-08,
"logits/chosen": 1.0899914503097534,
"logits/rejected": 1.1841706037521362,
"logps/chosen": -328.8277587890625,
"logps/ref_chosen": -300.8864440917969,
"logps/ref_rejected": -288.5682067871094,
"logps/rejected": -362.6290588378906,
"loss": 4.6325,
"margin_dpo/margin_mean": 46.119537353515625,
"margin_dpo/margin_std": 72.10877990722656,
"step": 397
},
{
"KL/chosen_KL_mean": -41.151214599609375,
"KL/mean": -58.75756072998047,
"KL/rejected_KL_mean": -76.3638916015625,
"KL/std": 65.13799285888672,
"epoch": 0.8335078534031414,
"fcm_dpo/beta": 0.007091197185218334,
"fcm_dpo/delta": 0.02512446977198124,
"fcm_dpo/margin": 35.21266174316406,
"fcm_dpo/q_t": 0.4427448511123657,
"grad_norm": 40.92946243286133,
"learning_rate": 4.168867517275806e-08,
"logits/chosen": 1.3057215213775635,
"logits/rejected": 1.3617304563522339,
"logps/chosen": -334.45355224609375,
"logps/ref_chosen": -293.3023681640625,
"logps/ref_rejected": -270.0457763671875,
"logps/rejected": -346.40966796875,
"loss": 4.9548,
"margin_dpo/margin_mean": 35.21266555786133,
"margin_dpo/margin_std": 82.94670867919922,
"step": 398
},
{
"KL/chosen_KL_mean": -35.81355285644531,
"KL/mean": -57.71054458618164,
"KL/rejected_KL_mean": -79.6075439453125,
"KL/std": 65.38770294189453,
"epoch": 0.8356020942408376,
"fcm_dpo/beta": 0.007297034841030836,
"fcm_dpo/delta": 0.05020921677350998,
"fcm_dpo/margin": 43.794002532958984,
"fcm_dpo/q_t": 0.4259675145149231,
"grad_norm": 44.98393630981445,
"learning_rate": 4.0682033438831584e-08,
"logits/chosen": 1.135213851928711,
"logits/rejected": 1.2031030654907227,
"logps/chosen": -319.2127990722656,
"logps/ref_chosen": -283.39923095703125,
"logps/ref_rejected": -264.5092468261719,
"logps/rejected": -344.1167907714844,
"loss": 4.6477,
"margin_dpo/margin_mean": 43.79399871826172,
"margin_dpo/margin_std": 71.06166076660156,
"step": 399
},
{
"KL/chosen_KL_mean": -30.493812561035156,
"KL/mean": -55.4969367980957,
"KL/rejected_KL_mean": -80.50005340576172,
"KL/std": 61.17449951171875,
"epoch": 0.837696335078534,
"fcm_dpo/beta": 0.007266245782375336,
"fcm_dpo/delta": -0.05055814981460571,
"fcm_dpo/margin": 50.00624084472656,
"fcm_dpo/q_t": 0.4153440296649933,
"grad_norm": 34.71428680419922,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": 1.3628064393997192,
"logits/rejected": 1.4308254718780518,
"logps/chosen": -284.9616394042969,
"logps/ref_chosen": -254.4678192138672,
"logps/ref_rejected": -252.00326538085938,
"logps/rejected": -332.5032958984375,
"loss": 4.5255,
"margin_dpo/margin_mean": 50.00624084472656,
"margin_dpo/margin_std": 72.89318084716797,
"step": 400
},
{
"epoch": 0.837696335078534,
"eval_KL/chosen_KL_mean": -33.885189056396484,
"eval_KL/mean": -57.455745697021484,
"eval_KL/rejected_KL_mean": -81.02631378173828,
"eval_KL/std": 62.96332550048828,
"eval_fcm_dpo/beta": 0.007154672406613827,
"eval_logits/chosen": 1.4236695766448975,
"eval_logits/rejected": 1.4793931245803833,
"eval_logps/chosen": -314.3018798828125,
"eval_logps/ref_chosen": -280.41668701171875,
"eval_logps/ref_rejected": -264.6455383300781,
"eval_logps/rejected": -345.6718444824219,
"eval_loss": 0.5765824317932129,
"eval_margin_dpo/margin_mean": 47.141117095947266,
"eval_margin_dpo/margin_std": 72.3220443725586,
"eval_runtime": 89.2477,
"eval_samples_per_second": 22.41,
"eval_steps_per_second": 2.801,
"step": 400
},
{
"KL/chosen_KL_mean": -27.291719436645508,
"KL/mean": -58.15824508666992,
"KL/rejected_KL_mean": -89.0247802734375,
"KL/std": 66.10443878173828,
"epoch": 0.8397905759162304,
"fcm_dpo/beta": 0.007040434516966343,
"fcm_dpo/delta": -0.036518603563308716,
"fcm_dpo/margin": 61.733055114746094,
"fcm_dpo/q_t": 0.398317813873291,
"grad_norm": 32.57052230834961,
"learning_rate": 3.8702478614051345e-08,
"logits/chosen": 1.1346800327301025,
"logits/rejected": 1.1599088907241821,
"logps/chosen": -337.89129638671875,
"logps/ref_chosen": -310.5995788574219,
"logps/ref_rejected": -292.7754821777344,
"logps/rejected": -381.8002624511719,
"loss": 4.2511,
"margin_dpo/margin_mean": 61.733055114746094,
"margin_dpo/margin_std": 72.1438980102539,
"step": 401
},
{
"KL/chosen_KL_mean": -29.142030715942383,
"KL/mean": -53.469818115234375,
"KL/rejected_KL_mean": -77.79762268066406,
"KL/std": 64.34076690673828,
"epoch": 0.8418848167539267,
"fcm_dpo/beta": 0.006985440384596586,
"fcm_dpo/delta": 0.031272128224372864,
"fcm_dpo/margin": 48.65558624267578,
"fcm_dpo/q_t": 0.4210313558578491,
"grad_norm": 36.524269104003906,
"learning_rate": 3.772967168071517e-08,
"logits/chosen": 1.4216690063476562,
"logits/rejected": 1.3480905294418335,
"logps/chosen": -328.993408203125,
"logps/ref_chosen": -299.8513488769531,
"logps/ref_rejected": -278.5664367675781,
"logps/rejected": -356.36407470703125,
"loss": 4.5789,
"margin_dpo/margin_mean": 48.65558624267578,
"margin_dpo/margin_std": 74.5606460571289,
"step": 402
},
{
"KL/chosen_KL_mean": -24.964340209960938,
"KL/mean": -58.60805892944336,
"KL/rejected_KL_mean": -92.25179290771484,
"KL/std": 66.777099609375,
"epoch": 0.8439790575916231,
"fcm_dpo/beta": 0.006979155354201794,
"fcm_dpo/delta": -0.07276086509227753,
"fcm_dpo/margin": 67.28743743896484,
"fcm_dpo/q_t": 0.38980188965797424,
"grad_norm": 27.25344467163086,
"learning_rate": 3.676824816087978e-08,
"logits/chosen": 1.4216952323913574,
"logits/rejected": 1.4643619060516357,
"logps/chosen": -327.39593505859375,
"logps/ref_chosen": -302.4316101074219,
"logps/ref_rejected": -270.951171875,
"logps/rejected": -363.20294189453125,
"loss": 4.1007,
"margin_dpo/margin_mean": 67.28743743896484,
"margin_dpo/margin_std": 65.54295349121094,
"step": 403
},
{
"KL/chosen_KL_mean": -37.125144958496094,
"KL/mean": -60.501609802246094,
"KL/rejected_KL_mean": -83.87809753417969,
"KL/std": 66.83379364013672,
"epoch": 0.8460732984293193,
"fcm_dpo/beta": 0.006721503101289272,
"fcm_dpo/delta": 0.018977034837007523,
"fcm_dpo/margin": 46.75294494628906,
"fcm_dpo/q_t": 0.42682117223739624,
"grad_norm": 30.03093147277832,
"learning_rate": 3.581825961277074e-08,
"logits/chosen": 1.3763082027435303,
"logits/rejected": 1.4138517379760742,
"logps/chosen": -327.02825927734375,
"logps/ref_chosen": -289.9031066894531,
"logps/ref_rejected": -252.84918212890625,
"logps/rejected": -336.7272644042969,
"loss": 4.6706,
"margin_dpo/margin_mean": 46.7529411315918,
"margin_dpo/margin_std": 77.15641021728516,
"step": 404
},
{
"KL/chosen_KL_mean": -28.577688217163086,
"KL/mean": -55.34074020385742,
"KL/rejected_KL_mean": -82.10379028320312,
"KL/std": 65.560302734375,
"epoch": 0.8481675392670157,
"fcm_dpo/beta": 0.006971847265958786,
"fcm_dpo/delta": 0.026871800422668457,
"fcm_dpo/margin": 53.52610397338867,
"fcm_dpo/q_t": 0.41349077224731445,
"grad_norm": 28.93546485900879,
"learning_rate": 3.487975698139084e-08,
"logits/chosen": 1.4755507707595825,
"logits/rejected": 1.6662366390228271,
"logps/chosen": -276.7517395019531,
"logps/ref_chosen": -248.17404174804688,
"logps/ref_rejected": -252.23764038085938,
"logps/rejected": -334.3414306640625,
"loss": 4.4438,
"margin_dpo/margin_mean": 53.52610397338867,
"margin_dpo/margin_std": 71.0295181274414,
"step": 405
},
{
"KL/chosen_KL_mean": -35.79462432861328,
"KL/mean": -57.796234130859375,
"KL/rejected_KL_mean": -79.79784393310547,
"KL/std": 65.24603271484375,
"epoch": 0.8502617801047121,
"fcm_dpo/beta": 0.006913081742823124,
"fcm_dpo/delta": -0.011782095767557621,
"fcm_dpo/margin": 44.00320816040039,
"fcm_dpo/q_t": 0.4291381239891052,
"grad_norm": 35.615928649902344,
"learning_rate": 3.3952790595787986e-08,
"logits/chosen": 1.3034807443618774,
"logits/rejected": 1.3731592893600464,
"logps/chosen": -312.21075439453125,
"logps/ref_chosen": -276.4161376953125,
"logps/ref_rejected": -264.0082702636719,
"logps/rejected": -343.80609130859375,
"loss": 4.7119,
"margin_dpo/margin_mean": 44.00320816040039,
"margin_dpo/margin_std": 75.86014556884766,
"step": 406
},
{
"KL/chosen_KL_mean": -42.82422637939453,
"KL/mean": -67.25929260253906,
"KL/rejected_KL_mean": -91.69435119628906,
"KL/std": 67.51566314697266,
"epoch": 0.8523560209424084,
"fcm_dpo/beta": 0.007086477242410183,
"fcm_dpo/delta": 0.05532386898994446,
"fcm_dpo/margin": 48.87013244628906,
"fcm_dpo/q_t": 0.42036789655685425,
"grad_norm": 31.873289108276367,
"learning_rate": 3.303741016635614e-08,
"logits/chosen": 1.3776185512542725,
"logits/rejected": 1.3903013467788696,
"logps/chosen": -300.2615966796875,
"logps/ref_chosen": -257.4373474121094,
"logps/ref_rejected": -219.66390991210938,
"logps/rejected": -311.3582458496094,
"loss": 4.5654,
"margin_dpo/margin_mean": 48.87013244628906,
"margin_dpo/margin_std": 76.1953125,
"step": 407
},
{
"KL/chosen_KL_mean": -31.4367733001709,
"KL/mean": -56.73390579223633,
"KL/rejected_KL_mean": -82.03103637695312,
"KL/std": 63.85590362548828,
"epoch": 0.8544502617801047,
"fcm_dpo/beta": 0.007128621451556683,
"fcm_dpo/delta": -0.04133890941739082,
"fcm_dpo/margin": 50.59425354003906,
"fcm_dpo/q_t": 0.4156496524810791,
"grad_norm": 29.58357048034668,
"learning_rate": 3.2133664782169944e-08,
"logits/chosen": 1.2520235776901245,
"logits/rejected": 1.363689661026001,
"logps/chosen": -320.0901794433594,
"logps/ref_chosen": -288.6533508300781,
"logps/ref_rejected": -291.54107666015625,
"logps/rejected": -373.5721130371094,
"loss": 4.5092,
"margin_dpo/margin_mean": 50.59425735473633,
"margin_dpo/margin_std": 70.98474884033203,
"step": 408
},
{
"KL/chosen_KL_mean": -34.46295166015625,
"KL/mean": -61.470436096191406,
"KL/rejected_KL_mean": -88.47792053222656,
"KL/std": 66.3189468383789,
"epoch": 0.856544502617801,
"fcm_dpo/beta": 0.006866151466965675,
"fcm_dpo/delta": -0.02705734595656395,
"fcm_dpo/margin": 54.014949798583984,
"fcm_dpo/q_t": 0.41403132677078247,
"grad_norm": 29.867446899414062,
"learning_rate": 3.12416029083514e-08,
"logits/chosen": 1.4281820058822632,
"logits/rejected": 1.5338895320892334,
"logps/chosen": -312.2003479003906,
"logps/ref_chosen": -277.7373962402344,
"logps/ref_rejected": -272.988525390625,
"logps/rejected": -361.4664306640625,
"loss": 4.5003,
"margin_dpo/margin_mean": 54.014949798583984,
"margin_dpo/margin_std": 79.91426086425781,
"step": 409
},
{
"KL/chosen_KL_mean": -33.24790573120117,
"KL/mean": -56.598358154296875,
"KL/rejected_KL_mean": -79.9488296508789,
"KL/std": 65.97520446777344,
"epoch": 0.8586387434554974,
"fcm_dpo/beta": 0.007001395337283611,
"fcm_dpo/delta": 0.03865911066532135,
"fcm_dpo/margin": 46.70093536376953,
"fcm_dpo/q_t": 0.42443960905075073,
"grad_norm": 45.927207946777344,
"learning_rate": 3.036127238347164e-08,
"logits/chosen": 1.6032644510269165,
"logits/rejected": 1.5830140113830566,
"logps/chosen": -307.468994140625,
"logps/ref_chosen": -274.2210998535156,
"logps/ref_rejected": -264.33465576171875,
"logps/rejected": -344.28350830078125,
"loss": 4.6541,
"margin_dpo/margin_mean": 46.700931549072266,
"margin_dpo/margin_std": 79.43925476074219,
"step": 410
},
{
"KL/chosen_KL_mean": -28.699905395507812,
"KL/mean": -55.76493835449219,
"KL/rejected_KL_mean": -82.82997131347656,
"KL/std": 66.32817077636719,
"epoch": 0.8607329842931937,
"fcm_dpo/beta": 0.006910680327564478,
"fcm_dpo/delta": -0.04498377442359924,
"fcm_dpo/margin": 54.130069732666016,
"fcm_dpo/q_t": 0.41248053312301636,
"grad_norm": 34.258750915527344,
"learning_rate": 2.9492720416985e-08,
"logits/chosen": 1.4129570722579956,
"logits/rejected": 1.47940194606781,
"logps/chosen": -301.5679016113281,
"logps/ref_chosen": -272.86798095703125,
"logps/ref_rejected": -250.35772705078125,
"logps/rejected": -333.18768310546875,
"loss": 4.4382,
"margin_dpo/margin_mean": 54.130069732666016,
"margin_dpo/margin_std": 68.8840103149414,
"step": 411
},
{
"KL/chosen_KL_mean": -40.81800842285156,
"KL/mean": -63.90482711791992,
"KL/rejected_KL_mean": -86.99164581298828,
"KL/std": 66.9229507446289,
"epoch": 0.86282722513089,
"fcm_dpo/beta": 0.006773846223950386,
"fcm_dpo/delta": 0.02097306028008461,
"fcm_dpo/margin": 46.17364501953125,
"fcm_dpo/q_t": 0.4274871349334717,
"grad_norm": 28.87379264831543,
"learning_rate": 2.863599358669755e-08,
"logits/chosen": 1.1528704166412354,
"logits/rejected": 1.2533339262008667,
"logps/chosen": -308.7372131347656,
"logps/ref_chosen": -267.919189453125,
"logps/ref_rejected": -271.73162841796875,
"logps/rejected": -358.7232360839844,
"loss": 4.6837,
"margin_dpo/margin_mean": 46.17365264892578,
"margin_dpo/margin_std": 78.83636474609375,
"step": 412
},
{
"KL/chosen_KL_mean": -29.904508590698242,
"KL/mean": -51.50232696533203,
"KL/rejected_KL_mean": -73.10014343261719,
"KL/std": 62.79712677001953,
"epoch": 0.8649214659685864,
"fcm_dpo/beta": 0.007038842886686325,
"fcm_dpo/delta": 0.048431336879730225,
"fcm_dpo/margin": 43.19563674926758,
"fcm_dpo/q_t": 0.4297616183757782,
"grad_norm": 32.770263671875,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": 1.2540414333343506,
"logits/rejected": 1.3082278966903687,
"logps/chosen": -294.8625793457031,
"logps/ref_chosen": -264.95806884765625,
"logps/ref_rejected": -267.40093994140625,
"logps/rejected": -340.5010986328125,
"loss": 4.6649,
"margin_dpo/margin_mean": 43.19563674926758,
"margin_dpo/margin_std": 71.008056640625,
"step": 413
},
{
"KL/chosen_KL_mean": -38.727088928222656,
"KL/mean": -63.03623962402344,
"KL/rejected_KL_mean": -87.34538269042969,
"KL/std": 66.72122192382812,
"epoch": 0.8670157068062827,
"fcm_dpo/beta": 0.00721154548227787,
"fcm_dpo/delta": -0.01244377251714468,
"fcm_dpo/margin": 48.6182861328125,
"fcm_dpo/q_t": 0.4196760654449463,
"grad_norm": 36.20530700683594,
"learning_rate": 2.6958198472749717e-08,
"logits/chosen": 1.3935970067977905,
"logits/rejected": 1.4582998752593994,
"logps/chosen": -328.73284912109375,
"logps/ref_chosen": -290.00579833984375,
"logps/ref_rejected": -266.8810119628906,
"logps/rejected": -354.22637939453125,
"loss": 4.6178,
"margin_dpo/margin_mean": 48.6182861328125,
"margin_dpo/margin_std": 81.58920288085938,
"step": 414
},
{
"KL/chosen_KL_mean": -27.320758819580078,
"KL/mean": -57.90191650390625,
"KL/rejected_KL_mean": -88.48309326171875,
"KL/std": 65.74607849121094,
"epoch": 0.8691099476439791,
"fcm_dpo/beta": 0.007084012031555176,
"fcm_dpo/delta": -0.03487085923552513,
"fcm_dpo/margin": 61.16233444213867,
"fcm_dpo/q_t": 0.3997090756893158,
"grad_norm": 34.59502029418945,
"learning_rate": 2.613722016414943e-08,
"logits/chosen": 1.477209210395813,
"logits/rejected": 1.5739178657531738,
"logps/chosen": -317.13177490234375,
"logps/ref_chosen": -289.81103515625,
"logps/ref_rejected": -278.2758483886719,
"logps/rejected": -366.7590026855469,
"loss": 4.2901,
"margin_dpo/margin_mean": 61.162330627441406,
"margin_dpo/margin_std": 76.23521423339844,
"step": 415
},
{
"KL/chosen_KL_mean": -27.77629280090332,
"KL/mean": -56.491310119628906,
"KL/rejected_KL_mean": -85.2063217163086,
"KL/std": 65.02974700927734,
"epoch": 0.8712041884816754,
"fcm_dpo/beta": 0.006733311805874109,
"fcm_dpo/delta": -0.02400980517268181,
"fcm_dpo/margin": 57.43003463745117,
"fcm_dpo/q_t": 0.40963587164878845,
"grad_norm": 38.45534896850586,
"learning_rate": 2.5328246937043525e-08,
"logits/chosen": 1.4707696437835693,
"logits/rejected": 1.4490132331848145,
"logps/chosen": -332.42510986328125,
"logps/ref_chosen": -304.6488037109375,
"logps/ref_rejected": -264.2703857421875,
"logps/rejected": -349.4767150878906,
"loss": 4.4291,
"margin_dpo/margin_mean": 57.430030822753906,
"margin_dpo/margin_std": 77.3693618774414,
"step": 416
},
{
"KL/chosen_KL_mean": -32.69208526611328,
"KL/mean": -57.48160934448242,
"KL/rejected_KL_mean": -82.27113342285156,
"KL/std": 64.19945526123047,
"epoch": 0.8732984293193717,
"fcm_dpo/beta": 0.006742675788700581,
"fcm_dpo/delta": -0.0234349574893713,
"fcm_dpo/margin": 49.579044342041016,
"fcm_dpo/q_t": 0.42191338539123535,
"grad_norm": 58.41111373901367,
"learning_rate": 2.4531322174210973e-08,
"logits/chosen": 1.5242427587509155,
"logits/rejected": 1.6251673698425293,
"logps/chosen": -334.58099365234375,
"logps/ref_chosen": -301.8889465332031,
"logps/ref_rejected": -275.0347595214844,
"logps/rejected": -357.305908203125,
"loss": 4.5991,
"margin_dpo/margin_mean": 49.579044342041016,
"margin_dpo/margin_std": 75.97627258300781,
"step": 417
},
{
"KL/chosen_KL_mean": -42.28622055053711,
"KL/mean": -63.07315444946289,
"KL/rejected_KL_mean": -83.86009216308594,
"KL/std": 65.60322570800781,
"epoch": 0.875392670157068,
"fcm_dpo/beta": 0.006690526846796274,
"fcm_dpo/delta": -0.006242621690034866,
"fcm_dpo/margin": 41.57387924194336,
"fcm_dpo/q_t": 0.4347190856933594,
"grad_norm": 55.8277587890625,
"learning_rate": 2.3746488612308295e-08,
"logits/chosen": 1.2606562376022339,
"logits/rejected": 1.276036262512207,
"logps/chosen": -311.5335693359375,
"logps/ref_chosen": -269.2473449707031,
"logps/ref_rejected": -275.5192565917969,
"logps/rejected": -359.37933349609375,
"loss": 4.7622,
"margin_dpo/margin_mean": 41.57387924194336,
"margin_dpo/margin_std": 70.28833770751953,
"step": 418
},
{
"KL/chosen_KL_mean": -39.38565444946289,
"KL/mean": -68.76937866210938,
"KL/rejected_KL_mean": -98.15310668945312,
"KL/std": 70.06245422363281,
"epoch": 0.8774869109947644,
"fcm_dpo/beta": 0.006503076758235693,
"fcm_dpo/delta": -0.03890489414334297,
"fcm_dpo/margin": 58.76746368408203,
"fcm_dpo/q_t": 0.4106443524360657,
"grad_norm": 28.051912307739258,
"learning_rate": 2.297378833957761e-08,
"logits/chosen": 1.5848643779754639,
"logits/rejected": 1.584818959236145,
"logps/chosen": -330.54608154296875,
"logps/ref_chosen": -291.1604309082031,
"logps/ref_rejected": -245.18325805664062,
"logps/rejected": -343.3363342285156,
"loss": 4.4578,
"margin_dpo/margin_mean": 58.7674674987793,
"margin_dpo/margin_std": 82.36515808105469,
"step": 419
},
{
"KL/chosen_KL_mean": -35.29539489746094,
"KL/mean": -63.34731674194336,
"KL/rejected_KL_mean": -91.39923858642578,
"KL/std": 68.02963256835938,
"epoch": 0.8795811518324608,
"fcm_dpo/beta": 0.006553223356604576,
"fcm_dpo/delta": 0.03296438604593277,
"fcm_dpo/margin": 56.10383224487305,
"fcm_dpo/q_t": 0.41389018297195435,
"grad_norm": 35.01494216918945,
"learning_rate": 2.2213262793589482e-08,
"logits/chosen": 1.1775184869766235,
"logits/rejected": 1.1260844469070435,
"logps/chosen": -293.7652282714844,
"logps/ref_chosen": -258.4698181152344,
"logps/ref_rejected": -242.916015625,
"logps/rejected": -334.31524658203125,
"loss": 4.4722,
"margin_dpo/margin_mean": 56.10383224487305,
"margin_dpo/margin_std": 78.17017364501953,
"step": 420
},
{
"KL/chosen_KL_mean": -25.225984573364258,
"KL/mean": -54.59425354003906,
"KL/rejected_KL_mean": -83.9625244140625,
"KL/std": 63.646209716796875,
"epoch": 0.881675392670157,
"fcm_dpo/beta": 0.0065411971881985664,
"fcm_dpo/delta": -0.014971577562391758,
"fcm_dpo/margin": 58.73655700683594,
"fcm_dpo/q_t": 0.4093979597091675,
"grad_norm": 28.420188903808594,
"learning_rate": 2.1464952759020856e-08,
"logits/chosen": 1.2410991191864014,
"logits/rejected": 1.2269824743270874,
"logps/chosen": -315.36676025390625,
"logps/ref_chosen": -290.1407775878906,
"logps/ref_rejected": -239.65676879882812,
"logps/rejected": -323.6192626953125,
"loss": 4.3795,
"margin_dpo/margin_mean": 58.73655319213867,
"margin_dpo/margin_std": 71.42195892333984,
"step": 421
},
{
"KL/chosen_KL_mean": -37.09423828125,
"KL/mean": -58.779449462890625,
"KL/rejected_KL_mean": -80.46466827392578,
"KL/std": 65.69966888427734,
"epoch": 0.8837696335078534,
"fcm_dpo/beta": 0.00664010364562273,
"fcm_dpo/delta": 0.034144721925258636,
"fcm_dpo/margin": 43.37041473388672,
"fcm_dpo/q_t": 0.4327212870121002,
"grad_norm": 29.3281192779541,
"learning_rate": 2.07288983654679e-08,
"logits/chosen": 1.3102211952209473,
"logits/rejected": 1.3857343196868896,
"logps/chosen": -314.0302734375,
"logps/ref_chosen": -276.93603515625,
"logps/ref_rejected": -250.398681640625,
"logps/rejected": -330.86334228515625,
"loss": 4.774,
"margin_dpo/margin_mean": 43.37041473388672,
"margin_dpo/margin_std": 80.89669799804688,
"step": 422
},
{
"KL/chosen_KL_mean": -30.344362258911133,
"KL/mean": -60.24177169799805,
"KL/rejected_KL_mean": -90.13917541503906,
"KL/std": 67.41627502441406,
"epoch": 0.8858638743455497,
"fcm_dpo/beta": 0.0066752671264112,
"fcm_dpo/delta": 0.0005025333957746625,
"fcm_dpo/margin": 59.79481887817383,
"fcm_dpo/q_t": 0.40733617544174194,
"grad_norm": 56.790443420410156,
"learning_rate": 2.0005139085293942e-08,
"logits/chosen": 1.2415132522583008,
"logits/rejected": 1.3153170347213745,
"logps/chosen": -315.7744445800781,
"logps/ref_chosen": -285.43011474609375,
"logps/ref_rejected": -258.0743408203125,
"logps/rejected": -348.2135009765625,
"loss": 4.399,
"margin_dpo/margin_mean": 59.79481887817383,
"margin_dpo/margin_std": 80.7657699584961,
"step": 423
},
{
"KL/chosen_KL_mean": -25.805326461791992,
"KL/mean": -54.49253845214844,
"KL/rejected_KL_mean": -83.17976379394531,
"KL/std": 63.28171920776367,
"epoch": 0.8879581151832461,
"fcm_dpo/beta": 0.006660280749201775,
"fcm_dpo/delta": -0.011748640798032284,
"fcm_dpo/margin": 57.37444305419922,
"fcm_dpo/q_t": 0.4099368453025818,
"grad_norm": 46.51303482055664,
"learning_rate": 1.9293713731512673e-08,
"logits/chosen": 1.2232285737991333,
"logits/rejected": 1.1983013153076172,
"logps/chosen": -325.1503601074219,
"logps/ref_chosen": -299.3450012207031,
"logps/ref_rejected": -244.60169982910156,
"logps/rejected": -327.78143310546875,
"loss": 4.3857,
"margin_dpo/margin_mean": 57.374446868896484,
"margin_dpo/margin_std": 69.36468505859375,
"step": 424
},
{
"KL/chosen_KL_mean": -32.66620635986328,
"KL/mean": -57.438941955566406,
"KL/rejected_KL_mean": -82.21167755126953,
"KL/std": 67.45172882080078,
"epoch": 0.8900523560209425,
"fcm_dpo/beta": 0.006631086580455303,
"fcm_dpo/delta": -0.012979666702449322,
"fcm_dpo/margin": 49.54546356201172,
"fcm_dpo/q_t": 0.42322254180908203,
"grad_norm": 40.446041107177734,
"learning_rate": 1.8594660455706763e-08,
"logits/chosen": 1.4363305568695068,
"logits/rejected": 1.5310245752334595,
"logps/chosen": -305.615966796875,
"logps/ref_chosen": -272.9497375488281,
"logps/ref_rejected": -274.6335754394531,
"logps/rejected": -356.84527587890625,
"loss": 4.6004,
"margin_dpo/margin_mean": 49.54546356201172,
"margin_dpo/margin_std": 73.35726165771484,
"step": 425
},
{
"KL/chosen_KL_mean": -28.64447784423828,
"KL/mean": -55.604576110839844,
"KL/rejected_KL_mean": -82.56466674804688,
"KL/std": 65.25291442871094,
"epoch": 0.8921465968586387,
"fcm_dpo/beta": 0.006651397328823805,
"fcm_dpo/delta": 0.012115703895688057,
"fcm_dpo/margin": 53.92020034790039,
"fcm_dpo/q_t": 0.41588619351387024,
"grad_norm": 22.520782470703125,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": 1.3158924579620361,
"logits/rejected": 1.3581820726394653,
"logps/chosen": -310.5860595703125,
"logps/ref_chosen": -281.9415588378906,
"logps/ref_rejected": -253.20486450195312,
"logps/rejected": -335.76953125,
"loss": 4.4642,
"margin_dpo/margin_mean": 53.92020034790039,
"margin_dpo/margin_std": 70.84208679199219,
"step": 426
},
{
"KL/chosen_KL_mean": -26.13115692138672,
"KL/mean": -61.82663345336914,
"KL/rejected_KL_mean": -97.52210998535156,
"KL/std": 67.89302062988281,
"epoch": 0.8942408376963351,
"fcm_dpo/beta": 0.006481249816715717,
"fcm_dpo/delta": -0.09418084472417831,
"fcm_dpo/margin": 71.39095306396484,
"fcm_dpo/q_t": 0.39159178733825684,
"grad_norm": 26.449934005737305,
"learning_rate": 1.7233819424956247e-08,
"logits/chosen": 1.446837067604065,
"logits/rejected": 1.4505691528320312,
"logps/chosen": -306.74774169921875,
"logps/ref_chosen": -280.6166076660156,
"logps/ref_rejected": -263.7532958984375,
"logps/rejected": -361.275390625,
"loss": 4.1412,
"margin_dpo/margin_mean": 71.39096069335938,
"margin_dpo/margin_std": 72.03829956054688,
"step": 427
},
{
"KL/chosen_KL_mean": -33.17949676513672,
"KL/mean": -66.57071685791016,
"KL/rejected_KL_mean": -99.96192169189453,
"KL/std": 68.0640869140625,
"epoch": 0.8963350785340314,
"fcm_dpo/beta": 0.006181970238685608,
"fcm_dpo/delta": -0.013892674818634987,
"fcm_dpo/margin": 66.78241729736328,
"fcm_dpo/q_t": 0.4024965763092041,
"grad_norm": 58.5177001953125,
"learning_rate": 1.6572104647786245e-08,
"logits/chosen": 1.4094966650009155,
"logits/rejected": 1.5273640155792236,
"logps/chosen": -319.34173583984375,
"logps/ref_chosen": -286.1622314453125,
"logps/ref_rejected": -273.3803405761719,
"logps/rejected": -373.34222412109375,
"loss": 4.2663,
"margin_dpo/margin_mean": 66.78242492675781,
"margin_dpo/margin_std": 70.78731536865234,
"step": 428
},
{
"KL/chosen_KL_mean": -33.086944580078125,
"KL/mean": -59.44505310058594,
"KL/rejected_KL_mean": -85.80316162109375,
"KL/std": 65.49327850341797,
"epoch": 0.8984293193717278,
"fcm_dpo/beta": 0.006138901691883802,
"fcm_dpo/delta": 0.019139215350151062,
"fcm_dpo/margin": 52.71621322631836,
"fcm_dpo/q_t": 0.42456236481666565,
"grad_norm": 30.656179428100586,
"learning_rate": 1.5922907900227017e-08,
"logits/chosen": 1.4257725477218628,
"logits/rejected": 1.4433765411376953,
"logps/chosen": -299.42352294921875,
"logps/ref_chosen": -266.3365783691406,
"logps/ref_rejected": -255.01956176757812,
"logps/rejected": -340.82275390625,
"loss": 4.6478,
"margin_dpo/margin_mean": 52.71621322631836,
"margin_dpo/margin_std": 85.40750122070312,
"step": 429
},
{
"KL/chosen_KL_mean": -33.892616271972656,
"KL/mean": -56.6580696105957,
"KL/rejected_KL_mean": -79.42350769042969,
"KL/std": 64.94258880615234,
"epoch": 0.900523560209424,
"fcm_dpo/beta": 0.006354113575071096,
"fcm_dpo/delta": 0.03459186851978302,
"fcm_dpo/margin": 45.53089141845703,
"fcm_dpo/q_t": 0.43239158391952515,
"grad_norm": 65.36149597167969,
"learning_rate": 1.5286263996730026e-08,
"logits/chosen": 1.452754020690918,
"logits/rejected": 1.4913965463638306,
"logps/chosen": -316.0439758300781,
"logps/ref_chosen": -282.1513977050781,
"logps/ref_rejected": -263.4792175292969,
"logps/rejected": -342.9027099609375,
"loss": 4.7431,
"margin_dpo/margin_mean": 45.53089141845703,
"margin_dpo/margin_std": 78.78829956054688,
"step": 430
},
{
"KL/chosen_KL_mean": -36.276119232177734,
"KL/mean": -58.02353286743164,
"KL/rejected_KL_mean": -79.77095031738281,
"KL/std": 65.9626693725586,
"epoch": 0.9026178010471204,
"fcm_dpo/beta": 0.00650543300434947,
"fcm_dpo/delta": 0.033233776688575745,
"fcm_dpo/margin": 43.49483108520508,
"fcm_dpo/q_t": 0.43427538871765137,
"grad_norm": 36.24375534057617,
"learning_rate": 1.4662207078575684e-08,
"logits/chosen": 1.3554952144622803,
"logits/rejected": 1.517827033996582,
"logps/chosen": -304.9892578125,
"logps/ref_chosen": -268.713134765625,
"logps/ref_rejected": -266.13446044921875,
"logps/rejected": -345.90545654296875,
"loss": 4.7386,
"margin_dpo/margin_mean": 43.49483108520508,
"margin_dpo/margin_std": 75.53327941894531,
"step": 431
},
{
"KL/chosen_KL_mean": -26.729814529418945,
"KL/mean": -55.14447021484375,
"KL/rejected_KL_mean": -83.55911254882812,
"KL/std": 66.13988494873047,
"epoch": 0.9047120418848168,
"fcm_dpo/beta": 0.006465256679803133,
"fcm_dpo/delta": 0.0010647280141711235,
"fcm_dpo/margin": 56.82931137084961,
"fcm_dpo/q_t": 0.41424909234046936,
"grad_norm": 35.50051498413086,
"learning_rate": 1.40507706120426e-08,
"logits/chosen": 1.6082763671875,
"logits/rejected": 1.6648359298706055,
"logps/chosen": -310.7140197753906,
"logps/ref_chosen": -283.9842224121094,
"logps/ref_rejected": -253.9568634033203,
"logps/rejected": -337.5159912109375,
"loss": 4.4608,
"margin_dpo/margin_mean": 56.82931137084961,
"margin_dpo/margin_std": 77.23971557617188,
"step": 432
},
{
"KL/chosen_KL_mean": -42.06611633300781,
"KL/mean": -64.81314086914062,
"KL/rejected_KL_mean": -87.56016540527344,
"KL/std": 65.60404205322266,
"epoch": 0.9068062827225131,
"fcm_dpo/beta": 0.006759243551641703,
"fcm_dpo/delta": 0.09632132947444916,
"fcm_dpo/margin": 45.49404525756836,
"fcm_dpo/q_t": 0.4279315173625946,
"grad_norm": 28.762327194213867,
"learning_rate": 1.345198738661285e-08,
"logits/chosen": 1.344286322593689,
"logits/rejected": 1.3697078227996826,
"logps/chosen": -284.5093078613281,
"logps/ref_chosen": -242.4431915283203,
"logps/ref_rejected": -252.87979125976562,
"logps/rejected": -340.4399719238281,
"loss": 4.6848,
"margin_dpo/margin_mean": 45.49404525756836,
"margin_dpo/margin_std": 76.451171875,
"step": 433
},
{
"KL/chosen_KL_mean": -35.92133712768555,
"KL/mean": -62.846778869628906,
"KL/rejected_KL_mean": -89.77222442626953,
"KL/std": 66.37069702148438,
"epoch": 0.9089005235602095,
"fcm_dpo/beta": 0.007066050078719854,
"fcm_dpo/delta": -0.013744387775659561,
"fcm_dpo/margin": 53.85087966918945,
"fcm_dpo/q_t": 0.4120463728904724,
"grad_norm": 35.42103958129883,
"learning_rate": 1.2865889513213628e-08,
"logits/chosen": 1.4268447160720825,
"logits/rejected": 1.3628696203231812,
"logps/chosen": -324.2055358886719,
"logps/ref_chosen": -288.2841796875,
"logps/ref_rejected": -252.9123992919922,
"logps/rejected": -342.6846008300781,
"loss": 4.4621,
"margin_dpo/margin_mean": 53.85087585449219,
"margin_dpo/margin_std": 76.14476776123047,
"step": 434
},
{
"KL/chosen_KL_mean": -29.39164924621582,
"KL/mean": -53.528568267822266,
"KL/rejected_KL_mean": -77.66548156738281,
"KL/std": 65.62640380859375,
"epoch": 0.9109947643979057,
"fcm_dpo/beta": 0.007086035795509815,
"fcm_dpo/delta": 0.030129510909318924,
"fcm_dpo/margin": 48.273834228515625,
"fcm_dpo/q_t": 0.4203599691390991,
"grad_norm": 55.130271911621094,
"learning_rate": 1.2292508422495157e-08,
"logits/chosen": 1.293613314628601,
"logits/rejected": 1.3832406997680664,
"logps/chosen": -278.406494140625,
"logps/ref_chosen": -249.01483154296875,
"logps/ref_rejected": -243.81192016601562,
"logps/rejected": -321.4773864746094,
"loss": 4.5701,
"margin_dpo/margin_mean": 48.273834228515625,
"margin_dpo/margin_std": 73.19772338867188,
"step": 435
},
{
"KL/chosen_KL_mean": -35.94551467895508,
"KL/mean": -52.956241607666016,
"KL/rejected_KL_mean": -69.96697998046875,
"KL/std": 66.97096252441406,
"epoch": 0.9130890052356021,
"fcm_dpo/beta": 0.007382245734333992,
"fcm_dpo/delta": 0.0650118738412857,
"fcm_dpo/margin": 34.02146911621094,
"fcm_dpo/q_t": 0.442565381526947,
"grad_norm": 40.14704895019531,
"learning_rate": 1.1731874863145142e-08,
"logits/chosen": 1.345801591873169,
"logits/rejected": 1.448590636253357,
"logps/chosen": -348.8756103515625,
"logps/ref_chosen": -312.9300537109375,
"logps/ref_rejected": -296.2721862792969,
"logps/rejected": -366.2391357421875,
"loss": 4.9019,
"margin_dpo/margin_mean": 34.02146530151367,
"margin_dpo/margin_std": 74.24811553955078,
"step": 436
},
{
"KL/chosen_KL_mean": -28.4786434173584,
"KL/mean": -57.46525955200195,
"KL/rejected_KL_mean": -86.45186614990234,
"KL/std": 68.7561264038086,
"epoch": 0.9151832460732985,
"fcm_dpo/beta": 0.007375634275376797,
"fcm_dpo/delta": -0.06073632091283798,
"fcm_dpo/margin": 57.973209381103516,
"fcm_dpo/q_t": 0.4013356864452362,
"grad_norm": 23.203168869018555,
"learning_rate": 1.118401890024001e-08,
"logits/chosen": 1.555214762687683,
"logits/rejected": 1.6089116334915161,
"logps/chosen": -296.7925720214844,
"logps/ref_chosen": -268.31390380859375,
"logps/ref_rejected": -268.7603759765625,
"logps/rejected": -355.2122497558594,
"loss": 4.3614,
"margin_dpo/margin_mean": 57.973209381103516,
"margin_dpo/margin_std": 78.40283966064453,
"step": 437
},
{
"KL/chosen_KL_mean": -43.95945358276367,
"KL/mean": -57.19584655761719,
"KL/rejected_KL_mean": -70.4322509765625,
"KL/std": 64.86258697509766,
"epoch": 0.9172774869109948,
"fcm_dpo/beta": 0.007335146423429251,
"fcm_dpo/delta": 0.05353505536913872,
"fcm_dpo/margin": 26.47280502319336,
"fcm_dpo/q_t": 0.4555557370185852,
"grad_norm": 98.37814331054688,
"learning_rate": 1.06489699136324e-08,
"logits/chosen": 1.3550448417663574,
"logits/rejected": 1.4457849264144897,
"logps/chosen": -299.19134521484375,
"logps/ref_chosen": -255.23190307617188,
"logps/ref_rejected": -241.1186981201172,
"logps/rejected": -311.55096435546875,
"loss": 5.1403,
"margin_dpo/margin_mean": 26.47280502319336,
"margin_dpo/margin_std": 78.69203186035156,
"step": 438
},
{
"KL/chosen_KL_mean": -35.10698318481445,
"KL/mean": -59.143768310546875,
"KL/rejected_KL_mean": -83.18055725097656,
"KL/std": 63.55727767944336,
"epoch": 0.9193717277486911,
"fcm_dpo/beta": 0.007670318242162466,
"fcm_dpo/delta": 0.03191578388214111,
"fcm_dpo/margin": 48.07355880737305,
"fcm_dpo/q_t": 0.41562914848327637,
"grad_norm": 33.346168518066406,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": 1.137187123298645,
"logits/rejected": 1.151048183441162,
"logps/chosen": -286.08734130859375,
"logps/ref_chosen": -250.9803466796875,
"logps/ref_rejected": -239.9314422607422,
"logps/rejected": -323.11199951171875,
"loss": 4.5363,
"margin_dpo/margin_mean": 48.07355880737305,
"margin_dpo/margin_std": 75.46719360351562,
"step": 439
},
{
"KL/chosen_KL_mean": -42.483943939208984,
"KL/mean": -61.24757766723633,
"KL/rejected_KL_mean": -80.01121520996094,
"KL/std": 63.740665435791016,
"epoch": 0.9214659685863874,
"fcm_dpo/beta": 0.007664527744054794,
"fcm_dpo/delta": -0.01909954845905304,
"fcm_dpo/margin": 37.52727127075195,
"fcm_dpo/q_t": 0.43392854928970337,
"grad_norm": 37.99457931518555,
"learning_rate": 9.617406953185136e-09,
"logits/chosen": 1.2969262599945068,
"logits/rejected": 1.2244068384170532,
"logps/chosen": -343.4866943359375,
"logps/ref_chosen": -301.0027770996094,
"logps/ref_rejected": -263.628662109375,
"logps/rejected": -343.639892578125,
"loss": 4.8331,
"margin_dpo/margin_mean": 37.52727127075195,
"margin_dpo/margin_std": 74.27835845947266,
"step": 440
},
{
"KL/chosen_KL_mean": -30.726608276367188,
"KL/mean": -59.48487854003906,
"KL/rejected_KL_mean": -88.24314880371094,
"KL/std": 66.66978454589844,
"epoch": 0.9235602094240838,
"fcm_dpo/beta": 0.007572128903120756,
"fcm_dpo/delta": -0.03766479343175888,
"fcm_dpo/margin": 57.516536712646484,
"fcm_dpo/q_t": 0.39927002787590027,
"grad_norm": 33.259742736816406,
"learning_rate": 9.12094829893642e-09,
"logits/chosen": 1.4339779615402222,
"logits/rejected": 1.5300768613815308,
"logps/chosen": -331.57537841796875,
"logps/ref_chosen": -300.8487854003906,
"logps/ref_rejected": -296.7118835449219,
"logps/rejected": -384.9550476074219,
"loss": 4.286,
"margin_dpo/margin_mean": 57.516536712646484,
"margin_dpo/margin_std": 68.08244323730469,
"step": 441
},
{
"KL/chosen_KL_mean": -44.049869537353516,
"KL/mean": -65.7066879272461,
"KL/rejected_KL_mean": -87.36351776123047,
"KL/std": 64.52679443359375,
"epoch": 0.9256544502617801,
"fcm_dpo/beta": 0.0074118864722549915,
"fcm_dpo/delta": 0.05567052215337753,
"fcm_dpo/margin": 43.31364440917969,
"fcm_dpo/q_t": 0.4267332851886749,
"grad_norm": 33.63667678833008,
"learning_rate": 8.637407257200496e-09,
"logits/chosen": 1.642959475517273,
"logits/rejected": 1.716474175453186,
"logps/chosen": -315.6982727050781,
"logps/ref_chosen": -271.6484069824219,
"logps/ref_rejected": -238.21197509765625,
"logps/rejected": -325.57550048828125,
"loss": 4.7617,
"margin_dpo/margin_mean": 43.31364440917969,
"margin_dpo/margin_std": 84.8089599609375,
"step": 442
},
{
"KL/chosen_KL_mean": -36.81932830810547,
"KL/mean": -59.354122161865234,
"KL/rejected_KL_mean": -81.88893127441406,
"KL/std": 64.16720581054688,
"epoch": 0.9277486910994764,
"fcm_dpo/beta": 0.007686508819460869,
"fcm_dpo/delta": -0.03169297054409981,
"fcm_dpo/margin": 45.06959915161133,
"fcm_dpo/q_t": 0.41995733976364136,
"grad_norm": 34.606407165527344,
"learning_rate": 8.166809758815895e-09,
"logits/chosen": 1.3172075748443604,
"logits/rejected": 1.4071624279022217,
"logps/chosen": -303.1158447265625,
"logps/ref_chosen": -266.2965087890625,
"logps/ref_rejected": -263.7326965332031,
"logps/rejected": -345.6216125488281,
"loss": 4.6215,
"margin_dpo/margin_mean": 45.06959915161133,
"margin_dpo/margin_std": 72.51322174072266,
"step": 443
},
{
"KL/chosen_KL_mean": -34.71281814575195,
"KL/mean": -59.032806396484375,
"KL/rejected_KL_mean": -83.35279846191406,
"KL/std": 67.0949478149414,
"epoch": 0.9298429319371728,
"fcm_dpo/beta": 0.007581553887575865,
"fcm_dpo/delta": 0.031975001096725464,
"fcm_dpo/margin": 48.639984130859375,
"fcm_dpo/q_t": 0.41638487577438354,
"grad_norm": 32.99140548706055,
"learning_rate": 7.709181040498253e-09,
"logits/chosen": 1.17976975440979,
"logits/rejected": 1.1861872673034668,
"logps/chosen": -327.07354736328125,
"logps/ref_chosen": -292.36077880859375,
"logps/ref_rejected": -264.35662841796875,
"logps/rejected": -347.70941162109375,
"loss": 4.5585,
"margin_dpo/margin_mean": 48.639984130859375,
"margin_dpo/margin_std": 79.28902435302734,
"step": 444
},
{
"KL/chosen_KL_mean": -34.34645080566406,
"KL/mean": -54.194400787353516,
"KL/rejected_KL_mean": -74.0423583984375,
"KL/std": 66.02217102050781,
"epoch": 0.9319371727748691,
"fcm_dpo/beta": 0.0076246075332164764,
"fcm_dpo/delta": -0.032751768827438354,
"fcm_dpo/margin": 39.69590759277344,
"fcm_dpo/q_t": 0.4303748905658722,
"grad_norm": 33.18951416015625,
"learning_rate": 7.2645456434869965e-09,
"logits/chosen": 1.361595869064331,
"logits/rejected": 1.4701972007751465,
"logps/chosen": -281.0662536621094,
"logps/ref_chosen": -246.71978759765625,
"logps/ref_rejected": -261.339599609375,
"logps/rejected": -335.3819885253906,
"loss": 4.7777,
"margin_dpo/margin_mean": 39.6959114074707,
"margin_dpo/margin_std": 73.78507995605469,
"step": 445
},
{
"KL/chosen_KL_mean": -28.681175231933594,
"KL/mean": -55.73102951049805,
"KL/rejected_KL_mean": -82.78089141845703,
"KL/std": 64.98340606689453,
"epoch": 0.9340314136125655,
"fcm_dpo/beta": 0.007325790822505951,
"fcm_dpo/delta": -0.04906530678272247,
"fcm_dpo/margin": 54.099708557128906,
"fcm_dpo/q_t": 0.4080902338027954,
"grad_norm": 31.139144897460938,
"learning_rate": 6.832927412229017e-09,
"logits/chosen": 1.2609220743179321,
"logits/rejected": 1.2591335773468018,
"logps/chosen": -327.2398986816406,
"logps/ref_chosen": -298.5587158203125,
"logps/ref_rejected": -264.5877685546875,
"logps/rejected": -347.36865234375,
"loss": 4.44,
"margin_dpo/margin_mean": 54.09970474243164,
"margin_dpo/margin_std": 76.38221740722656,
"step": 446
},
{
"KL/chosen_KL_mean": -23.04488182067871,
"KL/mean": -51.995361328125,
"KL/rejected_KL_mean": -80.94583892822266,
"KL/std": 63.761531829833984,
"epoch": 0.9361256544502617,
"fcm_dpo/beta": 0.0070063467137515545,
"fcm_dpo/delta": -0.038511306047439575,
"fcm_dpo/margin": 57.90095138549805,
"fcm_dpo/q_t": 0.40471604466438293,
"grad_norm": 32.2551383972168,
"learning_rate": 6.414349493100129e-09,
"logits/chosen": 1.236978530883789,
"logits/rejected": 1.2315425872802734,
"logps/chosen": -272.8981018066406,
"logps/ref_chosen": -249.8532257080078,
"logps/ref_rejected": -233.2117156982422,
"logps/rejected": -314.1575622558594,
"loss": 4.3238,
"margin_dpo/margin_mean": 57.90095138549805,
"margin_dpo/margin_std": 69.3619384765625,
"step": 447
},
{
"KL/chosen_KL_mean": -29.741437911987305,
"KL/mean": -55.09192657470703,
"KL/rejected_KL_mean": -80.4424057006836,
"KL/std": 65.35220336914062,
"epoch": 0.9382198952879581,
"fcm_dpo/beta": 0.0069551835767924786,
"fcm_dpo/delta": 0.005020767450332642,
"fcm_dpo/margin": 50.700958251953125,
"fcm_dpo/q_t": 0.41787856817245483,
"grad_norm": 34.935394287109375,
"learning_rate": 6.0088343331638756e-09,
"logits/chosen": 1.4007723331451416,
"logits/rejected": 1.5684510469436646,
"logps/chosen": -288.56304931640625,
"logps/ref_chosen": -258.8216247558594,
"logps/ref_rejected": -260.4953308105469,
"logps/rejected": -340.9377136230469,
"loss": 4.5225,
"margin_dpo/margin_mean": 50.70096206665039,
"margin_dpo/margin_std": 72.33600616455078,
"step": 448
},
{
"KL/chosen_KL_mean": -28.044179916381836,
"KL/mean": -56.71196746826172,
"KL/rejected_KL_mean": -85.3797607421875,
"KL/std": 64.18692016601562,
"epoch": 0.9403141361256544,
"fcm_dpo/beta": 0.006888584233820438,
"fcm_dpo/delta": -0.0480492040514946,
"fcm_dpo/margin": 57.33557891845703,
"fcm_dpo/q_t": 0.4075410068035126,
"grad_norm": 32.0770263671875,
"learning_rate": 5.616403678967624e-09,
"logits/chosen": 1.5268685817718506,
"logits/rejected": 1.429668664932251,
"logps/chosen": -352.2607116699219,
"logps/ref_chosen": -324.2165222167969,
"logps/ref_rejected": -237.05453491210938,
"logps/rejected": -322.4342956542969,
"loss": 4.3933,
"margin_dpo/margin_mean": 57.33557891845703,
"margin_dpo/margin_std": 74.86048889160156,
"step": 449
},
{
"KL/chosen_KL_mean": -40.39637756347656,
"KL/mean": -62.09858322143555,
"KL/rejected_KL_mean": -83.80079650878906,
"KL/std": 66.7348403930664,
"epoch": 0.9424083769633508,
"fcm_dpo/beta": 0.006802456919103861,
"fcm_dpo/delta": 0.038337331265211105,
"fcm_dpo/margin": 43.4044189453125,
"fcm_dpo/q_t": 0.43125033378601074,
"grad_norm": 47.554039001464844,
"learning_rate": 5.2370785753763356e-09,
"logits/chosen": 1.3350361585617065,
"logits/rejected": 1.3207178115844727,
"logps/chosen": -318.7976379394531,
"logps/ref_chosen": -278.4012451171875,
"logps/ref_rejected": -247.72671508789062,
"logps/rejected": -331.5274963378906,
"loss": 4.7082,
"margin_dpo/margin_mean": 43.4044189453125,
"margin_dpo/margin_std": 73.86143493652344,
"step": 450
},
{
"KL/chosen_KL_mean": -30.948543548583984,
"KL/mean": -56.13187789916992,
"KL/rejected_KL_mean": -81.31521606445312,
"KL/std": 63.70576095581055,
"epoch": 0.9445026178010472,
"fcm_dpo/beta": 0.006816812325268984,
"fcm_dpo/delta": -0.019252382218837738,
"fcm_dpo/margin": 50.366668701171875,
"fcm_dpo/q_t": 0.4198485314846039,
"grad_norm": 39.90604019165039,
"learning_rate": 4.8708793644441086e-09,
"logits/chosen": 1.2547870874404907,
"logits/rejected": 1.267533779144287,
"logps/chosen": -323.0431823730469,
"logps/ref_chosen": -292.0946350097656,
"logps/ref_rejected": -252.24237060546875,
"logps/rejected": -333.5575866699219,
"loss": 4.5659,
"margin_dpo/margin_mean": 50.366668701171875,
"margin_dpo/margin_std": 73.88423156738281,
"step": 451
},
{
"KL/chosen_KL_mean": -25.739990234375,
"KL/mean": -49.663002014160156,
"KL/rejected_KL_mean": -73.58600616455078,
"KL/std": 66.58033752441406,
"epoch": 0.9465968586387434,
"fcm_dpo/beta": 0.006885976064950228,
"fcm_dpo/delta": 0.03865534067153931,
"fcm_dpo/margin": 47.84601593017578,
"fcm_dpo/q_t": 0.423822820186615,
"grad_norm": 40.99658203125,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": 1.3628993034362793,
"logits/rejected": 1.4249635934829712,
"logps/chosen": -316.3507995605469,
"logps/ref_chosen": -290.61077880859375,
"logps/ref_rejected": -279.8963928222656,
"logps/rejected": -353.4823913574219,
"loss": 4.5878,
"margin_dpo/margin_mean": 47.84601593017578,
"margin_dpo/margin_std": 71.62435150146484,
"step": 452
},
{
"KL/chosen_KL_mean": -32.94917297363281,
"KL/mean": -59.938636779785156,
"KL/rejected_KL_mean": -86.9281005859375,
"KL/std": 65.81941223144531,
"epoch": 0.9486910994764398,
"fcm_dpo/beta": 0.007084976881742477,
"fcm_dpo/delta": 0.01817309856414795,
"fcm_dpo/margin": 53.97893142700195,
"fcm_dpo/q_t": 0.4109514057636261,
"grad_norm": 33.803531646728516,
"learning_rate": 4.1779364682113794e-09,
"logits/chosen": 1.623923897743225,
"logits/rejected": 1.6035189628601074,
"logps/chosen": -349.2644348144531,
"logps/ref_chosen": -316.3152770996094,
"logps/ref_rejected": -300.5086975097656,
"logps/rejected": -387.43682861328125,
"loss": 4.4253,
"margin_dpo/margin_mean": 53.97893142700195,
"margin_dpo/margin_std": 73.46149444580078,
"step": 453
},
{
"KL/chosen_KL_mean": -32.817420959472656,
"KL/mean": -57.83567428588867,
"KL/rejected_KL_mean": -82.85394287109375,
"KL/std": 68.1610107421875,
"epoch": 0.9507853403141361,
"fcm_dpo/beta": 0.007199456449598074,
"fcm_dpo/delta": 0.015694206580519676,
"fcm_dpo/margin": 50.036521911621094,
"fcm_dpo/q_t": 0.4167366623878479,
"grad_norm": 30.35013198852539,
"learning_rate": 3.851229943335393e-09,
"logits/chosen": 1.6498425006866455,
"logits/rejected": 1.7537225484848022,
"logps/chosen": -323.3627014160156,
"logps/ref_chosen": -290.5452575683594,
"logps/ref_rejected": -297.2102355957031,
"logps/rejected": -380.0641784667969,
"loss": 4.5173,
"margin_dpo/margin_mean": 50.036521911621094,
"margin_dpo/margin_std": 72.81925201416016,
"step": 454
},
{
"KL/chosen_KL_mean": -27.860380172729492,
"KL/mean": -48.78271484375,
"KL/rejected_KL_mean": -69.70503997802734,
"KL/std": 62.85211944580078,
"epoch": 0.9528795811518325,
"fcm_dpo/beta": 0.00735099334269762,
"fcm_dpo/delta": 0.02359689027070999,
"fcm_dpo/margin": 41.844661712646484,
"fcm_dpo/q_t": 0.4287131726741791,
"grad_norm": 35.1033821105957,
"learning_rate": 3.5377236299748147e-09,
"logits/chosen": 1.2487902641296387,
"logits/rejected": 1.3486948013305664,
"logps/chosen": -291.2807312011719,
"logps/ref_chosen": -263.4203186035156,
"logps/ref_rejected": -270.30291748046875,
"logps/rejected": -340.0079650878906,
"loss": 4.7048,
"margin_dpo/margin_mean": 41.844661712646484,
"margin_dpo/margin_std": 73.86407470703125,
"step": 455
},
{
"KL/chosen_KL_mean": -34.925262451171875,
"KL/mean": -64.77821350097656,
"KL/rejected_KL_mean": -94.63116455078125,
"KL/std": 69.82489013671875,
"epoch": 0.9549738219895288,
"fcm_dpo/beta": 0.007192356511950493,
"fcm_dpo/delta": -0.0638280063867569,
"fcm_dpo/margin": 59.705894470214844,
"fcm_dpo/q_t": 0.4018319845199585,
"grad_norm": 27.928911209106445,
"learning_rate": 3.2374343405217884e-09,
"logits/chosen": 1.2568565607070923,
"logits/rejected": 1.4191408157348633,
"logps/chosen": -346.9626159667969,
"logps/ref_chosen": -312.0373840332031,
"logps/ref_rejected": -290.50946044921875,
"logps/rejected": -385.140625,
"loss": 4.3587,
"margin_dpo/margin_mean": 59.705902099609375,
"margin_dpo/margin_std": 82.88359832763672,
"step": 456
},
{
"KL/chosen_KL_mean": -35.84705352783203,
"KL/mean": -63.143150329589844,
"KL/rejected_KL_mean": -90.43922424316406,
"KL/std": 68.66735076904297,
"epoch": 0.9570680628272251,
"fcm_dpo/beta": 0.007003502920269966,
"fcm_dpo/delta": -0.007018011063337326,
"fcm_dpo/margin": 54.59217834472656,
"fcm_dpo/q_t": 0.41150468587875366,
"grad_norm": 32.43729782104492,
"learning_rate": 2.9503781785795713e-09,
"logits/chosen": 1.2495930194854736,
"logits/rejected": 1.2768305540084839,
"logps/chosen": -333.01904296875,
"logps/ref_chosen": -297.1719970703125,
"logps/ref_rejected": -269.15557861328125,
"logps/rejected": -359.59478759765625,
"loss": 4.4655,
"margin_dpo/margin_mean": 54.59218215942383,
"margin_dpo/margin_std": 79.75926208496094,
"step": 457
},
{
"KL/chosen_KL_mean": -32.20441818237305,
"KL/mean": -56.76567459106445,
"KL/rejected_KL_mean": -81.32694244384766,
"KL/std": 69.28995513916016,
"epoch": 0.9591623036649215,
"fcm_dpo/beta": 0.006989713758230209,
"fcm_dpo/delta": -0.0033550746738910675,
"fcm_dpo/margin": 49.122520446777344,
"fcm_dpo/q_t": 0.420844703912735,
"grad_norm": 28.704254150390625,
"learning_rate": 2.6765705380989432e-09,
"logits/chosen": 1.3223989009857178,
"logits/rejected": 1.24267578125,
"logps/chosen": -319.5693359375,
"logps/ref_chosen": -287.3648986816406,
"logps/ref_rejected": -271.1472473144531,
"logps/rejected": -352.4742126464844,
"loss": 4.5797,
"margin_dpo/margin_mean": 49.122528076171875,
"margin_dpo/margin_std": 75.25064086914062,
"step": 458
},
{
"KL/chosen_KL_mean": -37.28789138793945,
"KL/mean": -56.56632614135742,
"KL/rejected_KL_mean": -75.84475708007812,
"KL/std": 65.46312713623047,
"epoch": 0.9612565445026178,
"fcm_dpo/beta": 0.007017179392278194,
"fcm_dpo/delta": 0.01031123474240303,
"fcm_dpo/margin": 38.55687713623047,
"fcm_dpo/q_t": 0.4377577304840088,
"grad_norm": 32.177268981933594,
"learning_rate": 2.416026102552732e-09,
"logits/chosen": 1.3852252960205078,
"logits/rejected": 1.47627854347229,
"logps/chosen": -325.8940124511719,
"logps/ref_chosen": -288.6061096191406,
"logps/ref_rejected": -285.6366271972656,
"logps/rejected": -361.48138427734375,
"loss": 4.8559,
"margin_dpo/margin_mean": 38.5568733215332,
"margin_dpo/margin_std": 78.94550323486328,
"step": 459
},
{
"KL/chosen_KL_mean": -36.56329345703125,
"KL/mean": -55.490535736083984,
"KL/rejected_KL_mean": -74.41775512695312,
"KL/std": 64.80435180664062,
"epoch": 0.9633507853403142,
"fcm_dpo/beta": 0.00710292998701334,
"fcm_dpo/delta": 0.03049231693148613,
"fcm_dpo/margin": 37.854469299316406,
"fcm_dpo/q_t": 0.4383412301540375,
"grad_norm": 50.06414031982422,
"learning_rate": 2.168758844148272e-09,
"logits/chosen": 1.137605905532837,
"logits/rejected": 1.060865879058838,
"logps/chosen": -346.8152160644531,
"logps/ref_chosen": -310.251953125,
"logps/ref_rejected": -252.962158203125,
"logps/rejected": -327.3798828125,
"loss": 4.8889,
"margin_dpo/margin_mean": 37.85446548461914,
"margin_dpo/margin_std": 81.85502624511719,
"step": 460
},
{
"KL/chosen_KL_mean": -33.92215347290039,
"KL/mean": -57.918094635009766,
"KL/rejected_KL_mean": -81.91403198242188,
"KL/std": 67.5086898803711,
"epoch": 0.9654450261780104,
"fcm_dpo/beta": 0.007304798811674118,
"fcm_dpo/delta": 0.017830081284046173,
"fcm_dpo/margin": 47.99187469482422,
"fcm_dpo/q_t": 0.4192803204059601,
"grad_norm": 34.50105285644531,
"learning_rate": 1.9347820230782295e-09,
"logits/chosen": 1.1698627471923828,
"logits/rejected": 1.2976014614105225,
"logps/chosen": -270.7026062011719,
"logps/ref_chosen": -236.7804412841797,
"logps/ref_rejected": -229.67550659179688,
"logps/rejected": -311.58953857421875,
"loss": 4.6163,
"margin_dpo/margin_mean": 47.991878509521484,
"margin_dpo/margin_std": 81.20181274414062,
"step": 461
},
{
"KL/chosen_KL_mean": -27.43246078491211,
"KL/mean": -57.93980026245117,
"KL/rejected_KL_mean": -88.4471435546875,
"KL/std": 66.90789794921875,
"epoch": 0.9675392670157068,
"fcm_dpo/beta": 0.007108363322913647,
"fcm_dpo/delta": -0.03596530854701996,
"fcm_dpo/margin": 61.01466369628906,
"fcm_dpo/q_t": 0.39972108602523804,
"grad_norm": 35.206180572509766,
"learning_rate": 1.7141081868094209e-09,
"logits/chosen": 1.2888528108596802,
"logits/rejected": 1.1797453165054321,
"logps/chosen": -362.7359619140625,
"logps/ref_chosen": -335.30352783203125,
"logps/ref_rejected": -250.6796875,
"logps/rejected": -339.1268005371094,
"loss": 4.3303,
"margin_dpo/margin_mean": 61.01466369628906,
"margin_dpo/margin_std": 82.29486083984375,
"step": 462
},
{
"KL/chosen_KL_mean": -35.447837829589844,
"KL/mean": -57.846675872802734,
"KL/rejected_KL_mean": -80.2455062866211,
"KL/std": 66.26893615722656,
"epoch": 0.9696335078534032,
"fcm_dpo/beta": 0.007227907422930002,
"fcm_dpo/delta": 0.04812032729387283,
"fcm_dpo/margin": 44.79767608642578,
"fcm_dpo/q_t": 0.4250917136669159,
"grad_norm": 43.74908447265625,
"learning_rate": 1.5067491694100153e-09,
"logits/chosen": 1.3669559955596924,
"logits/rejected": 1.2001994848251343,
"logps/chosen": -323.6680908203125,
"logps/ref_chosen": -288.22021484375,
"logps/ref_rejected": -227.96673583984375,
"logps/rejected": -308.21221923828125,
"loss": 4.638,
"margin_dpo/margin_mean": 44.79767608642578,
"margin_dpo/margin_std": 73.12406921386719,
"step": 463
},
{
"KL/chosen_KL_mean": -37.204872131347656,
"KL/mean": -60.64276123046875,
"KL/rejected_KL_mean": -84.08065795898438,
"KL/std": 65.3951187133789,
"epoch": 0.9717277486910995,
"fcm_dpo/beta": 0.007382219657301903,
"fcm_dpo/delta": 0.016385518014431,
"fcm_dpo/margin": 46.87579345703125,
"fcm_dpo/q_t": 0.4192740321159363,
"grad_norm": 43.08910369873047,
"learning_rate": 1.3127160909147672e-09,
"logits/chosen": 1.5911651849746704,
"logits/rejected": 1.6315275430679321,
"logps/chosen": -297.54180908203125,
"logps/ref_chosen": -260.3369140625,
"logps/ref_rejected": -254.2083740234375,
"logps/rejected": -338.2890319824219,
"loss": 4.592,
"margin_dpo/margin_mean": 46.87579345703125,
"margin_dpo/margin_std": 73.90274047851562,
"step": 464
},
{
"KL/chosen_KL_mean": -22.133838653564453,
"KL/mean": -52.00831604003906,
"KL/rejected_KL_mean": -81.88279724121094,
"KL/std": 66.21720886230469,
"epoch": 0.9738219895287958,
"fcm_dpo/beta": 0.007099932059645653,
"fcm_dpo/delta": -0.08648946136236191,
"fcm_dpo/margin": 59.74894714355469,
"fcm_dpo/q_t": 0.4017646610736847,
"grad_norm": 31.485698699951172,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": 1.4218883514404297,
"logits/rejected": 1.4663176536560059,
"logps/chosen": -305.2033386230469,
"logps/ref_chosen": -283.0694885253906,
"logps/ref_rejected": -289.1370544433594,
"logps/rejected": -371.0198669433594,
"loss": 4.3439,
"margin_dpo/margin_mean": 59.74894714355469,
"margin_dpo/margin_std": 73.79570770263672,
"step": 465
},
{
"KL/chosen_KL_mean": -23.852914810180664,
"KL/mean": -55.77590560913086,
"KL/rejected_KL_mean": -87.69889068603516,
"KL/std": 66.72920227050781,
"epoch": 0.9759162303664921,
"fcm_dpo/beta": 0.006767213344573975,
"fcm_dpo/delta": -0.034620895981788635,
"fcm_dpo/margin": 63.845977783203125,
"fcm_dpo/q_t": 0.3984140157699585,
"grad_norm": 30.930160522460938,
"learning_rate": 9.64668657069706e-10,
"logits/chosen": 0.9917327761650085,
"logits/rejected": 1.0448675155639648,
"logps/chosen": -280.7379455566406,
"logps/ref_chosen": -256.8850402832031,
"logps/ref_rejected": -247.0802459716797,
"logps/rejected": -334.77911376953125,
"loss": 4.2378,
"margin_dpo/margin_mean": 63.845977783203125,
"margin_dpo/margin_std": 71.75048828125,
"step": 466
},
{
"KL/chosen_KL_mean": -34.90070724487305,
"KL/mean": -57.99346160888672,
"KL/rejected_KL_mean": -81.08621215820312,
"KL/std": 66.7815933227539,
"epoch": 0.9780104712041885,
"fcm_dpo/beta": 0.006723516620695591,
"fcm_dpo/delta": -0.011592521332204342,
"fcm_dpo/margin": 46.18550109863281,
"fcm_dpo/q_t": 0.4279315769672394,
"grad_norm": 39.73870086669922,
"learning_rate": 8.106729664475176e-10,
"logits/chosen": 0.9874340295791626,
"logits/rejected": 1.1375449895858765,
"logps/chosen": -292.4247741699219,
"logps/ref_chosen": -257.5240783691406,
"logps/ref_rejected": -274.82122802734375,
"logps/rejected": -355.9074401855469,
"loss": 4.6955,
"margin_dpo/margin_mean": 46.18550109863281,
"margin_dpo/margin_std": 78.61346435546875,
"step": 467
},
{
"KL/chosen_KL_mean": -26.964384078979492,
"KL/mean": -48.01728439331055,
"KL/rejected_KL_mean": -69.07018280029297,
"KL/std": 64.16717529296875,
"epoch": 0.9801047120418848,
"fcm_dpo/beta": 0.00669306144118309,
"fcm_dpo/delta": -0.02148795872926712,
"fcm_dpo/margin": 42.105796813964844,
"fcm_dpo/q_t": 0.43420475721359253,
"grad_norm": 32.607810974121094,
"learning_rate": 6.700405431837585e-10,
"logits/chosen": 1.34190833568573,
"logits/rejected": 1.229614019393921,
"logps/chosen": -339.2240905761719,
"logps/ref_chosen": -312.2597351074219,
"logps/ref_rejected": -259.74383544921875,
"logps/rejected": -328.81396484375,
"loss": 4.7303,
"margin_dpo/margin_mean": 42.105796813964844,
"margin_dpo/margin_std": 69.45880889892578,
"step": 468
},
{
"KL/chosen_KL_mean": -23.92185401916504,
"KL/mean": -50.92070007324219,
"KL/rejected_KL_mean": -77.9195327758789,
"KL/std": 67.88216400146484,
"epoch": 0.9821989528795811,
"fcm_dpo/beta": 0.006680009420961142,
"fcm_dpo/delta": 0.014251125045120716,
"fcm_dpo/margin": 53.997684478759766,
"fcm_dpo/q_t": 0.41561082005500793,
"grad_norm": 36.88972091674805,
"learning_rate": 5.427789289685347e-10,
"logits/chosen": 1.1206728219985962,
"logits/rejected": 1.022853136062622,
"logps/chosen": -340.96160888671875,
"logps/ref_chosen": -317.03973388671875,
"logps/ref_rejected": -260.12628173828125,
"logps/rejected": -338.0457763671875,
"loss": 4.5264,
"margin_dpo/margin_mean": 53.997684478759766,
"margin_dpo/margin_std": 80.5838623046875,
"step": 469
},
{
"KL/chosen_KL_mean": -23.201324462890625,
"KL/mean": -53.28938293457031,
"KL/rejected_KL_mean": -83.3774185180664,
"KL/std": 67.57502746582031,
"epoch": 0.9842931937172775,
"fcm_dpo/beta": 0.006601809058338404,
"fcm_dpo/delta": -0.05593057721853256,
"fcm_dpo/margin": 60.17609405517578,
"fcm_dpo/q_t": 0.4073055684566498,
"grad_norm": 27.246021270751953,
"learning_rate": 4.288949484559934e-10,
"logits/chosen": 1.1400357484817505,
"logits/rejected": 1.0457278490066528,
"logps/chosen": -327.59527587890625,
"logps/ref_chosen": -304.3939514160156,
"logps/ref_rejected": -257.6587219238281,
"logps/rejected": -341.0361328125,
"loss": 4.4029,
"margin_dpo/margin_mean": 60.17609405517578,
"margin_dpo/margin_std": 78.45281982421875,
"step": 470
},
{
"KL/chosen_KL_mean": -34.258522033691406,
"KL/mean": -55.35606002807617,
"KL/rejected_KL_mean": -76.4535903930664,
"KL/std": 67.38356018066406,
"epoch": 0.9863874345549738,
"fcm_dpo/beta": 0.006539772264659405,
"fcm_dpo/delta": 0.0638960599899292,
"fcm_dpo/margin": 42.195072174072266,
"fcm_dpo/q_t": 0.4357315003871918,
"grad_norm": 50.0222053527832,
"learning_rate": 3.2839470889836627e-10,
"logits/chosen": 1.1337032318115234,
"logits/rejected": 1.2113916873931885,
"logps/chosen": -319.2023620605469,
"logps/ref_chosen": -284.94384765625,
"logps/ref_rejected": -297.4357604980469,
"logps/rejected": -373.88934326171875,
"loss": 4.7498,
"margin_dpo/margin_mean": 42.195072174072266,
"margin_dpo/margin_std": 73.13125610351562,
"step": 471
},
{
"KL/chosen_KL_mean": -25.31435775756836,
"KL/mean": -54.23596954345703,
"KL/rejected_KL_mean": -83.1575927734375,
"KL/std": 65.6238784790039,
"epoch": 0.9884816753926702,
"fcm_dpo/beta": 0.006686407141387463,
"fcm_dpo/delta": 0.01352601032704115,
"fcm_dpo/margin": 57.843223571777344,
"fcm_dpo/q_t": 0.4107360243797302,
"grad_norm": 40.486175537109375,
"learning_rate": 2.412835998185092e-10,
"logits/chosen": 1.2406048774719238,
"logits/rejected": 1.3722997903823853,
"logps/chosen": -257.66864013671875,
"logps/ref_chosen": -232.35427856445312,
"logps/ref_rejected": -245.6057891845703,
"logps/rejected": -328.7633972167969,
"loss": 4.419,
"margin_dpo/margin_mean": 57.843223571777344,
"margin_dpo/margin_std": 77.69920349121094,
"step": 472
},
{
"KL/chosen_KL_mean": -30.094152450561523,
"KL/mean": -59.20981979370117,
"KL/rejected_KL_mean": -88.32546997070312,
"KL/std": 66.05684661865234,
"epoch": 0.9905759162303664,
"fcm_dpo/beta": 0.006787514314055443,
"fcm_dpo/delta": -0.01905285194516182,
"fcm_dpo/margin": 58.23133087158203,
"fcm_dpo/q_t": 0.4079660177230835,
"grad_norm": 28.650922775268555,
"learning_rate": 1.6756629272085544e-10,
"logits/chosen": 1.3480632305145264,
"logits/rejected": 1.4126392602920532,
"logps/chosen": -308.10101318359375,
"logps/ref_chosen": -278.0068664550781,
"logps/ref_rejected": -257.19256591796875,
"logps/rejected": -345.51806640625,
"loss": 4.3959,
"margin_dpo/margin_mean": 58.23133087158203,
"margin_dpo/margin_std": 76.88653564453125,
"step": 473
},
{
"KL/chosen_KL_mean": -40.19635009765625,
"KL/mean": -61.384925842285156,
"KL/rejected_KL_mean": -82.57350158691406,
"KL/std": 64.97396087646484,
"epoch": 0.9926701570680628,
"fcm_dpo/beta": 0.0065875062718987465,
"fcm_dpo/delta": -0.00273904949426651,
"fcm_dpo/margin": 42.37714385986328,
"fcm_dpo/q_t": 0.43389174342155457,
"grad_norm": 62.54719924926758,
"learning_rate": 1.072467408408384e-10,
"logits/chosen": 1.2016754150390625,
"logits/rejected": 1.2527438402175903,
"logps/chosen": -320.953857421875,
"logps/ref_chosen": -280.75750732421875,
"logps/ref_rejected": -264.34295654296875,
"logps/rejected": -346.91644287109375,
"loss": 4.735,
"margin_dpo/margin_mean": 42.37714385986328,
"margin_dpo/margin_std": 68.2033462524414,
"step": 474
},
{
"KL/chosen_KL_mean": -28.445863723754883,
"KL/mean": -51.21686553955078,
"KL/rejected_KL_mean": -73.98786926269531,
"KL/std": 62.809024810791016,
"epoch": 0.9947643979057592,
"fcm_dpo/beta": 0.006702399346977472,
"fcm_dpo/delta": 0.014713454991579056,
"fcm_dpo/margin": 45.54199981689453,
"fcm_dpo/q_t": 0.4287365674972534,
"grad_norm": 34.98037338256836,
"learning_rate": 6.032817893297793e-11,
"logits/chosen": 1.175693392753601,
"logits/rejected": 1.3002114295959473,
"logps/chosen": -269.9172058105469,
"logps/ref_chosen": -241.4713592529297,
"logps/ref_rejected": -244.0782928466797,
"logps/rejected": -318.066162109375,
"loss": 4.6671,
"margin_dpo/margin_mean": 45.54199981689453,
"margin_dpo/margin_std": 73.11616516113281,
"step": 475
},
{
"KL/chosen_KL_mean": -19.955669403076172,
"KL/mean": -46.316307067871094,
"KL/rejected_KL_mean": -72.67695617675781,
"KL/std": 63.86593246459961,
"epoch": 0.9968586387434555,
"fcm_dpo/beta": 0.006906186696141958,
"fcm_dpo/delta": 0.03595799580216408,
"fcm_dpo/margin": 52.72127914428711,
"fcm_dpo/q_t": 0.4155328571796417,
"grad_norm": 29.642295837402344,
"learning_rate": 2.6813123097352287e-11,
"logits/chosen": 1.409799575805664,
"logits/rejected": 1.418018102645874,
"logps/chosen": -338.7300109863281,
"logps/ref_chosen": -318.7742919921875,
"logps/ref_rejected": -293.66595458984375,
"logps/rejected": -366.3429260253906,
"loss": 4.5041,
"margin_dpo/margin_mean": 52.72127914428711,
"margin_dpo/margin_std": 75.55950164794922,
"step": 476
},
{
"KL/chosen_KL_mean": -30.197856903076172,
"KL/mean": -58.40687942504883,
"KL/rejected_KL_mean": -86.61589050292969,
"KL/std": 67.73674011230469,
"epoch": 0.9989528795811519,
"fcm_dpo/beta": 0.007070903200656176,
"fcm_dpo/delta": 0.0004990696907043457,
"fcm_dpo/margin": 56.41804122924805,
"fcm_dpo/q_t": 0.40811702609062195,
"grad_norm": 33.42003631591797,
"learning_rate": 6.7033706447061635e-12,
"logits/chosen": 1.2682021856307983,
"logits/rejected": 1.3051879405975342,
"logps/chosen": -296.1335754394531,
"logps/ref_chosen": -265.9356994628906,
"logps/ref_rejected": -243.6986083984375,
"logps/rejected": -330.31451416015625,
"loss": 4.4395,
"margin_dpo/margin_mean": 56.41804122924805,
"margin_dpo/margin_std": 81.06288146972656,
"step": 477
},
{
"epoch": 0.9989528795811519,
"step": 477,
"total_flos": 0.0,
"train_loss": 4.806692141406941,
"train_runtime": 6998.3828,
"train_samples_per_second": 8.736,
"train_steps_per_second": 0.068
}
],
"logging_steps": 1,
"max_steps": 477,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink