W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.48
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
4d19c05e0b2e1ac2b3e4868e5bbe4f2590651c0b
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC 4d19c05e0b 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.48
Source: Original Platform
2026-05-26 11:58:18 +08:00

15707 lines
590 KiB
JSON
Raw Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.52754974365234,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.21381378173828,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": 0.012119293212890625,
"KL/mean": 0.023418381810188293,
"KL/rejected_KL_mean": 0.034717559814453125,
"KL/std": 0.2618991434574127,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02260264754295349,
"fcm_dpo/q_t": 0.5005650520324707,
"grad_norm": 70.9487533569336,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.48166990280151367,
"logits/rejected": -0.442186176776886,
"logps/chosen": -60.96947479248047,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.63787841796875,
"loss": 1.3889,
"margin_dpo/margin_mean": -0.022602319717407227,
"margin_dpo/margin_std": 0.3861812651157379,
"step": 3
},
{
"KL/chosen_KL_mean": -0.04089927673339844,
"KL/mean": 0.003831237554550171,
"KL/rejected_KL_mean": 0.04856109619140625,
"KL/std": 0.2447841912508011,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.08945882320404053,
"fcm_dpo/q_t": 0.5022358894348145,
"grad_norm": 72.29135131835938,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.46891048550605774,
"logits/rejected": -0.44123539328575134,
"logps/chosen": -56.80860900878906,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.59854888916016,
"loss": 1.3956,
"margin_dpo/margin_mean": -0.08945959806442261,
"margin_dpo/margin_std": 0.3571521043777466,
"step": 4
},
{
"KL/chosen_KL_mean": 0.04132270812988281,
"KL/mean": 0.02954573929309845,
"KL/rejected_KL_mean": 0.017772674560546875,
"KL/std": 0.2609584927558899,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0235537588596344,
"fcm_dpo/q_t": 0.49941110610961914,
"grad_norm": 89.83948516845703,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.5144683122634888,
"logits/rejected": -0.4707663655281067,
"logps/chosen": -53.81805419921875,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.13140869140625,
"loss": 1.3843,
"margin_dpo/margin_mean": 0.023554116487503052,
"margin_dpo/margin_std": 0.35333937406539917,
"step": 5
},
{
"KL/chosen_KL_mean": -0.00206756591796875,
"KL/mean": -0.015875771641731262,
"KL/rejected_KL_mean": -0.02968597412109375,
"KL/std": 0.24448521435260773,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.027621835470199585,
"fcm_dpo/q_t": 0.49930986762046814,
"grad_norm": 92.0187759399414,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.5049769282341003,
"logits/rejected": -0.4625431299209595,
"logps/chosen": -63.009552001953125,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.67503356933594,
"loss": 1.3839,
"margin_dpo/margin_mean": 0.02762138843536377,
"margin_dpo/margin_std": 0.3585347533226013,
"step": 6
},
{
"KL/chosen_KL_mean": 0.05124092102050781,
"KL/mean": 0.04420331120491028,
"KL/rejected_KL_mean": 0.03716278076171875,
"KL/std": 0.3070700764656067,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014078021049499512,
"fcm_dpo/q_t": 0.499647855758667,
"grad_norm": 82.254150390625,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.49819037318229675,
"logits/rejected": -0.4643648862838745,
"logps/chosen": -57.72357940673828,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.88343048095703,
"loss": 1.3854,
"margin_dpo/margin_mean": 0.014078229665756226,
"margin_dpo/margin_std": 0.46783095598220825,
"step": 7
},
{
"KL/chosen_KL_mean": -0.0028095245361328125,
"KL/mean": 0.0157100111246109,
"KL/rejected_KL_mean": 0.034236907958984375,
"KL/std": 0.2974522113800049,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03704729676246643,
"fcm_dpo/q_t": 0.5009254813194275,
"grad_norm": 79.46379852294922,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.5174297094345093,
"logits/rejected": -0.492509663105011,
"logps/chosen": -58.71884536743164,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.27719116210938,
"loss": 1.3904,
"margin_dpo/margin_mean": -0.037046968936920166,
"margin_dpo/margin_std": 0.3789815306663513,
"step": 8
},
{
"KL/chosen_KL_mean": 0.02559661865234375,
"KL/mean": 0.013599887490272522,
"KL/rejected_KL_mean": 0.0016021728515625,
"KL/std": 0.29548415541648865,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0239962637424469,
"fcm_dpo/q_t": 0.4993996322154999,
"grad_norm": 85.05180358886719,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.4923360347747803,
"logits/rejected": -0.4455389380455017,
"logps/chosen": -69.84124755859375,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.60105895996094,
"loss": 1.3843,
"margin_dpo/margin_mean": 0.023995906114578247,
"margin_dpo/margin_std": 0.4077387750148773,
"step": 9
},
{
"KL/chosen_KL_mean": -0.0124359130859375,
"KL/mean": 0.0032117292284965515,
"KL/rejected_KL_mean": 0.01885986328125,
"KL/std": 0.2599485218524933,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.031296178698539734,
"fcm_dpo/q_t": 0.5007817149162292,
"grad_norm": 71.18731689453125,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.48350363969802856,
"logits/rejected": -0.43980133533477783,
"logps/chosen": -48.370121002197266,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.35320281982422,
"loss": 1.3898,
"margin_dpo/margin_mean": -0.03129580616950989,
"margin_dpo/margin_std": 0.3704303503036499,
"step": 10
},
{
"KL/chosen_KL_mean": 0.01000213623046875,
"KL/mean": 0.0064017027616500854,
"KL/rejected_KL_mean": 0.002803802490234375,
"KL/std": 0.24205231666564941,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.007197290658950806,
"fcm_dpo/q_t": 0.4998197555541992,
"grad_norm": 68.69322204589844,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.45256781578063965,
"logits/rejected": -0.4272083044052124,
"logps/chosen": -53.00685119628906,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.77757263183594,
"loss": 1.3859,
"margin_dpo/margin_mean": 0.007197737693786621,
"margin_dpo/margin_std": 0.33006855845451355,
"step": 11
},
{
"KL/chosen_KL_mean": -0.028219223022460938,
"KL/mean": -0.04441457986831665,
"KL/rejected_KL_mean": -0.060604095458984375,
"KL/std": 0.2953580617904663,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03238654136657715,
"fcm_dpo/q_t": 0.4991909861564636,
"grad_norm": 89.39579772949219,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.5462939739227295,
"logits/rejected": -0.5103648900985718,
"logps/chosen": -61.833656311035156,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.91887664794922,
"loss": 1.3835,
"margin_dpo/margin_mean": 0.03238588571548462,
"margin_dpo/margin_std": 0.43585580587387085,
"step": 12
},
{
"KL/chosen_KL_mean": 0.0037994384765625,
"KL/mean": 0.034382060170173645,
"KL/rejected_KL_mean": 0.06496810913085938,
"KL/std": 0.27900394797325134,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06116214394569397,
"fcm_dpo/q_t": 0.5015291571617126,
"grad_norm": 79.63651275634766,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.470973938703537,
"logits/rejected": -0.44221845269203186,
"logps/chosen": -64.25655364990234,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.13811492919922,
"loss": 1.3927,
"margin_dpo/margin_mean": -0.06116211414337158,
"margin_dpo/margin_std": 0.3499847948551178,
"step": 13
},
{
"KL/chosen_KL_mean": -0.02565765380859375,
"KL/mean": -0.022577762603759766,
"KL/rejected_KL_mean": -0.019500732421875,
"KL/std": 0.24927328526973724,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.00615239143371582,
"fcm_dpo/q_t": 0.5001521706581116,
"grad_norm": 85.70947265625,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.48932701349258423,
"logits/rejected": -0.4504287838935852,
"logps/chosen": -58.135868072509766,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.06658172607422,
"loss": 1.3872,
"margin_dpo/margin_mean": -0.006151974201202393,
"margin_dpo/margin_std": 0.3624458312988281,
"step": 14
},
{
"KL/chosen_KL_mean": -0.018096923828125,
"KL/mean": -0.0443209707736969,
"KL/rejected_KL_mean": -0.07054519653320312,
"KL/std": 0.2593069076538086,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05244654417037964,
"fcm_dpo/q_t": 0.49868789315223694,
"grad_norm": 64.14149475097656,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.48956209421157837,
"logits/rejected": -0.47070273756980896,
"logps/chosen": -56.985008239746094,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.87918853759766,
"loss": 1.3815,
"margin_dpo/margin_mean": 0.05244651436805725,
"margin_dpo/margin_std": 0.39674949645996094,
"step": 15
},
{
"KL/chosen_KL_mean": -0.03409004211425781,
"KL/mean": -0.04335886240005493,
"KL/rejected_KL_mean": -0.052623748779296875,
"KL/std": 0.273632287979126,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01853311061859131,
"fcm_dpo/q_t": 0.49953699111938477,
"grad_norm": 84.44381713867188,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.5264363288879395,
"logits/rejected": -0.4855811595916748,
"logps/chosen": -61.773983001708984,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.4220962524414,
"loss": 1.3849,
"margin_dpo/margin_mean": 0.018532603979110718,
"margin_dpo/margin_std": 0.41814303398132324,
"step": 16
},
{
"KL/chosen_KL_mean": 0.052093505859375,
"KL/mean": 0.00013424456119537354,
"KL/rejected_KL_mean": -0.05182647705078125,
"KL/std": 0.24351900815963745,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1039208471775055,
"fcm_dpo/q_t": 0.49740248918533325,
"grad_norm": 78.52274322509766,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.48946413397789,
"logits/rejected": -0.450802206993103,
"logps/chosen": -67.65824127197266,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.43048095703125,
"loss": 1.3762,
"margin_dpo/margin_mean": 0.10392042994499207,
"margin_dpo/margin_std": 0.3155348598957062,
"step": 17
},
{
"KL/chosen_KL_mean": -0.03583717346191406,
"KL/mean": -0.04290063679218292,
"KL/rejected_KL_mean": -0.049968719482421875,
"KL/std": 0.2571260333061218,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01413002610206604,
"fcm_dpo/q_t": 0.49964603781700134,
"grad_norm": 81.87183380126953,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.4920946955680847,
"logits/rejected": -0.4363882541656494,
"logps/chosen": -47.77532958984375,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.52225494384766,
"loss": 1.3853,
"margin_dpo/margin_mean": 0.014129400253295898,
"margin_dpo/margin_std": 0.3615719676017761,
"step": 18
},
{
"KL/chosen_KL_mean": 0.031063079833984375,
"KL/mean": -0.03421039879322052,
"KL/rejected_KL_mean": -0.09947967529296875,
"KL/std": 0.24964235723018646,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1305374801158905,
"fcm_dpo/q_t": 0.4967383146286011,
"grad_norm": 72.95270538330078,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.489202618598938,
"logits/rejected": -0.4391380548477173,
"logps/chosen": -70.17430114746094,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.85707092285156,
"loss": 1.3736,
"margin_dpo/margin_mean": 0.13053762912750244,
"margin_dpo/margin_std": 0.32683902978897095,
"step": 19
},
{
"KL/chosen_KL_mean": 0.040554046630859375,
"KL/mean": -0.01780860126018524,
"KL/rejected_KL_mean": -0.07616806030273438,
"KL/std": 0.2401285469532013,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11672148108482361,
"fcm_dpo/q_t": 0.49708235263824463,
"grad_norm": 74.03209686279297,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5423327684402466,
"logits/rejected": -0.4851418733596802,
"logps/chosen": -50.76268768310547,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.89952087402344,
"loss": 1.3749,
"margin_dpo/margin_mean": 0.11672136187553406,
"margin_dpo/margin_std": 0.3035232722759247,
"step": 20
},
{
"KL/chosen_KL_mean": -0.004669189453125,
"KL/mean": -0.06208917498588562,
"KL/rejected_KL_mean": -0.11950302124023438,
"KL/std": 0.27453505992889404,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11483883857727051,
"fcm_dpo/q_t": 0.49713045358657837,
"grad_norm": 77.23396301269531,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.49135324358940125,
"logits/rejected": -0.46715474128723145,
"logps/chosen": -50.06768798828125,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -77.98829650878906,
"loss": 1.3751,
"margin_dpo/margin_mean": 0.11483970284461975,
"margin_dpo/margin_std": 0.3439750671386719,
"step": 21
},
{
"KL/chosen_KL_mean": 0.0177459716796875,
"KL/mean": -0.06365428864955902,
"KL/rejected_KL_mean": -0.14505386352539062,
"KL/std": 0.2868351936340332,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1628032624721527,
"fcm_dpo/q_t": 0.4959312379360199,
"grad_norm": 84.80654907226562,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.47701001167297363,
"logits/rejected": -0.43303921818733215,
"logps/chosen": -59.03989028930664,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.64972686767578,
"loss": 1.3704,
"margin_dpo/margin_mean": 0.1628030240535736,
"margin_dpo/margin_std": 0.35856860876083374,
"step": 22
},
{
"KL/chosen_KL_mean": 0.05772972106933594,
"KL/mean": -0.029708027839660645,
"KL/rejected_KL_mean": -0.1171417236328125,
"KL/std": 0.32750219106674194,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1748729944229126,
"fcm_dpo/q_t": 0.49562904238700867,
"grad_norm": 80.86152648925781,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.47440969944000244,
"logits/rejected": -0.45099538564682007,
"logps/chosen": -60.01996612548828,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.25669860839844,
"loss": 1.3694,
"margin_dpo/margin_mean": 0.17487305402755737,
"margin_dpo/margin_std": 0.4415150582790375,
"step": 23
},
{
"KL/chosen_KL_mean": 0.04306221008300781,
"KL/mean": -0.06060684472322464,
"KL/rejected_KL_mean": -0.16427993774414062,
"KL/std": 0.27947989106178284,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20734088122844696,
"fcm_dpo/q_t": 0.4948195815086365,
"grad_norm": 82.94523620605469,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5204753875732422,
"logits/rejected": -0.5036641955375671,
"logps/chosen": -44.24797058105469,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.28948974609375,
"loss": 1.3661,
"margin_dpo/margin_mean": 0.20734065771102905,
"margin_dpo/margin_std": 0.39581578969955444,
"step": 24
},
{
"KL/chosen_KL_mean": 0.007762908935546875,
"KL/mean": -0.0868898332118988,
"KL/rejected_KL_mean": -0.18154144287109375,
"KL/std": 0.332895964384079,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.18930190801620483,
"fcm_dpo/q_t": 0.4952710270881653,
"grad_norm": 73.9625244140625,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.48971864581108093,
"logits/rejected": -0.4594026803970337,
"logps/chosen": -52.52928924560547,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.52372741699219,
"loss": 1.368,
"margin_dpo/margin_mean": 0.18930193781852722,
"margin_dpo/margin_std": 0.4436342120170593,
"step": 25
},
{
"KL/chosen_KL_mean": 0.08230400085449219,
"KL/mean": -0.10930752754211426,
"KL/rejected_KL_mean": -0.3009147644042969,
"KL/std": 0.40695035457611084,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.383225679397583,
"fcm_dpo/q_t": 0.49042731523513794,
"grad_norm": 87.4775390625,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5517945289611816,
"logits/rejected": -0.52059006690979,
"logps/chosen": -53.84050369262695,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.66062927246094,
"loss": 1.3489,
"margin_dpo/margin_mean": 0.3832261562347412,
"margin_dpo/margin_std": 0.4770854711532593,
"step": 26
},
{
"KL/chosen_KL_mean": 0.12397003173828125,
"KL/mean": -0.1301194131374359,
"KL/rejected_KL_mean": -0.38420867919921875,
"KL/std": 0.47300222516059875,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5081788301467896,
"fcm_dpo/q_t": 0.48731520771980286,
"grad_norm": 92.92485046386719,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.5351539850234985,
"logits/rejected": -0.49897927045822144,
"logps/chosen": -42.774559020996094,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.10841369628906,
"loss": 1.337,
"margin_dpo/margin_mean": 0.5081784725189209,
"margin_dpo/margin_std": 0.578755795955658,
"step": 27
},
{
"KL/chosen_KL_mean": 0.037380218505859375,
"KL/mean": -0.13808989524841309,
"KL/rejected_KL_mean": -0.3135643005371094,
"KL/std": 0.395422101020813,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.35093891620635986,
"fcm_dpo/q_t": 0.49123209714889526,
"grad_norm": 76.57662963867188,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.5218425989151001,
"logits/rejected": -0.46810781955718994,
"logps/chosen": -60.51911926269531,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.71467590332031,
"loss": 1.3521,
"margin_dpo/margin_mean": 0.3509393632411957,
"margin_dpo/margin_std": 0.4972136616706848,
"step": 28
},
{
"KL/chosen_KL_mean": 0.07178878784179688,
"KL/mean": -0.212729811668396,
"KL/rejected_KL_mean": -0.4972419738769531,
"KL/std": 0.48936769366264343,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5690329670906067,
"fcm_dpo/q_t": 0.48579180240631104,
"grad_norm": 90.49282836914062,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.5380536317825317,
"logits/rejected": -0.4905041456222534,
"logps/chosen": -57.73599624633789,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.89159393310547,
"loss": 1.331,
"margin_dpo/margin_mean": 0.5690339803695679,
"margin_dpo/margin_std": 0.5614117980003357,
"step": 29
},
{
"KL/chosen_KL_mean": 0.12009620666503906,
"KL/mean": -0.20123910903930664,
"KL/rejected_KL_mean": -0.5225753784179688,
"KL/std": 0.5947461724281311,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6426688432693481,
"fcm_dpo/q_t": 0.48396098613739014,
"grad_norm": 87.52836608886719,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.47559118270874023,
"logits/rejected": -0.4440673589706421,
"logps/chosen": -52.457275390625,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -99.01177978515625,
"loss": 1.3242,
"margin_dpo/margin_mean": 0.642669141292572,
"margin_dpo/margin_std": 0.6606372594833374,
"step": 30
},
{
"KL/chosen_KL_mean": 0.09145545959472656,
"KL/mean": -0.1492789387702942,
"KL/rejected_KL_mean": -0.3900146484375,
"KL/std": 0.5775303840637207,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4814666509628296,
"fcm_dpo/q_t": 0.4879956841468811,
"grad_norm": 67.94648742675781,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.4888034462928772,
"logits/rejected": -0.4421996772289276,
"logps/chosen": -63.7154655456543,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.28401947021484,
"loss": 1.3403,
"margin_dpo/margin_mean": 0.4814665913581848,
"margin_dpo/margin_std": 0.7748575210571289,
"step": 31
},
{
"KL/chosen_KL_mean": 0.15587234497070312,
"KL/mean": -0.18827323615550995,
"KL/rejected_KL_mean": -0.5324172973632812,
"KL/std": 0.6637758612632751,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6882903575897217,
"fcm_dpo/q_t": 0.48283424973487854,
"grad_norm": 83.79432678222656,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.5092083215713501,
"logits/rejected": -0.4677237272262573,
"logps/chosen": -62.58365249633789,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.84991455078125,
"loss": 1.3202,
"margin_dpo/margin_mean": 0.6882905960083008,
"margin_dpo/margin_std": 0.776940643787384,
"step": 32
},
{
"KL/chosen_KL_mean": 0.12227249145507812,
"KL/mean": -0.19456741213798523,
"KL/rejected_KL_mean": -0.5114059448242188,
"KL/std": 0.5843777656555176,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6336773633956909,
"fcm_dpo/q_t": 0.48418158292770386,
"grad_norm": 71.57157135009766,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.4876636266708374,
"logits/rejected": -0.46141698956489563,
"logps/chosen": -53.138702392578125,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.39654541015625,
"loss": 1.325,
"margin_dpo/margin_mean": 0.6336771249771118,
"margin_dpo/margin_std": 0.6494045257568359,
"step": 33
},
{
"KL/chosen_KL_mean": 0.09919548034667969,
"KL/mean": -0.3045317530632019,
"KL/rejected_KL_mean": -0.7082595825195312,
"KL/std": 0.7005865573883057,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10320889204740524,
"fcm_dpo/delta": 0.1579238921403885,
"fcm_dpo/margin": 0.8074537515640259,
"fcm_dpo/q_t": 0.4795566201210022,
"grad_norm": 81.28727722167969,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.525603175163269,
"logits/rejected": -0.5081762671470642,
"logps/chosen": -50.71813201904297,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.6301040649414,
"loss": 1.3065,
"margin_dpo/margin_mean": 0.8074527382850647,
"margin_dpo/margin_std": 0.8205959796905518,
"step": 34
},
{
"KL/chosen_KL_mean": 0.16864395141601562,
"KL/mean": -0.4522685408592224,
"KL/rejected_KL_mean": -1.0731773376464844,
"KL/std": 0.9934526681900024,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10746243596076965,
"fcm_dpo/delta": 0.2701945900917053,
"fcm_dpo/margin": 1.2418255805969238,
"fcm_dpo/q_t": 0.4677559733390808,
"grad_norm": 87.83883666992188,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.5124801397323608,
"logits/rejected": -0.47577357292175293,
"logps/chosen": -50.8558464050293,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.8976058959961,
"loss": 1.2608,
"margin_dpo/margin_mean": 1.2418255805969238,
"margin_dpo/margin_std": 1.0855079889297485,
"step": 35
},
{
"KL/chosen_KL_mean": 0.013975143432617188,
"KL/mean": -0.5303745865821838,
"KL/rejected_KL_mean": -1.0747222900390625,
"KL/std": 1.0487756729125977,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.11381839960813522,
"fcm_dpo/delta": 0.27891629934310913,
"fcm_dpo/margin": 1.0886973142623901,
"fcm_dpo/q_t": 0.46992403268814087,
"grad_norm": 81.70454406738281,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5460728406906128,
"logits/rejected": -0.5091354846954346,
"logps/chosen": -51.977516174316406,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.11534881591797,
"loss": 1.2707,
"margin_dpo/margin_mean": 1.0886969566345215,
"margin_dpo/margin_std": 1.171996831893921,
"step": 36
},
{
"KL/chosen_KL_mean": 0.023801803588867188,
"KL/mean": -0.5040315985679626,
"KL/rejected_KL_mean": -1.0318717956542969,
"KL/std": 1.0373797416687012,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.1198909655213356,
"fcm_dpo/delta": 0.2773377299308777,
"fcm_dpo/margin": 1.0556702613830566,
"fcm_dpo/q_t": 0.46964770555496216,
"grad_norm": 72.01265716552734,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.5176362991333008,
"logits/rejected": -0.4745715260505676,
"logps/chosen": -62.78330612182617,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.92694091796875,
"loss": 1.2708,
"margin_dpo/margin_mean": 1.0556700229644775,
"margin_dpo/margin_std": 1.3116538524627686,
"step": 37
},
{
"KL/chosen_KL_mean": 0.17965316772460938,
"KL/mean": -0.5454572439193726,
"KL/rejected_KL_mean": -1.2705650329589844,
"KL/std": 1.3868043422698975,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.12598319351673126,
"fcm_dpo/delta": 0.22110190987586975,
"fcm_dpo/margin": 1.4502217769622803,
"fcm_dpo/q_t": 0.45613712072372437,
"grad_norm": 84.13441467285156,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5286298394203186,
"logits/rejected": -0.49596402049064636,
"logps/chosen": -48.21086883544922,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.1830062866211,
"loss": 1.2227,
"margin_dpo/margin_mean": 1.4502220153808594,
"margin_dpo/margin_std": 1.6653861999511719,
"step": 38
},
{
"KL/chosen_KL_mean": 0.07242012023925781,
"KL/mean": -0.7552523016929626,
"KL/rejected_KL_mean": -1.5829238891601562,
"KL/std": 1.3359938859939575,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.13071495294570923,
"fcm_dpo/delta": 0.18790489435195923,
"fcm_dpo/margin": 1.6553417444229126,
"fcm_dpo/q_t": 0.44766077399253845,
"grad_norm": 111.4832534790039,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.5785598754882812,
"logits/rejected": -0.5401351451873779,
"logps/chosen": -50.67805480957031,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.15243530273438,
"loss": 1.1908,
"margin_dpo/margin_mean": 1.6553418636322021,
"margin_dpo/margin_std": 1.4348034858703613,
"step": 39
},
{
"KL/chosen_KL_mean": 0.14400482177734375,
"KL/mean": -0.60335373878479,
"KL/rejected_KL_mean": -1.3507156372070312,
"KL/std": 1.3741190433502197,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.13639971613883972,
"fcm_dpo/delta": 0.19976407289505005,
"fcm_dpo/margin": 1.494720220565796,
"fcm_dpo/q_t": 0.45102059841156006,
"grad_norm": 78.08142852783203,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5232949256896973,
"logits/rejected": -0.4935886859893799,
"logps/chosen": -57.841064453125,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.65078735351562,
"loss": 1.2059,
"margin_dpo/margin_mean": 1.4947205781936646,
"margin_dpo/margin_std": 1.6571329832077026,
"step": 40
},
{
"KL/chosen_KL_mean": 0.04059600830078125,
"KL/mean": -0.9012515544891357,
"KL/rejected_KL_mean": -1.8430938720703125,
"KL/std": 1.8287923336029053,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.1405719816684723,
"fcm_dpo/delta": 0.1386878788471222,
"fcm_dpo/margin": 1.8836982250213623,
"fcm_dpo/q_t": 0.43659090995788574,
"grad_norm": 89.06941223144531,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.5429688692092896,
"logits/rejected": -0.5066103339195251,
"logps/chosen": -62.65522003173828,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.86662292480469,
"loss": 1.1586,
"margin_dpo/margin_mean": 1.8836987018585205,
"margin_dpo/margin_std": 1.968984842300415,
"step": 41
},
{
"KL/chosen_KL_mean": 0.21233558654785156,
"KL/mean": -1.0596989393234253,
"KL/rejected_KL_mean": -2.3317337036132812,
"KL/std": 2.1389851570129395,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.14226916432380676,
"fcm_dpo/delta": 0.03947313129901886,
"fcm_dpo/margin": 2.5440683364868164,
"fcm_dpo/q_t": 0.4139750599861145,
"grad_norm": 101.21937561035156,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.5372546315193176,
"logits/rejected": -0.49066978693008423,
"logps/chosen": -58.75408935546875,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.2401123046875,
"loss": 1.084,
"margin_dpo/margin_mean": 2.5440685749053955,
"margin_dpo/margin_std": 2.3744425773620605,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5405921936035156,
"KL/mean": -0.6949355602264404,
"KL/rejected_KL_mean": -1.9304618835449219,
"KL/std": 1.8301477432250977,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.14327695965766907,
"fcm_dpo/delta": 0.047662679105997086,
"fcm_dpo/margin": 2.4710512161254883,
"fcm_dpo/q_t": 0.4148353338241577,
"grad_norm": 93.92181396484375,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.5310695171356201,
"logits/rejected": -0.5056588649749756,
"logps/chosen": -53.61540222167969,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.41065216064453,
"loss": 1.0826,
"margin_dpo/margin_mean": 2.47105073928833,
"margin_dpo/margin_std": 1.9682235717773438,
"step": 43
},
{
"KL/chosen_KL_mean": 0.2531108856201172,
"KL/mean": -1.1532858610153198,
"KL/rejected_KL_mean": -2.559680938720703,
"KL/std": 2.2799549102783203,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.14362391829490662,
"fcm_dpo/delta": -0.004183035343885422,
"fcm_dpo/margin": 2.8127894401550293,
"fcm_dpo/q_t": 0.4032197594642639,
"grad_norm": 101.86164855957031,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.4643116593360901,
"logits/rejected": -0.4446447789669037,
"logps/chosen": -49.82538604736328,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.34344482421875,
"loss": 1.0471,
"margin_dpo/margin_mean": 2.812788963317871,
"margin_dpo/margin_std": 2.241828203201294,
"step": 44
},
{
"KL/chosen_KL_mean": 0.11113929748535156,
"KL/mean": -0.9870209693908691,
"KL/rejected_KL_mean": -2.0851821899414062,
"KL/std": 2.0045013427734375,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.14574244618415833,
"fcm_dpo/delta": 0.08238682150840759,
"fcm_dpo/margin": 2.1963233947753906,
"fcm_dpo/q_t": 0.4246715307235718,
"grad_norm": 82.72820281982422,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.4895831346511841,
"logits/rejected": -0.47717034816741943,
"logps/chosen": -48.30378723144531,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.0216064453125,
"loss": 1.1231,
"margin_dpo/margin_mean": 2.1963229179382324,
"margin_dpo/margin_std": 2.4882490634918213,
"step": 45
},
{
"KL/chosen_KL_mean": 0.18288421630859375,
"KL/mean": -1.276892900466919,
"KL/rejected_KL_mean": -2.736675262451172,
"KL/std": 2.590786933898926,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.14583346247673035,
"fcm_dpo/delta": -0.026936061680316925,
"fcm_dpo/margin": 2.9195570945739746,
"fcm_dpo/q_t": 0.4010394215583801,
"grad_norm": 87.75068664550781,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.5153135657310486,
"logits/rejected": -0.463581919670105,
"logps/chosen": -55.81654357910156,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.38926696777344,
"loss": 1.0532,
"margin_dpo/margin_mean": 2.9195570945739746,
"margin_dpo/margin_std": 3.142143726348877,
"step": 46
},
{
"KL/chosen_KL_mean": 0.387847900390625,
"KL/mean": -1.0973193645477295,
"KL/rejected_KL_mean": -2.582489013671875,
"KL/std": 2.5982861518859863,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.14528387784957886,
"fcm_dpo/delta": -0.03321390599012375,
"fcm_dpo/margin": 2.970334529876709,
"fcm_dpo/q_t": 0.39802086353302,
"grad_norm": 84.18961334228516,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.5865252614021301,
"logits/rejected": -0.5347657799720764,
"logps/chosen": -57.538230895996094,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.2616958618164,
"loss": 1.0386,
"margin_dpo/margin_mean": 2.9703352451324463,
"margin_dpo/margin_std": 2.69755482673645,
"step": 47
},
{
"KL/chosen_KL_mean": 0.10560417175292969,
"KL/mean": -1.441537857055664,
"KL/rejected_KL_mean": -2.988677978515625,
"KL/std": 2.643078565597534,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.14288130402565002,
"fcm_dpo/delta": -0.04455402493476868,
"fcm_dpo/margin": 3.0942819118499756,
"fcm_dpo/q_t": 0.39582157135009766,
"grad_norm": 93.56952667236328,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.5885308980941772,
"logits/rejected": -0.5303641557693481,
"logps/chosen": -57.08246994018555,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -91.00527954101562,
"loss": 1.0395,
"margin_dpo/margin_mean": 3.0942819118499756,
"margin_dpo/margin_std": 2.9218857288360596,
"step": 48
},
{
"KL/chosen_KL_mean": 0.33449745178222656,
"KL/mean": -1.5316673517227173,
"KL/rejected_KL_mean": -3.3978309631347656,
"KL/std": 3.353222370147705,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.1409154236316681,
"fcm_dpo/delta": -0.13273698091506958,
"fcm_dpo/margin": 3.732323169708252,
"fcm_dpo/q_t": 0.38071030378341675,
"grad_norm": 92.1107177734375,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.5483835935592651,
"logits/rejected": -0.4899318218231201,
"logps/chosen": -61.350772857666016,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.1653060913086,
"loss": 0.9959,
"margin_dpo/margin_mean": 3.7323226928710938,
"margin_dpo/margin_std": 3.8772082328796387,
"step": 49
},
{
"KL/chosen_KL_mean": -0.16253089904785156,
"KL/mean": -2.2148306369781494,
"KL/rejected_KL_mean": -4.267131805419922,
"KL/std": 3.460838794708252,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.13572102785110474,
"fcm_dpo/delta": -0.16670596599578857,
"fcm_dpo/margin": 4.104592800140381,
"fcm_dpo/q_t": 0.371456503868103,
"grad_norm": 75.59302520751953,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5448007583618164,
"logits/rejected": -0.5087927579879761,
"logps/chosen": -58.88666915893555,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.62527465820312,
"loss": 0.9677,
"margin_dpo/margin_mean": 4.104592800140381,
"margin_dpo/margin_std": 3.9096031188964844,
"step": 50
},
{
"KL/chosen_KL_mean": -0.28087806701660156,
"KL/mean": -2.270071506500244,
"KL/rejected_KL_mean": -4.259269714355469,
"KL/std": 4.135315895080566,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.1313726007938385,
"fcm_dpo/delta": -0.1310398280620575,
"fcm_dpo/margin": 3.978391408920288,
"fcm_dpo/q_t": 0.3860105872154236,
"grad_norm": 61.80961990356445,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.5152701735496521,
"logits/rejected": -0.4818602204322815,
"logps/chosen": -61.65454864501953,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.26127624511719,
"loss": 1.0265,
"margin_dpo/margin_mean": 3.978390693664551,
"margin_dpo/margin_std": 5.0646772384643555,
"step": 51
},
{
"KL/chosen_KL_mean": 0.36811256408691406,
"KL/mean": -2.65940523147583,
"KL/rejected_KL_mean": -5.686923980712891,
"KL/std": 4.684706687927246,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.12406842410564423,
"fcm_dpo/delta": -0.3800222873687744,
"fcm_dpo/margin": 6.0550360679626465,
"fcm_dpo/q_t": 0.33053886890411377,
"grad_norm": 65.30122375488281,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.5419540405273438,
"logits/rejected": -0.48573756217956543,
"logps/chosen": -51.96924591064453,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.66084289550781,
"loss": 0.8506,
"margin_dpo/margin_mean": 6.055036544799805,
"margin_dpo/margin_std": 4.9989800453186035,
"step": 52
},
{
"KL/chosen_KL_mean": -0.18677711486816406,
"KL/mean": -3.213822841644287,
"KL/rejected_KL_mean": -6.240867614746094,
"KL/std": 5.253150939941406,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.11692243814468384,
"fcm_dpo/delta": -0.3300025165081024,
"fcm_dpo/margin": 6.054093360900879,
"fcm_dpo/q_t": 0.34951359033584595,
"grad_norm": 64.69686126708984,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.6237785816192627,
"logits/rejected": -0.6024104356765747,
"logps/chosen": -53.501426696777344,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.02445983886719,
"loss": 0.9121,
"margin_dpo/margin_mean": 6.054093360900879,
"margin_dpo/margin_std": 6.0293989181518555,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3544273376464844,
"KL/mean": -3.032489538192749,
"KL/rejected_KL_mean": -5.710548400878906,
"KL/std": 5.030124664306641,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.1107134222984314,
"fcm_dpo/delta": -0.2048681676387787,
"fcm_dpo/margin": 5.356122970581055,
"fcm_dpo/q_t": 0.36526209115982056,
"grad_norm": 58.66542053222656,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.5997574329376221,
"logits/rejected": -0.5465754270553589,
"logps/chosen": -51.0430908203125,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.42594146728516,
"loss": 0.9456,
"margin_dpo/margin_mean": 5.3561224937438965,
"margin_dpo/margin_std": 5.040410995483398,
"step": 54
},
{
"KL/chosen_KL_mean": -0.8778724670410156,
"KL/mean": -4.0574259757995605,
"KL/rejected_KL_mean": -7.236972808837891,
"KL/std": 6.0431365966796875,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.1051725447177887,
"fcm_dpo/delta": -0.2873228192329407,
"fcm_dpo/margin": 6.359100341796875,
"fcm_dpo/q_t": 0.3560294508934021,
"grad_norm": 55.624202728271484,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6527860760688782,
"logits/rejected": -0.5910372734069824,
"logps/chosen": -63.493106842041016,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.23047637939453,
"loss": 0.9494,
"margin_dpo/margin_mean": 6.359099388122559,
"margin_dpo/margin_std": 7.310431480407715,
"step": 55
},
{
"KL/chosen_KL_mean": -0.6478157043457031,
"KL/mean": -3.6710574626922607,
"KL/rejected_KL_mean": -6.694305419921875,
"KL/std": 5.903786659240723,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.10002656280994415,
"fcm_dpo/delta": -0.21787334978580475,
"fcm_dpo/margin": 6.0464887619018555,
"fcm_dpo/q_t": 0.3679868280887604,
"grad_norm": 50.561851501464844,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.6278807520866394,
"logits/rejected": -0.586266040802002,
"logps/chosen": -58.580543518066406,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -100.86874389648438,
"loss": 0.9811,
"margin_dpo/margin_mean": 6.0464887619018555,
"margin_dpo/margin_std": 7.142303466796875,
"step": 56
},
{
"KL/chosen_KL_mean": -0.682220458984375,
"KL/mean": -4.073773384094238,
"KL/rejected_KL_mean": -7.46533203125,
"KL/std": 5.824618339538574,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.0954844132065773,
"fcm_dpo/delta": -0.2640697658061981,
"fcm_dpo/margin": 6.783110618591309,
"fcm_dpo/q_t": 0.3528316020965576,
"grad_norm": 53.0193977355957,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.5914736986160278,
"logits/rejected": -0.563991129398346,
"logps/chosen": -71.17750549316406,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -103.03079223632812,
"loss": 0.9247,
"margin_dpo/margin_mean": 6.78311014175415,
"margin_dpo/margin_std": 6.405632972717285,
"step": 57
},
{
"KL/chosen_KL_mean": -0.9351482391357422,
"KL/mean": -4.74384069442749,
"KL/rejected_KL_mean": -8.552532196044922,
"KL/std": 6.671572208404541,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.0899924710392952,
"fcm_dpo/delta": -0.3056134581565857,
"fcm_dpo/margin": 7.6173810958862305,
"fcm_dpo/q_t": 0.3502853810787201,
"grad_norm": 54.50084686279297,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.6031397581100464,
"logits/rejected": -0.5278700590133667,
"logps/chosen": -63.06808853149414,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -93.16983032226562,
"loss": 0.9236,
"margin_dpo/margin_mean": 7.617381572723389,
"margin_dpo/margin_std": 7.944563865661621,
"step": 58
},
{
"KL/chosen_KL_mean": -1.396148681640625,
"KL/mean": -5.524988174438477,
"KL/rejected_KL_mean": -9.653823852539062,
"KL/std": 7.2611308097839355,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.08342745900154114,
"fcm_dpo/delta": -0.3148040175437927,
"fcm_dpo/margin": 8.257675170898438,
"fcm_dpo/q_t": 0.3485872745513916,
"grad_norm": 51.20856475830078,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.6402565836906433,
"logits/rejected": -0.6007430553436279,
"logps/chosen": -53.32867431640625,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.53902435302734,
"loss": 0.9173,
"margin_dpo/margin_mean": 8.257675170898438,
"margin_dpo/margin_std": 8.405501365661621,
"step": 59
},
{
"KL/chosen_KL_mean": -2.5095481872558594,
"KL/mean": -5.817296028137207,
"KL/rejected_KL_mean": -9.12503433227539,
"KL/std": 6.605817794799805,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.08078856766223907,
"fcm_dpo/delta": -0.14282643795013428,
"fcm_dpo/margin": 6.615485191345215,
"fcm_dpo/q_t": 0.37698274850845337,
"grad_norm": 54.96247100830078,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.6214213371276855,
"logits/rejected": -0.5622453093528748,
"logps/chosen": -63.45173645019531,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.5184326171875,
"loss": 1.0065,
"margin_dpo/margin_mean": 6.615485191345215,
"margin_dpo/margin_std": 7.516720771789551,
"step": 60
},
{
"KL/chosen_KL_mean": -1.354644775390625,
"KL/mean": -5.456735610961914,
"KL/rejected_KL_mean": -9.558830261230469,
"KL/std": 8.855253219604492,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.07721322774887085,
"fcm_dpo/delta": -0.25074315071105957,
"fcm_dpo/margin": 8.20418930053711,
"fcm_dpo/q_t": 0.36832770705223083,
"grad_norm": 45.67434310913086,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.597309947013855,
"logits/rejected": -0.5627981424331665,
"logps/chosen": -61.98816680908203,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -99.41133117675781,
"loss": 0.9848,
"margin_dpo/margin_mean": 8.20418930053711,
"margin_dpo/margin_std": 10.933375358581543,
"step": 61
},
{
"KL/chosen_KL_mean": -1.5534076690673828,
"KL/mean": -4.591529369354248,
"KL/rejected_KL_mean": -7.629646301269531,
"KL/std": 6.465494155883789,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.07611910998821259,
"fcm_dpo/delta": -0.06562487781047821,
"fcm_dpo/margin": 6.076238632202148,
"fcm_dpo/q_t": 0.39654144644737244,
"grad_norm": 44.995994567871094,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.6019065380096436,
"logits/rejected": -0.5675424337387085,
"logps/chosen": -57.70417785644531,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -83.19584655761719,
"loss": 1.0601,
"margin_dpo/margin_mean": 6.076239109039307,
"margin_dpo/margin_std": 7.934353351593018,
"step": 62
},
{
"KL/chosen_KL_mean": -2.580839157104492,
"KL/mean": -6.7052154541015625,
"KL/rejected_KL_mean": -10.82958984375,
"KL/std": 8.269981384277344,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.07275725901126862,
"fcm_dpo/delta": -0.21487507224082947,
"fcm_dpo/margin": 8.248749732971191,
"fcm_dpo/q_t": 0.3637925684452057,
"grad_norm": 45.84245681762695,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.6152663826942444,
"logits/rejected": -0.5709649920463562,
"logps/chosen": -75.72823333740234,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -108.43965148925781,
"loss": 0.9641,
"margin_dpo/margin_mean": 8.248749732971191,
"margin_dpo/margin_std": 8.703377723693848,
"step": 63
},
{
"KL/chosen_KL_mean": -0.9510021209716797,
"KL/mean": -5.819749355316162,
"KL/rejected_KL_mean": -10.688499450683594,
"KL/std": 8.974802017211914,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.06858538091182709,
"fcm_dpo/delta": -0.2923140823841095,
"fcm_dpo/margin": 9.737493515014648,
"fcm_dpo/q_t": 0.3547326326370239,
"grad_norm": 42.15247344970703,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.5690383911132812,
"logits/rejected": -0.5363543033599854,
"logps/chosen": -54.94960021972656,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -104.21870422363281,
"loss": 0.9457,
"margin_dpo/margin_mean": 9.737493515014648,
"margin_dpo/margin_std": 11.002016067504883,
"step": 64
},
{
"KL/chosen_KL_mean": -3.15191650390625,
"KL/mean": -8.072813034057617,
"KL/rejected_KL_mean": -12.99371337890625,
"KL/std": 10.051557540893555,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.06531103700399399,
"fcm_dpo/delta": -0.2614961266517639,
"fcm_dpo/margin": 9.841796875,
"fcm_dpo/q_t": 0.3564883768558502,
"grad_norm": 42.087825775146484,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6592050790786743,
"logits/rejected": -0.646178662776947,
"logps/chosen": -67.9879150390625,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -122.94017028808594,
"loss": 0.9588,
"margin_dpo/margin_mean": 9.841795921325684,
"margin_dpo/margin_std": 11.036256790161133,
"step": 65
},
{
"KL/chosen_KL_mean": -2.923826217651367,
"KL/mean": -7.444736480712891,
"KL/rejected_KL_mean": -11.965641021728516,
"KL/std": 9.111772537231445,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.06262894719839096,
"fcm_dpo/delta": -0.1786663979291916,
"fcm_dpo/margin": 9.041810989379883,
"fcm_dpo/q_t": 0.3762148320674896,
"grad_norm": 37.77988815307617,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.6611512899398804,
"logits/rejected": -0.6294161081314087,
"logps/chosen": -54.36735153198242,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -87.60194396972656,
"loss": 0.9981,
"margin_dpo/margin_mean": 9.041810989379883,
"margin_dpo/margin_std": 11.0140380859375,
"step": 66
},
{
"KL/chosen_KL_mean": -2.5404319763183594,
"KL/mean": -7.237859725952148,
"KL/rejected_KL_mean": -11.935287475585938,
"KL/std": 9.200343132019043,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.061006784439086914,
"fcm_dpo/delta": -0.1833253800868988,
"fcm_dpo/margin": 9.394853591918945,
"fcm_dpo/q_t": 0.3741205930709839,
"grad_norm": 37.25181579589844,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.5752357244491577,
"logits/rejected": -0.5304913520812988,
"logps/chosen": -61.8812370300293,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -84.72257232666016,
"loss": 0.991,
"margin_dpo/margin_mean": 9.394853591918945,
"margin_dpo/margin_std": 11.157440185546875,
"step": 67
},
{
"KL/chosen_KL_mean": -3.007139205932617,
"KL/mean": -7.437971115112305,
"KL/rejected_KL_mean": -11.86880111694336,
"KL/std": 8.222919464111328,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.05923429876565933,
"fcm_dpo/delta": -0.13163843750953674,
"fcm_dpo/margin": 8.861660957336426,
"fcm_dpo/q_t": 0.3803149461746216,
"grad_norm": 37.139434814453125,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6381473541259766,
"logits/rejected": -0.5780563354492188,
"logps/chosen": -68.21297454833984,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -89.0760498046875,
"loss": 0.9965,
"margin_dpo/margin_mean": 8.86166000366211,
"margin_dpo/margin_std": 9.258892059326172,
"step": 68
},
{
"KL/chosen_KL_mean": -3.585367202758789,
"KL/mean": -9.017387390136719,
"KL/rejected_KL_mean": -14.449409484863281,
"KL/std": 9.787298202514648,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.056480493396520615,
"fcm_dpo/delta": -0.22901439666748047,
"fcm_dpo/margin": 10.864038467407227,
"fcm_dpo/q_t": 0.3623312711715698,
"grad_norm": 39.13850402832031,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.6019381284713745,
"logits/rejected": -0.5787031650543213,
"logps/chosen": -63.40460968017578,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -117.83827209472656,
"loss": 0.9433,
"margin_dpo/margin_mean": 10.864038467407227,
"margin_dpo/margin_std": 10.782807350158691,
"step": 69
},
{
"KL/chosen_KL_mean": -5.221946716308594,
"KL/mean": -11.105976104736328,
"KL/rejected_KL_mean": -16.990005493164062,
"KL/std": 12.004703521728516,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.054179318249225616,
"fcm_dpo/delta": -0.2531682252883911,
"fcm_dpo/margin": 11.768058776855469,
"fcm_dpo/q_t": 0.3599928617477417,
"grad_norm": 40.161224365234375,
"learning_rate": 5e-07,
"logits/chosen": -0.5996398329734802,
"logits/rejected": -0.5637354850769043,
"logps/chosen": -67.152587890625,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -108.05078887939453,
"loss": 0.951,
"margin_dpo/margin_mean": 11.768059730529785,
"margin_dpo/margin_std": 12.794807434082031,
"step": 70
},
{
"KL/chosen_KL_mean": -5.026617050170898,
"KL/mean": -11.98879623413086,
"KL/rejected_KL_mean": -18.95098114013672,
"KL/std": 12.658889770507812,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.050611525774002075,
"fcm_dpo/delta": -0.3290429711341858,
"fcm_dpo/margin": 13.924360275268555,
"fcm_dpo/q_t": 0.34701576828956604,
"grad_norm": 36.80656814575195,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.6452410221099854,
"logits/rejected": -0.6052732467651367,
"logps/chosen": -66.7769546508789,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -116.28759765625,
"loss": 0.9036,
"margin_dpo/margin_mean": 13.924360275268555,
"margin_dpo/margin_std": 13.787056922912598,
"step": 71
},
{
"KL/chosen_KL_mean": -5.833978652954102,
"KL/mean": -13.214212417602539,
"KL/rejected_KL_mean": -20.59444808959961,
"KL/std": 13.704126358032227,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.047448836266994476,
"fcm_dpo/delta": -0.3239469826221466,
"fcm_dpo/margin": 14.76047420501709,
"fcm_dpo/q_t": 0.34694284200668335,
"grad_norm": 37.849456787109375,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.6686690449714661,
"logits/rejected": -0.6333932280540466,
"logps/chosen": -71.88739013671875,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -115.88143920898438,
"loss": 0.9216,
"margin_dpo/margin_mean": 14.76047420501709,
"margin_dpo/margin_std": 15.948001861572266,
"step": 72
},
{
"KL/chosen_KL_mean": -7.895063400268555,
"KL/mean": -15.281990051269531,
"KL/rejected_KL_mean": -22.668926239013672,
"KL/std": 17.924169540405273,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.04465475678443909,
"fcm_dpo/delta": -0.27992942929267883,
"fcm_dpo/margin": 14.7738618850708,
"fcm_dpo/q_t": 0.36867302656173706,
"grad_norm": 36.75870132446289,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.6891875267028809,
"logits/rejected": -0.6485644578933716,
"logps/chosen": -74.15133666992188,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -113.12506103515625,
"loss": 1.0298,
"margin_dpo/margin_mean": 14.773862838745117,
"margin_dpo/margin_std": 22.38170623779297,
"step": 73
},
{
"KL/chosen_KL_mean": -8.47895622253418,
"KL/mean": -16.591014862060547,
"KL/rejected_KL_mean": -24.70307159423828,
"KL/std": 19.87729263305664,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.04214803874492645,
"fcm_dpo/delta": -0.30570995807647705,
"fcm_dpo/margin": 16.224117279052734,
"fcm_dpo/q_t": 0.36257338523864746,
"grad_norm": 38.15811538696289,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.6253733038902283,
"logits/rejected": -0.6167181730270386,
"logps/chosen": -61.903839111328125,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -120.65000915527344,
"loss": 0.9935,
"margin_dpo/margin_mean": 16.224117279052734,
"margin_dpo/margin_std": 24.648841857910156,
"step": 74
},
{
"KL/chosen_KL_mean": -8.299543380737305,
"KL/mean": -19.795719146728516,
"KL/rejected_KL_mean": -31.291885375976562,
"KL/std": 20.560977935791016,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.03871320188045502,
"fcm_dpo/delta": -0.5353958010673523,
"fcm_dpo/margin": 22.992351531982422,
"fcm_dpo/q_t": 0.317760705947876,
"grad_norm": 32.1464729309082,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6416128873825073,
"logits/rejected": -0.6341755390167236,
"logps/chosen": -60.16120529174805,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -142.54586791992188,
"loss": 0.8396,
"margin_dpo/margin_mean": 22.992347717285156,
"margin_dpo/margin_std": 23.244129180908203,
"step": 75
},
{
"KL/chosen_KL_mean": -10.1375732421875,
"KL/mean": -17.43459701538086,
"KL/rejected_KL_mean": -24.731616973876953,
"KL/std": 16.729257583618164,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.036620307713747025,
"fcm_dpo/delta": -0.14195583760738373,
"fcm_dpo/margin": 14.594053268432617,
"fcm_dpo/q_t": 0.3785336911678314,
"grad_norm": 32.55298614501953,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.6936876773834229,
"logits/rejected": -0.6513668894767761,
"logps/chosen": -63.40361022949219,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -102.9482421875,
"loss": 1.0197,
"margin_dpo/margin_mean": 14.594053268432617,
"margin_dpo/margin_std": 18.254350662231445,
"step": 76
},
{
"KL/chosen_KL_mean": -9.1678466796875,
"KL/mean": -21.043556213378906,
"KL/rejected_KL_mean": -32.91926574707031,
"KL/std": 22.885698318481445,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.033799584954977036,
"fcm_dpo/delta": -0.44206297397613525,
"fcm_dpo/margin": 23.751419067382812,
"fcm_dpo/q_t": 0.32698968052864075,
"grad_norm": 33.31752395629883,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.6848742961883545,
"logits/rejected": -0.6468466520309448,
"logps/chosen": -67.2645263671875,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -126.69287872314453,
"loss": 0.8878,
"margin_dpo/margin_mean": 23.751419067382812,
"margin_dpo/margin_std": 25.314167022705078,
"step": 77
},
{
"KL/chosen_KL_mean": -9.164955139160156,
"KL/mean": -18.119544982910156,
"KL/rejected_KL_mean": -27.074134826660156,
"KL/std": 18.446874618530273,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.03241837024688721,
"fcm_dpo/delta": -0.19134587049484253,
"fcm_dpo/margin": 17.909175872802734,
"fcm_dpo/q_t": 0.37250542640686035,
"grad_norm": 30.10005760192871,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.6356874704360962,
"logits/rejected": -0.6102886199951172,
"logps/chosen": -64.77873992919922,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -112.00849914550781,
"loss": 0.9917,
"margin_dpo/margin_mean": 17.909175872802734,
"margin_dpo/margin_std": 21.184471130371094,
"step": 78
},
{
"KL/chosen_KL_mean": -9.568305969238281,
"KL/mean": -19.395065307617188,
"KL/rejected_KL_mean": -29.221824645996094,
"KL/std": 20.979488372802734,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.030958333984017372,
"fcm_dpo/delta": -0.22205796837806702,
"fcm_dpo/margin": 19.65351676940918,
"fcm_dpo/q_t": 0.3720153868198395,
"grad_norm": 26.722917556762695,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6731311082839966,
"logits/rejected": -0.6446127891540527,
"logps/chosen": -65.01879119873047,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -116.869384765625,
"loss": 1.0015,
"margin_dpo/margin_mean": 19.653514862060547,
"margin_dpo/margin_std": 25.67258071899414,
"step": 79
},
{
"KL/chosen_KL_mean": -12.641990661621094,
"KL/mean": -21.817380905151367,
"KL/rejected_KL_mean": -30.99276351928711,
"KL/std": 22.139293670654297,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.02968379482626915,
"fcm_dpo/delta": -0.155572310090065,
"fcm_dpo/margin": 18.35076904296875,
"fcm_dpo/q_t": 0.3862488269805908,
"grad_norm": 29.31393814086914,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.708533763885498,
"logits/rejected": -0.6655611991882324,
"logps/chosen": -71.16128540039062,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -118.54026794433594,
"loss": 1.0441,
"margin_dpo/margin_mean": 18.35076904296875,
"margin_dpo/margin_std": 25.68282699584961,
"step": 80
},
{
"KL/chosen_KL_mean": -11.637100219726562,
"KL/mean": -24.316699981689453,
"KL/rejected_KL_mean": -36.996299743652344,
"KL/std": 26.420780181884766,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.02805560827255249,
"fcm_dpo/delta": -0.33825811743736267,
"fcm_dpo/margin": 25.35919761657715,
"fcm_dpo/q_t": 0.351850688457489,
"grad_norm": 29.6393985748291,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.6757654547691345,
"logits/rejected": -0.6454110145568848,
"logps/chosen": -78.08596801757812,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -166.65899658203125,
"loss": 0.9463,
"margin_dpo/margin_mean": 25.35919761657715,
"margin_dpo/margin_std": 31.365787506103516,
"step": 81
},
{
"KL/chosen_KL_mean": -14.165742874145508,
"KL/mean": -24.178695678710938,
"KL/rejected_KL_mean": -34.19164276123047,
"KL/std": 24.08298110961914,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.027383368462324142,
"fcm_dpo/delta": -0.15754976868629456,
"fcm_dpo/margin": 20.025901794433594,
"fcm_dpo/q_t": 0.38632169365882874,
"grad_norm": 31.63576316833496,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.6542444229125977,
"logits/rejected": -0.6435472965240479,
"logps/chosen": -66.39813232421875,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -124.93489074707031,
"loss": 1.0894,
"margin_dpo/margin_mean": 20.025901794433594,
"margin_dpo/margin_std": 33.26299285888672,
"step": 82
},
{
"KL/chosen_KL_mean": -14.488626480102539,
"KL/mean": -26.423978805541992,
"KL/rejected_KL_mean": -38.35934066772461,
"KL/std": 25.066070556640625,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.025875981897115707,
"fcm_dpo/delta": -0.23306897282600403,
"fcm_dpo/margin": 23.870704650878906,
"fcm_dpo/q_t": 0.36692357063293457,
"grad_norm": 29.748979568481445,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.6859611868858337,
"logits/rejected": -0.6524330377578735,
"logps/chosen": -70.31600952148438,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -142.0752410888672,
"loss": 0.9878,
"margin_dpo/margin_mean": 23.870704650878906,
"margin_dpo/margin_std": 29.676780700683594,
"step": 83
},
{
"KL/chosen_KL_mean": -13.53270149230957,
"KL/mean": -23.642658233642578,
"KL/rejected_KL_mean": -33.75260925292969,
"KL/std": 21.36370849609375,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.025026695802807808,
"fcm_dpo/delta": -0.11385335773229599,
"fcm_dpo/margin": 20.219905853271484,
"fcm_dpo/q_t": 0.38283252716064453,
"grad_norm": 26.598854064941406,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.6387369632720947,
"logits/rejected": -0.5981898307800293,
"logps/chosen": -80.70886993408203,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -121.05120849609375,
"loss": 1.0144,
"margin_dpo/margin_mean": 20.21990394592285,
"margin_dpo/margin_std": 21.734418869018555,
"step": 84
},
{
"KL/chosen_KL_mean": -13.872922897338867,
"KL/mean": -24.366085052490234,
"KL/rejected_KL_mean": -34.85926055908203,
"KL/std": 21.958358764648438,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.02479904145002365,
"fcm_dpo/delta": -0.12728425860404968,
"fcm_dpo/margin": 20.98633575439453,
"fcm_dpo/q_t": 0.38485896587371826,
"grad_norm": 26.617475509643555,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6399697065353394,
"logits/rejected": -0.6071797013282776,
"logps/chosen": -72.279541015625,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -113.49806213378906,
"loss": 1.0344,
"margin_dpo/margin_mean": 20.98633575439453,
"margin_dpo/margin_std": 26.402019500732422,
"step": 85
},
{
"KL/chosen_KL_mean": -18.034120559692383,
"KL/mean": -29.181171417236328,
"KL/rejected_KL_mean": -40.32822036743164,
"KL/std": 29.792917251586914,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.024033017456531525,
"fcm_dpo/delta": -0.14324147999286652,
"fcm_dpo/margin": 22.294099807739258,
"fcm_dpo/q_t": 0.39599549770355225,
"grad_norm": 30.38920021057129,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.6997466683387756,
"logits/rejected": -0.6884916424751282,
"logps/chosen": -74.17158508300781,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -128.4498748779297,
"loss": 1.1146,
"margin_dpo/margin_mean": 22.29410171508789,
"margin_dpo/margin_std": 41.11421203613281,
"step": 86
},
{
"KL/chosen_KL_mean": -17.486053466796875,
"KL/mean": -30.587915420532227,
"KL/rejected_KL_mean": -43.68976974487305,
"KL/std": 29.706398010253906,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.022885756567120552,
"fcm_dpo/delta": -0.21425558626651764,
"fcm_dpo/margin": 26.203716278076172,
"fcm_dpo/q_t": 0.3703950047492981,
"grad_norm": 25.73578453063965,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.7155952453613281,
"logits/rejected": -0.6913902163505554,
"logps/chosen": -73.12214660644531,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -139.1573486328125,
"loss": 1.0227,
"margin_dpo/margin_mean": 26.203716278076172,
"margin_dpo/margin_std": 36.01057052612305,
"step": 87
},
{
"KL/chosen_KL_mean": -22.227901458740234,
"KL/mean": -32.0677375793457,
"KL/rejected_KL_mean": -41.90757369995117,
"KL/std": 30.92605972290039,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.022482173517346382,
"fcm_dpo/delta": -0.0452876091003418,
"fcm_dpo/margin": 19.679677963256836,
"fcm_dpo/q_t": 0.40572047233581543,
"grad_norm": 26.90105438232422,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.670428991317749,
"logits/rejected": -0.6643358469009399,
"logps/chosen": -95.8990478515625,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -148.61607360839844,
"loss": 1.1257,
"margin_dpo/margin_mean": 19.679677963256836,
"margin_dpo/margin_std": 32.507869720458984,
"step": 88
},
{
"KL/chosen_KL_mean": -13.619379043579102,
"KL/mean": -24.656768798828125,
"KL/rejected_KL_mean": -35.69416046142578,
"KL/std": 26.05047607421875,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.022331606596708298,
"fcm_dpo/delta": -0.09776569902896881,
"fcm_dpo/margin": 22.074787139892578,
"fcm_dpo/q_t": 0.3905707895755768,
"grad_norm": 24.52141571044922,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.7306783199310303,
"logits/rejected": -0.6993080973625183,
"logps/chosen": -74.24429321289062,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -117.7777099609375,
"loss": 1.0475,
"margin_dpo/margin_mean": 22.074787139892578,
"margin_dpo/margin_std": 29.647079467773438,
"step": 89
},
{
"KL/chosen_KL_mean": -17.485197067260742,
"KL/mean": -32.5256233215332,
"KL/rejected_KL_mean": -47.5660514831543,
"KL/std": 35.56610870361328,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.021488668397068977,
"fcm_dpo/delta": -0.26283973455429077,
"fcm_dpo/margin": 30.08085823059082,
"fcm_dpo/q_t": 0.37226438522338867,
"grad_norm": 26.66068458557129,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.6556218862533569,
"logits/rejected": -0.6689947843551636,
"logps/chosen": -70.7705078125,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -159.1107635498047,
"loss": 1.0278,
"margin_dpo/margin_mean": 30.080860137939453,
"margin_dpo/margin_std": 44.20441818237305,
"step": 90
},
{
"KL/chosen_KL_mean": -18.215688705444336,
"KL/mean": -30.59687042236328,
"KL/rejected_KL_mean": -42.978057861328125,
"KL/std": 28.502140045166016,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.02069549262523651,
"fcm_dpo/delta": -0.11838565766811371,
"fcm_dpo/margin": 24.762371063232422,
"fcm_dpo/q_t": 0.39080309867858887,
"grad_norm": 25.125621795654297,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.7024148106575012,
"logits/rejected": -0.6713524460792542,
"logps/chosen": -80.01864624023438,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -130.85202026367188,
"loss": 1.0666,
"margin_dpo/margin_mean": 24.762371063232422,
"margin_dpo/margin_std": 37.02970886230469,
"step": 91
},
{
"KL/chosen_KL_mean": -16.36530113220215,
"KL/mean": -28.872386932373047,
"KL/rejected_KL_mean": -41.37947082519531,
"KL/std": 29.21990203857422,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.02026374265551567,
"fcm_dpo/delta": -0.1125737801194191,
"fcm_dpo/margin": 25.01416778564453,
"fcm_dpo/q_t": 0.39040741324424744,
"grad_norm": 22.92947769165039,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.6997867822647095,
"logits/rejected": -0.676429271697998,
"logps/chosen": -68.00607299804688,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -119.26065063476562,
"loss": 1.0572,
"margin_dpo/margin_mean": 25.01416778564453,
"margin_dpo/margin_std": 35.404052734375,
"step": 92
},
{
"KL/chosen_KL_mean": -18.400798797607422,
"KL/mean": -30.97686004638672,
"KL/rejected_KL_mean": -43.552921295166016,
"KL/std": 27.59107780456543,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.019758395850658417,
"fcm_dpo/delta": -0.10195006430149078,
"fcm_dpo/margin": 25.15212631225586,
"fcm_dpo/q_t": 0.38974255323410034,
"grad_norm": 23.7645320892334,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.7242586612701416,
"logits/rejected": -0.6749112606048584,
"logps/chosen": -70.93003845214844,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -120.71366882324219,
"loss": 1.0348,
"margin_dpo/margin_mean": 25.15212631225586,
"margin_dpo/margin_std": 31.98870086669922,
"step": 93
},
{
"KL/chosen_KL_mean": -19.79575538635254,
"KL/mean": -34.32554626464844,
"KL/rejected_KL_mean": -48.8553352355957,
"KL/std": 31.645320892333984,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.019052118062973022,
"fcm_dpo/delta": -0.1642438918352127,
"fcm_dpo/margin": 29.0595760345459,
"fcm_dpo/q_t": 0.37634679675102234,
"grad_norm": 23.290563583374023,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.6614178419113159,
"logits/rejected": -0.6468695402145386,
"logps/chosen": -81.01837158203125,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -148.45437622070312,
"loss": 0.997,
"margin_dpo/margin_mean": 29.05957794189453,
"margin_dpo/margin_std": 33.09284591674805,
"step": 94
},
{
"KL/chosen_KL_mean": -18.84189796447754,
"KL/mean": -30.522144317626953,
"KL/rejected_KL_mean": -42.202392578125,
"KL/std": 30.20883560180664,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.018812362104654312,
"fcm_dpo/delta": -0.04170980304479599,
"fcm_dpo/margin": 23.360492706298828,
"fcm_dpo/q_t": 0.3993641436100006,
"grad_norm": 21.823162078857422,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.6893196105957031,
"logits/rejected": -0.6614104509353638,
"logps/chosen": -71.36553955078125,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -118.0827407836914,
"loss": 1.0766,
"margin_dpo/margin_mean": 23.360492706298828,
"margin_dpo/margin_std": 31.601409912109375,
"step": 95
},
{
"KL/chosen_KL_mean": -18.52091407775879,
"KL/mean": -34.392066955566406,
"KL/rejected_KL_mean": -50.263206481933594,
"KL/std": 34.390037536621094,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.01818418689072132,
"fcm_dpo/delta": -0.19050875306129456,
"fcm_dpo/margin": 31.74229621887207,
"fcm_dpo/q_t": 0.3741267919540405,
"grad_norm": 22.391244888305664,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.702680766582489,
"logits/rejected": -0.6744130849838257,
"logps/chosen": -80.67788696289062,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -146.85922241210938,
"loss": 0.9906,
"margin_dpo/margin_mean": 31.742298126220703,
"margin_dpo/margin_std": 37.9951171875,
"step": 96
},
{
"KL/chosen_KL_mean": -19.829116821289062,
"KL/mean": -32.06022262573242,
"KL/rejected_KL_mean": -44.29132843017578,
"KL/std": 28.707332611083984,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.01816154085099697,
"fcm_dpo/delta": -0.04735935106873512,
"fcm_dpo/margin": 24.46221351623535,
"fcm_dpo/q_t": 0.39719265699386597,
"grad_norm": 23.29534339904785,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.6568596363067627,
"logits/rejected": -0.6160587668418884,
"logps/chosen": -74.47547912597656,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -121.2560806274414,
"loss": 1.0603,
"margin_dpo/margin_mean": 24.46221351623535,
"margin_dpo/margin_std": 28.667625427246094,
"step": 97
},
{
"KL/chosen_KL_mean": -24.416702270507812,
"KL/mean": -38.547080993652344,
"KL/rejected_KL_mean": -52.677459716796875,
"KL/std": 32.80169677734375,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.01765655353665352,
"fcm_dpo/delta": -0.10444936156272888,
"fcm_dpo/margin": 28.260757446289062,
"fcm_dpo/q_t": 0.38620907068252563,
"grad_norm": 24.261281967163086,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.6890856027603149,
"logits/rejected": -0.6740926504135132,
"logps/chosen": -89.67533111572266,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -149.2049560546875,
"loss": 1.0602,
"margin_dpo/margin_mean": 28.260757446289062,
"margin_dpo/margin_std": 39.32505798339844,
"step": 98
},
{
"KL/chosen_KL_mean": -19.188392639160156,
"KL/mean": -36.50630187988281,
"KL/rejected_KL_mean": -53.82420349121094,
"KL/std": 35.373687744140625,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.017020825296640396,
"fcm_dpo/delta": -0.20224523544311523,
"fcm_dpo/margin": 34.63581848144531,
"fcm_dpo/q_t": 0.3741142749786377,
"grad_norm": 21.056440353393555,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.6723522543907166,
"logits/rejected": -0.6737344861030579,
"logps/chosen": -64.82687377929688,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -140.26214599609375,
"loss": 0.994,
"margin_dpo/margin_mean": 34.63581848144531,
"margin_dpo/margin_std": 43.95652389526367,
"step": 99
},
{
"KL/chosen_KL_mean": -21.610239028930664,
"KL/mean": -31.642812728881836,
"KL/rejected_KL_mean": -41.675384521484375,
"KL/std": 28.504859924316406,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.017114732414484024,
"fcm_dpo/delta": 0.0580127090215683,
"fcm_dpo/margin": 20.065147399902344,
"fcm_dpo/q_t": 0.4225125014781952,
"grad_norm": 23.509201049804688,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6509604454040527,
"logits/rejected": -0.6082332730293274,
"logps/chosen": -79.2042236328125,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -115.735595703125,
"loss": 1.1693,
"margin_dpo/margin_mean": 20.065147399902344,
"margin_dpo/margin_std": 36.753334045410156,
"step": 100
},
{
"KL/chosen_KL_mean": -26.565208435058594,
"KL/mean": -38.76583480834961,
"KL/rejected_KL_mean": -50.96645736694336,
"KL/std": 35.17367935180664,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.01696072332561016,
"fcm_dpo/delta": -0.015185698866844177,
"fcm_dpo/margin": 24.401235580444336,
"fcm_dpo/q_t": 0.41169631481170654,
"grad_norm": 23.37774658203125,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.6405997276306152,
"logits/rejected": -0.6082361936569214,
"logps/chosen": -88.21406555175781,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -134.15614318847656,
"loss": 1.1319,
"margin_dpo/margin_mean": 24.401235580444336,
"margin_dpo/margin_std": 42.36262512207031,
"step": 101
},
{
"KL/chosen_KL_mean": -28.67186164855957,
"KL/mean": -37.669090270996094,
"KL/rejected_KL_mean": -46.66631317138672,
"KL/std": 32.99787139892578,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.01739579439163208,
"fcm_dpo/delta": 0.08802524209022522,
"fcm_dpo/margin": 17.994457244873047,
"fcm_dpo/q_t": 0.43094637989997864,
"grad_norm": 27.00950813293457,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.6696982383728027,
"logits/rejected": -0.6241832971572876,
"logps/chosen": -92.75074005126953,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -114.8533935546875,
"loss": 1.2095,
"margin_dpo/margin_mean": 17.994457244873047,
"margin_dpo/margin_std": 39.09446716308594,
"step": 102
},
{
"KL/chosen_KL_mean": -22.722579956054688,
"KL/mean": -40.756980895996094,
"KL/rejected_KL_mean": -58.7913932800293,
"KL/std": 36.61017608642578,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.016756968572735786,
"fcm_dpo/delta": -0.2181204855442047,
"fcm_dpo/margin": 36.068809509277344,
"fcm_dpo/q_t": 0.3694770336151123,
"grad_norm": 22.91532325744629,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.6852983236312866,
"logits/rejected": -0.6532962322235107,
"logps/chosen": -84.02185821533203,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -152.36410522460938,
"loss": 0.9828,
"margin_dpo/margin_mean": 36.068809509277344,
"margin_dpo/margin_std": 44.13578796386719,
"step": 103
},
{
"KL/chosen_KL_mean": -24.786264419555664,
"KL/mean": -41.86579895019531,
"KL/rejected_KL_mean": -58.945335388183594,
"KL/std": 40.4488639831543,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.01623380184173584,
"fcm_dpo/delta": -0.1633920669555664,
"fcm_dpo/margin": 34.15907669067383,
"fcm_dpo/q_t": 0.3847202956676483,
"grad_norm": 22.349903106689453,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.6932718753814697,
"logits/rejected": -0.664044976234436,
"logps/chosen": -79.15904235839844,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -148.5100555419922,
"loss": 1.0435,
"margin_dpo/margin_mean": 34.159080505371094,
"margin_dpo/margin_std": 49.494911193847656,
"step": 104
},
{
"KL/chosen_KL_mean": -23.31475257873535,
"KL/mean": -47.551963806152344,
"KL/rejected_KL_mean": -71.7891845703125,
"KL/std": 39.350242614746094,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.015241267159581184,
"fcm_dpo/delta": -0.36551716923713684,
"fcm_dpo/margin": 48.474430084228516,
"fcm_dpo/q_t": 0.3338563144207001,
"grad_norm": 22.70827293395996,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.6466401219367981,
"logits/rejected": -0.6229440569877625,
"logps/chosen": -77.95369720458984,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -169.7626953125,
"loss": 0.8715,
"margin_dpo/margin_mean": 48.474430084228516,
"margin_dpo/margin_std": 43.423583984375,
"step": 105
},
{
"KL/chosen_KL_mean": -26.223730087280273,
"KL/mean": -41.38634490966797,
"KL/rejected_KL_mean": -56.54896545410156,
"KL/std": 34.416595458984375,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.014795205555856228,
"fcm_dpo/delta": -0.05113736912608147,
"fcm_dpo/margin": 30.32522964477539,
"fcm_dpo/q_t": 0.3969998359680176,
"grad_norm": 21.981822967529297,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.6268575191497803,
"logits/rejected": -0.597021222114563,
"logps/chosen": -81.05662536621094,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -141.77359008789062,
"loss": 1.0591,
"margin_dpo/margin_mean": 30.32522964477539,
"margin_dpo/margin_std": 37.44782638549805,
"step": 106
},
{
"KL/chosen_KL_mean": -31.174667358398438,
"KL/mean": -48.2926025390625,
"KL/rejected_KL_mean": -65.41053771972656,
"KL/std": 41.864192962646484,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.014499913901090622,
"fcm_dpo/delta": -0.10227993130683899,
"fcm_dpo/margin": 34.235870361328125,
"fcm_dpo/q_t": 0.38768211007118225,
"grad_norm": 21.1258487701416,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.6795566082000732,
"logits/rejected": -0.6355477571487427,
"logps/chosen": -100.88247680664062,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -160.15005493164062,
"loss": 1.0454,
"margin_dpo/margin_mean": 34.235870361328125,
"margin_dpo/margin_std": 44.14703369140625,
"step": 107
},
{
"KL/chosen_KL_mean": -27.941110610961914,
"KL/mean": -48.69300079345703,
"KL/rejected_KL_mean": -69.44490051269531,
"KL/std": 46.46894836425781,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.014074774459004402,
"fcm_dpo/delta": -0.1955530345439911,
"fcm_dpo/margin": 41.5037956237793,
"fcm_dpo/q_t": 0.37944066524505615,
"grad_norm": 21.240692138671875,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.6599795818328857,
"logits/rejected": -0.6397461891174316,
"logps/chosen": -83.95099639892578,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -165.24090576171875,
"loss": 1.0361,
"margin_dpo/margin_mean": 41.50379180908203,
"margin_dpo/margin_std": 60.09625244140625,
"step": 108
},
{
"KL/chosen_KL_mean": -26.06736946105957,
"KL/mean": -48.12309646606445,
"KL/rejected_KL_mean": -70.1788101196289,
"KL/std": 44.416900634765625,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.013461511582136154,
"fcm_dpo/delta": -0.20704127848148346,
"fcm_dpo/margin": 44.111446380615234,
"fcm_dpo/q_t": 0.3681620955467224,
"grad_norm": 22.13717269897461,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.59981369972229,
"logits/rejected": -0.578811526298523,
"logps/chosen": -88.9528579711914,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -168.8645477294922,
"loss": 0.9887,
"margin_dpo/margin_mean": 44.1114501953125,
"margin_dpo/margin_std": 52.54209518432617,
"step": 109
},
{
"KL/chosen_KL_mean": -26.961872100830078,
"KL/mean": -46.808937072753906,
"KL/rejected_KL_mean": -66.65599822998047,
"KL/std": 44.2524528503418,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.01295989565551281,
"fcm_dpo/delta": -0.12524738907814026,
"fcm_dpo/margin": 39.694129943847656,
"fcm_dpo/q_t": 0.3869009017944336,
"grad_norm": 19.041017532348633,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.6018489003181458,
"logits/rejected": -0.566160261631012,
"logps/chosen": -85.71556091308594,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -146.406005859375,
"loss": 1.0523,
"margin_dpo/margin_mean": 39.694129943847656,
"margin_dpo/margin_std": 53.206546783447266,
"step": 110
},
{
"KL/chosen_KL_mean": -30.57050132751465,
"KL/mean": -53.029212951660156,
"KL/rejected_KL_mean": -75.48793029785156,
"KL/std": 50.006378173828125,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.012622907757759094,
"fcm_dpo/delta": -0.17867523431777954,
"fcm_dpo/margin": 44.91743469238281,
"fcm_dpo/q_t": 0.37669897079467773,
"grad_norm": 21.740154266357422,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.6179641485214233,
"logits/rejected": -0.594010591506958,
"logps/chosen": -99.19461059570312,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -173.9167938232422,
"loss": 1.0337,
"margin_dpo/margin_mean": 44.91743469238281,
"margin_dpo/margin_std": 61.63525390625,
"step": 111
},
{
"KL/chosen_KL_mean": -28.242313385009766,
"KL/mean": -42.492279052734375,
"KL/rejected_KL_mean": -56.74223709106445,
"KL/std": 34.65959930419922,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.012602494098246098,
"fcm_dpo/delta": 0.04215133190155029,
"fcm_dpo/margin": 28.499923706054688,
"fcm_dpo/q_t": 0.41772669553756714,
"grad_norm": 19.57350730895996,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.6415656805038452,
"logits/rejected": -0.6061877608299255,
"logps/chosen": -78.49195861816406,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -121.51666259765625,
"loss": 1.1268,
"margin_dpo/margin_mean": 28.499921798706055,
"margin_dpo/margin_std": 41.986968994140625,
"step": 112
},
{
"KL/chosen_KL_mean": -35.24995803833008,
"KL/mean": -50.625701904296875,
"KL/rejected_KL_mean": -66.00144958496094,
"KL/std": 36.226314544677734,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.012705144472420216,
"fcm_dpo/delta": 0.009673002175986767,
"fcm_dpo/margin": 30.751483917236328,
"fcm_dpo/q_t": 0.4100167751312256,
"grad_norm": 19.772661209106445,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.6053025722503662,
"logits/rejected": -0.5635230541229248,
"logps/chosen": -101.96290588378906,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -143.97015380859375,
"loss": 1.0939,
"margin_dpo/margin_mean": 30.751482009887695,
"margin_dpo/margin_std": 40.09980773925781,
"step": 113
},
{
"KL/chosen_KL_mean": -32.449310302734375,
"KL/mean": -56.777923583984375,
"KL/rejected_KL_mean": -81.10653686523438,
"KL/std": 54.06468963623047,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.01223907433450222,
"fcm_dpo/delta": -0.21012035012245178,
"fcm_dpo/margin": 48.65721893310547,
"fcm_dpo/q_t": 0.37521958351135254,
"grad_norm": 21.462278366088867,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.6051023006439209,
"logits/rejected": -0.5736086964607239,
"logps/chosen": -90.23439025878906,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -168.2161865234375,
"loss": 1.0021,
"margin_dpo/margin_mean": 48.65721893310547,
"margin_dpo/margin_std": 65.560791015625,
"step": 114
},
{
"KL/chosen_KL_mean": -42.36359786987305,
"KL/mean": -58.933746337890625,
"KL/rejected_KL_mean": -75.50389099121094,
"KL/std": 51.648048400878906,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.012213023379445076,
"fcm_dpo/delta": -0.005111444741487503,
"fcm_dpo/margin": 33.140296936035156,
"fcm_dpo/q_t": 0.4153473973274231,
"grad_norm": 27.328691482543945,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.6272869110107422,
"logits/rejected": -0.6080690622329712,
"logps/chosen": -107.94624328613281,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -174.0694122314453,
"loss": 1.1709,
"margin_dpo/margin_mean": 33.140296936035156,
"margin_dpo/margin_std": 68.08973693847656,
"step": 115
},
{
"KL/chosen_KL_mean": -31.53169822692871,
"KL/mean": -49.997901916503906,
"KL/rejected_KL_mean": -68.464111328125,
"KL/std": 43.47674560546875,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.012136522680521011,
"fcm_dpo/delta": -0.0506804920732975,
"fcm_dpo/margin": 36.932403564453125,
"fcm_dpo/q_t": 0.3989343047142029,
"grad_norm": 21.574514389038086,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.6011572480201721,
"logits/rejected": -0.5915130376815796,
"logps/chosen": -82.9320068359375,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -148.9859619140625,
"loss": 1.0625,
"margin_dpo/margin_mean": 36.932403564453125,
"margin_dpo/margin_std": 47.14096450805664,
"step": 116
},
{
"KL/chosen_KL_mean": -40.54469299316406,
"KL/mean": -55.65740966796875,
"KL/rejected_KL_mean": -70.77013397216797,
"KL/std": 46.44588851928711,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.012116730213165283,
"fcm_dpo/delta": 0.035039689391851425,
"fcm_dpo/margin": 30.225439071655273,
"fcm_dpo/q_t": 0.4179823696613312,
"grad_norm": 27.370105743408203,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.5658215284347534,
"logits/rejected": -0.5278164744377136,
"logps/chosen": -109.84310150146484,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -137.3541259765625,
"loss": 1.146,
"margin_dpo/margin_mean": 30.225440979003906,
"margin_dpo/margin_std": 52.66984176635742,
"step": 117
},
{
"KL/chosen_KL_mean": -32.08608627319336,
"KL/mean": -50.51774597167969,
"KL/rejected_KL_mean": -68.94939422607422,
"KL/std": 41.92464065551758,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.012059303931891918,
"fcm_dpo/delta": -0.04659657180309296,
"fcm_dpo/margin": 36.863319396972656,
"fcm_dpo/q_t": 0.4000033140182495,
"grad_norm": 20.587656021118164,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.6026766300201416,
"logits/rejected": -0.5824264287948608,
"logps/chosen": -87.72706604003906,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -144.6184539794922,
"loss": 1.0657,
"margin_dpo/margin_mean": 36.86331558227539,
"margin_dpo/margin_std": 47.96271514892578,
"step": 118
},
{
"KL/chosen_KL_mean": -43.64602279663086,
"KL/mean": -61.894630432128906,
"KL/rejected_KL_mean": -80.14323425292969,
"KL/std": 47.0084228515625,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.01187710091471672,
"fcm_dpo/delta": -0.03619098290801048,
"fcm_dpo/margin": 36.49721145629883,
"fcm_dpo/q_t": 0.40584415197372437,
"grad_norm": 23.66024398803711,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.6127524375915527,
"logits/rejected": -0.6005865931510925,
"logps/chosen": -117.15621948242188,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -183.1205291748047,
"loss": 1.1046,
"margin_dpo/margin_mean": 36.49720764160156,
"margin_dpo/margin_std": 57.2733154296875,
"step": 119
},
{
"KL/chosen_KL_mean": -45.010643005371094,
"KL/mean": -70.27703857421875,
"KL/rejected_KL_mean": -95.54344177246094,
"KL/std": 58.90607452392578,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.011582771316170692,
"fcm_dpo/delta": -0.19700661301612854,
"fcm_dpo/margin": 50.532806396484375,
"fcm_dpo/q_t": 0.3717060089111328,
"grad_norm": 21.907379150390625,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.6179412603378296,
"logits/rejected": -0.5869693160057068,
"logps/chosen": -121.79147338867188,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -203.5671844482422,
"loss": 0.9953,
"margin_dpo/margin_mean": 50.532806396484375,
"margin_dpo/margin_std": 61.917076110839844,
"step": 120
},
{
"KL/chosen_KL_mean": -43.67778015136719,
"KL/mean": -71.53456115722656,
"KL/rejected_KL_mean": -99.39134216308594,
"KL/std": 57.339202880859375,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.011032504960894585,
"fcm_dpo/delta": -0.23044228553771973,
"fcm_dpo/margin": 55.71354675292969,
"fcm_dpo/q_t": 0.3678060472011566,
"grad_norm": 23.938167572021484,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.5758655071258545,
"logits/rejected": -0.572211742401123,
"logps/chosen": -105.46768188476562,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -209.3859100341797,
"loss": 0.9973,
"margin_dpo/margin_mean": 55.71355056762695,
"margin_dpo/margin_std": 71.77114868164062,
"step": 121
},
{
"KL/chosen_KL_mean": -39.53837203979492,
"KL/mean": -74.14775848388672,
"KL/rejected_KL_mean": -108.75715637207031,
"KL/std": 65.19731140136719,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.010416809469461441,
"fcm_dpo/delta": -0.34584206342697144,
"fcm_dpo/margin": 69.2187728881836,
"fcm_dpo/q_t": 0.34151148796081543,
"grad_norm": 23.229158401489258,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.5348167419433594,
"logits/rejected": -0.5510457158088684,
"logps/chosen": -86.44058227539062,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -215.47134399414062,
"loss": 0.8958,
"margin_dpo/margin_mean": 69.2187728881836,
"margin_dpo/margin_std": 68.12580871582031,
"step": 122
},
{
"KL/chosen_KL_mean": -43.5894775390625,
"KL/mean": -66.2512435913086,
"KL/rejected_KL_mean": -88.91300964355469,
"KL/std": 55.7255859375,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.010076490230858326,
"fcm_dpo/delta": -0.060425400733947754,
"fcm_dpo/margin": 45.323524475097656,
"fcm_dpo/q_t": 0.4018062949180603,
"grad_norm": 20.704429626464844,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.5436292290687561,
"logits/rejected": -0.5264102220535278,
"logps/chosen": -104.92811584472656,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -176.6884002685547,
"loss": 1.1086,
"margin_dpo/margin_mean": 45.32352828979492,
"margin_dpo/margin_std": 74.68168640136719,
"step": 123
},
{
"KL/chosen_KL_mean": -50.04446029663086,
"KL/mean": -83.56463623046875,
"KL/rejected_KL_mean": -117.08482360839844,
"KL/std": 76.33003234863281,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.009682193398475647,
"fcm_dpo/delta": -0.26734763383865356,
"fcm_dpo/margin": 67.04035949707031,
"fcm_dpo/q_t": 0.3692913055419922,
"grad_norm": 23.1252498626709,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.5410699248313904,
"logits/rejected": -0.5437477827072144,
"logps/chosen": -121.4927978515625,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -234.66539001464844,
"loss": 1.0151,
"margin_dpo/margin_mean": 67.04035949707031,
"margin_dpo/margin_std": 95.87313842773438,
"step": 124
},
{
"KL/chosen_KL_mean": -41.3752326965332,
"KL/mean": -71.00807189941406,
"KL/rejected_KL_mean": -100.64091491699219,
"KL/std": 64.5465316772461,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.009364111348986626,
"fcm_dpo/delta": -0.16384468972682953,
"fcm_dpo/margin": 59.265682220458984,
"fcm_dpo/q_t": 0.38098064064979553,
"grad_norm": 19.144956588745117,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.5046694278717041,
"logits/rejected": -0.507087230682373,
"logps/chosen": -91.51217651367188,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -184.62953186035156,
"loss": 1.0301,
"margin_dpo/margin_mean": 59.26567840576172,
"margin_dpo/margin_std": 80.23193359375,
"step": 125
},
{
"KL/chosen_KL_mean": -44.953895568847656,
"KL/mean": -72.60568237304688,
"KL/rejected_KL_mean": -100.2574691772461,
"KL/std": 58.28239440917969,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.009100959636271,
"fcm_dpo/delta": -0.10908694565296173,
"fcm_dpo/margin": 55.30357360839844,
"fcm_dpo/q_t": 0.38956084847450256,
"grad_norm": 20.653635025024414,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.5091671943664551,
"logits/rejected": -0.503969132900238,
"logps/chosen": -100.62096405029297,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -198.38723754882812,
"loss": 1.0457,
"margin_dpo/margin_mean": 55.30357360839844,
"margin_dpo/margin_std": 74.20457458496094,
"step": 126
},
{
"KL/chosen_KL_mean": -43.32256317138672,
"KL/mean": -63.5421142578125,
"KL/rejected_KL_mean": -83.76165771484375,
"KL/std": 50.99864959716797,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.00911656953394413,
"fcm_dpo/delta": 0.032512567937374115,
"fcm_dpo/margin": 40.43910217285156,
"fcm_dpo/q_t": 0.41489607095718384,
"grad_norm": 20.905221939086914,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.5238068103790283,
"logits/rejected": -0.5045550465583801,
"logps/chosen": -99.87724304199219,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -160.55743408203125,
"loss": 1.1317,
"margin_dpo/margin_mean": 40.43910217285156,
"margin_dpo/margin_std": 64.11962127685547,
"step": 127
},
{
"KL/chosen_KL_mean": -46.23556137084961,
"KL/mean": -67.52534484863281,
"KL/rejected_KL_mean": -88.81513977050781,
"KL/std": 59.858497619628906,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.009177174419164658,
"fcm_dpo/delta": 0.009238209575414658,
"fcm_dpo/margin": 42.5795783996582,
"fcm_dpo/q_t": 0.4133646786212921,
"grad_norm": 29.049198150634766,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.5192145109176636,
"logits/rejected": -0.4998999834060669,
"logps/chosen": -104.35652160644531,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -165.25408935546875,
"loss": 1.1434,
"margin_dpo/margin_mean": 42.5795783996582,
"margin_dpo/margin_std": 75.051513671875,
"step": 128
},
{
"KL/chosen_KL_mean": -56.05470657348633,
"KL/mean": -78.45681762695312,
"KL/rejected_KL_mean": -100.85891723632812,
"KL/std": 65.97745513916016,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.009138467721641064,
"fcm_dpo/delta": -0.009851408191025257,
"fcm_dpo/margin": 44.80420684814453,
"fcm_dpo/q_t": 0.41433221101760864,
"grad_norm": 20.67636489868164,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.5321705341339111,
"logits/rejected": -0.5218693614006042,
"logps/chosen": -122.97108459472656,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -197.50112915039062,
"loss": 1.15,
"margin_dpo/margin_mean": 44.80420684814453,
"margin_dpo/margin_std": 85.82166290283203,
"step": 129
},
{
"KL/chosen_KL_mean": -41.49655532836914,
"KL/mean": -75.23775482177734,
"KL/rejected_KL_mean": -108.97895050048828,
"KL/std": 67.84368896484375,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.008932676166296005,
"fcm_dpo/delta": -0.21578986942768097,
"fcm_dpo/margin": 67.48239135742188,
"fcm_dpo/q_t": 0.37061649560928345,
"grad_norm": 21.26862907409668,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.49152904748916626,
"logits/rejected": -0.4860014319419861,
"logps/chosen": -86.16340637207031,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -191.76060485839844,
"loss": 0.9985,
"margin_dpo/margin_mean": 67.48239135742188,
"margin_dpo/margin_std": 82.64620971679688,
"step": 130
},
{
"KL/chosen_KL_mean": -36.96665954589844,
"KL/mean": -70.82868194580078,
"KL/rejected_KL_mean": -104.69070434570312,
"KL/std": 67.29373168945312,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.008490066044032574,
"fcm_dpo/delta": -0.1859792321920395,
"fcm_dpo/margin": 67.72404479980469,
"fcm_dpo/q_t": 0.36927857995033264,
"grad_norm": 29.203166961669922,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.5128257274627686,
"logits/rejected": -0.5198142528533936,
"logps/chosen": -81.89125061035156,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -193.13470458984375,
"loss": 0.9808,
"margin_dpo/margin_mean": 67.72404479980469,
"margin_dpo/margin_std": 73.8796615600586,
"step": 131
},
{
"KL/chosen_KL_mean": -49.28675079345703,
"KL/mean": -76.96174621582031,
"KL/rejected_KL_mean": -104.63676452636719,
"KL/std": 67.005615234375,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.008325127884745598,
"fcm_dpo/delta": -0.06394054740667343,
"fcm_dpo/margin": 55.35000228881836,
"fcm_dpo/q_t": 0.40161028504371643,
"grad_norm": 19.584449768066406,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.5270289778709412,
"logits/rejected": -0.5260351896286011,
"logps/chosen": -108.28783416748047,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -192.52891540527344,
"loss": 1.0894,
"margin_dpo/margin_mean": 55.350006103515625,
"margin_dpo/margin_std": 87.00371551513672,
"step": 132
},
{
"KL/chosen_KL_mean": -57.79051208496094,
"KL/mean": -82.15497589111328,
"KL/rejected_KL_mean": -106.51943969726562,
"KL/std": 60.27544403076172,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.008292624726891518,
"fcm_dpo/delta": -0.004351507872343063,
"fcm_dpo/margin": 48.728919982910156,
"fcm_dpo/q_t": 0.4111335277557373,
"grad_norm": 27.549100875854492,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.511991024017334,
"logits/rejected": -0.4981684684753418,
"logps/chosen": -124.39501190185547,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -202.85299682617188,
"loss": 1.1121,
"margin_dpo/margin_mean": 48.728919982910156,
"margin_dpo/margin_std": 76.42304229736328,
"step": 133
},
{
"KL/chosen_KL_mean": -46.12035369873047,
"KL/mean": -72.27875518798828,
"KL/rejected_KL_mean": -98.43716430664062,
"KL/std": 58.89271926879883,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.008291056379675865,
"fcm_dpo/delta": -0.03539953753352165,
"fcm_dpo/margin": 52.316802978515625,
"fcm_dpo/q_t": 0.40254372358322144,
"grad_norm": 18.665939331054688,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.4636070132255554,
"logits/rejected": -0.44918179512023926,
"logps/chosen": -98.18960571289062,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -186.0916748046875,
"loss": 1.0695,
"margin_dpo/margin_mean": 52.316802978515625,
"margin_dpo/margin_std": 68.16081237792969,
"step": 134
},
{
"KL/chosen_KL_mean": -50.93290710449219,
"KL/mean": -88.6656265258789,
"KL/rejected_KL_mean": -126.39834594726562,
"KL/std": 78.58366394042969,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.007986115291714668,
"fcm_dpo/delta": -0.21575535833835602,
"fcm_dpo/margin": 75.46543884277344,
"fcm_dpo/q_t": 0.3711055815219879,
"grad_norm": 22.26523780822754,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.4720418155193329,
"logits/rejected": -0.5066670775413513,
"logps/chosen": -101.28677368164062,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -242.3780975341797,
"loss": 0.9955,
"margin_dpo/margin_mean": 75.4654312133789,
"margin_dpo/margin_std": 94.92060089111328,
"step": 135
},
{
"KL/chosen_KL_mean": -60.29087829589844,
"KL/mean": -85.43303680419922,
"KL/rejected_KL_mean": -110.5751953125,
"KL/std": 70.91878509521484,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.00785889569669962,
"fcm_dpo/delta": 0.004331991076469421,
"fcm_dpo/margin": 50.2843132019043,
"fcm_dpo/q_t": 0.41889041662216187,
"grad_norm": 20.16181182861328,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.49389830231666565,
"logits/rejected": -0.4865124225616455,
"logps/chosen": -125.36338806152344,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -206.89642333984375,
"loss": 1.1418,
"margin_dpo/margin_mean": 50.2843132019043,
"margin_dpo/margin_std": 90.57024383544922,
"step": 136
},
{
"KL/chosen_KL_mean": -57.76277542114258,
"KL/mean": -97.14057922363281,
"KL/rejected_KL_mean": -136.5183868408203,
"KL/std": 94.00605010986328,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.0076486412435770035,
"fcm_dpo/delta": -0.21565069258213043,
"fcm_dpo/margin": 78.755615234375,
"fcm_dpo/q_t": 0.37695467472076416,
"grad_norm": 18.521162033081055,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.4430182874202728,
"logits/rejected": -0.4700878858566284,
"logps/chosen": -106.52188873291016,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -250.3821563720703,
"loss": 1.0248,
"margin_dpo/margin_mean": 78.755615234375,
"margin_dpo/margin_std": 112.62461853027344,
"step": 137
},
{
"KL/chosen_KL_mean": -60.64649963378906,
"KL/mean": -90.4873275756836,
"KL/rejected_KL_mean": -120.32815551757812,
"KL/std": 71.90509033203125,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.0075285714119672775,
"fcm_dpo/delta": -0.05161011964082718,
"fcm_dpo/margin": 59.68164825439453,
"fcm_dpo/q_t": 0.39715737104415894,
"grad_norm": 20.577062606811523,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.42421677708625793,
"logits/rejected": -0.41229647397994995,
"logps/chosen": -121.16615295410156,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -213.5251007080078,
"loss": 1.0575,
"margin_dpo/margin_mean": 59.68164825439453,
"margin_dpo/margin_std": 73.65308380126953,
"step": 138
},
{
"KL/chosen_KL_mean": -50.748390197753906,
"KL/mean": -86.53669738769531,
"KL/rejected_KL_mean": -122.32501220703125,
"KL/std": 68.32363891601562,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.007364056073129177,
"fcm_dpo/delta": -0.13397160172462463,
"fcm_dpo/margin": 71.57662200927734,
"fcm_dpo/q_t": 0.38309505581855774,
"grad_norm": 18.44870948791504,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.4444272220134735,
"logits/rejected": -0.43558651208877563,
"logps/chosen": -97.6397705078125,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -202.05299377441406,
"loss": 1.0191,
"margin_dpo/margin_mean": 71.57662963867188,
"margin_dpo/margin_std": 88.49024963378906,
"step": 139
},
{
"KL/chosen_KL_mean": -61.39569091796875,
"KL/mean": -93.86390686035156,
"KL/rejected_KL_mean": -126.33212280273438,
"KL/std": 76.08689880371094,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.007215442135930061,
"fcm_dpo/delta": -0.07196357846260071,
"fcm_dpo/margin": 64.9364242553711,
"fcm_dpo/q_t": 0.395868718624115,
"grad_norm": 21.35501480102539,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.4337591230869293,
"logits/rejected": -0.42049670219421387,
"logps/chosen": -120.37040710449219,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -209.61622619628906,
"loss": 1.0684,
"margin_dpo/margin_mean": 64.93641662597656,
"margin_dpo/margin_std": 91.21418762207031,
"step": 140
},
{
"KL/chosen_KL_mean": -69.03340148925781,
"KL/mean": -99.73199462890625,
"KL/rejected_KL_mean": -130.4305877685547,
"KL/std": 83.39578247070312,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.007154383696615696,
"fcm_dpo/delta": -0.041043903678655624,
"fcm_dpo/margin": 61.39719009399414,
"fcm_dpo/q_t": 0.40136831998825073,
"grad_norm": 25.930021286010742,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.4134112596511841,
"logits/rejected": -0.40002214908599854,
"logps/chosen": -144.10906982421875,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -228.6228485107422,
"loss": 1.1017,
"margin_dpo/margin_mean": 61.397193908691406,
"margin_dpo/margin_std": 96.28628540039062,
"step": 141
},
{
"KL/chosen_KL_mean": -70.42542266845703,
"KL/mean": -105.56607055664062,
"KL/rejected_KL_mean": -140.70672607421875,
"KL/std": 91.09803009033203,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.0070582120679318905,
"fcm_dpo/delta": -0.10101625323295593,
"fcm_dpo/margin": 70.28129577636719,
"fcm_dpo/q_t": 0.39278340339660645,
"grad_norm": 26.267784118652344,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.42163795232772827,
"logits/rejected": -0.4213171601295471,
"logps/chosen": -128.45335388183594,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -235.2889404296875,
"loss": 1.0825,
"margin_dpo/margin_mean": 70.28129577636719,
"margin_dpo/margin_std": 107.12841796875,
"step": 142
},
{
"KL/chosen_KL_mean": -74.68773651123047,
"KL/mean": -97.2497787475586,
"KL/rejected_KL_mean": -119.81182098388672,
"KL/std": 80.62870788574219,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.0070522744208574295,
"fcm_dpo/delta": 0.08447334915399551,
"fcm_dpo/margin": 45.12409210205078,
"fcm_dpo/q_t": 0.43191662430763245,
"grad_norm": 23.529150009155273,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.3745364546775818,
"logits/rejected": -0.3672657012939453,
"logps/chosen": -132.2841796875,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -198.8114013671875,
"loss": 1.1956,
"margin_dpo/margin_mean": 45.12409210205078,
"margin_dpo/margin_std": 94.2938003540039,
"step": 143
},
{
"KL/chosen_KL_mean": -66.37725067138672,
"KL/mean": -93.94914245605469,
"KL/rejected_KL_mean": -121.52104187011719,
"KL/std": 68.17041015625,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.007112512364983559,
"fcm_dpo/delta": 0.008096275851130486,
"fcm_dpo/margin": 55.14379119873047,
"fcm_dpo/q_t": 0.4112512767314911,
"grad_norm": 20.934486389160156,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.4165929853916168,
"logits/rejected": -0.40653660893440247,
"logps/chosen": -126.28361511230469,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -203.52130126953125,
"loss": 1.1075,
"margin_dpo/margin_mean": 55.14379119873047,
"margin_dpo/margin_std": 79.93777465820312,
"step": 144
},
{
"KL/chosen_KL_mean": -64.00596618652344,
"KL/mean": -93.86626434326172,
"KL/rejected_KL_mean": -123.72657775878906,
"KL/std": 68.75868225097656,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.0070916153490543365,
"fcm_dpo/delta": -0.02458004653453827,
"fcm_dpo/margin": 59.720611572265625,
"fcm_dpo/q_t": 0.4040692448616028,
"grad_norm": 24.859176635742188,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.4160414934158325,
"logits/rejected": -0.4003496766090393,
"logps/chosen": -120.60662841796875,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -201.5928955078125,
"loss": 1.087,
"margin_dpo/margin_mean": 59.720611572265625,
"margin_dpo/margin_std": 83.28811645507812,
"step": 145
},
{
"KL/chosen_KL_mean": -86.9118881225586,
"KL/mean": -109.9712142944336,
"KL/rejected_KL_mean": -133.03054809570312,
"KL/std": 74.28648376464844,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.0071436744183301926,
"fcm_dpo/delta": 0.07298312336206436,
"fcm_dpo/margin": 46.11865997314453,
"fcm_dpo/q_t": 0.4252440631389618,
"grad_norm": 26.424964904785156,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.39881205558776855,
"logits/rejected": -0.3725563585758209,
"logps/chosen": -152.91233825683594,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -214.73333740234375,
"loss": 1.1818,
"margin_dpo/margin_mean": 46.11865997314453,
"margin_dpo/margin_std": 89.22908020019531,
"step": 146
},
{
"KL/chosen_KL_mean": -63.9739990234375,
"KL/mean": -95.299072265625,
"KL/rejected_KL_mean": -126.6241455078125,
"KL/std": 75.37117004394531,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.007126647979021072,
"fcm_dpo/delta": -0.0488114133477211,
"fcm_dpo/margin": 62.65013122558594,
"fcm_dpo/q_t": 0.4014296233654022,
"grad_norm": 20.018695831298828,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.45268571376800537,
"logits/rejected": -0.4357437789440155,
"logps/chosen": -117.37948608398438,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -198.01475524902344,
"loss": 1.0888,
"margin_dpo/margin_mean": 62.65013122558594,
"margin_dpo/margin_std": 93.385009765625,
"step": 147
},
{
"KL/chosen_KL_mean": -62.910282135009766,
"KL/mean": -89.76136016845703,
"KL/rejected_KL_mean": -116.61244201660156,
"KL/std": 75.14965057373047,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.00720440037548542,
"fcm_dpo/delta": 0.01156807690858841,
"fcm_dpo/margin": 53.70217514038086,
"fcm_dpo/q_t": 0.41519132256507874,
"grad_norm": 19.996816635131836,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.46612483263015747,
"logits/rejected": -0.45894235372543335,
"logps/chosen": -127.84736633300781,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -219.706298828125,
"loss": 1.1206,
"margin_dpo/margin_mean": 53.70217514038086,
"margin_dpo/margin_std": 79.02860260009766,
"step": 148
},
{
"KL/chosen_KL_mean": -60.414554595947266,
"KL/mean": -93.79571533203125,
"KL/rejected_KL_mean": -127.1768569946289,
"KL/std": 67.92532348632812,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.007002708502113819,
"fcm_dpo/delta": -0.07285458594560623,
"fcm_dpo/margin": 66.7623062133789,
"fcm_dpo/q_t": 0.39336636662483215,
"grad_norm": 18.49657440185547,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.4581308960914612,
"logits/rejected": -0.4425523281097412,
"logps/chosen": -118.88832092285156,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -226.49160766601562,
"loss": 1.0507,
"margin_dpo/margin_mean": 66.7623062133789,
"margin_dpo/margin_std": 80.3822021484375,
"step": 149
},
{
"KL/chosen_KL_mean": -52.977989196777344,
"KL/mean": -85.69601440429688,
"KL/rejected_KL_mean": -118.41404724121094,
"KL/std": 79.58135223388672,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.006925541907548904,
"fcm_dpo/delta": -0.057165369391441345,
"fcm_dpo/margin": 65.43606567382812,
"fcm_dpo/q_t": 0.40269482135772705,
"grad_norm": 18.195709228515625,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.4748048186302185,
"logits/rejected": -0.4793627858161926,
"logps/chosen": -98.68379974365234,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -201.7616424560547,
"loss": 1.0769,
"margin_dpo/margin_mean": 65.4360580444336,
"margin_dpo/margin_std": 95.0787582397461,
"step": 150
},
{
"KL/chosen_KL_mean": -66.27272033691406,
"KL/mean": -99.1185531616211,
"KL/rejected_KL_mean": -131.96438598632812,
"KL/std": 75.18177032470703,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.006895341910421848,
"fcm_dpo/delta": -0.05550111457705498,
"fcm_dpo/margin": 65.69166564941406,
"fcm_dpo/q_t": 0.3965303897857666,
"grad_norm": 20.076454162597656,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.547601580619812,
"logits/rejected": -0.5237374901771545,
"logps/chosen": -136.84356689453125,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -232.42820739746094,
"loss": 1.0574,
"margin_dpo/margin_mean": 65.69166564941406,
"margin_dpo/margin_std": 82.30619812011719,
"step": 151
},
{
"KL/chosen_KL_mean": -57.727760314941406,
"KL/mean": -97.53009796142578,
"KL/rejected_KL_mean": -137.33242797851562,
"KL/std": 77.0035400390625,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.006748279090970755,
"fcm_dpo/delta": -0.14475369453430176,
"fcm_dpo/margin": 79.60468292236328,
"fcm_dpo/q_t": 0.3807300627231598,
"grad_norm": 20.364458084106445,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.5026451945304871,
"logits/rejected": -0.5023806095123291,
"logps/chosen": -117.89215087890625,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -243.472900390625,
"loss": 1.0132,
"margin_dpo/margin_mean": 79.60467529296875,
"margin_dpo/margin_std": 94.24575805664062,
"step": 152
},
{
"KL/chosen_KL_mean": -58.48396301269531,
"KL/mean": -92.23340606689453,
"KL/rejected_KL_mean": -125.98284912109375,
"KL/std": 83.61231231689453,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.006644603796303272,
"fcm_dpo/delta": -0.05075981840491295,
"fcm_dpo/margin": 67.49888610839844,
"fcm_dpo/q_t": 0.40270525217056274,
"grad_norm": 15.50756549835205,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.4666762948036194,
"logits/rejected": -0.4615291357040405,
"logps/chosen": -114.79924011230469,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -211.638671875,
"loss": 1.0883,
"margin_dpo/margin_mean": 67.49888610839844,
"margin_dpo/margin_std": 102.39913940429688,
"step": 153
},
{
"KL/chosen_KL_mean": -70.078369140625,
"KL/mean": -102.3911361694336,
"KL/rejected_KL_mean": -134.70388793945312,
"KL/std": 84.00700378417969,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.006613458506762981,
"fcm_dpo/delta": -0.02885277569293976,
"fcm_dpo/margin": 64.62553405761719,
"fcm_dpo/q_t": 0.40623587369918823,
"grad_norm": 19.01382827758789,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.4539533853530884,
"logits/rejected": -0.4565533995628357,
"logps/chosen": -132.82093811035156,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -238.94808959960938,
"loss": 1.1195,
"margin_dpo/margin_mean": 64.62552642822266,
"margin_dpo/margin_std": 106.87834167480469,
"step": 154
},
{
"KL/chosen_KL_mean": -63.6666374206543,
"KL/mean": -96.20714569091797,
"KL/rejected_KL_mean": -128.74766540527344,
"KL/std": 77.72993469238281,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.006566190160810947,
"fcm_dpo/delta": -0.028627503663301468,
"fcm_dpo/margin": 65.0810317993164,
"fcm_dpo/q_t": 0.40416377782821655,
"grad_norm": 19.003984451293945,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.466948926448822,
"logits/rejected": -0.44710594415664673,
"logps/chosen": -124.31982421875,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -206.2398681640625,
"loss": 1.0947,
"margin_dpo/margin_mean": 65.0810317993164,
"margin_dpo/margin_std": 96.49385070800781,
"step": 155
},
{
"KL/chosen_KL_mean": -89.43757629394531,
"KL/mean": -108.11923217773438,
"KL/rejected_KL_mean": -126.8009033203125,
"KL/std": 82.73648071289062,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.006675416603684425,
"fcm_dpo/delta": 0.15469492971897125,
"fcm_dpo/margin": 37.36332702636719,
"fcm_dpo/q_t": 0.4441591501235962,
"grad_norm": 28.448909759521484,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.42967504262924194,
"logits/rejected": -0.42389848828315735,
"logps/chosen": -158.929443359375,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -203.97018432617188,
"loss": 1.2757,
"margin_dpo/margin_mean": 37.36332702636719,
"margin_dpo/margin_std": 108.8001708984375,
"step": 156
},
{
"KL/chosen_KL_mean": -80.1298828125,
"KL/mean": -121.85565185546875,
"KL/rejected_KL_mean": -163.5814208984375,
"KL/std": 90.1138916015625,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.00655252393335104,
"fcm_dpo/delta": -0.1568107306957245,
"fcm_dpo/margin": 83.45154571533203,
"fcm_dpo/q_t": 0.3757891058921814,
"grad_norm": 23.798870086669922,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.38265031576156616,
"logits/rejected": -0.3891392946243286,
"logps/chosen": -141.49832153320312,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -271.227783203125,
"loss": 1.0205,
"margin_dpo/margin_mean": 83.45154571533203,
"margin_dpo/margin_std": 102.42288208007812,
"step": 157
},
{
"KL/chosen_KL_mean": -80.79515075683594,
"KL/mean": -125.07054138183594,
"KL/rejected_KL_mean": -169.345947265625,
"KL/std": 111.43318176269531,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.006371090654283762,
"fcm_dpo/delta": -0.17393946647644043,
"fcm_dpo/margin": 88.55078125,
"fcm_dpo/q_t": 0.3846416473388672,
"grad_norm": 20.157474517822266,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.37623435258865356,
"logits/rejected": -0.39367592334747314,
"logps/chosen": -138.4080810546875,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -283.0406188964844,
"loss": 1.0498,
"margin_dpo/margin_mean": 88.55077362060547,
"margin_dpo/margin_std": 134.24615478515625,
"step": 158
},
{
"KL/chosen_KL_mean": -91.09815979003906,
"KL/mean": -121.51444244384766,
"KL/rejected_KL_mean": -151.93072509765625,
"KL/std": 98.36724853515625,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.0063539366237819195,
"fcm_dpo/delta": 0.013539198786020279,
"fcm_dpo/margin": 60.832557678222656,
"fcm_dpo/q_t": 0.4142197370529175,
"grad_norm": 21.495512008666992,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -0.40772944688796997,
"logits/rejected": -0.3778040409088135,
"logps/chosen": -172.65850830078125,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -240.82943725585938,
"loss": 1.1449,
"margin_dpo/margin_mean": 60.832557678222656,
"margin_dpo/margin_std": 106.78931427001953,
"step": 159
},
{
"KL/chosen_KL_mean": -92.82463073730469,
"KL/mean": -133.309326171875,
"KL/rejected_KL_mean": -173.79403686523438,
"KL/std": 104.54073333740234,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.0061985463835299015,
"fcm_dpo/delta": -0.10827823728322983,
"fcm_dpo/margin": 80.96941375732422,
"fcm_dpo/q_t": 0.39520615339279175,
"grad_norm": 23.35076332092285,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.36829066276550293,
"logits/rejected": -0.35965633392333984,
"logps/chosen": -158.55551147460938,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -271.0118408203125,
"loss": 1.0896,
"margin_dpo/margin_mean": 80.96941375732422,
"margin_dpo/margin_std": 131.15676879882812,
"step": 160
},
{
"KL/chosen_KL_mean": -79.95537567138672,
"KL/mean": -115.55949401855469,
"KL/rejected_KL_mean": -151.1636199951172,
"KL/std": 82.77009582519531,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.006156083196401596,
"fcm_dpo/delta": -0.040130577981472015,
"fcm_dpo/margin": 71.20826721191406,
"fcm_dpo/q_t": 0.40316489338874817,
"grad_norm": 22.19649887084961,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.37738293409347534,
"logits/rejected": -0.37736430764198303,
"logps/chosen": -132.391845703125,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -234.59457397460938,
"loss": 1.0892,
"margin_dpo/margin_mean": 71.20826721191406,
"margin_dpo/margin_std": 104.3830795288086,
"step": 161
},
{
"KL/chosen_KL_mean": -76.72035217285156,
"KL/mean": -112.05348205566406,
"KL/rejected_KL_mean": -147.3865966796875,
"KL/std": 91.60896301269531,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.0060644932091236115,
"fcm_dpo/delta": -0.031555503606796265,
"fcm_dpo/margin": 70.66625213623047,
"fcm_dpo/q_t": 0.40564799308776855,
"grad_norm": 22.75583267211914,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.42984259128570557,
"logits/rejected": -0.414145290851593,
"logps/chosen": -139.3309326171875,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -236.77719116210938,
"loss": 1.1054,
"margin_dpo/margin_mean": 70.666259765625,
"margin_dpo/margin_std": 108.65666961669922,
"step": 162
},
{
"KL/chosen_KL_mean": -86.96597290039062,
"KL/mean": -121.1256103515625,
"KL/rejected_KL_mean": -155.28524780273438,
"KL/std": 92.2059326171875,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.006092265248298645,
"fcm_dpo/delta": -0.016920365393161774,
"fcm_dpo/margin": 68.31927490234375,
"fcm_dpo/q_t": 0.40940380096435547,
"grad_norm": 22.025920867919922,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.3410716652870178,
"logits/rejected": -0.3217606544494629,
"logps/chosen": -141.98760986328125,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -230.70347595214844,
"loss": 1.1158,
"margin_dpo/margin_mean": 68.31927490234375,
"margin_dpo/margin_std": 111.27782440185547,
"step": 163
},
{
"KL/chosen_KL_mean": -78.96578979492188,
"KL/mean": -120.85997009277344,
"KL/rejected_KL_mean": -162.754150390625,
"KL/std": 89.56776428222656,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.005949662998318672,
"fcm_dpo/delta": -0.1053713783621788,
"fcm_dpo/margin": 83.78836822509766,
"fcm_dpo/q_t": 0.3873593211174011,
"grad_norm": 21.903921127319336,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.3370760381221771,
"logits/rejected": -0.32179197669029236,
"logps/chosen": -134.60646057128906,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -242.41879272460938,
"loss": 1.0341,
"margin_dpo/margin_mean": 83.78837585449219,
"margin_dpo/margin_std": 102.59002685546875,
"step": 164
},
{
"KL/chosen_KL_mean": -84.470703125,
"KL/mean": -111.83903503417969,
"KL/rejected_KL_mean": -139.20736694335938,
"KL/std": 75.84541320800781,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.006008328404277563,
"fcm_dpo/delta": 0.0735950842499733,
"fcm_dpo/margin": 54.73666000366211,
"fcm_dpo/q_t": 0.4263015389442444,
"grad_norm": 22.944101333618164,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.3864282965660095,
"logits/rejected": -0.36028358340263367,
"logps/chosen": -145.78138732910156,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -212.8779754638672,
"loss": 1.1705,
"margin_dpo/margin_mean": 54.736656188964844,
"margin_dpo/margin_std": 100.56073760986328,
"step": 165
},
{
"KL/chosen_KL_mean": -74.94708251953125,
"KL/mean": -125.40420532226562,
"KL/rejected_KL_mean": -175.861328125,
"KL/std": 110.8086929321289,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.005840961821377277,
"fcm_dpo/delta": -0.20264847576618195,
"fcm_dpo/margin": 100.91423034667969,
"fcm_dpo/q_t": 0.37943559885025024,
"grad_norm": 17.69853401184082,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.33663398027420044,
"logits/rejected": -0.3471418023109436,
"logps/chosen": -125.9306869506836,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -273.9564514160156,
"loss": 1.0146,
"margin_dpo/margin_mean": 100.91423034667969,
"margin_dpo/margin_std": 139.43954467773438,
"step": 166
},
{
"KL/chosen_KL_mean": -75.11039733886719,
"KL/mean": -128.34188842773438,
"KL/rejected_KL_mean": -181.57339477539062,
"KL/std": 100.28507232666016,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.005624993704259396,
"fcm_dpo/delta": -0.21160906553268433,
"fcm_dpo/margin": 106.46297454833984,
"fcm_dpo/q_t": 0.3662090003490448,
"grad_norm": 23.289794921875,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.33210843801498413,
"logits/rejected": -0.33472272753715515,
"logps/chosen": -125.53449249267578,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -277.60382080078125,
"loss": 0.968,
"margin_dpo/margin_mean": 106.46297454833984,
"margin_dpo/margin_std": 115.56710815429688,
"step": 167
},
{
"KL/chosen_KL_mean": -81.955078125,
"KL/mean": -120.9576187133789,
"KL/rejected_KL_mean": -159.9601593017578,
"KL/std": 94.59892272949219,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.005527859553694725,
"fcm_dpo/delta": -0.03275269269943237,
"fcm_dpo/margin": 78.00506591796875,
"fcm_dpo/q_t": 0.40346774458885193,
"grad_norm": 20.936565399169922,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.3593074083328247,
"logits/rejected": -0.35054898262023926,
"logps/chosen": -131.41790771484375,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -235.26870727539062,
"loss": 1.078,
"margin_dpo/margin_mean": 78.00506591796875,
"margin_dpo/margin_std": 105.96997833251953,
"step": 168
},
{
"KL/chosen_KL_mean": -83.62703704833984,
"KL/mean": -117.71241760253906,
"KL/rejected_KL_mean": -151.7978057861328,
"KL/std": 92.74655151367188,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.005502847954630852,
"fcm_dpo/delta": 0.02455396205186844,
"fcm_dpo/margin": 68.17076110839844,
"fcm_dpo/q_t": 0.41967761516571045,
"grad_norm": 20.715503692626953,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.36550360918045044,
"logits/rejected": -0.34896135330200195,
"logps/chosen": -143.43048095703125,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -235.1435546875,
"loss": 1.1596,
"margin_dpo/margin_mean": 68.17076110839844,
"margin_dpo/margin_std": 127.58938598632812,
"step": 169
},
{
"KL/chosen_KL_mean": -73.78776550292969,
"KL/mean": -114.02574157714844,
"KL/rejected_KL_mean": -154.26370239257812,
"KL/std": 88.0094985961914,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.005487416870892048,
"fcm_dpo/delta": -0.04451485723257065,
"fcm_dpo/margin": 80.47593688964844,
"fcm_dpo/q_t": 0.39898285269737244,
"grad_norm": 18.808998107910156,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.3425253629684448,
"logits/rejected": -0.3321423828601837,
"logps/chosen": -123.25953674316406,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -230.18104553222656,
"loss": 1.0729,
"margin_dpo/margin_mean": 80.47593688964844,
"margin_dpo/margin_std": 105.5477066040039,
"step": 170
},
{
"KL/chosen_KL_mean": -107.6456298828125,
"KL/mean": -140.05889892578125,
"KL/rejected_KL_mean": -172.47218322753906,
"KL/std": 101.28620910644531,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.005539648234844208,
"fcm_dpo/delta": 0.04242313653230667,
"fcm_dpo/margin": 64.82654571533203,
"fcm_dpo/q_t": 0.4240008294582367,
"grad_norm": 29.54085922241211,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.34802329540252686,
"logits/rejected": -0.32956576347351074,
"logps/chosen": -192.1449432373047,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -281.8542785644531,
"loss": 1.1712,
"margin_dpo/margin_mean": 64.82654571533203,
"margin_dpo/margin_std": 129.8333282470703,
"step": 171
},
{
"KL/chosen_KL_mean": -94.89369201660156,
"KL/mean": -128.6036376953125,
"KL/rejected_KL_mean": -162.31356811523438,
"KL/std": 97.58683776855469,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.005568898282945156,
"fcm_dpo/delta": 0.025504067540168762,
"fcm_dpo/margin": 67.41987609863281,
"fcm_dpo/q_t": 0.41584065556526184,
"grad_norm": 24.327449798583984,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.3986096680164337,
"logits/rejected": -0.3816487789154053,
"logps/chosen": -163.547607421875,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -247.750244140625,
"loss": 1.1322,
"margin_dpo/margin_mean": 67.41987609863281,
"margin_dpo/margin_std": 110.49188232421875,
"step": 172
},
{
"KL/chosen_KL_mean": -87.27731323242188,
"KL/mean": -121.42498779296875,
"KL/rejected_KL_mean": -155.57266235351562,
"KL/std": 92.45211791992188,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.005617057904601097,
"fcm_dpo/delta": 0.016708562150597572,
"fcm_dpo/margin": 68.29533386230469,
"fcm_dpo/q_t": 0.41303884983062744,
"grad_norm": 20.406827926635742,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.41884762048721313,
"logits/rejected": -0.39233148097991943,
"logps/chosen": -150.32818603515625,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -234.256591796875,
"loss": 1.1075,
"margin_dpo/margin_mean": 68.29534149169922,
"margin_dpo/margin_std": 96.916748046875,
"step": 173
},
{
"KL/chosen_KL_mean": -83.62928771972656,
"KL/mean": -124.2885971069336,
"KL/rejected_KL_mean": -164.94790649414062,
"KL/std": 97.81199645996094,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.005588578060269356,
"fcm_dpo/delta": -0.05747688561677933,
"fcm_dpo/margin": 81.31861114501953,
"fcm_dpo/q_t": 0.40022996068000793,
"grad_norm": 21.319252014160156,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.3324238061904907,
"logits/rejected": -0.34984150528907776,
"logps/chosen": -136.99224853515625,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -266.859130859375,
"loss": 1.0818,
"margin_dpo/margin_mean": 81.31861114501953,
"margin_dpo/margin_std": 116.7562026977539,
"step": 174
},
{
"KL/chosen_KL_mean": -77.55169677734375,
"KL/mean": -133.1820068359375,
"KL/rejected_KL_mean": -188.81231689453125,
"KL/std": 105.05592346191406,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.0053607383742928505,
"fcm_dpo/delta": -0.20923927426338196,
"fcm_dpo/margin": 111.2606201171875,
"fcm_dpo/q_t": 0.3642469048500061,
"grad_norm": 26.155202865600586,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.28884798288345337,
"logits/rejected": -0.2741377353668213,
"logps/chosen": -122.96946716308594,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -278.318115234375,
"loss": 0.9525,
"margin_dpo/margin_mean": 111.2606201171875,
"margin_dpo/margin_std": 109.48503875732422,
"step": 175
},
{
"KL/chosen_KL_mean": -84.14366149902344,
"KL/mean": -130.95176696777344,
"KL/rejected_KL_mean": -177.7598876953125,
"KL/std": 104.69883728027344,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.005244849249720573,
"fcm_dpo/delta": -0.09559769183397293,
"fcm_dpo/margin": 93.61622619628906,
"fcm_dpo/q_t": 0.3924563527107239,
"grad_norm": 19.009029388427734,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.3173462450504303,
"logits/rejected": -0.3352587819099426,
"logps/chosen": -134.59649658203125,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -273.31884765625,
"loss": 1.0433,
"margin_dpo/margin_mean": 93.61622619628906,
"margin_dpo/margin_std": 121.37356567382812,
"step": 176
},
{
"KL/chosen_KL_mean": -98.31048583984375,
"KL/mean": -144.74362182617188,
"KL/rejected_KL_mean": -191.17674255371094,
"KL/std": 113.88668823242188,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.005161338951438665,
"fcm_dpo/delta": -0.08324076235294342,
"fcm_dpo/margin": 92.86624145507812,
"fcm_dpo/q_t": 0.3939231038093567,
"grad_norm": 26.65350914001465,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.316158264875412,
"logits/rejected": -0.30725324153900146,
"logps/chosen": -159.52694702148438,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -287.0705261230469,
"loss": 1.0487,
"margin_dpo/margin_mean": 92.86624145507812,
"margin_dpo/margin_std": 120.88250732421875,
"step": 177
},
{
"KL/chosen_KL_mean": -109.49505615234375,
"KL/mean": -168.2081298828125,
"KL/rejected_KL_mean": -226.9211883544922,
"KL/std": 134.9578094482422,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.004957615397870541,
"fcm_dpo/delta": -0.19456958770751953,
"fcm_dpo/margin": 117.42613983154297,
"fcm_dpo/q_t": 0.3746492266654968,
"grad_norm": 25.899173736572266,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.247529536485672,
"logits/rejected": -0.23764248192310333,
"logps/chosen": -167.7598419189453,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -332.2865295410156,
"loss": 1.0001,
"margin_dpo/margin_mean": 117.42613983154297,
"margin_dpo/margin_std": 148.1576385498047,
"step": 178
},
{
"KL/chosen_KL_mean": -113.17926025390625,
"KL/mean": -154.49534606933594,
"KL/rejected_KL_mean": -195.81143188476562,
"KL/std": 114.84059143066406,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.004900245927274227,
"fcm_dpo/delta": -0.005644813179969788,
"fcm_dpo/margin": 82.63217163085938,
"fcm_dpo/q_t": 0.4111664593219757,
"grad_norm": 32.68442153930664,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.27366673946380615,
"logits/rejected": -0.28250646591186523,
"logps/chosen": -174.23757934570312,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -286.3392639160156,
"loss": 1.1353,
"margin_dpo/margin_mean": 82.63216400146484,
"margin_dpo/margin_std": 142.4680633544922,
"step": 179
},
{
"KL/chosen_KL_mean": -98.1256103515625,
"KL/mean": -151.30715942382812,
"KL/rejected_KL_mean": -204.48873901367188,
"KL/std": 103.5050048828125,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.004829989746212959,
"fcm_dpo/delta": -0.12006718665361404,
"fcm_dpo/margin": 106.36312866210938,
"fcm_dpo/q_t": 0.38355523347854614,
"grad_norm": 20.321218490600586,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.30043309926986694,
"logits/rejected": -0.2955361604690552,
"logps/chosen": -152.46832275390625,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -302.7005615234375,
"loss": 1.0233,
"margin_dpo/margin_mean": 106.36312866210938,
"margin_dpo/margin_std": 128.10833740234375,
"step": 180
},
{
"KL/chosen_KL_mean": -84.74370574951172,
"KL/mean": -117.97935485839844,
"KL/rejected_KL_mean": -151.21499633789062,
"KL/std": 94.81138610839844,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.0048685576766729355,
"fcm_dpo/delta": 0.07881682366132736,
"fcm_dpo/margin": 66.4712905883789,
"fcm_dpo/q_t": 0.42652463912963867,
"grad_norm": 18.8144474029541,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.2951802611351013,
"logits/rejected": -0.2693043351173401,
"logps/chosen": -139.74417114257812,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -212.87115478515625,
"loss": 1.1676,
"margin_dpo/margin_mean": 66.4712905883789,
"margin_dpo/margin_std": 119.32975006103516,
"step": 181
},
{
"KL/chosen_KL_mean": -79.9010009765625,
"KL/mean": -137.60540771484375,
"KL/rejected_KL_mean": -195.309814453125,
"KL/std": 114.05545043945312,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.004749820567667484,
"fcm_dpo/delta": -0.157757967710495,
"fcm_dpo/margin": 115.4088363647461,
"fcm_dpo/q_t": 0.37575432658195496,
"grad_norm": 18.484952926635742,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.2895713448524475,
"logits/rejected": -0.29972368478775024,
"logps/chosen": -121.00885009765625,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -284.8314208984375,
"loss": 1.0091,
"margin_dpo/margin_mean": 115.4088363647461,
"margin_dpo/margin_std": 137.26278686523438,
"step": 182
},
{
"KL/chosen_KL_mean": -117.18058776855469,
"KL/mean": -148.19338989257812,
"KL/rejected_KL_mean": -179.2061767578125,
"KL/std": 94.16122436523438,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.004760707728564739,
"fcm_dpo/delta": 0.10700409859418869,
"fcm_dpo/margin": 62.02560043334961,
"fcm_dpo/q_t": 0.4322977662086487,
"grad_norm": 21.09385108947754,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.2902984619140625,
"logits/rejected": -0.2783670127391815,
"logps/chosen": -174.7051544189453,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -255.18190002441406,
"loss": 1.1813,
"margin_dpo/margin_mean": 62.02560043334961,
"margin_dpo/margin_std": 103.5242691040039,
"step": 183
},
{
"KL/chosen_KL_mean": -97.4403076171875,
"KL/mean": -130.1512908935547,
"KL/rejected_KL_mean": -162.86228942871094,
"KL/std": 88.96430969238281,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.004917686805129051,
"fcm_dpo/delta": 0.07984302192926407,
"fcm_dpo/margin": 65.42196655273438,
"fcm_dpo/q_t": 0.42779117822647095,
"grad_norm": 19.217758178710938,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.34908443689346313,
"logits/rejected": -0.3400712013244629,
"logps/chosen": -155.98526000976562,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -239.496337890625,
"loss": 1.1633,
"margin_dpo/margin_mean": 65.42196655273438,
"margin_dpo/margin_std": 111.91808319091797,
"step": 184
},
{
"KL/chosen_KL_mean": -104.18730926513672,
"KL/mean": -129.8303680419922,
"KL/rejected_KL_mean": -155.47341918945312,
"KL/std": 100.83888244628906,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.00501694530248642,
"fcm_dpo/delta": 0.14654436707496643,
"fcm_dpo/margin": 51.286109924316406,
"fcm_dpo/q_t": 0.44505059719085693,
"grad_norm": 21.227956771850586,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.29932302236557007,
"logits/rejected": -0.27377188205718994,
"logps/chosen": -166.21316528320312,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -229.23602294921875,
"loss": 1.2302,
"margin_dpo/margin_mean": 51.286109924316406,
"margin_dpo/margin_std": 121.88388061523438,
"step": 185
},
{
"KL/chosen_KL_mean": -95.56036376953125,
"KL/mean": -144.005126953125,
"KL/rejected_KL_mean": -192.44989013671875,
"KL/std": 101.79664611816406,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.004984917119145393,
"fcm_dpo/delta": -0.08735480904579163,
"fcm_dpo/margin": 96.88951873779297,
"fcm_dpo/q_t": 0.3907889723777771,
"grad_norm": 28.150287628173828,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.3715486228466034,
"logits/rejected": -0.3490146994590759,
"logps/chosen": -164.91383361816406,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -280.5223388671875,
"loss": 1.0364,
"margin_dpo/margin_mean": 96.88951873779297,
"margin_dpo/margin_std": 116.3585205078125,
"step": 186
},
{
"KL/chosen_KL_mean": -89.00492858886719,
"KL/mean": -129.27084350585938,
"KL/rejected_KL_mean": -169.53675842285156,
"KL/std": 96.89251708984375,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.004988536238670349,
"fcm_dpo/delta": -0.0021523088216781616,
"fcm_dpo/margin": 80.53184509277344,
"fcm_dpo/q_t": 0.40814200043678284,
"grad_norm": 23.734664916992188,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.3111017942428589,
"logits/rejected": -0.3041296601295471,
"logps/chosen": -141.76138305664062,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -251.505859375,
"loss": 1.0907,
"margin_dpo/margin_mean": 80.5318374633789,
"margin_dpo/margin_std": 105.92333984375,
"step": 187
},
{
"KL/chosen_KL_mean": -81.93511962890625,
"KL/mean": -130.56826782226562,
"KL/rejected_KL_mean": -179.20138549804688,
"KL/std": 106.78981018066406,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.0049115014262497425,
"fcm_dpo/delta": -0.0815688967704773,
"fcm_dpo/margin": 97.26626586914062,
"fcm_dpo/q_t": 0.3923574984073639,
"grad_norm": 31.012367248535156,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.3215259909629822,
"logits/rejected": -0.3264486491680145,
"logps/chosen": -131.35060119628906,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -268.7418212890625,
"loss": 1.039,
"margin_dpo/margin_mean": 97.26626586914062,
"margin_dpo/margin_std": 116.94786071777344,
"step": 188
},
{
"KL/chosen_KL_mean": -94.69155883789062,
"KL/mean": -135.35601806640625,
"KL/rejected_KL_mean": -176.02047729492188,
"KL/std": 106.91317749023438,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.004893161356449127,
"fcm_dpo/delta": 0.002108026295900345,
"fcm_dpo/margin": 81.32891082763672,
"fcm_dpo/q_t": 0.41269636154174805,
"grad_norm": 29.160770416259766,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.34372690320014954,
"logits/rejected": -0.3232707381248474,
"logps/chosen": -147.09051513671875,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -248.1878204345703,
"loss": 1.1171,
"margin_dpo/margin_mean": 81.32891845703125,
"margin_dpo/margin_std": 131.2293701171875,
"step": 189
},
{
"KL/chosen_KL_mean": -101.5174560546875,
"KL/mean": -149.2252197265625,
"KL/rejected_KL_mean": -196.93295288085938,
"KL/std": 114.80913543701172,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.004881042987108231,
"fcm_dpo/delta": -0.06988224387168884,
"fcm_dpo/margin": 95.41551971435547,
"fcm_dpo/q_t": 0.3969564437866211,
"grad_norm": 19.110261917114258,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.3549377918243408,
"logits/rejected": -0.3403692841529846,
"logps/chosen": -166.20053100585938,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -299.4834899902344,
"loss": 1.075,
"margin_dpo/margin_mean": 95.41551971435547,
"margin_dpo/margin_std": 132.25625610351562,
"step": 190
},
{
"KL/chosen_KL_mean": -94.4166259765625,
"KL/mean": -162.89239501953125,
"KL/rejected_KL_mean": -231.3681640625,
"KL/std": 133.07308959960938,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.004625319503247738,
"fcm_dpo/delta": -0.250690221786499,
"fcm_dpo/margin": 136.9515380859375,
"fcm_dpo/q_t": 0.3601779639720917,
"grad_norm": 20.274974822998047,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.34608668088912964,
"logits/rejected": -0.3248242735862732,
"logps/chosen": -163.07550048828125,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -341.5078125,
"loss": 0.9469,
"margin_dpo/margin_mean": 136.9515380859375,
"margin_dpo/margin_std": 146.31927490234375,
"step": 191
},
{
"KL/chosen_KL_mean": -122.44120788574219,
"KL/mean": -167.78265380859375,
"KL/rejected_KL_mean": -213.12411499023438,
"KL/std": 115.74710845947266,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.004585623275488615,
"fcm_dpo/delta": -0.016613848507404327,
"fcm_dpo/margin": 90.68290710449219,
"fcm_dpo/q_t": 0.4083861708641052,
"grad_norm": 23.432592391967773,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.32005661725997925,
"logits/rejected": -0.3190295696258545,
"logps/chosen": -192.16812133789062,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -316.4454650878906,
"loss": 1.1089,
"margin_dpo/margin_mean": 90.68290710449219,
"margin_dpo/margin_std": 142.26962280273438,
"step": 192
},
{
"KL/chosen_KL_mean": -123.967041015625,
"KL/mean": -150.6930694580078,
"KL/rejected_KL_mean": -177.4190673828125,
"KL/std": 106.99697875976562,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.004696063697338104,
"fcm_dpo/delta": 0.15172256529331207,
"fcm_dpo/margin": 53.45201873779297,
"fcm_dpo/q_t": 0.4412054121494293,
"grad_norm": 27.434032440185547,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.3420638144016266,
"logits/rejected": -0.3330894708633423,
"logps/chosen": -184.1575469970703,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -253.82662963867188,
"loss": 1.249,
"margin_dpo/margin_mean": 53.4520263671875,
"margin_dpo/margin_std": 136.81741333007812,
"step": 193
},
{
"KL/chosen_KL_mean": -77.48358154296875,
"KL/mean": -120.73272705078125,
"KL/rejected_KL_mean": -163.98187255859375,
"KL/std": 89.46562957763672,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.004693003837019205,
"fcm_dpo/delta": -0.006373826414346695,
"fcm_dpo/margin": 86.498291015625,
"fcm_dpo/q_t": 0.40509384870529175,
"grad_norm": 18.315134048461914,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.25825223326683044,
"logits/rejected": -0.24261374771595,
"logps/chosen": -115.32395935058594,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -224.66665649414062,
"loss": 1.0718,
"margin_dpo/margin_mean": 86.498291015625,
"margin_dpo/margin_std": 98.0810775756836,
"step": 194
},
{
"KL/chosen_KL_mean": -122.55229949951172,
"KL/mean": -169.00955200195312,
"KL/rejected_KL_mean": -215.466796875,
"KL/std": 110.25206756591797,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.004678967874497175,
"fcm_dpo/delta": -0.03632781654596329,
"fcm_dpo/margin": 92.91448974609375,
"fcm_dpo/q_t": 0.4003722071647644,
"grad_norm": 21.463722229003906,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.28670287132263184,
"logits/rejected": -0.3056129813194275,
"logps/chosen": -177.44387817382812,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -312.23773193359375,
"loss": 1.0636,
"margin_dpo/margin_mean": 92.91448974609375,
"margin_dpo/margin_std": 114.20263671875,
"step": 195
},
{
"KL/chosen_KL_mean": -96.20404815673828,
"KL/mean": -149.17095947265625,
"KL/rejected_KL_mean": -202.1378631591797,
"KL/std": 113.6821060180664,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.004583077039569616,
"fcm_dpo/delta": -0.09096772968769073,
"fcm_dpo/margin": 105.9338150024414,
"fcm_dpo/q_t": 0.39174580574035645,
"grad_norm": 18.06390380859375,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.32519471645355225,
"logits/rejected": -0.31019407510757446,
"logps/chosen": -149.4492950439453,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -278.1907958984375,
"loss": 1.0516,
"margin_dpo/margin_mean": 105.9338150024414,
"margin_dpo/margin_std": 136.15292358398438,
"step": 196
},
{
"KL/chosen_KL_mean": -99.3115234375,
"KL/mean": -140.2286376953125,
"KL/rejected_KL_mean": -181.14573669433594,
"KL/std": 98.75188446044922,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.004617801867425442,
"fcm_dpo/delta": 0.022662952542304993,
"fcm_dpo/margin": 81.83421325683594,
"fcm_dpo/q_t": 0.41381216049194336,
"grad_norm": 18.633390426635742,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.31022053956985474,
"logits/rejected": -0.30425357818603516,
"logps/chosen": -159.7318572998047,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -258.3546447753906,
"loss": 1.1065,
"margin_dpo/margin_mean": 81.83421325683594,
"margin_dpo/margin_std": 111.92158508300781,
"step": 197
},
{
"KL/chosen_KL_mean": -112.12559509277344,
"KL/mean": -164.85427856445312,
"KL/rejected_KL_mean": -217.58297729492188,
"KL/std": 124.64435577392578,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.004552255384624004,
"fcm_dpo/delta": -0.08407382667064667,
"fcm_dpo/margin": 105.45738220214844,
"fcm_dpo/q_t": 0.3944292962551117,
"grad_norm": 22.357629776000977,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.32955610752105713,
"logits/rejected": -0.32924240827560425,
"logps/chosen": -167.16177368164062,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -314.82623291015625,
"loss": 1.0521,
"margin_dpo/margin_mean": 105.45738983154297,
"margin_dpo/margin_std": 141.28903198242188,
"step": 198
},
{
"KL/chosen_KL_mean": -107.11283874511719,
"KL/mean": -159.80538940429688,
"KL/rejected_KL_mean": -212.49794006347656,
"KL/std": 113.97561645507812,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.004454015754163265,
"fcm_dpo/delta": -0.07400257885456085,
"fcm_dpo/margin": 105.38510131835938,
"fcm_dpo/q_t": 0.3943500220775604,
"grad_norm": 21.528413772583008,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.3109109401702881,
"logits/rejected": -0.2927241027355194,
"logps/chosen": -163.94166564941406,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -297.1461486816406,
"loss": 1.0586,
"margin_dpo/margin_mean": 105.38510131835938,
"margin_dpo/margin_std": 137.27755737304688,
"step": 199
},
{
"KL/chosen_KL_mean": -106.08656311035156,
"KL/mean": -158.6329345703125,
"KL/rejected_KL_mean": -211.1792755126953,
"KL/std": 122.60960388183594,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.004403954371809959,
"fcm_dpo/delta": -0.06639470160007477,
"fcm_dpo/margin": 105.09272003173828,
"fcm_dpo/q_t": 0.3984246253967285,
"grad_norm": 21.341712951660156,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.27302879095077515,
"logits/rejected": -0.2546115517616272,
"logps/chosen": -159.15362548828125,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -291.7877197265625,
"loss": 1.0826,
"margin_dpo/margin_mean": 105.09272003173828,
"margin_dpo/margin_std": 156.76052856445312,
"step": 200
},
{
"KL/chosen_KL_mean": -112.1373062133789,
"KL/mean": -163.8036651611328,
"KL/rejected_KL_mean": -215.47003173828125,
"KL/std": 128.93569946289062,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.0043801432475447655,
"fcm_dpo/delta": -0.05509566143155098,
"fcm_dpo/margin": 103.33273315429688,
"fcm_dpo/q_t": 0.39835184812545776,
"grad_norm": 21.340782165527344,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.37157952785491943,
"logits/rejected": -0.36827552318573,
"logps/chosen": -187.53952026367188,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -330.27825927734375,
"loss": 1.0749,
"margin_dpo/margin_mean": 103.33273315429688,
"margin_dpo/margin_std": 143.89541625976562,
"step": 201
},
{
"KL/chosen_KL_mean": -114.76101684570312,
"KL/mean": -151.6962127685547,
"KL/rejected_KL_mean": -188.63140869140625,
"KL/std": 110.6797866821289,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.004410090856254101,
"fcm_dpo/delta": 0.07666227966547012,
"fcm_dpo/margin": 73.87037658691406,
"fcm_dpo/q_t": 0.42786508798599243,
"grad_norm": 22.023473739624023,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.28071317076683044,
"logits/rejected": -0.2946799397468567,
"logps/chosen": -164.86233520507812,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -275.616455078125,
"loss": 1.1785,
"margin_dpo/margin_mean": 73.87037658691406,
"margin_dpo/margin_std": 142.80360412597656,
"step": 202
},
{
"KL/chosen_KL_mean": -114.09391784667969,
"KL/mean": -159.502197265625,
"KL/rejected_KL_mean": -204.91046142578125,
"KL/std": 113.20623779296875,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.004417533054947853,
"fcm_dpo/delta": -0.0012725908309221268,
"fcm_dpo/margin": 90.81655883789062,
"fcm_dpo/q_t": 0.4084164500236511,
"grad_norm": 23.425310134887695,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.3172285556793213,
"logits/rejected": -0.3071226179599762,
"logps/chosen": -174.70361328125,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -290.80645751953125,
"loss": 1.0936,
"margin_dpo/margin_mean": 90.81655883789062,
"margin_dpo/margin_std": 123.70480346679688,
"step": 203
},
{
"KL/chosen_KL_mean": -127.60630798339844,
"KL/mean": -169.1259002685547,
"KL/rejected_KL_mean": -210.64549255371094,
"KL/std": 124.28507995605469,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.0044425297528505325,
"fcm_dpo/delta": 0.032297275960445404,
"fcm_dpo/margin": 83.0391845703125,
"fcm_dpo/q_t": 0.4190390110015869,
"grad_norm": 23.64316749572754,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.4175642132759094,
"logits/rejected": -0.38105836510658264,
"logps/chosen": -207.77127075195312,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -298.3414001464844,
"loss": 1.1456,
"margin_dpo/margin_mean": 83.0391845703125,
"margin_dpo/margin_std": 144.03652954101562,
"step": 204
},
{
"KL/chosen_KL_mean": -121.0059814453125,
"KL/mean": -176.66131591796875,
"KL/rejected_KL_mean": -232.31663513183594,
"KL/std": 126.79728698730469,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.004353957716375589,
"fcm_dpo/delta": -0.09127674996852875,
"fcm_dpo/margin": 111.31065368652344,
"fcm_dpo/q_t": 0.3891245722770691,
"grad_norm": 20.96511459350586,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.3102046549320221,
"logits/rejected": -0.27934738993644714,
"logps/chosen": -180.39071655273438,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -317.4416809082031,
"loss": 1.0485,
"margin_dpo/margin_mean": 111.31065368652344,
"margin_dpo/margin_std": 137.294677734375,
"step": 205
},
{
"KL/chosen_KL_mean": -110.99642181396484,
"KL/mean": -170.51837158203125,
"KL/rejected_KL_mean": -230.0403594970703,
"KL/std": 118.66337585449219,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.004280811175704002,
"fcm_dpo/delta": -0.1161835789680481,
"fcm_dpo/margin": 119.0439224243164,
"fcm_dpo/q_t": 0.38180607557296753,
"grad_norm": 25.16000747680664,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.2330382764339447,
"logits/rejected": -0.23527702689170837,
"logps/chosen": -157.96092224121094,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -328.99383544921875,
"loss": 1.0108,
"margin_dpo/margin_mean": 119.0439224243164,
"margin_dpo/margin_std": 128.36184692382812,
"step": 206
},
{
"KL/chosen_KL_mean": -102.92005920410156,
"KL/mean": -171.51202392578125,
"KL/rejected_KL_mean": -240.10397338867188,
"KL/std": 136.4534912109375,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.004162299912422895,
"fcm_dpo/delta": -0.18118005990982056,
"fcm_dpo/margin": 137.18392944335938,
"fcm_dpo/q_t": 0.3742104470729828,
"grad_norm": 23.807788848876953,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.33577364683151245,
"logits/rejected": -0.3098195493221283,
"logps/chosen": -158.976318359375,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -324.5517578125,
"loss": 0.9884,
"margin_dpo/margin_mean": 137.18392944335938,
"margin_dpo/margin_std": 158.6349334716797,
"step": 207
},
{
"KL/chosen_KL_mean": -158.25042724609375,
"KL/mean": -212.32925415039062,
"KL/rejected_KL_mean": -266.4080810546875,
"KL/std": 131.32470703125,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.00408403854817152,
"fcm_dpo/delta": -0.043813541531562805,
"fcm_dpo/margin": 108.15763092041016,
"fcm_dpo/q_t": 0.4003239870071411,
"grad_norm": 26.89968490600586,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.3362833261489868,
"logits/rejected": -0.3170960247516632,
"logps/chosen": -225.31805419921875,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -360.6949462890625,
"loss": 1.0899,
"margin_dpo/margin_mean": 108.15763092041016,
"margin_dpo/margin_std": 159.3949737548828,
"step": 208
},
{
"KL/chosen_KL_mean": -132.40557861328125,
"KL/mean": -179.732421875,
"KL/rejected_KL_mean": -227.05926513671875,
"KL/std": 117.1972427368164,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.004092153161764145,
"fcm_dpo/delta": 0.013167150318622589,
"fcm_dpo/margin": 94.65370178222656,
"fcm_dpo/q_t": 0.41229110956192017,
"grad_norm": 28.013294219970703,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.24393108487129211,
"logits/rejected": -0.23185241222381592,
"logps/chosen": -188.58726501464844,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -308.00079345703125,
"loss": 1.1249,
"margin_dpo/margin_mean": 94.6537094116211,
"margin_dpo/margin_std": 148.58241271972656,
"step": 209
},
{
"KL/chosen_KL_mean": -121.67991638183594,
"KL/mean": -176.57435607910156,
"KL/rejected_KL_mean": -231.46881103515625,
"KL/std": 121.48208618164062,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.004080562852323055,
"fcm_dpo/delta": -0.0504092276096344,
"fcm_dpo/margin": 109.78889465332031,
"fcm_dpo/q_t": 0.39947184920310974,
"grad_norm": 26.426563262939453,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.27577272057533264,
"logits/rejected": -0.26780596375465393,
"logps/chosen": -168.05172729492188,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -308.1504211425781,
"loss": 1.0716,
"margin_dpo/margin_mean": 109.78888702392578,
"margin_dpo/margin_std": 147.1390838623047,
"step": 210
},
{
"KL/chosen_KL_mean": -172.16531372070312,
"KL/mean": -219.18504333496094,
"KL/rejected_KL_mean": -266.20477294921875,
"KL/std": 139.02566528320312,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.004065372049808502,
"fcm_dpo/delta": 0.018387429416179657,
"fcm_dpo/margin": 94.03945922851562,
"fcm_dpo/q_t": 0.4178032875061035,
"grad_norm": 36.138675689697266,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.31168001890182495,
"logits/rejected": -0.27302902936935425,
"logps/chosen": -251.09767150878906,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -353.0257568359375,
"loss": 1.1431,
"margin_dpo/margin_mean": 94.03945922851562,
"margin_dpo/margin_std": 163.8000030517578,
"step": 211
},
{
"KL/chosen_KL_mean": -140.9364776611328,
"KL/mean": -207.66456604003906,
"KL/rejected_KL_mean": -274.39263916015625,
"KL/std": 148.56857299804688,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.003955514170229435,
"fcm_dpo/delta": -0.13721655309200287,
"fcm_dpo/margin": 133.4561767578125,
"fcm_dpo/q_t": 0.382011741399765,
"grad_norm": 24.687803268432617,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.31145745515823364,
"logits/rejected": -0.30414634943008423,
"logps/chosen": -199.13348388671875,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -377.45050048828125,
"loss": 1.0306,
"margin_dpo/margin_mean": 133.4561767578125,
"margin_dpo/margin_std": 164.7611846923828,
"step": 212
},
{
"KL/chosen_KL_mean": -132.23367309570312,
"KL/mean": -195.90211486816406,
"KL/rejected_KL_mean": -259.570556640625,
"KL/std": 128.409912109375,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.0038843729998916388,
"fcm_dpo/delta": -0.10020861029624939,
"fcm_dpo/margin": 127.33690643310547,
"fcm_dpo/q_t": 0.3873175084590912,
"grad_norm": 31.009477615356445,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.30835479497909546,
"logits/rejected": -0.2814680337905884,
"logps/chosen": -199.74636840820312,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -353.48529052734375,
"loss": 1.0333,
"margin_dpo/margin_mean": 127.33689880371094,
"margin_dpo/margin_std": 152.68551635742188,
"step": 213
},
{
"KL/chosen_KL_mean": -113.12945556640625,
"KL/mean": -170.78250122070312,
"KL/rejected_KL_mean": -228.435546875,
"KL/std": 127.31915283203125,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.003852786961942911,
"fcm_dpo/delta": -0.046301376074552536,
"fcm_dpo/margin": 115.30607604980469,
"fcm_dpo/q_t": 0.39957842230796814,
"grad_norm": 23.4969425201416,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.21872687339782715,
"logits/rejected": -0.22276321053504944,
"logps/chosen": -154.73434448242188,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -305.95294189453125,
"loss": 1.0659,
"margin_dpo/margin_mean": 115.30607604980469,
"margin_dpo/margin_std": 148.88754272460938,
"step": 214
},
{
"KL/chosen_KL_mean": -133.77352905273438,
"KL/mean": -193.89248657226562,
"KL/rejected_KL_mean": -254.01144409179688,
"KL/std": 130.903076171875,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.0037993849255144596,
"fcm_dpo/delta": -0.059836942702531815,
"fcm_dpo/margin": 120.23790740966797,
"fcm_dpo/q_t": 0.39470916986465454,
"grad_norm": 25.146631240844727,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.2989373803138733,
"logits/rejected": -0.29156550765037537,
"logps/chosen": -187.05279541015625,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -343.97607421875,
"loss": 1.0433,
"margin_dpo/margin_mean": 120.23790740966797,
"margin_dpo/margin_std": 136.16165161132812,
"step": 215
},
{
"KL/chosen_KL_mean": -137.1045379638672,
"KL/mean": -197.48190307617188,
"KL/rejected_KL_mean": -257.85931396484375,
"KL/std": 137.61212158203125,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.0037784408777952194,
"fcm_dpo/delta": -0.059020306915044785,
"fcm_dpo/margin": 120.75474548339844,
"fcm_dpo/q_t": 0.39792197942733765,
"grad_norm": 22.765789031982422,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.2852722406387329,
"logits/rejected": -0.26953035593032837,
"logps/chosen": -185.99234008789062,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -335.0582275390625,
"loss": 1.0761,
"margin_dpo/margin_mean": 120.75474548339844,
"margin_dpo/margin_std": 169.55313110351562,
"step": 216
},
{
"KL/chosen_KL_mean": -138.76242065429688,
"KL/mean": -209.293701171875,
"KL/rejected_KL_mean": -279.824951171875,
"KL/std": 140.5864715576172,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.003676855470985174,
"fcm_dpo/delta": -0.12545619904994965,
"fcm_dpo/margin": 141.06256103515625,
"fcm_dpo/q_t": 0.3828177750110626,
"grad_norm": 20.623205184936523,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.2778016924858093,
"logits/rejected": -0.27289605140686035,
"logps/chosen": -188.60772705078125,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -379.9032897949219,
"loss": 1.0089,
"margin_dpo/margin_mean": 141.06256103515625,
"margin_dpo/margin_std": 158.33660888671875,
"step": 217
},
{
"KL/chosen_KL_mean": -147.40609741210938,
"KL/mean": -201.98513793945312,
"KL/rejected_KL_mean": -256.5641784667969,
"KL/std": 138.4207763671875,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.003653950057923794,
"fcm_dpo/delta": 0.0011254064738750458,
"fcm_dpo/margin": 109.15806579589844,
"fcm_dpo/q_t": 0.410250723361969,
"grad_norm": 21.11243438720703,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.2692360579967499,
"logits/rejected": -0.2633015513420105,
"logps/chosen": -205.9827880859375,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -344.41058349609375,
"loss": 1.108,
"margin_dpo/margin_mean": 109.15806579589844,
"margin_dpo/margin_std": 161.23675537109375,
"step": 218
},
{
"KL/chosen_KL_mean": -157.17605590820312,
"KL/mean": -204.95626831054688,
"KL/rejected_KL_mean": -252.73646545410156,
"KL/std": 144.8588104248047,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.003704340197145939,
"fcm_dpo/delta": 0.04699729382991791,
"fcm_dpo/margin": 95.56039428710938,
"fcm_dpo/q_t": 0.420939564704895,
"grad_norm": 30.59574317932129,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.3211911916732788,
"logits/rejected": -0.3160223066806793,
"logps/chosen": -218.25991821289062,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -338.56689453125,
"loss": 1.1644,
"margin_dpo/margin_mean": 95.56039428710938,
"margin_dpo/margin_std": 177.19955444335938,
"step": 219
},
{
"KL/chosen_KL_mean": -177.92498779296875,
"KL/mean": -217.5970001220703,
"KL/rejected_KL_mean": -257.26898193359375,
"KL/std": 125.33465576171875,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.003746958449482918,
"fcm_dpo/delta": 0.10597775876522064,
"fcm_dpo/margin": 79.34402465820312,
"fcm_dpo/q_t": 0.4305725693702698,
"grad_norm": 28.42041015625,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.30295485258102417,
"logits/rejected": -0.28097742795944214,
"logps/chosen": -247.95626831054688,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -344.95452880859375,
"loss": 1.1672,
"margin_dpo/margin_mean": 79.34402465820312,
"margin_dpo/margin_std": 129.00704956054688,
"step": 220
},
{
"KL/chosen_KL_mean": -154.29933166503906,
"KL/mean": -240.99725341796875,
"KL/rejected_KL_mean": -327.6951904296875,
"KL/std": 159.48818969726562,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.0036492447834461927,
"fcm_dpo/delta": -0.24793776869773865,
"fcm_dpo/margin": 173.39584350585938,
"fcm_dpo/q_t": 0.35594797134399414,
"grad_norm": 23.717262268066406,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.270913690328598,
"logits/rejected": -0.2767089307308197,
"logps/chosen": -206.45404052734375,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -434.162841796875,
"loss": 0.9334,
"margin_dpo/margin_mean": 173.39584350585938,
"margin_dpo/margin_std": 163.9197998046875,
"step": 221
},
{
"KL/chosen_KL_mean": -159.87034606933594,
"KL/mean": -225.4850616455078,
"KL/rejected_KL_mean": -291.09979248046875,
"KL/std": 147.8228759765625,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.0035684143658727407,
"fcm_dpo/delta": -0.07168504595756531,
"fcm_dpo/margin": 131.2294464111328,
"fcm_dpo/q_t": 0.3936772048473358,
"grad_norm": 20.67112922668457,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.3012784719467163,
"logits/rejected": -0.30357640981674194,
"logps/chosen": -220.84144592285156,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -391.1009521484375,
"loss": 1.0517,
"margin_dpo/margin_mean": 131.22943115234375,
"margin_dpo/margin_std": 164.2476348876953,
"step": 222
},
{
"KL/chosen_KL_mean": -169.24302673339844,
"KL/mean": -224.17355346679688,
"KL/rejected_KL_mean": -279.1040954589844,
"KL/std": 143.19537353515625,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.003542605321854353,
"fcm_dpo/delta": 0.011229310184717178,
"fcm_dpo/margin": 109.86106872558594,
"fcm_dpo/q_t": 0.412454217672348,
"grad_norm": 24.183177947998047,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.22766795754432678,
"logits/rejected": -0.22294780611991882,
"logps/chosen": -221.88360595703125,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -361.92913818359375,
"loss": 1.1302,
"margin_dpo/margin_mean": 109.86106872558594,
"margin_dpo/margin_std": 182.3267364501953,
"step": 223
},
{
"KL/chosen_KL_mean": -147.57431030273438,
"KL/mean": -209.97825622558594,
"KL/rejected_KL_mean": -272.3822021484375,
"KL/std": 158.82989501953125,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.003500790800899267,
"fcm_dpo/delta": -0.04011113941669464,
"fcm_dpo/margin": 124.80789184570312,
"fcm_dpo/q_t": 0.40373706817626953,
"grad_norm": 25.53409767150879,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.25661876797676086,
"logits/rejected": -0.23753324151039124,
"logps/chosen": -196.1697235107422,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -349.4986572265625,
"loss": 1.0905,
"margin_dpo/margin_mean": 124.8078842163086,
"margin_dpo/margin_std": 181.3145294189453,
"step": 224
},
{
"KL/chosen_KL_mean": -164.8527069091797,
"KL/mean": -239.45616149902344,
"KL/rejected_KL_mean": -314.05963134765625,
"KL/std": 152.40457153320312,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.003453275188803673,
"fcm_dpo/delta": -0.12168119102716446,
"fcm_dpo/margin": 149.2069091796875,
"fcm_dpo/q_t": 0.38449984788894653,
"grad_norm": 24.14275550842285,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.27758195996284485,
"logits/rejected": -0.266676127910614,
"logps/chosen": -222.85317993164062,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -413.9625244140625,
"loss": 1.0292,
"margin_dpo/margin_mean": 149.2069091796875,
"margin_dpo/margin_std": 187.06063842773438,
"step": 225
},
{
"KL/chosen_KL_mean": -138.45205688476562,
"KL/mean": -197.54412841796875,
"KL/rejected_KL_mean": -256.63623046875,
"KL/std": 149.68759155273438,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.0034129605628550053,
"fcm_dpo/delta": -0.004264920949935913,
"fcm_dpo/margin": 118.18417358398438,
"fcm_dpo/q_t": 0.41034334897994995,
"grad_norm": 29.583045959472656,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.38879233598709106,
"logits/rejected": -0.3535831570625305,
"logps/chosen": -197.35086059570312,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -335.323974609375,
"loss": 1.1057,
"margin_dpo/margin_mean": 118.18417358398438,
"margin_dpo/margin_std": 173.61318969726562,
"step": 226
},
{
"KL/chosen_KL_mean": -158.3021697998047,
"KL/mean": -232.66543579101562,
"KL/rejected_KL_mean": -307.0286865234375,
"KL/std": 167.4959716796875,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.003365355310961604,
"fcm_dpo/delta": -0.10656450688838959,
"fcm_dpo/margin": 148.72653198242188,
"fcm_dpo/q_t": 0.3860970139503479,
"grad_norm": 20.811311721801758,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.3414561152458191,
"logits/rejected": -0.3285638093948364,
"logps/chosen": -217.3743438720703,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -406.4410705566406,
"loss": 1.0252,
"margin_dpo/margin_mean": 148.72653198242188,
"margin_dpo/margin_std": 174.76016235351562,
"step": 227
},
{
"KL/chosen_KL_mean": -167.0030975341797,
"KL/mean": -219.13742065429688,
"KL/rejected_KL_mean": -271.271728515625,
"KL/std": 138.55203247070312,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.0033898483961820602,
"fcm_dpo/delta": 0.04812644422054291,
"fcm_dpo/margin": 104.26863098144531,
"fcm_dpo/q_t": 0.4190364480018616,
"grad_norm": 23.566688537597656,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.31667008996009827,
"logits/rejected": -0.29874318838119507,
"logps/chosen": -232.89437866210938,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -362.32049560546875,
"loss": 1.141,
"margin_dpo/margin_mean": 104.26863098144531,
"margin_dpo/margin_std": 169.02606201171875,
"step": 228
},
{
"KL/chosen_KL_mean": -172.6579132080078,
"KL/mean": -228.57550048828125,
"KL/rejected_KL_mean": -284.4930419921875,
"KL/std": 159.468017578125,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.003405486699193716,
"fcm_dpo/delta": 0.01987878978252411,
"fcm_dpo/margin": 111.83513641357422,
"fcm_dpo/q_t": 0.41233646869659424,
"grad_norm": 26.473695755004883,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.41149044036865234,
"logits/rejected": -0.38208431005477905,
"logps/chosen": -243.36428833007812,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -369.0204772949219,
"loss": 1.1162,
"margin_dpo/margin_mean": 111.83513641357422,
"margin_dpo/margin_std": 166.87164306640625,
"step": 229
},
{
"KL/chosen_KL_mean": -133.82962036132812,
"KL/mean": -219.05960083007812,
"KL/rejected_KL_mean": -304.2895812988281,
"KL/std": 151.04354858398438,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.0033145418856292963,
"fcm_dpo/delta": -0.1751500815153122,
"fcm_dpo/margin": 170.4599609375,
"fcm_dpo/q_t": 0.36932283639907837,
"grad_norm": 25.753311157226562,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.2865590751171112,
"logits/rejected": -0.29057538509368896,
"logps/chosen": -173.1116180419922,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -389.9114990234375,
"loss": 0.9682,
"margin_dpo/margin_mean": 170.4599609375,
"margin_dpo/margin_std": 166.1871337890625,
"step": 230
},
{
"KL/chosen_KL_mean": -153.423828125,
"KL/mean": -212.75845336914062,
"KL/rejected_KL_mean": -272.0930480957031,
"KL/std": 134.63912963867188,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.0032932045869529247,
"fcm_dpo/delta": 0.00957314670085907,
"fcm_dpo/margin": 118.6692123413086,
"fcm_dpo/q_t": 0.41134652495384216,
"grad_norm": 20.769832611083984,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.35538873076438904,
"logits/rejected": -0.3243277072906494,
"logps/chosen": -216.70028686523438,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -346.2169494628906,
"loss": 1.0945,
"margin_dpo/margin_mean": 118.66921997070312,
"margin_dpo/margin_std": 156.93490600585938,
"step": 231
},
{
"KL/chosen_KL_mean": -189.22994995117188,
"KL/mean": -235.84271240234375,
"KL/rejected_KL_mean": -282.4554443359375,
"KL/std": 156.42123413085938,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.003335074055939913,
"fcm_dpo/delta": 0.09204444289207458,
"fcm_dpo/margin": 93.22552490234375,
"fcm_dpo/q_t": 0.43068164587020874,
"grad_norm": 26.37661361694336,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.3877544403076172,
"logits/rejected": -0.36684519052505493,
"logps/chosen": -259.97869873046875,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -366.4325256347656,
"loss": 1.1658,
"margin_dpo/margin_mean": 93.22552490234375,
"margin_dpo/margin_std": 160.78085327148438,
"step": 232
},
{
"KL/chosen_KL_mean": -168.43820190429688,
"KL/mean": -242.55459594726562,
"KL/rejected_KL_mean": -316.6710205078125,
"KL/std": 165.54281616210938,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.003311819862574339,
"fcm_dpo/delta": -0.09555429965257645,
"fcm_dpo/margin": 148.23281860351562,
"fcm_dpo/q_t": 0.3932535648345947,
"grad_norm": 24.77134132385254,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.3793497085571289,
"logits/rejected": -0.38402658700942993,
"logps/chosen": -223.32113647460938,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -424.15106201171875,
"loss": 1.0634,
"margin_dpo/margin_mean": 148.23281860351562,
"margin_dpo/margin_std": 211.1150665283203,
"step": 233
},
{
"KL/chosen_KL_mean": -164.03988647460938,
"KL/mean": -241.57525634765625,
"KL/rejected_KL_mean": -319.110595703125,
"KL/std": 147.59521484375,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.003218206809833646,
"fcm_dpo/delta": -0.10609423369169235,
"fcm_dpo/margin": 155.07070922851562,
"fcm_dpo/q_t": 0.3857956826686859,
"grad_norm": 33.62549591064453,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.36793646216392517,
"logits/rejected": -0.3934020400047302,
"logps/chosen": -208.13441467285156,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -419.11724853515625,
"loss": 1.0175,
"margin_dpo/margin_mean": 155.07069396972656,
"margin_dpo/margin_std": 166.5502471923828,
"step": 234
},
{
"KL/chosen_KL_mean": -194.43746948242188,
"KL/mean": -243.66107177734375,
"KL/rejected_KL_mean": -292.8846435546875,
"KL/std": 138.89715576171875,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.0032716416753828526,
"fcm_dpo/delta": 0.08014119416475296,
"fcm_dpo/margin": 98.44720458984375,
"fcm_dpo/q_t": 0.4257309138774872,
"grad_norm": 23.623655319213867,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.3998695909976959,
"logits/rejected": -0.3791996240615845,
"logps/chosen": -256.6753845214844,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -383.27972412109375,
"loss": 1.1598,
"margin_dpo/margin_mean": 98.44720458984375,
"margin_dpo/margin_std": 164.85972595214844,
"step": 235
},
{
"KL/chosen_KL_mean": -138.26724243164062,
"KL/mean": -217.79342651367188,
"KL/rejected_KL_mean": -297.31964111328125,
"KL/std": 148.53077697753906,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.0032193802762776613,
"fcm_dpo/delta": -0.1180877834558487,
"fcm_dpo/margin": 159.05239868164062,
"fcm_dpo/q_t": 0.3797944188117981,
"grad_norm": 44.529022216796875,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.39753884077072144,
"logits/rejected": -0.3979169726371765,
"logps/chosen": -187.6085968017578,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -400.8312683105469,
"loss": 0.9862,
"margin_dpo/margin_mean": 159.05239868164062,
"margin_dpo/margin_std": 140.80845642089844,
"step": 236
},
{
"KL/chosen_KL_mean": -186.890380859375,
"KL/mean": -246.773193359375,
"KL/rejected_KL_mean": -306.656005859375,
"KL/std": 148.77780151367188,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.0032119196839630604,
"fcm_dpo/delta": 0.015867752954363823,
"fcm_dpo/margin": 119.76559448242188,
"fcm_dpo/q_t": 0.41194605827331543,
"grad_norm": 26.484649658203125,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.37552815675735474,
"logits/rejected": -0.3661985993385315,
"logps/chosen": -241.05850219726562,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -401.43634033203125,
"loss": 1.107,
"margin_dpo/margin_mean": 119.76559448242188,
"margin_dpo/margin_std": 169.92410278320312,
"step": 237
},
{
"KL/chosen_KL_mean": -174.848388671875,
"KL/mean": -235.38360595703125,
"KL/rejected_KL_mean": -295.91888427734375,
"KL/std": 152.73675537109375,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.0032016821205615997,
"fcm_dpo/delta": 0.012528903782367706,
"fcm_dpo/margin": 121.0704574584961,
"fcm_dpo/q_t": 0.41281700134277344,
"grad_norm": 28.03104019165039,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.35810092091560364,
"logits/rejected": -0.37316906452178955,
"logps/chosen": -228.82150268554688,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -385.3367919921875,
"loss": 1.102,
"margin_dpo/margin_mean": 121.0704574584961,
"margin_dpo/margin_std": 166.73324584960938,
"step": 238
},
{
"KL/chosen_KL_mean": -192.3235321044922,
"KL/mean": -261.5285339355469,
"KL/rejected_KL_mean": -330.7334899902344,
"KL/std": 148.68975830078125,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.003175197634845972,
"fcm_dpo/delta": -0.04259705916047096,
"fcm_dpo/margin": 138.40997314453125,
"fcm_dpo/q_t": 0.3980611562728882,
"grad_norm": 35.199913024902344,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.3678056597709656,
"logits/rejected": -0.3555825352668762,
"logps/chosen": -250.42135620117188,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -424.3264465332031,
"loss": 1.0604,
"margin_dpo/margin_mean": 138.40997314453125,
"margin_dpo/margin_std": 161.06385803222656,
"step": 239
},
{
"KL/chosen_KL_mean": -206.96327209472656,
"KL/mean": -264.1934814453125,
"KL/rejected_KL_mean": -321.42364501953125,
"KL/std": 160.36300659179688,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.0031964327208697796,
"fcm_dpo/delta": 0.035312630236148834,
"fcm_dpo/margin": 114.46038818359375,
"fcm_dpo/q_t": 0.41712719202041626,
"grad_norm": 34.83024597167969,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.40968841314315796,
"logits/rejected": -0.3899762034416199,
"logps/chosen": -267.5777587890625,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -395.54217529296875,
"loss": 1.145,
"margin_dpo/margin_mean": 114.46038818359375,
"margin_dpo/margin_std": 193.00048828125,
"step": 240
},
{
"KL/chosen_KL_mean": -178.86370849609375,
"KL/mean": -267.1542663574219,
"KL/rejected_KL_mean": -355.44482421875,
"KL/std": 176.8538818359375,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.0031412378884851933,
"fcm_dpo/delta": -0.16355636715888977,
"fcm_dpo/margin": 176.58108520507812,
"fcm_dpo/q_t": 0.3761065602302551,
"grad_norm": 29.53580665588379,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.3840961456298828,
"logits/rejected": -0.35689496994018555,
"logps/chosen": -244.9547882080078,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -443.50567626953125,
"loss": 0.9973,
"margin_dpo/margin_mean": 176.5811004638672,
"margin_dpo/margin_std": 199.89828491210938,
"step": 241
},
{
"KL/chosen_KL_mean": -196.65576171875,
"KL/mean": -260.2139892578125,
"KL/rejected_KL_mean": -323.7721862792969,
"KL/std": 153.70497131347656,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.0031248326413333416,
"fcm_dpo/delta": 0.0024800747632980347,
"fcm_dpo/margin": 127.11642456054688,
"fcm_dpo/q_t": 0.4107505977153778,
"grad_norm": 27.897132873535156,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.3300558924674988,
"logits/rejected": -0.3024280071258545,
"logps/chosen": -264.5196838378906,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -407.13250732421875,
"loss": 1.1016,
"margin_dpo/margin_mean": 127.11642456054688,
"margin_dpo/margin_std": 180.156005859375,
"step": 242
},
{
"KL/chosen_KL_mean": -192.8182830810547,
"KL/mean": -271.3682861328125,
"KL/rejected_KL_mean": -349.918212890625,
"KL/std": 168.34677124023438,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.003067805664613843,
"fcm_dpo/delta": -0.08624139428138733,
"fcm_dpo/margin": 157.09994506835938,
"fcm_dpo/q_t": 0.39058151841163635,
"grad_norm": 24.1042537689209,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.3527655601501465,
"logits/rejected": -0.3175613582134247,
"logps/chosen": -255.90252685546875,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -426.25384521484375,
"loss": 1.0386,
"margin_dpo/margin_mean": 157.09994506835938,
"margin_dpo/margin_std": 191.3821563720703,
"step": 243
},
{
"KL/chosen_KL_mean": -164.286376953125,
"KL/mean": -245.10650634765625,
"KL/rejected_KL_mean": -325.9266357421875,
"KL/std": 160.13262939453125,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.003011933993548155,
"fcm_dpo/delta": -0.09149331599473953,
"fcm_dpo/margin": 161.64024353027344,
"fcm_dpo/q_t": 0.38753992319107056,
"grad_norm": 32.73558044433594,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.4291718900203705,
"logits/rejected": -0.42000579833984375,
"logps/chosen": -225.4270782470703,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -420.81854248046875,
"loss": 1.0157,
"margin_dpo/margin_mean": 161.6402587890625,
"margin_dpo/margin_std": 165.7441864013672,
"step": 244
},
{
"KL/chosen_KL_mean": -187.01638793945312,
"KL/mean": -250.25192260742188,
"KL/rejected_KL_mean": -313.4874267578125,
"KL/std": 156.05523681640625,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.0029890513978898525,
"fcm_dpo/delta": 0.02175918221473694,
"fcm_dpo/margin": 126.47102355957031,
"fcm_dpo/q_t": 0.41278743743896484,
"grad_norm": 26.198579788208008,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.4349704682826996,
"logits/rejected": -0.40197300910949707,
"logps/chosen": -254.2786865234375,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -401.12750244140625,
"loss": 1.1113,
"margin_dpo/margin_mean": 126.47102355957031,
"margin_dpo/margin_std": 172.0400390625,
"step": 245
},
{
"KL/chosen_KL_mean": -181.31773376464844,
"KL/mean": -244.63754272460938,
"KL/rejected_KL_mean": -307.95733642578125,
"KL/std": 169.26904296875,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.003016488626599312,
"fcm_dpo/delta": 0.01862441748380661,
"fcm_dpo/margin": 126.63958740234375,
"fcm_dpo/q_t": 0.4137783348560333,
"grad_norm": 25.956335067749023,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.45792657136917114,
"logits/rejected": -0.4178639054298401,
"logps/chosen": -248.01470947265625,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -392.3036804199219,
"loss": 1.1079,
"margin_dpo/margin_mean": 126.63959503173828,
"margin_dpo/margin_std": 180.6471405029297,
"step": 246
},
{
"KL/chosen_KL_mean": -212.8636474609375,
"KL/mean": -301.8062438964844,
"KL/rejected_KL_mean": -390.7489013671875,
"KL/std": 178.96267700195312,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.002963733160868287,
"fcm_dpo/delta": -0.1344645619392395,
"fcm_dpo/margin": 177.8852081298828,
"fcm_dpo/q_t": 0.37880799174308777,
"grad_norm": 27.184240341186523,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.41092449426651,
"logits/rejected": -0.4103108048439026,
"logps/chosen": -269.468994140625,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -497.0421447753906,
"loss": 1.0015,
"margin_dpo/margin_mean": 177.8852081298828,
"margin_dpo/margin_std": 190.83111572265625,
"step": 247
},
{
"KL/chosen_KL_mean": -184.45071411132812,
"KL/mean": -263.3221740722656,
"KL/rejected_KL_mean": -342.19366455078125,
"KL/std": 146.73001098632812,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.002908772323280573,
"fcm_dpo/delta": -0.06217579543590546,
"fcm_dpo/margin": 157.74295043945312,
"fcm_dpo/q_t": 0.3912145495414734,
"grad_norm": 22.271547317504883,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.37712520360946655,
"logits/rejected": -0.3957890272140503,
"logps/chosen": -228.49392700195312,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -434.050537109375,
"loss": 1.0207,
"margin_dpo/margin_mean": 157.74295043945312,
"margin_dpo/margin_std": 142.253173828125,
"step": 248
},
{
"KL/chosen_KL_mean": -246.08953857421875,
"KL/mean": -288.27056884765625,
"KL/rejected_KL_mean": -330.45159912109375,
"KL/std": 161.197265625,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.0029767826199531555,
"fcm_dpo/delta": 0.15274295210838318,
"fcm_dpo/margin": 84.36204528808594,
"fcm_dpo/q_t": 0.4428468942642212,
"grad_norm": 45.0234260559082,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.4169383645057678,
"logits/rejected": -0.39167478680610657,
"logps/chosen": -308.5318908691406,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -410.9196472167969,
"loss": 1.2249,
"margin_dpo/margin_mean": 84.36204528808594,
"margin_dpo/margin_std": 186.65567016601562,
"step": 249
},
{
"KL/chosen_KL_mean": -216.86962890625,
"KL/mean": -296.193115234375,
"KL/rejected_KL_mean": -375.5166015625,
"KL/std": 161.82952880859375,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.002979879267513752,
"fcm_dpo/delta": -0.07642242312431335,
"fcm_dpo/margin": 158.64694213867188,
"fcm_dpo/q_t": 0.38990771770477295,
"grad_norm": 39.645294189453125,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.3875870704650879,
"logits/rejected": -0.32922881841659546,
"logps/chosen": -282.50634765625,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -449.388427734375,
"loss": 1.0248,
"margin_dpo/margin_mean": 158.64694213867188,
"margin_dpo/margin_std": 162.55426025390625,
"step": 250
},
{
"KL/chosen_KL_mean": -228.77435302734375,
"KL/mean": -284.546875,
"KL/rejected_KL_mean": -340.31939697265625,
"KL/std": 171.0687255859375,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.0029769130051136017,
"fcm_dpo/delta": 0.0702948123216629,
"fcm_dpo/margin": 111.5450210571289,
"fcm_dpo/q_t": 0.42472678422927856,
"grad_norm": 27.457468032836914,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.3817696273326874,
"logits/rejected": -0.3497239947319031,
"logps/chosen": -285.95709228515625,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -417.98284912109375,
"loss": 1.1561,
"margin_dpo/margin_mean": 111.5450210571289,
"margin_dpo/margin_std": 183.6023406982422,
"step": 251
},
{
"KL/chosen_KL_mean": -225.74676513671875,
"KL/mean": -299.53564453125,
"KL/rejected_KL_mean": -373.324462890625,
"KL/std": 150.81161499023438,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.0029721325263381004,
"fcm_dpo/delta": -0.040585629642009735,
"fcm_dpo/margin": 147.57772827148438,
"fcm_dpo/q_t": 0.3952495753765106,
"grad_norm": 26.062089920043945,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.4397411346435547,
"logits/rejected": -0.39890581369400024,
"logps/chosen": -297.43243408203125,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -458.08245849609375,
"loss": 1.0434,
"margin_dpo/margin_mean": 147.57772827148438,
"margin_dpo/margin_std": 150.42495727539062,
"step": 252
},
{
"KL/chosen_KL_mean": -182.36927795410156,
"KL/mean": -253.0843505859375,
"KL/rejected_KL_mean": -323.7994079589844,
"KL/std": 155.97808837890625,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.0029799549374729395,
"fcm_dpo/delta": -0.02296675369143486,
"fcm_dpo/margin": 141.4301300048828,
"fcm_dpo/q_t": 0.40165138244628906,
"grad_norm": 22.8338680267334,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.4326644539833069,
"logits/rejected": -0.42504405975341797,
"logps/chosen": -251.50320434570312,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -422.501953125,
"loss": 1.0752,
"margin_dpo/margin_mean": 141.4301300048828,
"margin_dpo/margin_std": 173.86398315429688,
"step": 253
},
{
"KL/chosen_KL_mean": -176.87899780273438,
"KL/mean": -238.10438537597656,
"KL/rejected_KL_mean": -299.32977294921875,
"KL/std": 163.08511352539062,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.0029733264818787575,
"fcm_dpo/delta": 0.03728723153471947,
"fcm_dpo/margin": 122.45077514648438,
"fcm_dpo/q_t": 0.41952478885650635,
"grad_norm": 22.23654556274414,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.4106729328632355,
"logits/rejected": -0.39602309465408325,
"logps/chosen": -231.03399658203125,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -389.6374206542969,
"loss": 1.1379,
"margin_dpo/margin_mean": 122.45077514648438,
"margin_dpo/margin_std": 204.29995727539062,
"step": 254
},
{
"KL/chosen_KL_mean": -181.30291748046875,
"KL/mean": -246.3142547607422,
"KL/rejected_KL_mean": -311.32562255859375,
"KL/std": 142.95404052734375,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.0029753465205430984,
"fcm_dpo/delta": 0.01340005174279213,
"fcm_dpo/margin": 130.02267456054688,
"fcm_dpo/q_t": 0.4105185270309448,
"grad_norm": 18.779788970947266,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.3276970386505127,
"logits/rejected": -0.3166770339012146,
"logps/chosen": -238.44459533691406,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -401.5341796875,
"loss": 1.1068,
"margin_dpo/margin_mean": 130.02267456054688,
"margin_dpo/margin_std": 181.08340454101562,
"step": 255
},
{
"KL/chosen_KL_mean": -158.61541748046875,
"KL/mean": -228.71539306640625,
"KL/rejected_KL_mean": -298.81536865234375,
"KL/std": 156.46206665039062,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.0029811577405780554,
"fcm_dpo/delta": -0.01873961091041565,
"fcm_dpo/margin": 140.199951171875,
"fcm_dpo/q_t": 0.4038965106010437,
"grad_norm": 23.397459030151367,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.30310070514678955,
"logits/rejected": -0.3115818500518799,
"logps/chosen": -213.77891540527344,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -391.3782958984375,
"loss": 1.0792,
"margin_dpo/margin_mean": 140.199951171875,
"margin_dpo/margin_std": 181.5375213623047,
"step": 256
},
{
"KL/chosen_KL_mean": -152.9769287109375,
"KL/mean": -225.80526733398438,
"KL/rejected_KL_mean": -298.6336364746094,
"KL/std": 159.47927856445312,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.0029455325566232204,
"fcm_dpo/delta": -0.03134532645344734,
"fcm_dpo/margin": 145.6566925048828,
"fcm_dpo/q_t": 0.4020821452140808,
"grad_norm": 19.92885971069336,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.3385705053806305,
"logits/rejected": -0.3252190947532654,
"logps/chosen": -202.40061950683594,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -378.1715393066406,
"loss": 1.0739,
"margin_dpo/margin_mean": 145.6566925048828,
"margin_dpo/margin_std": 181.281982421875,
"step": 257
},
{
"KL/chosen_KL_mean": -198.8714599609375,
"KL/mean": -269.2030944824219,
"KL/rejected_KL_mean": -339.53472900390625,
"KL/std": 167.5731201171875,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.0029567554593086243,
"fcm_dpo/delta": -0.016622664406895638,
"fcm_dpo/margin": 140.6632843017578,
"fcm_dpo/q_t": 0.4036473035812378,
"grad_norm": 20.93349838256836,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.3120572865009308,
"logits/rejected": -0.31056955456733704,
"logps/chosen": -258.2555847167969,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -435.52484130859375,
"loss": 1.0943,
"margin_dpo/margin_mean": 140.66329956054688,
"margin_dpo/margin_std": 199.73696899414062,
"step": 258
},
{
"KL/chosen_KL_mean": -186.13726806640625,
"KL/mean": -246.82554626464844,
"KL/rejected_KL_mean": -307.51385498046875,
"KL/std": 158.20399475097656,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.0029596006497740746,
"fcm_dpo/delta": 0.04224412143230438,
"fcm_dpo/margin": 121.37655639648438,
"fcm_dpo/q_t": 0.417808473110199,
"grad_norm": 24.50770378112793,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.3589467704296112,
"logits/rejected": -0.35707467794418335,
"logps/chosen": -238.96560668945312,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -396.70550537109375,
"loss": 1.1253,
"margin_dpo/margin_mean": 121.37657165527344,
"margin_dpo/margin_std": 178.23170471191406,
"step": 259
},
{
"KL/chosen_KL_mean": -195.42129516601562,
"KL/mean": -275.30938720703125,
"KL/rejected_KL_mean": -355.197509765625,
"KL/std": 164.61087036132812,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.0029522283002734184,
"fcm_dpo/delta": -0.07524219155311584,
"fcm_dpo/margin": 159.77621459960938,
"fcm_dpo/q_t": 0.3926173746585846,
"grad_norm": 28.655542373657227,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.3197569251060486,
"logits/rejected": -0.33596664667129517,
"logps/chosen": -242.83897399902344,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -450.28729248046875,
"loss": 1.0311,
"margin_dpo/margin_mean": 159.77621459960938,
"margin_dpo/margin_std": 177.47177124023438,
"step": 260
},
{
"KL/chosen_KL_mean": -201.64346313476562,
"KL/mean": -277.6656494140625,
"KL/rejected_KL_mean": -353.6878662109375,
"KL/std": 180.62509155273438,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.00290453783236444,
"fcm_dpo/delta": -0.04374488443136215,
"fcm_dpo/margin": 152.04437255859375,
"fcm_dpo/q_t": 0.4000317454338074,
"grad_norm": 22.516796112060547,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.29051870107650757,
"logits/rejected": -0.2918134927749634,
"logps/chosen": -254.67483520507812,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -442.20281982421875,
"loss": 1.0725,
"margin_dpo/margin_mean": 152.04437255859375,
"margin_dpo/margin_std": 203.62240600585938,
"step": 261
},
{
"KL/chosen_KL_mean": -240.88412475585938,
"KL/mean": -304.42376708984375,
"KL/rejected_KL_mean": -367.96343994140625,
"KL/std": 165.10328674316406,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.0029267538338899612,
"fcm_dpo/delta": 0.028837885707616806,
"fcm_dpo/margin": 127.07933044433594,
"fcm_dpo/q_t": 0.4135780334472656,
"grad_norm": 24.18539047241211,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.3253632187843323,
"logits/rejected": -0.3014422059059143,
"logps/chosen": -300.5042724609375,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -454.3819885253906,
"loss": 1.1037,
"margin_dpo/margin_mean": 127.07933044433594,
"margin_dpo/margin_std": 165.640869140625,
"step": 262
},
{
"KL/chosen_KL_mean": -227.05230712890625,
"KL/mean": -316.44732666015625,
"KL/rejected_KL_mean": -405.8423156738281,
"KL/std": 193.70480346679688,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.0028580743819475174,
"fcm_dpo/delta": -0.1178855374455452,
"fcm_dpo/margin": 178.79000854492188,
"fcm_dpo/q_t": 0.3831760883331299,
"grad_norm": 25.265499114990234,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.3177950382232666,
"logits/rejected": -0.29410669207572937,
"logps/chosen": -286.4732666015625,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -502.69952392578125,
"loss": 1.018,
"margin_dpo/margin_mean": 178.7899932861328,
"margin_dpo/margin_std": 203.7642822265625,
"step": 263
},
{
"KL/chosen_KL_mean": -241.2745361328125,
"KL/mean": -318.86224365234375,
"KL/rejected_KL_mean": -396.44989013671875,
"KL/std": 177.32969665527344,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.002816273830831051,
"fcm_dpo/delta": -0.04003193974494934,
"fcm_dpo/margin": 155.17535400390625,
"fcm_dpo/q_t": 0.40031126141548157,
"grad_norm": 24.550264358520508,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.35513001680374146,
"logits/rejected": -0.3457460403442383,
"logps/chosen": -303.99664306640625,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -490.30609130859375,
"loss": 1.0724,
"margin_dpo/margin_mean": 155.1753692626953,
"margin_dpo/margin_std": 197.9132080078125,
"step": 264
},
{
"KL/chosen_KL_mean": -272.23443603515625,
"KL/mean": -345.8155517578125,
"KL/rejected_KL_mean": -419.3966064453125,
"KL/std": 205.12991333007812,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.0028229644522070885,
"fcm_dpo/delta": -0.01610906980931759,
"fcm_dpo/margin": 147.16217041015625,
"fcm_dpo/q_t": 0.4075104594230652,
"grad_norm": 32.928955078125,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.2887270450592041,
"logits/rejected": -0.27040165662765503,
"logps/chosen": -334.2059020996094,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -507.417236328125,
"loss": 1.1221,
"margin_dpo/margin_mean": 147.16217041015625,
"margin_dpo/margin_std": 246.03475952148438,
"step": 265
},
{
"KL/chosen_KL_mean": -277.2935485839844,
"KL/mean": -338.205322265625,
"KL/rejected_KL_mean": -399.11712646484375,
"KL/std": 174.17181396484375,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.0028457948938012123,
"fcm_dpo/delta": 0.05520811304450035,
"fcm_dpo/margin": 121.82360076904297,
"fcm_dpo/q_t": 0.4216112792491913,
"grad_norm": 60.411834716796875,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.3695454001426697,
"logits/rejected": -0.32945775985717773,
"logps/chosen": -344.3932189941406,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -467.08837890625,
"loss": 1.1455,
"margin_dpo/margin_mean": 121.82359313964844,
"margin_dpo/margin_std": 201.16543579101562,
"step": 266
},
{
"KL/chosen_KL_mean": -251.60833740234375,
"KL/mean": -329.51861572265625,
"KL/rejected_KL_mean": -407.428955078125,
"KL/std": 190.1357421875,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.002824952360242605,
"fcm_dpo/delta": -0.042434047907590866,
"fcm_dpo/margin": 155.82058715820312,
"fcm_dpo/q_t": 0.39909815788269043,
"grad_norm": 35.148372650146484,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.3738306760787964,
"logits/rejected": -0.34012115001678467,
"logps/chosen": -320.5791015625,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -497.5973815917969,
"loss": 1.0761,
"margin_dpo/margin_mean": 155.82058715820312,
"margin_dpo/margin_std": 211.06903076171875,
"step": 267
},
{
"KL/chosen_KL_mean": -259.2694396972656,
"KL/mean": -333.34063720703125,
"KL/rejected_KL_mean": -407.41180419921875,
"KL/std": 180.32666015625,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.002812296850606799,
"fcm_dpo/delta": -0.0176074281334877,
"fcm_dpo/margin": 148.14236450195312,
"fcm_dpo/q_t": 0.40650928020477295,
"grad_norm": 36.6395378112793,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.34564322233200073,
"logits/rejected": -0.35396748781204224,
"logps/chosen": -315.16973876953125,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -509.0594482421875,
"loss": 1.0918,
"margin_dpo/margin_mean": 148.14236450195312,
"margin_dpo/margin_std": 211.45758056640625,
"step": 268
},
{
"KL/chosen_KL_mean": -271.8604431152344,
"KL/mean": -358.281982421875,
"KL/rejected_KL_mean": -444.70355224609375,
"KL/std": 175.954345703125,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.002785563003271818,
"fcm_dpo/delta": -0.08552718162536621,
"fcm_dpo/margin": 172.84307861328125,
"fcm_dpo/q_t": 0.38992932438850403,
"grad_norm": 30.89337158203125,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.4149805009365082,
"logits/rejected": -0.39908021688461304,
"logps/chosen": -341.8999938964844,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -552.0529174804688,
"loss": 1.0491,
"margin_dpo/margin_mean": 172.84307861328125,
"margin_dpo/margin_std": 220.37840270996094,
"step": 269
},
{
"KL/chosen_KL_mean": -228.65283203125,
"KL/mean": -294.26617431640625,
"KL/rejected_KL_mean": -359.8794860839844,
"KL/std": 157.0207977294922,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.002793004736304283,
"fcm_dpo/delta": 0.03450280427932739,
"fcm_dpo/margin": 131.22665405273438,
"fcm_dpo/q_t": 0.41584137082099915,
"grad_norm": 34.00802230834961,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.4382932186126709,
"logits/rejected": -0.4344383478164673,
"logps/chosen": -298.1863098144531,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -469.8081359863281,
"loss": 1.1246,
"margin_dpo/margin_mean": 131.22665405273438,
"margin_dpo/margin_std": 197.15127563476562,
"step": 270
},
{
"KL/chosen_KL_mean": -203.8043975830078,
"KL/mean": -284.2559509277344,
"KL/rejected_KL_mean": -364.70745849609375,
"KL/std": 154.94090270996094,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.002762623131275177,
"fcm_dpo/delta": -0.0468977689743042,
"fcm_dpo/margin": 160.903076171875,
"fcm_dpo/q_t": 0.3969581723213196,
"grad_norm": 27.79316520690918,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.3980172276496887,
"logits/rejected": -0.38150689005851746,
"logps/chosen": -260.5689697265625,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -457.2213134765625,
"loss": 1.0442,
"margin_dpo/margin_mean": 160.903076171875,
"margin_dpo/margin_std": 173.694580078125,
"step": 271
},
{
"KL/chosen_KL_mean": -182.1749725341797,
"KL/mean": -278.8963623046875,
"KL/rejected_KL_mean": -375.6177978515625,
"KL/std": 174.50149536132812,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.0027207238599658012,
"fcm_dpo/delta": -0.13311508297920227,
"fcm_dpo/margin": 193.4428253173828,
"fcm_dpo/q_t": 0.3770027756690979,
"grad_norm": 39.809661865234375,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.37144580483436584,
"logits/rejected": -0.3810126781463623,
"logps/chosen": -231.672119140625,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -481.16058349609375,
"loss": 0.9857,
"margin_dpo/margin_mean": 193.4428253173828,
"margin_dpo/margin_std": 180.5559844970703,
"step": 272
},
{
"KL/chosen_KL_mean": -212.50933837890625,
"KL/mean": -307.0111083984375,
"KL/rejected_KL_mean": -401.51287841796875,
"KL/std": 174.13787841796875,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.0026398475747555494,
"fcm_dpo/delta": -0.10482463985681534,
"fcm_dpo/margin": 189.00352478027344,
"fcm_dpo/q_t": 0.3841009736061096,
"grad_norm": 25.719608306884766,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.3823262155056,
"logits/rejected": -0.36422067880630493,
"logps/chosen": -275.4847412109375,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -494.0114440917969,
"loss": 1.0094,
"margin_dpo/margin_mean": 189.00352478027344,
"margin_dpo/margin_std": 193.2362823486328,
"step": 273
},
{
"KL/chosen_KL_mean": -262.0535888671875,
"KL/mean": -333.47491455078125,
"KL/rejected_KL_mean": -404.896240234375,
"KL/std": 162.6813201904297,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.0026513845659792423,
"fcm_dpo/delta": 0.02189909853041172,
"fcm_dpo/margin": 142.8426513671875,
"fcm_dpo/q_t": 0.4122008681297302,
"grad_norm": 22.938400268554688,
"learning_rate": 3.75e-07,
"logits/chosen": -0.3189348578453064,
"logits/rejected": -0.3026675581932068,
"logps/chosen": -317.7213134765625,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -482.2293395996094,
"loss": 1.0992,
"margin_dpo/margin_mean": 142.8426513671875,
"margin_dpo/margin_std": 183.34677124023438,
"step": 274
},
{
"KL/chosen_KL_mean": -207.0484619140625,
"KL/mean": -285.15045166015625,
"KL/rejected_KL_mean": -363.25244140625,
"KL/std": 170.67037963867188,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.0026485861744731665,
"fcm_dpo/delta": -0.014418797567486763,
"fcm_dpo/margin": 156.20401000976562,
"fcm_dpo/q_t": 0.40412938594818115,
"grad_norm": 26.63812255859375,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.35331815481185913,
"logits/rejected": -0.35876357555389404,
"logps/chosen": -255.64315795898438,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -456.55615234375,
"loss": 1.0773,
"margin_dpo/margin_mean": 156.2040252685547,
"margin_dpo/margin_std": 193.69981384277344,
"step": 275
},
{
"KL/chosen_KL_mean": -230.7656707763672,
"KL/mean": -310.7945861816406,
"KL/rejected_KL_mean": -390.82354736328125,
"KL/std": 179.71499633789062,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.0026255841366946697,
"fcm_dpo/delta": -0.021273206919431686,
"fcm_dpo/margin": 160.05784606933594,
"fcm_dpo/q_t": 0.40462201833724976,
"grad_norm": 26.980419158935547,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.3262450098991394,
"logits/rejected": -0.3033270239830017,
"logps/chosen": -287.34307861328125,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -461.189208984375,
"loss": 1.084,
"margin_dpo/margin_mean": 160.05784606933594,
"margin_dpo/margin_std": 216.21392822265625,
"step": 276
},
{
"KL/chosen_KL_mean": -258.88909912109375,
"KL/mean": -339.9987487792969,
"KL/rejected_KL_mean": -421.1083679199219,
"KL/std": 182.54701232910156,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.0026227221824228764,
"fcm_dpo/delta": -0.026605986058712006,
"fcm_dpo/margin": 162.21926879882812,
"fcm_dpo/q_t": 0.4024323523044586,
"grad_norm": 28.496627807617188,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.3310853838920593,
"logits/rejected": -0.31346580386161804,
"logps/chosen": -315.16064453125,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -513.9896240234375,
"loss": 1.0807,
"margin_dpo/margin_mean": 162.2192840576172,
"margin_dpo/margin_std": 215.76904296875,
"step": 277
},
{
"KL/chosen_KL_mean": -231.9629364013672,
"KL/mean": -326.51513671875,
"KL/rejected_KL_mean": -421.0672607421875,
"KL/std": 190.9918975830078,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.0025728708133101463,
"fcm_dpo/delta": -0.09131142497062683,
"fcm_dpo/margin": 189.1043243408203,
"fcm_dpo/q_t": 0.38871318101882935,
"grad_norm": 23.81670570373535,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.31821274757385254,
"logits/rejected": -0.32499927282333374,
"logps/chosen": -284.9048767089844,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -512.3208618164062,
"loss": 1.0258,
"margin_dpo/margin_mean": 189.1043243408203,
"margin_dpo/margin_std": 212.00811767578125,
"step": 278
},
{
"KL/chosen_KL_mean": -274.34991455078125,
"KL/mean": -368.72027587890625,
"KL/rejected_KL_mean": -463.09063720703125,
"KL/std": 205.50079345703125,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.002521257847547531,
"fcm_dpo/delta": -0.0808742344379425,
"fcm_dpo/margin": 188.74072265625,
"fcm_dpo/q_t": 0.39357665181159973,
"grad_norm": 27.415855407714844,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.27504926919937134,
"logits/rejected": -0.27438250184059143,
"logps/chosen": -322.9912109375,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -550.9420776367188,
"loss": 1.0599,
"margin_dpo/margin_mean": 188.74072265625,
"margin_dpo/margin_std": 252.65435791015625,
"step": 279
},
{
"KL/chosen_KL_mean": -273.220458984375,
"KL/mean": -370.7410583496094,
"KL/rejected_KL_mean": -468.26165771484375,
"KL/std": 192.02523803710938,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.002490004990249872,
"fcm_dpo/delta": -0.09012575447559357,
"fcm_dpo/margin": 195.04116821289062,
"fcm_dpo/q_t": 0.38661879301071167,
"grad_norm": 24.108890533447266,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.2909424304962158,
"logits/rejected": -0.2941422462463379,
"logps/chosen": -332.0176086425781,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -566.8804931640625,
"loss": 1.0206,
"margin_dpo/margin_mean": 195.04116821289062,
"margin_dpo/margin_std": 207.14418029785156,
"step": 280
},
{
"KL/chosen_KL_mean": -244.1348876953125,
"KL/mean": -332.23809814453125,
"KL/rejected_KL_mean": -420.34130859375,
"KL/std": 186.86834716796875,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.00247568404302001,
"fcm_dpo/delta": -0.03801019489765167,
"fcm_dpo/margin": 176.2063751220703,
"fcm_dpo/q_t": 0.3967971205711365,
"grad_norm": 23.407094955444336,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.3744785487651825,
"logits/rejected": -0.35385334491729736,
"logps/chosen": -299.6234130859375,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -501.223876953125,
"loss": 1.058,
"margin_dpo/margin_mean": 176.2063751220703,
"margin_dpo/margin_std": 202.88414001464844,
"step": 281
},
{
"KL/chosen_KL_mean": -265.88043212890625,
"KL/mean": -335.734619140625,
"KL/rejected_KL_mean": -405.58880615234375,
"KL/std": 192.28700256347656,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.00249258684925735,
"fcm_dpo/delta": 0.05299558490514755,
"fcm_dpo/margin": 139.70834350585938,
"fcm_dpo/q_t": 0.4222215414047241,
"grad_norm": 23.37168312072754,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.403659462928772,
"logits/rejected": -0.3835230767726898,
"logps/chosen": -338.9505615234375,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -500.93975830078125,
"loss": 1.1354,
"margin_dpo/margin_mean": 139.70834350585938,
"margin_dpo/margin_std": 219.26756286621094,
"step": 282
},
{
"KL/chosen_KL_mean": -278.4122314453125,
"KL/mean": -378.302490234375,
"KL/rejected_KL_mean": -478.1927795410156,
"KL/std": 221.9765625,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.002461041323840618,
"fcm_dpo/delta": -0.09631384164094925,
"fcm_dpo/margin": 199.78057861328125,
"fcm_dpo/q_t": 0.3858485817909241,
"grad_norm": 34.04173278808594,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.3402835726737976,
"logits/rejected": -0.3434496521949768,
"logps/chosen": -340.3106689453125,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -575.1793212890625,
"loss": 1.0199,
"margin_dpo/margin_mean": 199.78057861328125,
"margin_dpo/margin_std": 213.38031005859375,
"step": 283
},
{
"KL/chosen_KL_mean": -261.8716125488281,
"KL/mean": -354.55419921875,
"KL/rejected_KL_mean": -447.2367858886719,
"KL/std": 204.1056671142578,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.0024192240089178085,
"fcm_dpo/delta": -0.05081501603126526,
"fcm_dpo/margin": 185.36517333984375,
"fcm_dpo/q_t": 0.3958737552165985,
"grad_norm": 36.13024139404297,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.41608667373657227,
"logits/rejected": -0.40421050786972046,
"logps/chosen": -320.30712890625,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -540.7060546875,
"loss": 1.0411,
"margin_dpo/margin_mean": 185.36517333984375,
"margin_dpo/margin_std": 199.91265869140625,
"step": 284
},
{
"KL/chosen_KL_mean": -288.56036376953125,
"KL/mean": -369.67523193359375,
"KL/rejected_KL_mean": -450.7901306152344,
"KL/std": 178.81466674804688,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.0024310359731316566,
"fcm_dpo/delta": 0.005226004868745804,
"fcm_dpo/margin": 162.2297821044922,
"fcm_dpo/q_t": 0.4067595601081848,
"grad_norm": 25.638004302978516,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.37708234786987305,
"logits/rejected": -0.3645085096359253,
"logps/chosen": -354.79254150390625,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -549.9169921875,
"loss": 1.0887,
"margin_dpo/margin_mean": 162.2297821044922,
"margin_dpo/margin_std": 199.06863403320312,
"step": 285
},
{
"KL/chosen_KL_mean": -281.560791015625,
"KL/mean": -375.9154357910156,
"KL/rejected_KL_mean": -470.27008056640625,
"KL/std": 201.82308959960938,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.0024013579823076725,
"fcm_dpo/delta": -0.05564514547586441,
"fcm_dpo/margin": 188.70924377441406,
"fcm_dpo/q_t": 0.39494001865386963,
"grad_norm": 21.66739845275879,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.360429048538208,
"logits/rejected": -0.3457929491996765,
"logps/chosen": -354.5118103027344,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -558.8585205078125,
"loss": 1.0474,
"margin_dpo/margin_mean": 188.70924377441406,
"margin_dpo/margin_std": 217.59698486328125,
"step": 286
},
{
"KL/chosen_KL_mean": -260.7840576171875,
"KL/mean": -335.9070129394531,
"KL/rejected_KL_mean": -411.02996826171875,
"KL/std": 174.77919006347656,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.0024088136851787567,
"fcm_dpo/delta": 0.03944006562232971,
"fcm_dpo/margin": 150.24591064453125,
"fcm_dpo/q_t": 0.41572195291519165,
"grad_norm": 26.050071716308594,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.3475659489631653,
"logits/rejected": -0.32449665665626526,
"logps/chosen": -322.3251953125,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -488.7260437011719,
"loss": 1.1087,
"margin_dpo/margin_mean": 150.24591064453125,
"margin_dpo/margin_std": 195.06973266601562,
"step": 287
},
{
"KL/chosen_KL_mean": -248.489990234375,
"KL/mean": -345.380126953125,
"KL/rejected_KL_mean": -442.270263671875,
"KL/std": 185.03494262695312,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.0023706420324742794,
"fcm_dpo/delta": -0.06380210071802139,
"fcm_dpo/margin": 193.78025817871094,
"fcm_dpo/q_t": 0.39201951026916504,
"grad_norm": 19.817523956298828,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.3824455738067627,
"logits/rejected": -0.36902564764022827,
"logps/chosen": -305.1512145996094,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -529.60595703125,
"loss": 1.029,
"margin_dpo/margin_mean": 193.7802734375,
"margin_dpo/margin_std": 186.56927490234375,
"step": 288
},
{
"KL/chosen_KL_mean": -244.011474609375,
"KL/mean": -348.7763671875,
"KL/rejected_KL_mean": -453.54119873046875,
"KL/std": 203.1474609375,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.0023524933494627476,
"fcm_dpo/delta": -0.09762175381183624,
"fcm_dpo/margin": 209.52972412109375,
"fcm_dpo/q_t": 0.3861865997314453,
"grad_norm": 24.97647476196289,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.30401328206062317,
"logits/rejected": -0.3052119314670563,
"logps/chosen": -289.24188232421875,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -541.183837890625,
"loss": 1.0157,
"margin_dpo/margin_mean": 209.52969360351562,
"margin_dpo/margin_std": 219.72787475585938,
"step": 289
},
{
"KL/chosen_KL_mean": -264.4161071777344,
"KL/mean": -370.1914367675781,
"KL/rejected_KL_mean": -475.966796875,
"KL/std": 215.30389404296875,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.002299440558999777,
"fcm_dpo/delta": -0.0910457968711853,
"fcm_dpo/margin": 211.55064392089844,
"fcm_dpo/q_t": 0.3896278738975525,
"grad_norm": 22.546796798706055,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.3152487874031067,
"logits/rejected": -0.3381202518939972,
"logps/chosen": -319.8876037597656,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -592.6753540039062,
"loss": 1.0352,
"margin_dpo/margin_mean": 211.5506591796875,
"margin_dpo/margin_std": 254.94503784179688,
"step": 290
},
{
"KL/chosen_KL_mean": -228.40325927734375,
"KL/mean": -325.32037353515625,
"KL/rejected_KL_mean": -422.2374572753906,
"KL/std": 190.18911743164062,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.002286990638822317,
"fcm_dpo/delta": -0.04551897943019867,
"fcm_dpo/margin": 193.83419799804688,
"fcm_dpo/q_t": 0.39684972167015076,
"grad_norm": 24.015735626220703,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.3624478578567505,
"logits/rejected": -0.35021036863327026,
"logps/chosen": -291.66363525390625,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -511.5345458984375,
"loss": 1.0569,
"margin_dpo/margin_mean": 193.83419799804688,
"margin_dpo/margin_std": 227.18531799316406,
"step": 291
},
{
"KL/chosen_KL_mean": -259.12725830078125,
"KL/mean": -364.222412109375,
"KL/rejected_KL_mean": -469.3175048828125,
"KL/std": 228.16139221191406,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.0022533410228788853,
"fcm_dpo/delta": -0.077273428440094,
"fcm_dpo/margin": 210.19024658203125,
"fcm_dpo/q_t": 0.39149677753448486,
"grad_norm": 24.348356246948242,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.3501250445842743,
"logits/rejected": -0.3385492265224457,
"logps/chosen": -313.0458068847656,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -559.2788696289062,
"loss": 1.0506,
"margin_dpo/margin_mean": 210.19024658203125,
"margin_dpo/margin_std": 265.28570556640625,
"step": 292
},
{
"KL/chosen_KL_mean": -262.6152648925781,
"KL/mean": -341.5986328125,
"KL/rejected_KL_mean": -420.58203125,
"KL/std": 223.62734985351562,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.002254858613014221,
"fcm_dpo/delta": 0.045318976044654846,
"fcm_dpo/margin": 157.96676635742188,
"fcm_dpo/q_t": 0.42001551389694214,
"grad_norm": 35.9681510925293,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.3428913354873657,
"logits/rejected": -0.3345603942871094,
"logps/chosen": -322.99127197265625,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -498.4344482421875,
"loss": 1.1362,
"margin_dpo/margin_mean": 157.96676635742188,
"margin_dpo/margin_std": 253.5558319091797,
"step": 293
},
{
"KL/chosen_KL_mean": -239.33828735351562,
"KL/mean": -329.43267822265625,
"KL/rejected_KL_mean": -419.52703857421875,
"KL/std": 200.11917114257812,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.0022502196952700615,
"fcm_dpo/delta": -0.005795244127511978,
"fcm_dpo/margin": 180.18875122070312,
"fcm_dpo/q_t": 0.40808266401290894,
"grad_norm": 22.653549194335938,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.33727580308914185,
"logits/rejected": -0.34320569038391113,
"logps/chosen": -287.42584228515625,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -501.4240417480469,
"loss": 1.0933,
"margin_dpo/margin_mean": 180.18875122070312,
"margin_dpo/margin_std": 248.12245178222656,
"step": 294
},
{
"KL/chosen_KL_mean": -292.0207824707031,
"KL/mean": -394.706298828125,
"KL/rejected_KL_mean": -497.391845703125,
"KL/std": 232.46151733398438,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.002237509936094284,
"fcm_dpo/delta": -0.062341004610061646,
"fcm_dpo/margin": 205.37106323242188,
"fcm_dpo/q_t": 0.39664924144744873,
"grad_norm": 32.122962951660156,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.329035222530365,
"logits/rejected": -0.33049476146698,
"logps/chosen": -341.9454345703125,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -584.84814453125,
"loss": 1.0669,
"margin_dpo/margin_mean": 205.37107849121094,
"margin_dpo/margin_std": 275.5567321777344,
"step": 295
},
{
"KL/chosen_KL_mean": -373.27532958984375,
"KL/mean": -442.79229736328125,
"KL/rejected_KL_mean": -512.3092651367188,
"KL/std": 223.4410400390625,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.002242644317448139,
"fcm_dpo/delta": 0.09080183506011963,
"fcm_dpo/margin": 139.03390502929688,
"fcm_dpo/q_t": 0.42925071716308594,
"grad_norm": 37.06538009643555,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.3686225414276123,
"logits/rejected": -0.3517499566078186,
"logps/chosen": -438.7666015625,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -605.3983154296875,
"loss": 1.1995,
"margin_dpo/margin_mean": 139.03392028808594,
"margin_dpo/margin_std": 286.92205810546875,
"step": 296
},
{
"KL/chosen_KL_mean": -318.09564208984375,
"KL/mean": -401.1370849609375,
"KL/rejected_KL_mean": -484.1784973144531,
"KL/std": 211.25762939453125,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.002271018922328949,
"fcm_dpo/delta": 0.023693598806858063,
"fcm_dpo/margin": 166.08282470703125,
"fcm_dpo/q_t": 0.41268670558929443,
"grad_norm": 33.412715911865234,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.40956276655197144,
"logits/rejected": -0.41560155153274536,
"logps/chosen": -374.5726318359375,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -579.3170166015625,
"loss": 1.1004,
"margin_dpo/margin_mean": 166.08282470703125,
"margin_dpo/margin_std": 214.5130615234375,
"step": 297
},
{
"KL/chosen_KL_mean": -343.67779541015625,
"KL/mean": -453.3380432128906,
"KL/rejected_KL_mean": -562.998291015625,
"KL/std": 277.4004211425781,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.002254146384075284,
"fcm_dpo/delta": -0.09923798590898514,
"fcm_dpo/margin": 219.32052612304688,
"fcm_dpo/q_t": 0.3934746980667114,
"grad_norm": 27.909835815429688,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.4167177379131317,
"logits/rejected": -0.4354039132595062,
"logps/chosen": -411.0029296875,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -679.6604614257812,
"loss": 1.0671,
"margin_dpo/margin_mean": 219.32052612304688,
"margin_dpo/margin_std": 325.619873046875,
"step": 298
},
{
"KL/chosen_KL_mean": -277.94677734375,
"KL/mean": -368.5576171875,
"KL/rejected_KL_mean": -459.16839599609375,
"KL/std": 209.79385375976562,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.002244377974420786,
"fcm_dpo/delta": -0.0077451057732105255,
"fcm_dpo/margin": 181.2216339111328,
"fcm_dpo/q_t": 0.40871721506118774,
"grad_norm": 34.998390197753906,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.3771274983882904,
"logits/rejected": -0.38257941603660583,
"logps/chosen": -326.90887451171875,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -543.4966430664062,
"loss": 1.1056,
"margin_dpo/margin_mean": 181.22164916992188,
"margin_dpo/margin_std": 266.51416015625,
"step": 299
},
{
"KL/chosen_KL_mean": -348.21954345703125,
"KL/mean": -457.40155029296875,
"KL/rejected_KL_mean": -566.5836181640625,
"KL/std": 245.39015197753906,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.0022067520767450333,
"fcm_dpo/delta": -0.08593274652957916,
"fcm_dpo/margin": 218.36404418945312,
"fcm_dpo/q_t": 0.39113539457321167,
"grad_norm": 38.547908782958984,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.41613638401031494,
"logits/rejected": -0.4217377305030823,
"logps/chosen": -407.29327392578125,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -662.550048828125,
"loss": 1.0635,
"margin_dpo/margin_mean": 218.36404418945312,
"margin_dpo/margin_std": 299.53173828125,
"step": 300
},
{
"KL/chosen_KL_mean": -289.61981201171875,
"KL/mean": -399.01641845703125,
"KL/rejected_KL_mean": -508.4130554199219,
"KL/std": 224.61441040039062,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.002160984091460705,
"fcm_dpo/delta": -0.0767727866768837,
"fcm_dpo/margin": 218.79324340820312,
"fcm_dpo/q_t": 0.3935708999633789,
"grad_norm": 22.94151496887207,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.4170186519622803,
"logits/rejected": -0.41876429319381714,
"logps/chosen": -346.8691711425781,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -600.7666015625,
"loss": 1.0439,
"margin_dpo/margin_mean": 218.79324340820312,
"margin_dpo/margin_std": 268.8450012207031,
"step": 301
},
{
"KL/chosen_KL_mean": -249.56222534179688,
"KL/mean": -341.4454345703125,
"KL/rejected_KL_mean": -433.32861328125,
"KL/std": 198.3855438232422,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.0021501141600310802,
"fcm_dpo/delta": 0.004691842943429947,
"fcm_dpo/margin": 183.76638793945312,
"fcm_dpo/q_t": 0.4091954827308655,
"grad_norm": 24.27311134338379,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.38404256105422974,
"logits/rejected": -0.3903312683105469,
"logps/chosen": -300.7602233886719,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -530.5549926757812,
"loss": 1.0904,
"margin_dpo/margin_mean": 183.76638793945312,
"margin_dpo/margin_std": 234.3460693359375,
"step": 302
},
{
"KL/chosen_KL_mean": -257.18695068359375,
"KL/mean": -340.093994140625,
"KL/rejected_KL_mean": -423.0010070800781,
"KL/std": 207.89459228515625,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.002169003477320075,
"fcm_dpo/delta": 0.04184654355049133,
"fcm_dpo/margin": 165.81407165527344,
"fcm_dpo/q_t": 0.41749584674835205,
"grad_norm": 22.197315216064453,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.4156029224395752,
"logits/rejected": -0.4042498767375946,
"logps/chosen": -323.90087890625,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -509.9464416503906,
"loss": 1.127,
"margin_dpo/margin_mean": 165.8140869140625,
"margin_dpo/margin_std": 246.3565216064453,
"step": 303
},
{
"KL/chosen_KL_mean": -225.50621032714844,
"KL/mean": -318.69146728515625,
"KL/rejected_KL_mean": -411.8767395019531,
"KL/std": 186.12899780273438,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.002184029668569565,
"fcm_dpo/delta": -0.007586956024169922,
"fcm_dpo/margin": 186.37051391601562,
"fcm_dpo/q_t": 0.4025821089744568,
"grad_norm": 34.385562896728516,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.4139406085014343,
"logits/rejected": -0.4038363993167877,
"logps/chosen": -297.4569091796875,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -502.3487548828125,
"loss": 1.0524,
"margin_dpo/margin_mean": 186.37051391601562,
"margin_dpo/margin_std": 163.2909393310547,
"step": 304
},
{
"KL/chosen_KL_mean": -241.08352661132812,
"KL/mean": -322.9089050292969,
"KL/rejected_KL_mean": -404.7342834472656,
"KL/std": 209.6416015625,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.002178459893912077,
"fcm_dpo/delta": 0.044787101447582245,
"fcm_dpo/margin": 163.65077209472656,
"fcm_dpo/q_t": 0.4187085032463074,
"grad_norm": 21.086244583129883,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.45557549595832825,
"logits/rejected": -0.4429972171783447,
"logps/chosen": -307.8787841796875,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -497.4888916015625,
"loss": 1.121,
"margin_dpo/margin_mean": 163.65077209472656,
"margin_dpo/margin_std": 227.79843139648438,
"step": 305
},
{
"KL/chosen_KL_mean": -230.5140380859375,
"KL/mean": -311.62640380859375,
"KL/rejected_KL_mean": -392.73876953125,
"KL/std": 184.592529296875,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.0022113663144409657,
"fcm_dpo/delta": 0.04271689057350159,
"fcm_dpo/margin": 162.22476196289062,
"fcm_dpo/q_t": 0.4160733222961426,
"grad_norm": 26.13888168334961,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.39813679456710815,
"logits/rejected": -0.3748471140861511,
"logps/chosen": -300.19793701171875,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -477.89794921875,
"loss": 1.1031,
"margin_dpo/margin_mean": 162.22474670410156,
"margin_dpo/margin_std": 195.34988403320312,
"step": 306
},
{
"KL/chosen_KL_mean": -216.44622802734375,
"KL/mean": -298.306396484375,
"KL/rejected_KL_mean": -380.1665344238281,
"KL/std": 168.08889770507812,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.0022216022480279207,
"fcm_dpo/delta": 0.037642501294612885,
"fcm_dpo/margin": 163.72030639648438,
"fcm_dpo/q_t": 0.41505366563796997,
"grad_norm": 25.627588272094727,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.4154399633407593,
"logits/rejected": -0.3966638445854187,
"logps/chosen": -286.61163330078125,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -467.13885498046875,
"loss": 1.1052,
"margin_dpo/margin_mean": 163.72030639648438,
"margin_dpo/margin_std": 206.83914184570312,
"step": 307
},
{
"KL/chosen_KL_mean": -226.32872009277344,
"KL/mean": -319.50994873046875,
"KL/rejected_KL_mean": -412.69122314453125,
"KL/std": 194.25296020507812,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.0022258548997342587,
"fcm_dpo/delta": -0.015454130247235298,
"fcm_dpo/margin": 186.36245727539062,
"fcm_dpo/q_t": 0.4018438756465912,
"grad_norm": 30.474700927734375,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.38194745779037476,
"logits/rejected": -0.3681604266166687,
"logps/chosen": -281.57373046875,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -492.0634765625,
"loss": 1.0564,
"margin_dpo/margin_mean": 186.36245727539062,
"margin_dpo/margin_std": 188.6147918701172,
"step": 308
},
{
"KL/chosen_KL_mean": -239.09133911132812,
"KL/mean": -332.51025390625,
"KL/rejected_KL_mean": -425.92913818359375,
"KL/std": 211.6947479248047,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.002228262834250927,
"fcm_dpo/delta": -0.017414994537830353,
"fcm_dpo/margin": 186.83782958984375,
"fcm_dpo/q_t": 0.4025392532348633,
"grad_norm": 27.617185592651367,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.3523610830307007,
"logits/rejected": -0.34558966755867004,
"logps/chosen": -288.0504150390625,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -508.2698669433594,
"loss": 1.0739,
"margin_dpo/margin_mean": 186.83782958984375,
"margin_dpo/margin_std": 224.22433471679688,
"step": 309
},
{
"KL/chosen_KL_mean": -277.80377197265625,
"KL/mean": -365.20257568359375,
"KL/rejected_KL_mean": -452.6014099121094,
"KL/std": 186.077880859375,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.0022279408294707537,
"fcm_dpo/delta": 0.010630182921886444,
"fcm_dpo/margin": 174.79759216308594,
"fcm_dpo/q_t": 0.40788841247558594,
"grad_norm": 24.81958770751953,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.3655666410923004,
"logits/rejected": -0.3523295223712921,
"logps/chosen": -340.54559326171875,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -532.53173828125,
"loss": 1.0754,
"margin_dpo/margin_mean": 174.79759216308594,
"margin_dpo/margin_std": 181.52981567382812,
"step": 310
},
{
"KL/chosen_KL_mean": -288.7535095214844,
"KL/mean": -389.1260681152344,
"KL/rejected_KL_mean": -489.4986572265625,
"KL/std": 242.73895263671875,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.002205129014328122,
"fcm_dpo/delta": -0.04465709626674652,
"fcm_dpo/margin": 200.74514770507812,
"fcm_dpo/q_t": 0.39941084384918213,
"grad_norm": 23.147109985351562,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.34936434030532837,
"logits/rejected": -0.3253268897533417,
"logps/chosen": -341.781494140625,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -566.9368896484375,
"loss": 1.064,
"margin_dpo/margin_mean": 200.74514770507812,
"margin_dpo/margin_std": 254.4142303466797,
"step": 311
},
{
"KL/chosen_KL_mean": -274.93408203125,
"KL/mean": -365.4719543457031,
"KL/rejected_KL_mean": -456.00982666015625,
"KL/std": 207.9627685546875,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.0022081173956394196,
"fcm_dpo/delta": -6.601214408874512e-05,
"fcm_dpo/margin": 181.0757293701172,
"fcm_dpo/q_t": 0.4093562364578247,
"grad_norm": 25.84002685546875,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -0.3078731298446655,
"logits/rejected": -0.29903510212898254,
"logps/chosen": -324.3262939453125,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -531.8026123046875,
"loss": 1.0932,
"margin_dpo/margin_mean": 181.07574462890625,
"margin_dpo/margin_std": 246.26904296875,
"step": 312
},
{
"KL/chosen_KL_mean": -258.62164306640625,
"KL/mean": -353.00494384765625,
"KL/rejected_KL_mean": -447.38824462890625,
"KL/std": 222.29296875,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.002209829166531563,
"fcm_dpo/delta": -0.018744412809610367,
"fcm_dpo/margin": 188.7665557861328,
"fcm_dpo/q_t": 0.4038567841053009,
"grad_norm": 23.65576171875,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -0.3879469037055969,
"logits/rejected": -0.3879814147949219,
"logps/chosen": -308.7744140625,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -533.79443359375,
"loss": 1.0832,
"margin_dpo/margin_mean": 188.7665557861328,
"margin_dpo/margin_std": 242.31149291992188,
"step": 313
},
{
"KL/chosen_KL_mean": -279.6490783691406,
"KL/mean": -368.64263916015625,
"KL/rejected_KL_mean": -457.63616943359375,
"KL/std": 213.9251708984375,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.002186702797189355,
"fcm_dpo/delta": 0.010914817452430725,
"fcm_dpo/margin": 177.98709106445312,
"fcm_dpo/q_t": 0.41287338733673096,
"grad_norm": 21.331357955932617,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.4344632625579834,
"logits/rejected": -0.43607085943222046,
"logps/chosen": -336.88665771484375,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -555.232666015625,
"loss": 1.1167,
"margin_dpo/margin_mean": 177.98707580566406,
"margin_dpo/margin_std": 270.8641662597656,
"step": 314
},
{
"KL/chosen_KL_mean": -259.404052734375,
"KL/mean": -342.66021728515625,
"KL/rejected_KL_mean": -425.91644287109375,
"KL/std": 190.9168701171875,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.0022047478705644608,
"fcm_dpo/delta": 0.03413639962673187,
"fcm_dpo/margin": 166.5124053955078,
"fcm_dpo/q_t": 0.41494593024253845,
"grad_norm": 22.818117141723633,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.337954044342041,
"logits/rejected": -0.34216612577438354,
"logps/chosen": -308.4736328125,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -511.59735107421875,
"loss": 1.1044,
"margin_dpo/margin_mean": 166.51239013671875,
"margin_dpo/margin_std": 212.81405639648438,
"step": 315
},
{
"KL/chosen_KL_mean": -262.956298828125,
"KL/mean": -379.14678955078125,
"KL/rejected_KL_mean": -495.3372497558594,
"KL/std": 229.5782470703125,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.002165478654205799,
"fcm_dpo/delta": -0.10975506901741028,
"fcm_dpo/margin": 232.38096618652344,
"fcm_dpo/q_t": 0.38564929366111755,
"grad_norm": 27.47342300415039,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.3632145822048187,
"logits/rejected": -0.3669757843017578,
"logps/chosen": -317.217041015625,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -596.6187133789062,
"loss": 1.0149,
"margin_dpo/margin_mean": 232.3809814453125,
"margin_dpo/margin_std": 254.15489196777344,
"step": 316
},
{
"KL/chosen_KL_mean": -273.8968811035156,
"KL/mean": -402.15899658203125,
"KL/rejected_KL_mean": -530.421142578125,
"KL/std": 214.53253173828125,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.002123830374330282,
"fcm_dpo/delta": -0.152874156832695,
"fcm_dpo/margin": 256.52423095703125,
"fcm_dpo/q_t": 0.3734695017337799,
"grad_norm": 23.029985427856445,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.37268176674842834,
"logits/rejected": -0.36425209045410156,
"logps/chosen": -329.9910888671875,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -631.1201782226562,
"loss": 0.9752,
"margin_dpo/margin_mean": 256.52423095703125,
"margin_dpo/margin_std": 236.38705444335938,
"step": 317
},
{
"KL/chosen_KL_mean": -306.3728942871094,
"KL/mean": -398.26910400390625,
"KL/rejected_KL_mean": -490.16534423828125,
"KL/std": 222.15777587890625,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.0021082893945276737,
"fcm_dpo/delta": 0.012857386842370033,
"fcm_dpo/margin": 183.79248046875,
"fcm_dpo/q_t": 0.410823255777359,
"grad_norm": 26.980371475219727,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.43110066652297974,
"logits/rejected": -0.4063284993171692,
"logps/chosen": -371.0185852050781,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -572.9295654296875,
"loss": 1.0961,
"margin_dpo/margin_mean": 183.79248046875,
"margin_dpo/margin_std": 241.28457641601562,
"step": 318
},
{
"KL/chosen_KL_mean": -274.31536865234375,
"KL/mean": -389.5890197753906,
"KL/rejected_KL_mean": -504.8626708984375,
"KL/std": 245.08279418945312,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.002078516175970435,
"fcm_dpo/delta": -0.08318348973989487,
"fcm_dpo/margin": 230.54730224609375,
"fcm_dpo/q_t": 0.3892419934272766,
"grad_norm": 24.00583267211914,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.4252866208553314,
"logits/rejected": -0.4395906329154968,
"logps/chosen": -323.69915771484375,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -618.7691650390625,
"loss": 1.0327,
"margin_dpo/margin_mean": 230.54730224609375,
"margin_dpo/margin_std": 264.1103515625,
"step": 319
},
{
"KL/chosen_KL_mean": -281.8694152832031,
"KL/mean": -408.36529541015625,
"KL/rejected_KL_mean": -534.8611450195312,
"KL/std": 252.33741760253906,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.002026338130235672,
"fcm_dpo/delta": -0.11938206106424332,
"fcm_dpo/margin": 252.99169921875,
"fcm_dpo/q_t": 0.382274866104126,
"grad_norm": 31.98560333251953,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.4378474950790405,
"logits/rejected": -0.4452739953994751,
"logps/chosen": -341.37432861328125,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -632.5283203125,
"loss": 1.0061,
"margin_dpo/margin_mean": 252.99171447753906,
"margin_dpo/margin_std": 269.489013671875,
"step": 320
},
{
"KL/chosen_KL_mean": -346.11138916015625,
"KL/mean": -450.9620666503906,
"KL/rejected_KL_mean": -555.812744140625,
"KL/std": 253.70281982421875,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.002002749592065811,
"fcm_dpo/delta": -0.021664388477802277,
"fcm_dpo/margin": 209.70138549804688,
"fcm_dpo/q_t": 0.40391805768013,
"grad_norm": 22.900249481201172,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -0.3867838978767395,
"logits/rejected": -0.3807498812675476,
"logps/chosen": -407.6600646972656,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -647.4537963867188,
"loss": 1.0809,
"margin_dpo/margin_mean": 209.7013702392578,
"margin_dpo/margin_std": 269.6270751953125,
"step": 321
},
{
"KL/chosen_KL_mean": -296.47418212890625,
"KL/mean": -411.1878662109375,
"KL/rejected_KL_mean": -525.9015502929688,
"KL/std": 223.96456909179688,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.001987969968467951,
"fcm_dpo/delta": -0.05918703228235245,
"fcm_dpo/margin": 229.42739868164062,
"fcm_dpo/q_t": 0.3931780457496643,
"grad_norm": 23.179550170898438,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -0.41528087854385376,
"logits/rejected": -0.41565677523612976,
"logps/chosen": -353.76422119140625,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -621.6514892578125,
"loss": 1.0349,
"margin_dpo/margin_mean": 229.42739868164062,
"margin_dpo/margin_std": 237.61573791503906,
"step": 322
},
{
"KL/chosen_KL_mean": -306.5753173828125,
"KL/mean": -404.47186279296875,
"KL/rejected_KL_mean": -502.368408203125,
"KL/std": 220.96112060546875,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.00198989175260067,
"fcm_dpo/delta": 0.010807945393025875,
"fcm_dpo/margin": 195.7930908203125,
"fcm_dpo/q_t": 0.4091640114784241,
"grad_norm": 34.06732940673828,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -0.43661847710609436,
"logits/rejected": -0.4351438283920288,
"logps/chosen": -357.80926513671875,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -577.4302978515625,
"loss": 1.0912,
"margin_dpo/margin_mean": 195.7930908203125,
"margin_dpo/margin_std": 242.7722930908203,
"step": 323
},
{
"KL/chosen_KL_mean": -353.3026123046875,
"KL/mean": -450.3382568359375,
"KL/rejected_KL_mean": -547.3738403320312,
"KL/std": 240.54566955566406,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.002010114025324583,
"fcm_dpo/delta": 0.009144224226474762,
"fcm_dpo/margin": 194.07122802734375,
"fcm_dpo/q_t": 0.4112786650657654,
"grad_norm": 37.594154357910156,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -0.4559288024902344,
"logits/rejected": -0.4483965337276459,
"logps/chosen": -418.43780517578125,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -633.851318359375,
"loss": 1.1081,
"margin_dpo/margin_mean": 194.0712432861328,
"margin_dpo/margin_std": 265.3919677734375,
"step": 324
},
{
"KL/chosen_KL_mean": -268.78985595703125,
"KL/mean": -350.67626953125,
"KL/rejected_KL_mean": -432.562744140625,
"KL/std": 201.10546875,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.0020195613615214825,
"fcm_dpo/delta": 0.07154245674610138,
"fcm_dpo/margin": 163.77285766601562,
"fcm_dpo/q_t": 0.422224223613739,
"grad_norm": 25.190866470336914,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.495430052280426,
"logits/rejected": -0.4804334044456482,
"logps/chosen": -325.00543212890625,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -502.648681640625,
"loss": 1.1267,
"margin_dpo/margin_mean": 163.7728729248047,
"margin_dpo/margin_std": 205.93533325195312,
"step": 325
},
{
"KL/chosen_KL_mean": -294.3157958984375,
"KL/mean": -368.50244140625,
"KL/rejected_KL_mean": -442.6891174316406,
"KL/std": 189.423095703125,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.0020513960625976324,
"fcm_dpo/delta": 0.09874353557825089,
"fcm_dpo/margin": 148.37332153320312,
"fcm_dpo/q_t": 0.4277682900428772,
"grad_norm": 50.60403823852539,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.47400808334350586,
"logits/rejected": -0.45543599128723145,
"logps/chosen": -367.040771484375,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -522.535888671875,
"loss": 1.1407,
"margin_dpo/margin_mean": 148.37332153320312,
"margin_dpo/margin_std": 184.3029022216797,
"step": 326
},
{
"KL/chosen_KL_mean": -258.0245056152344,
"KL/mean": -365.32470703125,
"KL/rejected_KL_mean": -472.6249084472656,
"KL/std": 198.78585815429688,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.0020476686768233776,
"fcm_dpo/delta": -0.04147752374410629,
"fcm_dpo/margin": 214.6004180908203,
"fcm_dpo/q_t": 0.39583897590637207,
"grad_norm": 48.44367218017578,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.4509025812149048,
"logits/rejected": -0.4456767737865448,
"logps/chosen": -327.158935546875,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -584.5587158203125,
"loss": 1.0387,
"margin_dpo/margin_mean": 214.6004180908203,
"margin_dpo/margin_std": 209.97634887695312,
"step": 327
},
{
"KL/chosen_KL_mean": -267.9549865722656,
"KL/mean": -367.7935791015625,
"KL/rejected_KL_mean": -467.6322021484375,
"KL/std": 218.40206909179688,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.002048112917691469,
"fcm_dpo/delta": -0.009362122975289822,
"fcm_dpo/margin": 199.6772003173828,
"fcm_dpo/q_t": 0.40534746646881104,
"grad_norm": 34.26046371459961,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.4282514452934265,
"logits/rejected": -0.43282201886177063,
"logps/chosen": -327.6421813964844,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -558.4871826171875,
"loss": 1.0743,
"margin_dpo/margin_mean": 199.67721557617188,
"margin_dpo/margin_std": 239.71859741210938,
"step": 328
},
{
"KL/chosen_KL_mean": -314.18572998046875,
"KL/mean": -422.6015625,
"KL/rejected_KL_mean": -531.017333984375,
"KL/std": 242.86639404296875,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.002023177221417427,
"fcm_dpo/delta": -0.041052184998989105,
"fcm_dpo/margin": 216.8316192626953,
"fcm_dpo/q_t": 0.3982122540473938,
"grad_norm": 27.216005325317383,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -0.4262320399284363,
"logits/rejected": -0.4210492968559265,
"logps/chosen": -379.4319152832031,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -631.715087890625,
"loss": 1.0655,
"margin_dpo/margin_mean": 216.83163452148438,
"margin_dpo/margin_std": 263.0550231933594,
"step": 329
},
{
"KL/chosen_KL_mean": -271.18463134765625,
"KL/mean": -386.3155822753906,
"KL/rejected_KL_mean": -501.446533203125,
"KL/std": 238.26080322265625,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.002008675830438733,
"fcm_dpo/delta": -0.06554098427295685,
"fcm_dpo/margin": 230.26187133789062,
"fcm_dpo/q_t": 0.39291393756866455,
"grad_norm": 26.819442749023438,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -0.3993980884552002,
"logits/rejected": -0.3997897505760193,
"logps/chosen": -318.1829833984375,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -588.3233642578125,
"loss": 1.0381,
"margin_dpo/margin_mean": 230.26187133789062,
"margin_dpo/margin_std": 257.1680908203125,
"step": 330
},
{
"KL/chosen_KL_mean": -301.6651306152344,
"KL/mean": -418.078125,
"KL/rejected_KL_mean": -534.4911499023438,
"KL/std": 204.872314453125,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.001982024870812893,
"fcm_dpo/delta": -0.06445472687482834,
"fcm_dpo/margin": 232.82598876953125,
"fcm_dpo/q_t": 0.390749990940094,
"grad_norm": 30.03919792175293,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -0.41282713413238525,
"logits/rejected": -0.40853530168533325,
"logps/chosen": -352.18939208984375,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -623.506591796875,
"loss": 1.0137,
"margin_dpo/margin_mean": 232.82598876953125,
"margin_dpo/margin_std": 201.24801635742188,
"step": 331
},
{
"KL/chosen_KL_mean": -300.2237243652344,
"KL/mean": -383.75494384765625,
"KL/rejected_KL_mean": -467.2861328125,
"KL/std": 211.57131958007812,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.0019986871629953384,
"fcm_dpo/delta": 0.06825611740350723,
"fcm_dpo/margin": 167.06240844726562,
"fcm_dpo/q_t": 0.4234786927700043,
"grad_norm": 21.762527465820312,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -0.4211878478527069,
"logits/rejected": -0.4023297131061554,
"logps/chosen": -349.40399169921875,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -543.7713012695312,
"loss": 1.1323,
"margin_dpo/margin_mean": 167.0624237060547,
"margin_dpo/margin_std": 237.78700256347656,
"step": 332
},
{
"KL/chosen_KL_mean": -331.29693603515625,
"KL/mean": -439.12548828125,
"KL/rejected_KL_mean": -546.9540405273438,
"KL/std": 251.02923583984375,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.0019800434820353985,
"fcm_dpo/delta": -0.029480591416358948,
"fcm_dpo/margin": 215.6570281982422,
"fcm_dpo/q_t": 0.4032571315765381,
"grad_norm": 23.11104965209961,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -0.42099469900131226,
"logits/rejected": -0.4137647747993469,
"logps/chosen": -395.0527038574219,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -641.9981689453125,
"loss": 1.0919,
"margin_dpo/margin_mean": 215.6570281982422,
"margin_dpo/margin_std": 301.61212158203125,
"step": 333
},
{
"KL/chosen_KL_mean": -313.2780456542969,
"KL/mean": -411.32177734375,
"KL/rejected_KL_mean": -509.3655090332031,
"KL/std": 270.41351318359375,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.001991923665627837,
"fcm_dpo/delta": 0.009776053950190544,
"fcm_dpo/margin": 196.08746337890625,
"fcm_dpo/q_t": 0.4119870066642761,
"grad_norm": 22.336389541625977,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.4335794150829315,
"logits/rejected": -0.43230512738227844,
"logps/chosen": -380.2578125,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -604.6824340820312,
"loss": 1.121,
"margin_dpo/margin_mean": 196.08746337890625,
"margin_dpo/margin_std": 306.477294921875,
"step": 334
},
{
"KL/chosen_KL_mean": -335.70196533203125,
"KL/mean": -414.1585998535156,
"KL/rejected_KL_mean": -492.6152038574219,
"KL/std": 244.75711059570312,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.0020306003279983997,
"fcm_dpo/delta": 0.08306629955768585,
"fcm_dpo/margin": 156.9132080078125,
"fcm_dpo/q_t": 0.4281030297279358,
"grad_norm": 29.569700241088867,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -0.4642852246761322,
"logits/rejected": -0.4429335594177246,
"logps/chosen": -398.24444580078125,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -580.23291015625,
"loss": 1.1652,
"margin_dpo/margin_mean": 156.9132080078125,
"margin_dpo/margin_std": 268.2879333496094,
"step": 335
},
{
"KL/chosen_KL_mean": -330.2154541015625,
"KL/mean": -455.31500244140625,
"KL/rejected_KL_mean": -580.41455078125,
"KL/std": 285.79803466796875,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.0020034818444401026,
"fcm_dpo/delta": -0.10648685693740845,
"fcm_dpo/margin": 250.1991424560547,
"fcm_dpo/q_t": 0.38840019702911377,
"grad_norm": 31.805938720703125,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -0.4141591787338257,
"logits/rejected": -0.41369566321372986,
"logps/chosen": -384.74658203125,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -684.8187866210938,
"loss": 1.0525,
"margin_dpo/margin_mean": 250.1991424560547,
"margin_dpo/margin_std": 338.71246337890625,
"step": 336
},
{
"KL/chosen_KL_mean": -295.72308349609375,
"KL/mean": -417.3601379394531,
"KL/rejected_KL_mean": -538.9971923828125,
"KL/std": 227.58993530273438,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.001958114095032215,
"fcm_dpo/delta": -0.08043714612722397,
"fcm_dpo/margin": 243.27413940429688,
"fcm_dpo/q_t": 0.3905276656150818,
"grad_norm": 29.914546966552734,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -0.42889440059661865,
"logits/rejected": -0.41781020164489746,
"logps/chosen": -360.85174560546875,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -640.7242431640625,
"loss": 1.0313,
"margin_dpo/margin_mean": 243.27413940429688,
"margin_dpo/margin_std": 274.2711486816406,
"step": 337
},
{
"KL/chosen_KL_mean": -267.119384765625,
"KL/mean": -383.24462890625,
"KL/rejected_KL_mean": -499.36981201171875,
"KL/std": 221.5072021484375,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.0019371946109458804,
"fcm_dpo/delta": -0.05233225226402283,
"fcm_dpo/margin": 232.25042724609375,
"fcm_dpo/q_t": 0.3943594694137573,
"grad_norm": 32.328243255615234,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -0.43586158752441406,
"logits/rejected": -0.4274016320705414,
"logps/chosen": -325.5421142578125,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -588.4383544921875,
"loss": 1.0337,
"margin_dpo/margin_mean": 232.25042724609375,
"margin_dpo/margin_std": 232.92489624023438,
"step": 338
},
{
"KL/chosen_KL_mean": -294.05352783203125,
"KL/mean": -407.55291748046875,
"KL/rejected_KL_mean": -521.05224609375,
"KL/std": 231.62216186523438,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.0019379344303160906,
"fcm_dpo/delta": -0.04284397512674332,
"fcm_dpo/margin": 226.99868774414062,
"fcm_dpo/q_t": 0.3972986936569214,
"grad_norm": 29.154096603393555,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -0.4619607925415039,
"logits/rejected": -0.44948840141296387,
"logps/chosen": -354.0488586425781,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -624.963134765625,
"loss": 1.0493,
"margin_dpo/margin_mean": 226.99868774414062,
"margin_dpo/margin_std": 233.55419921875,
"step": 339
},
{
"KL/chosen_KL_mean": -315.4582824707031,
"KL/mean": -416.0123291015625,
"KL/rejected_KL_mean": -516.5663452148438,
"KL/std": 227.33596801757812,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.0019145121332257986,
"fcm_dpo/delta": 0.015345301479101181,
"fcm_dpo/margin": 201.10806274414062,
"fcm_dpo/q_t": 0.41142672300338745,
"grad_norm": 28.22606658935547,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -0.3919587731361389,
"logits/rejected": -0.37107688188552856,
"logps/chosen": -368.28851318359375,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -589.673583984375,
"loss": 1.104,
"margin_dpo/margin_mean": 201.10806274414062,
"margin_dpo/margin_std": 275.59588623046875,
"step": 340
},
{
"KL/chosen_KL_mean": -324.9350280761719,
"KL/mean": -440.717041015625,
"KL/rejected_KL_mean": -556.4990234375,
"KL/std": 254.97799682617188,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.0019036681624129415,
"fcm_dpo/delta": -0.04316044598817825,
"fcm_dpo/margin": 231.56407165527344,
"fcm_dpo/q_t": 0.3988724946975708,
"grad_norm": 25.336105346679688,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.40650486946105957,
"logits/rejected": -0.405281662940979,
"logps/chosen": -372.8348388671875,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -658.3089599609375,
"loss": 1.0573,
"margin_dpo/margin_mean": 231.56407165527344,
"margin_dpo/margin_std": 270.78424072265625,
"step": 341
},
{
"KL/chosen_KL_mean": -327.50567626953125,
"KL/mean": -430.53558349609375,
"KL/rejected_KL_mean": -533.5655517578125,
"KL/std": 242.9896697998047,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.0019156227353960276,
"fcm_dpo/delta": 0.005151804536581039,
"fcm_dpo/margin": 206.05987548828125,
"fcm_dpo/q_t": 0.4093712568283081,
"grad_norm": 26.062049865722656,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.4568237066268921,
"logits/rejected": -0.43443238735198975,
"logps/chosen": -399.5023193359375,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -626.1551513671875,
"loss": 1.101,
"margin_dpo/margin_mean": 206.05987548828125,
"margin_dpo/margin_std": 287.116943359375,
"step": 342
},
{
"KL/chosen_KL_mean": -322.3934326171875,
"KL/mean": -443.8787536621094,
"KL/rejected_KL_mean": -565.3640747070312,
"KL/std": 246.6431884765625,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.0018897566478699446,
"fcm_dpo/delta": -0.062056764960289,
"fcm_dpo/margin": 242.97064208984375,
"fcm_dpo/q_t": 0.3914814591407776,
"grad_norm": 25.208070755004883,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.4649735689163208,
"logits/rejected": -0.46656423807144165,
"logps/chosen": -376.799072265625,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -676.405517578125,
"loss": 1.0218,
"margin_dpo/margin_mean": 242.97064208984375,
"margin_dpo/margin_std": 221.15011596679688,
"step": 343
},
{
"KL/chosen_KL_mean": -315.85748291015625,
"KL/mean": -426.89324951171875,
"KL/rejected_KL_mean": -537.9290161132812,
"KL/std": 271.2740173339844,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.0018969422671943903,
"fcm_dpo/delta": -0.023906365036964417,
"fcm_dpo/margin": 222.07159423828125,
"fcm_dpo/q_t": 0.4061945974826813,
"grad_norm": 32.413124084472656,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.4314219653606415,
"logits/rejected": -0.4362150728702545,
"logps/chosen": -369.8221435546875,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -628.5523681640625,
"loss": 1.0819,
"margin_dpo/margin_mean": 222.0715789794922,
"margin_dpo/margin_std": 282.13812255859375,
"step": 344
},
{
"KL/chosen_KL_mean": -371.65093994140625,
"KL/mean": -492.77777099609375,
"KL/rejected_KL_mean": -613.904541015625,
"KL/std": 254.54132080078125,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.001865363446995616,
"fcm_dpo/delta": -0.05433148890733719,
"fcm_dpo/margin": 242.25363159179688,
"fcm_dpo/q_t": 0.395372211933136,
"grad_norm": 22.87287139892578,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.4728688597679138,
"logits/rejected": -0.47000813484191895,
"logps/chosen": -433.3366394042969,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -713.39501953125,
"loss": 1.0481,
"margin_dpo/margin_mean": 242.25364685058594,
"margin_dpo/margin_std": 279.2045593261719,
"step": 345
},
{
"KL/chosen_KL_mean": -372.73223876953125,
"KL/mean": -485.113525390625,
"KL/rejected_KL_mean": -597.494873046875,
"KL/std": 246.56961059570312,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.0018515766132622957,
"fcm_dpo/delta": -0.016864469274878502,
"fcm_dpo/margin": 224.76260375976562,
"fcm_dpo/q_t": 0.40249505639076233,
"grad_norm": 20.88530158996582,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -0.43277889490127563,
"logits/rejected": -0.4222702980041504,
"logps/chosen": -427.988525390625,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -674.91015625,
"loss": 1.0697,
"margin_dpo/margin_mean": 224.76263427734375,
"margin_dpo/margin_std": 265.6940002441406,
"step": 346
},
{
"KL/chosen_KL_mean": -361.72607421875,
"KL/mean": -476.1159362792969,
"KL/rejected_KL_mean": -590.5057373046875,
"KL/std": 262.36614990234375,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.0018435390666127205,
"fcm_dpo/delta": -0.022720448672771454,
"fcm_dpo/margin": 228.77969360351562,
"fcm_dpo/q_t": 0.40304189920425415,
"grad_norm": 22.485294342041016,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -0.43125152587890625,
"logits/rejected": -0.42406296730041504,
"logps/chosen": -419.2923278808594,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -682.86083984375,
"loss": 1.0752,
"margin_dpo/margin_mean": 228.77969360351562,
"margin_dpo/margin_std": 290.30322265625,
"step": 347
},
{
"KL/chosen_KL_mean": -323.0507507324219,
"KL/mean": -417.467041015625,
"KL/rejected_KL_mean": -511.8833312988281,
"KL/std": 221.05445861816406,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.0018442126456648111,
"fcm_dpo/delta": 0.053140588104724884,
"fcm_dpo/margin": 188.83261108398438,
"fcm_dpo/q_t": 0.41966158151626587,
"grad_norm": 22.27375602722168,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -0.47590553760528564,
"logits/rejected": -0.47713130712509155,
"logps/chosen": -379.36846923828125,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -601.021728515625,
"loss": 1.1241,
"margin_dpo/margin_mean": 188.83261108398438,
"margin_dpo/margin_std": 255.472900390625,
"step": 348
},
{
"KL/chosen_KL_mean": -299.5309143066406,
"KL/mean": -407.94049072265625,
"KL/rejected_KL_mean": -516.3500366210938,
"KL/std": 246.6162109375,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.0018662457587197423,
"fcm_dpo/delta": -0.005084618926048279,
"fcm_dpo/margin": 216.81912231445312,
"fcm_dpo/q_t": 0.4084588885307312,
"grad_norm": 21.248226165771484,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -0.41005945205688477,
"logits/rejected": -0.4089745283126831,
"logps/chosen": -357.5564270019531,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -613.855224609375,
"loss": 1.0995,
"margin_dpo/margin_mean": 216.81912231445312,
"margin_dpo/margin_std": 308.3006896972656,
"step": 349
},
{
"KL/chosen_KL_mean": -315.78411865234375,
"KL/mean": -420.061767578125,
"KL/rejected_KL_mean": -524.3394775390625,
"KL/std": 230.697021484375,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.0018557433504611254,
"fcm_dpo/delta": 0.013263138011097908,
"fcm_dpo/margin": 208.5552978515625,
"fcm_dpo/q_t": 0.40980684757232666,
"grad_norm": 33.216182708740234,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -0.47512534260749817,
"logits/rejected": -0.4778461754322052,
"logps/chosen": -380.1146240234375,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -614.2110595703125,
"loss": 1.1033,
"margin_dpo/margin_mean": 208.5552978515625,
"margin_dpo/margin_std": 284.32781982421875,
"step": 350
},
{
"KL/chosen_KL_mean": -295.28631591796875,
"KL/mean": -421.78533935546875,
"KL/rejected_KL_mean": -548.2843017578125,
"KL/std": 270.7213134765625,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.0018403417197987437,
"fcm_dpo/delta": -0.06909814476966858,
"fcm_dpo/margin": 252.99801635742188,
"fcm_dpo/q_t": 0.39455610513687134,
"grad_norm": 22.35758399963379,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -0.4860556423664093,
"logits/rejected": -0.48423847556114197,
"logps/chosen": -355.9584655761719,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -649.8497314453125,
"loss": 1.0492,
"margin_dpo/margin_mean": 252.99801635742188,
"margin_dpo/margin_std": 310.3662414550781,
"step": 351
},
{
"KL/chosen_KL_mean": -352.4783020019531,
"KL/mean": -432.419921875,
"KL/rejected_KL_mean": -512.3615112304688,
"KL/std": 252.67904663085938,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.001861312659457326,
"fcm_dpo/delta": 0.10567327588796616,
"fcm_dpo/margin": 159.88320922851562,
"fcm_dpo/q_t": 0.4318529963493347,
"grad_norm": 29.07950210571289,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -0.46844974160194397,
"logits/rejected": -0.4366309642791748,
"logps/chosen": -423.4217529296875,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -589.0034790039062,
"loss": 1.1751,
"margin_dpo/margin_mean": 159.88320922851562,
"margin_dpo/margin_std": 276.7126159667969,
"step": 352
},
{
"KL/chosen_KL_mean": -332.9418640136719,
"KL/mean": -436.4351806640625,
"KL/rejected_KL_mean": -539.928466796875,
"KL/std": 252.6245574951172,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.0018715719925239682,
"fcm_dpo/delta": 0.01264815405011177,
"fcm_dpo/margin": 206.98663330078125,
"fcm_dpo/q_t": 0.41147467494010925,
"grad_norm": 32.52266311645508,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -0.5107035636901855,
"logits/rejected": -0.5004223585128784,
"logps/chosen": -391.33721923828125,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -620.2640380859375,
"loss": 1.1022,
"margin_dpo/margin_mean": 206.98663330078125,
"margin_dpo/margin_std": 278.4654541015625,
"step": 353
},
{
"KL/chosen_KL_mean": -288.1324462890625,
"KL/mean": -404.28253173828125,
"KL/rejected_KL_mean": -520.4326171875,
"KL/std": 234.51644897460938,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.001870082225650549,
"fcm_dpo/delta": -0.03605414927005768,
"fcm_dpo/margin": 232.30015563964844,
"fcm_dpo/q_t": 0.3978366255760193,
"grad_norm": 22.32083511352539,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -0.47122490406036377,
"logits/rejected": -0.4608303904533386,
"logps/chosen": -347.9354248046875,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -609.1900634765625,
"loss": 1.0421,
"margin_dpo/margin_mean": 232.3001708984375,
"margin_dpo/margin_std": 231.21661376953125,
"step": 354
},
{
"KL/chosen_KL_mean": -286.2269287109375,
"KL/mean": -399.2731018066406,
"KL/rejected_KL_mean": -512.3192749023438,
"KL/std": 236.06967163085938,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.0018591878470033407,
"fcm_dpo/delta": -0.021364610642194748,
"fcm_dpo/margin": 226.09228515625,
"fcm_dpo/q_t": 0.4009643793106079,
"grad_norm": 33.417415618896484,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -0.5266305208206177,
"logits/rejected": -0.5164577960968018,
"logps/chosen": -340.35546875,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -594.725341796875,
"loss": 1.056,
"margin_dpo/margin_mean": 226.09228515625,
"margin_dpo/margin_std": 236.9951171875,
"step": 355
},
{
"KL/chosen_KL_mean": -351.47430419921875,
"KL/mean": -416.1058654785156,
"KL/rejected_KL_mean": -480.7374267578125,
"KL/std": 247.41590881347656,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.0019163393881171942,
"fcm_dpo/delta": 0.1543731987476349,
"fcm_dpo/margin": 129.26312255859375,
"fcm_dpo/q_t": 0.44436901807785034,
"grad_norm": 38.0467643737793,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -0.4860239028930664,
"logits/rejected": -0.46484631299972534,
"logps/chosen": -416.14813232421875,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -556.63671875,
"loss": 1.2369,
"margin_dpo/margin_mean": 129.26312255859375,
"margin_dpo/margin_std": 304.29168701171875,
"step": 356
},
{
"KL/chosen_KL_mean": -321.70184326171875,
"KL/mean": -426.21234130859375,
"KL/rejected_KL_mean": -530.7227783203125,
"KL/std": 239.44512939453125,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.0019112075679004192,
"fcm_dpo/delta": 0.00017703324556350708,
"fcm_dpo/margin": 209.02096557617188,
"fcm_dpo/q_t": 0.406343549489975,
"grad_norm": 24.745952606201172,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -0.47398656606674194,
"logits/rejected": -0.4626181721687317,
"logps/chosen": -374.4276123046875,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -617.56396484375,
"loss": 1.0775,
"margin_dpo/margin_mean": 209.02096557617188,
"margin_dpo/margin_std": 240.1793212890625,
"step": 357
},
{
"KL/chosen_KL_mean": -295.3409729003906,
"KL/mean": -391.8931884765625,
"KL/rejected_KL_mean": -488.4454345703125,
"KL/std": 233.13546752929688,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.0019269022159278393,
"fcm_dpo/delta": 0.028992902487516403,
"fcm_dpo/margin": 193.10446166992188,
"fcm_dpo/q_t": 0.4144141972064972,
"grad_norm": 27.003738403320312,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.5157039165496826,
"logits/rejected": -0.5015970468521118,
"logps/chosen": -358.5464172363281,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -576.8187255859375,
"loss": 1.1041,
"margin_dpo/margin_mean": 193.10446166992188,
"margin_dpo/margin_std": 253.48260498046875,
"step": 358
},
{
"KL/chosen_KL_mean": -329.5625305175781,
"KL/mean": -441.9998474121094,
"KL/rejected_KL_mean": -554.4371948242188,
"KL/std": 232.54946899414062,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.0019179590744897723,
"fcm_dpo/delta": -0.03281618282198906,
"fcm_dpo/margin": 224.8746337890625,
"fcm_dpo/q_t": 0.40181970596313477,
"grad_norm": 28.059730529785156,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -0.4727786183357239,
"logits/rejected": -0.4688911736011505,
"logps/chosen": -385.9327392578125,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -636.6109619140625,
"loss": 1.0721,
"margin_dpo/margin_mean": 224.8746337890625,
"margin_dpo/margin_std": 285.3248596191406,
"step": 359
},
{
"KL/chosen_KL_mean": -333.458740234375,
"KL/mean": -414.25750732421875,
"KL/rejected_KL_mean": -495.05621337890625,
"KL/std": 196.81692504882812,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.0019466321682557464,
"fcm_dpo/delta": 0.0879821926355362,
"fcm_dpo/margin": 161.5974578857422,
"fcm_dpo/q_t": 0.4257877469062805,
"grad_norm": 42.480712890625,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -0.4350992441177368,
"logits/rejected": -0.42406588792800903,
"logps/chosen": -384.91912841796875,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -564.8951416015625,
"loss": 1.1332,
"margin_dpo/margin_mean": 161.59744262695312,
"margin_dpo/margin_std": 199.5988311767578,
"step": 360
},
{
"KL/chosen_KL_mean": -343.19219970703125,
"KL/mean": -426.57568359375,
"KL/rejected_KL_mean": -509.95916748046875,
"KL/std": 232.37318420410156,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.0019770534709095955,
"fcm_dpo/delta": 0.07246075570583344,
"fcm_dpo/margin": 166.7669677734375,
"fcm_dpo/q_t": 0.42386797070503235,
"grad_norm": 34.09297180175781,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -0.43081650137901306,
"logits/rejected": -0.4273498058319092,
"logps/chosen": -397.06170654296875,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -600.7283935546875,
"loss": 1.1452,
"margin_dpo/margin_mean": 166.7669677734375,
"margin_dpo/margin_std": 257.58953857421875,
"step": 361
},
{
"KL/chosen_KL_mean": -295.250244140625,
"KL/mean": -429.8717346191406,
"KL/rejected_KL_mean": -564.4932861328125,
"KL/std": 243.06881713867188,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.001937607303261757,
"fcm_dpo/delta": -0.12887535989284515,
"fcm_dpo/margin": 269.2430114746094,
"fcm_dpo/q_t": 0.3792804479598999,
"grad_norm": 29.63429069519043,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -0.43117934465408325,
"logits/rejected": -0.4422128200531006,
"logps/chosen": -353.8893127441406,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -670.0751953125,
"loss": 0.9869,
"margin_dpo/margin_mean": 269.2430114746094,
"margin_dpo/margin_std": 257.28125,
"step": 362
},
{
"KL/chosen_KL_mean": -264.6348571777344,
"KL/mean": -406.2776184082031,
"KL/rejected_KL_mean": -547.92041015625,
"KL/std": 246.21340942382812,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.0018945279298350215,
"fcm_dpo/delta": -0.1442004144191742,
"fcm_dpo/margin": 283.2854919433594,
"fcm_dpo/q_t": 0.375781774520874,
"grad_norm": 24.405099868774414,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -0.41785454750061035,
"logits/rejected": -0.4117404818534851,
"logps/chosen": -309.1932373046875,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -622.6153564453125,
"loss": 0.9783,
"margin_dpo/margin_mean": 283.2854919433594,
"margin_dpo/margin_std": 266.176025390625,
"step": 363
},
{
"KL/chosen_KL_mean": -288.89862060546875,
"KL/mean": -392.5069580078125,
"KL/rejected_KL_mean": -496.11529541015625,
"KL/std": 236.96873474121094,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.001885156612843275,
"fcm_dpo/delta": 0.0093899667263031,
"fcm_dpo/margin": 207.2166748046875,
"fcm_dpo/q_t": 0.4092685282230377,
"grad_norm": 21.653566360473633,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.44824981689453125,
"logits/rejected": -0.45862144231796265,
"logps/chosen": -337.793212890625,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -587.5110473632812,
"loss": 1.097,
"margin_dpo/margin_mean": 207.21669006347656,
"margin_dpo/margin_std": 273.45452880859375,
"step": 364
},
{
"KL/chosen_KL_mean": -273.8106689453125,
"KL/mean": -381.73956298828125,
"KL/rejected_KL_mean": -489.6685485839844,
"KL/std": 248.11196899414062,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.0018767972942441702,
"fcm_dpo/delta": -0.0053392937406897545,
"fcm_dpo/margin": 215.85787963867188,
"fcm_dpo/q_t": 0.4069938659667969,
"grad_norm": 21.937294006347656,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -0.4212133288383484,
"logits/rejected": -0.43191930651664734,
"logps/chosen": -325.30340576171875,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -582.3702392578125,
"loss": 1.0753,
"margin_dpo/margin_mean": 215.85787963867188,
"margin_dpo/margin_std": 255.764892578125,
"step": 365
},
{
"KL/chosen_KL_mean": -245.94454956054688,
"KL/mean": -353.47900390625,
"KL/rejected_KL_mean": -461.013427734375,
"KL/std": 230.1700897216797,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.001868388382717967,
"fcm_dpo/delta": -0.002255551517009735,
"fcm_dpo/margin": 215.06887817382812,
"fcm_dpo/q_t": 0.4071570634841919,
"grad_norm": 21.38697052001953,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -0.4181390404701233,
"logits/rejected": -0.4320718050003052,
"logps/chosen": -290.6651306152344,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -544.3238525390625,
"loss": 1.0873,
"margin_dpo/margin_mean": 215.06887817382812,
"margin_dpo/margin_std": 274.4052429199219,
"step": 366
},
{
"KL/chosen_KL_mean": -257.902099609375,
"KL/mean": -347.42755126953125,
"KL/rejected_KL_mean": -436.95306396484375,
"KL/std": 208.62741088867188,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.001896706409752369,
"fcm_dpo/delta": 0.062337495386600494,
"fcm_dpo/margin": 179.05096435546875,
"fcm_dpo/q_t": 0.4197365641593933,
"grad_norm": 22.023393630981445,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -0.4201430678367615,
"logits/rejected": -0.4021303057670593,
"logps/chosen": -316.3074951171875,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -513.7044067382812,
"loss": 1.1196,
"margin_dpo/margin_mean": 179.05096435546875,
"margin_dpo/margin_std": 226.32080078125,
"step": 367
},
{
"KL/chosen_KL_mean": -236.77053833007812,
"KL/mean": -378.0199890136719,
"KL/rejected_KL_mean": -519.2694091796875,
"KL/std": 237.63888549804688,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.0018626508535817266,
"fcm_dpo/delta": -0.1331697702407837,
"fcm_dpo/margin": 282.4989013671875,
"fcm_dpo/q_t": 0.37490114569664,
"grad_norm": 36.5501823425293,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -0.4193364977836609,
"logits/rejected": -0.42513972520828247,
"logps/chosen": -281.2230529785156,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -617.82470703125,
"loss": 0.9663,
"margin_dpo/margin_mean": 282.4989013671875,
"margin_dpo/margin_std": 217.8269500732422,
"step": 368
},
{
"KL/chosen_KL_mean": -326.0372314453125,
"KL/mean": -400.8064880371094,
"KL/rejected_KL_mean": -475.5757141113281,
"KL/std": 242.97593688964844,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.001889625797048211,
"fcm_dpo/delta": 0.12015080451965332,
"fcm_dpo/margin": 149.5384979248047,
"fcm_dpo/q_t": 0.4347541332244873,
"grad_norm": 25.68787384033203,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -0.44025710225105286,
"logits/rejected": -0.42140746116638184,
"logps/chosen": -397.41876220703125,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -566.87158203125,
"loss": 1.19,
"margin_dpo/margin_mean": 149.5384979248047,
"margin_dpo/margin_std": 279.17926025390625,
"step": 369
},
{
"KL/chosen_KL_mean": -332.7804870605469,
"KL/mean": -408.59906005859375,
"KL/rejected_KL_mean": -484.41766357421875,
"KL/std": 246.52499389648438,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.001930559752508998,
"fcm_dpo/delta": 0.11008969694375992,
"fcm_dpo/margin": 151.63714599609375,
"fcm_dpo/q_t": 0.4327510893344879,
"grad_norm": 27.80760955810547,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -0.4499947726726532,
"logits/rejected": -0.44359490275382996,
"logps/chosen": -404.38800048828125,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -581.677490234375,
"loss": 1.1832,
"margin_dpo/margin_mean": 151.63714599609375,
"margin_dpo/margin_std": 275.9014892578125,
"step": 370
},
{
"KL/chosen_KL_mean": -326.5951843261719,
"KL/mean": -433.36822509765625,
"KL/rejected_KL_mean": -540.1412963867188,
"KL/std": 257.112548828125,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.0019381106831133366,
"fcm_dpo/delta": -0.014613218605518341,
"fcm_dpo/margin": 213.54608154296875,
"fcm_dpo/q_t": 0.4060685932636261,
"grad_norm": 21.661701202392578,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -0.5168248414993286,
"logits/rejected": -0.5063666105270386,
"logps/chosen": -396.0096740722656,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -639.3134765625,
"loss": 1.0939,
"margin_dpo/margin_mean": 213.54608154296875,
"margin_dpo/margin_std": 299.3140869140625,
"step": 371
},
{
"KL/chosen_KL_mean": -314.97894287109375,
"KL/mean": -441.3173522949219,
"KL/rejected_KL_mean": -567.65576171875,
"KL/std": 292.46392822265625,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.0019034836441278458,
"fcm_dpo/delta": -0.08524809032678604,
"fcm_dpo/margin": 252.6768341064453,
"fcm_dpo/q_t": 0.3927803039550781,
"grad_norm": 20.759563446044922,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -0.46310439705848694,
"logits/rejected": -0.4459260106086731,
"logps/chosen": -376.79693603515625,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -646.1952514648438,
"loss": 1.0493,
"margin_dpo/margin_mean": 252.6768341064453,
"margin_dpo/margin_std": 329.3980407714844,
"step": 372
},
{
"KL/chosen_KL_mean": -346.1135559082031,
"KL/mean": -465.76202392578125,
"KL/rejected_KL_mean": -585.4105224609375,
"KL/std": 286.3425598144531,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.001893439912237227,
"fcm_dpo/delta": -0.05597112700343132,
"fcm_dpo/margin": 239.29693603515625,
"fcm_dpo/q_t": 0.39477214217185974,
"grad_norm": 37.95475769042969,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -0.4648551344871521,
"logits/rejected": -0.44541704654693604,
"logps/chosen": -410.3306884765625,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -671.3701171875,
"loss": 1.05,
"margin_dpo/margin_mean": 239.29693603515625,
"margin_dpo/margin_std": 269.4728698730469,
"step": 373
},
{
"KL/chosen_KL_mean": -320.47308349609375,
"KL/mean": -428.29461669921875,
"KL/rejected_KL_mean": -536.1160888671875,
"KL/std": 311.81317138671875,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.00186735182069242,
"fcm_dpo/delta": -0.003267388790845871,
"fcm_dpo/margin": 215.6429901123047,
"fcm_dpo/q_t": 0.411865770816803,
"grad_norm": 22.822025299072266,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -0.4975176453590393,
"logits/rejected": -0.4788419008255005,
"logps/chosen": -391.123291015625,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -629.7562255859375,
"loss": 1.1191,
"margin_dpo/margin_mean": 215.6429901123047,
"margin_dpo/margin_std": 348.92791748046875,
"step": 374
},
{
"KL/chosen_KL_mean": -333.2713623046875,
"KL/mean": -435.57318115234375,
"KL/rejected_KL_mean": -537.8748779296875,
"KL/std": 250.6235809326172,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.0018701537046581507,
"fcm_dpo/delta": 0.017490137368440628,
"fcm_dpo/margin": 204.60354614257812,
"fcm_dpo/q_t": 0.41193336248397827,
"grad_norm": 35.44966125488281,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -0.48623552918434143,
"logits/rejected": -0.4854326844215393,
"logps/chosen": -393.35162353515625,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -626.813232421875,
"loss": 1.1195,
"margin_dpo/margin_mean": 204.60354614257812,
"margin_dpo/margin_std": 305.8067626953125,
"step": 375
},
{
"KL/chosen_KL_mean": -329.8739929199219,
"KL/mean": -463.758544921875,
"KL/rejected_KL_mean": -597.6431884765625,
"KL/std": 283.9222106933594,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.001852883491665125,
"fcm_dpo/delta": -0.10125644505023956,
"fcm_dpo/margin": 267.7691650390625,
"fcm_dpo/q_t": 0.38710707426071167,
"grad_norm": 24.405960083007812,
"learning_rate": 2.5e-07,
"logits/chosen": -0.4748380184173584,
"logits/rejected": -0.4657718539237976,
"logps/chosen": -392.5343017578125,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -703.1697998046875,
"loss": 1.0372,
"margin_dpo/margin_mean": 267.7691650390625,
"margin_dpo/margin_std": 331.33099365234375,
"step": 376
},
{
"KL/chosen_KL_mean": -334.75823974609375,
"KL/mean": -461.15093994140625,
"KL/rejected_KL_mean": -587.543701171875,
"KL/std": 287.4979248046875,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.0018305475823581219,
"fcm_dpo/delta": -0.06573105603456497,
"fcm_dpo/margin": 252.7854461669922,
"fcm_dpo/q_t": 0.3940247893333435,
"grad_norm": 23.72755241394043,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -0.3885685205459595,
"logits/rejected": -0.385770320892334,
"logps/chosen": -389.2369384765625,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -686.2470092773438,
"loss": 1.0482,
"margin_dpo/margin_mean": 252.7854461669922,
"margin_dpo/margin_std": 304.465087890625,
"step": 377
},
{
"KL/chosen_KL_mean": -306.72894287109375,
"KL/mean": -441.921875,
"KL/rejected_KL_mean": -577.1148681640625,
"KL/std": 269.91107177734375,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.0017946755979210138,
"fcm_dpo/delta": -0.08971542119979858,
"fcm_dpo/margin": 270.3858642578125,
"fcm_dpo/q_t": 0.3866649270057678,
"grad_norm": 21.382219314575195,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -0.40874114632606506,
"logits/rejected": -0.41893163323402405,
"logps/chosen": -351.7494812011719,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -665.1618041992188,
"loss": 1.0224,
"margin_dpo/margin_mean": 270.3858642578125,
"margin_dpo/margin_std": 291.0148010253906,
"step": 378
},
{
"KL/chosen_KL_mean": -351.1424560546875,
"KL/mean": -483.84320068359375,
"KL/rejected_KL_mean": -616.5439453125,
"KL/std": 272.0304260253906,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.0017617056146264076,
"fcm_dpo/delta": -0.07156138122081757,
"fcm_dpo/margin": 265.40155029296875,
"fcm_dpo/q_t": 0.39362668991088867,
"grad_norm": 24.216798782348633,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -0.4043707847595215,
"logits/rejected": -0.4257460832595825,
"logps/chosen": -404.32452392578125,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -730.8441162109375,
"loss": 1.0483,
"margin_dpo/margin_mean": 265.4015197753906,
"margin_dpo/margin_std": 324.57073974609375,
"step": 379
},
{
"KL/chosen_KL_mean": -344.0395812988281,
"KL/mean": -488.787109375,
"KL/rejected_KL_mean": -633.53466796875,
"KL/std": 300.50885009765625,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.0017366987885907292,
"fcm_dpo/delta": -0.10811088979244232,
"fcm_dpo/margin": 289.4950866699219,
"fcm_dpo/q_t": 0.3833885192871094,
"grad_norm": 24.989524841308594,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -0.41599875688552856,
"logits/rejected": -0.44077515602111816,
"logps/chosen": -395.3926086425781,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -737.726318359375,
"loss": 1.0199,
"margin_dpo/margin_mean": 289.4950866699219,
"margin_dpo/margin_std": 327.95684814453125,
"step": 380
},
{
"KL/chosen_KL_mean": -352.6854248046875,
"KL/mean": -451.4104309082031,
"KL/rejected_KL_mean": -550.135498046875,
"KL/std": 252.61831665039062,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.0017375698080286384,
"fcm_dpo/delta": 0.05898090451955795,
"fcm_dpo/margin": 197.4500732421875,
"fcm_dpo/q_t": 0.42169779539108276,
"grad_norm": 25.88216781616211,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -0.4213838577270508,
"logits/rejected": -0.4027274549007416,
"logps/chosen": -410.48846435546875,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -629.3548583984375,
"loss": 1.1561,
"margin_dpo/margin_mean": 197.4500732421875,
"margin_dpo/margin_std": 342.58575439453125,
"step": 381
},
{
"KL/chosen_KL_mean": -350.3328857421875,
"KL/mean": -471.0789489746094,
"KL/rejected_KL_mean": -591.8250122070312,
"KL/std": 237.4173583984375,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.0017401074292138219,
"fcm_dpo/delta": -0.021109461784362793,
"fcm_dpo/margin": 241.49209594726562,
"fcm_dpo/q_t": 0.40183016657829285,
"grad_norm": 32.546287536621094,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -0.4856158494949341,
"logits/rejected": -0.4916708171367645,
"logps/chosen": -416.35321044921875,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -702.53515625,
"loss": 1.0602,
"margin_dpo/margin_mean": 241.49209594726562,
"margin_dpo/margin_std": 266.0589904785156,
"step": 382
},
{
"KL/chosen_KL_mean": -351.69720458984375,
"KL/mean": -473.2536926269531,
"KL/rejected_KL_mean": -594.8101806640625,
"KL/std": 264.37579345703125,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.0017301496118307114,
"fcm_dpo/delta": -0.021601904183626175,
"fcm_dpo/margin": 243.11300659179688,
"fcm_dpo/q_t": 0.40308916568756104,
"grad_norm": 23.824296951293945,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -0.4409335255622864,
"logits/rejected": -0.45157590508461,
"logps/chosen": -402.08868408203125,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -688.5260009765625,
"loss": 1.091,
"margin_dpo/margin_mean": 243.11300659179688,
"margin_dpo/margin_std": 342.62615966796875,
"step": 383
},
{
"KL/chosen_KL_mean": -353.783447265625,
"KL/mean": -453.2200927734375,
"KL/rejected_KL_mean": -552.65673828125,
"KL/std": 246.05821228027344,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.0017507347511127591,
"fcm_dpo/delta": 0.05325193330645561,
"fcm_dpo/margin": 198.87332153320312,
"fcm_dpo/q_t": 0.4177956283092499,
"grad_norm": 26.645626068115234,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -0.49639368057250977,
"logits/rejected": -0.4920317530632019,
"logps/chosen": -405.82952880859375,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -638.4176635742188,
"loss": 1.109,
"margin_dpo/margin_mean": 198.8733367919922,
"margin_dpo/margin_std": 234.27737426757812,
"step": 384
},
{
"KL/chosen_KL_mean": -335.99951171875,
"KL/mean": -452.42498779296875,
"KL/rejected_KL_mean": -568.8504638671875,
"KL/std": 211.4859619140625,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.0017469192389398813,
"fcm_dpo/delta": -0.007062938995659351,
"fcm_dpo/margin": 232.8509521484375,
"fcm_dpo/q_t": 0.4036387503147125,
"grad_norm": 32.45009231567383,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -0.4798526167869568,
"logits/rejected": -0.4585825204849243,
"logps/chosen": -401.5516662597656,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -646.6783447265625,
"loss": 1.0606,
"margin_dpo/margin_mean": 232.8509521484375,
"margin_dpo/margin_std": 235.7921142578125,
"step": 385
},
{
"KL/chosen_KL_mean": -356.20208740234375,
"KL/mean": -485.5333251953125,
"KL/rejected_KL_mean": -614.864501953125,
"KL/std": 275.59869384765625,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.0017344809602946043,
"fcm_dpo/delta": -0.050910089164972305,
"fcm_dpo/margin": 258.6624450683594,
"fcm_dpo/q_t": 0.3971731662750244,
"grad_norm": 24.357194900512695,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -0.47782373428344727,
"logits/rejected": -0.4760916233062744,
"logps/chosen": -414.4239196777344,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -707.19189453125,
"loss": 1.0564,
"margin_dpo/margin_mean": 258.6624450683594,
"margin_dpo/margin_std": 317.303466796875,
"step": 386
},
{
"KL/chosen_KL_mean": -365.7936096191406,
"KL/mean": -475.52777099609375,
"KL/rejected_KL_mean": -585.2620239257812,
"KL/std": 252.08740234375,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.001727396622300148,
"fcm_dpo/delta": 0.021443448960781097,
"fcm_dpo/margin": 219.4683837890625,
"fcm_dpo/q_t": 0.4107184112071991,
"grad_norm": 25.59358024597168,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -0.4887543320655823,
"logits/rejected": -0.4731796383857727,
"logps/chosen": -432.21307373046875,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -677.43115234375,
"loss": 1.0906,
"margin_dpo/margin_mean": 219.4683837890625,
"margin_dpo/margin_std": 254.8131103515625,
"step": 387
},
{
"KL/chosen_KL_mean": -350.8273010253906,
"KL/mean": -490.6996154785156,
"KL/rejected_KL_mean": -630.5719604492188,
"KL/std": 284.7591552734375,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.0017110585467889905,
"fcm_dpo/delta": -0.08294045180082321,
"fcm_dpo/margin": 279.74468994140625,
"fcm_dpo/q_t": 0.39071470499038696,
"grad_norm": 25.492565155029297,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -0.3969477415084839,
"logits/rejected": -0.40795570611953735,
"logps/chosen": -400.9567565917969,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -735.0050048828125,
"loss": 1.0314,
"margin_dpo/margin_mean": 279.7446594238281,
"margin_dpo/margin_std": 319.7544250488281,
"step": 388
},
{
"KL/chosen_KL_mean": -381.65643310546875,
"KL/mean": -502.360595703125,
"KL/rejected_KL_mean": -623.0647583007812,
"KL/std": 286.53057861328125,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.0017027594149112701,
"fcm_dpo/delta": -0.011586458422243595,
"fcm_dpo/margin": 241.40829467773438,
"fcm_dpo/q_t": 0.40512824058532715,
"grad_norm": 28.0815486907959,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -0.4462489187717438,
"logits/rejected": -0.43941187858581543,
"logps/chosen": -439.56304931640625,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -700.9793090820312,
"loss": 1.0814,
"margin_dpo/margin_mean": 241.40829467773438,
"margin_dpo/margin_std": 308.45904541015625,
"step": 389
},
{
"KL/chosen_KL_mean": -375.6517333984375,
"KL/mean": -493.81524658203125,
"KL/rejected_KL_mean": -611.978759765625,
"KL/std": 276.1775207519531,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.001697941217571497,
"fcm_dpo/delta": -0.001557283103466034,
"fcm_dpo/margin": 236.3270263671875,
"fcm_dpo/q_t": 0.4103020429611206,
"grad_norm": 24.159801483154297,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -0.4154917895793915,
"logits/rejected": -0.40859878063201904,
"logps/chosen": -424.8776550292969,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -697.5069580078125,
"loss": 1.1063,
"margin_dpo/margin_mean": 236.32704162597656,
"margin_dpo/margin_std": 351.1764221191406,
"step": 390
},
{
"KL/chosen_KL_mean": -365.83709716796875,
"KL/mean": -436.49969482421875,
"KL/rejected_KL_mean": -507.162353515625,
"KL/std": 257.5408020019531,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.0017448479775339365,
"fcm_dpo/delta": 0.1572088599205017,
"fcm_dpo/margin": 141.32522583007812,
"fcm_dpo/q_t": 0.44321030378341675,
"grad_norm": 39.628028869628906,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -0.45562559366226196,
"logits/rejected": -0.4495752155780792,
"logps/chosen": -430.166748046875,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -593.9005126953125,
"loss": 1.2098,
"margin_dpo/margin_mean": 141.32522583007812,
"margin_dpo/margin_std": 270.3441467285156,
"step": 391
},
{
"KL/chosen_KL_mean": -317.95721435546875,
"KL/mean": -458.79046630859375,
"KL/rejected_KL_mean": -599.6236572265625,
"KL/std": 278.5746154785156,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.0017387724947184324,
"fcm_dpo/delta": -0.09427893161773682,
"fcm_dpo/margin": 281.6664123535156,
"fcm_dpo/q_t": 0.38599228858947754,
"grad_norm": 25.61530876159668,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -0.4354037046432495,
"logits/rejected": -0.42951297760009766,
"logps/chosen": -371.4612121582031,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -701.969482421875,
"loss": 1.0072,
"margin_dpo/margin_mean": 281.6664123535156,
"margin_dpo/margin_std": 269.19384765625,
"step": 392
},
{
"KL/chosen_KL_mean": -308.1512451171875,
"KL/mean": -415.21820068359375,
"KL/rejected_KL_mean": -522.2850952148438,
"KL/std": 262.9591064453125,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.0017301738262176514,
"fcm_dpo/delta": 0.030642738565802574,
"fcm_dpo/margin": 214.13385009765625,
"fcm_dpo/q_t": 0.4171079993247986,
"grad_norm": 21.046979904174805,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -0.41253232955932617,
"logits/rejected": -0.4099976420402527,
"logps/chosen": -354.62518310546875,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -594.2539672851562,
"loss": 1.123,
"margin_dpo/margin_mean": 214.13385009765625,
"margin_dpo/margin_std": 329.9991455078125,
"step": 393
},
{
"KL/chosen_KL_mean": -352.0980224609375,
"KL/mean": -468.20489501953125,
"KL/rejected_KL_mean": -584.311767578125,
"KL/std": 285.9789733886719,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.0017325542867183685,
"fcm_dpo/delta": -0.0024658460170030594,
"fcm_dpo/margin": 232.21368408203125,
"fcm_dpo/q_t": 0.4075622856616974,
"grad_norm": 25.002628326416016,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -0.4068412184715271,
"logits/rejected": -0.406298965215683,
"logps/chosen": -405.00958251953125,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -675.1343994140625,
"loss": 1.084,
"margin_dpo/margin_mean": 232.21368408203125,
"margin_dpo/margin_std": 293.77191162109375,
"step": 394
},
{
"KL/chosen_KL_mean": -350.91033935546875,
"KL/mean": -474.3104248046875,
"KL/rejected_KL_mean": -597.7105712890625,
"KL/std": 282.8669738769531,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.001723103690892458,
"fcm_dpo/delta": -0.026621468365192413,
"fcm_dpo/margin": 246.8002166748047,
"fcm_dpo/q_t": 0.40307846665382385,
"grad_norm": 26.7552433013916,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -0.46250438690185547,
"logits/rejected": -0.46052122116088867,
"logps/chosen": -413.4564208984375,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -681.4931640625,
"loss": 1.0793,
"margin_dpo/margin_mean": 246.80023193359375,
"margin_dpo/margin_std": 326.7712097167969,
"step": 395
},
{
"KL/chosen_KL_mean": -351.205078125,
"KL/mean": -466.0959777832031,
"KL/rejected_KL_mean": -580.9869384765625,
"KL/std": 271.4552307128906,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.0017233972903341055,
"fcm_dpo/delta": 0.004093697294592857,
"fcm_dpo/margin": 229.7818145751953,
"fcm_dpo/q_t": 0.40719085931777954,
"grad_norm": 22.856327056884766,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.49015119671821594,
"logits/rejected": -0.47574925422668457,
"logps/chosen": -420.20098876953125,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -669.633544921875,
"loss": 1.0721,
"margin_dpo/margin_mean": 229.7818145751953,
"margin_dpo/margin_std": 249.69090270996094,
"step": 396
},
{
"KL/chosen_KL_mean": -333.80255126953125,
"KL/mean": -477.57720947265625,
"KL/rejected_KL_mean": -621.3518676757812,
"KL/std": 273.3784484863281,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.001697493251413107,
"fcm_dpo/delta": -0.0931856781244278,
"fcm_dpo/margin": 287.54931640625,
"fcm_dpo/q_t": 0.3860146105289459,
"grad_norm": 30.467397689819336,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -0.40329790115356445,
"logits/rejected": -0.40474754571914673,
"logps/chosen": -395.0797119140625,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -724.468017578125,
"loss": 1.0114,
"margin_dpo/margin_mean": 287.54931640625,
"margin_dpo/margin_std": 285.5010681152344,
"step": 397
},
{
"KL/chosen_KL_mean": -361.37225341796875,
"KL/mean": -491.65283203125,
"KL/rejected_KL_mean": -621.9334716796875,
"KL/std": 285.1490478515625,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.0016844018828123808,
"fcm_dpo/delta": -0.040688835084438324,
"fcm_dpo/margin": 260.5611877441406,
"fcm_dpo/q_t": 0.40008544921875,
"grad_norm": 21.616304397583008,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.48934173583984375,
"logits/rejected": -0.4850488007068634,
"logps/chosen": -429.5238037109375,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -730.45703125,
"loss": 1.0645,
"margin_dpo/margin_mean": 260.5611877441406,
"margin_dpo/margin_std": 329.8893127441406,
"step": 398
},
{
"KL/chosen_KL_mean": -320.106201171875,
"KL/mean": -435.54656982421875,
"KL/rejected_KL_mean": -550.9869384765625,
"KL/std": 252.38217163085938,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.0016788092907518148,
"fcm_dpo/delta": 0.012697199359536171,
"fcm_dpo/margin": 230.88070678710938,
"fcm_dpo/q_t": 0.4103388786315918,
"grad_norm": 26.99171257019043,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -0.47362208366394043,
"logits/rejected": -0.43797028064727783,
"logps/chosen": -380.9960021972656,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -628.9525146484375,
"loss": 1.1009,
"margin_dpo/margin_mean": 230.8806915283203,
"margin_dpo/margin_std": 310.9993591308594,
"step": 399
},
{
"KL/chosen_KL_mean": -306.8255615234375,
"KL/mean": -475.8355712890625,
"KL/rejected_KL_mean": -644.8455810546875,
"KL/std": 272.01824951171875,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.0016412277473136783,
"fcm_dpo/delta": -0.16435033082962036,
"fcm_dpo/margin": 338.02001953125,
"fcm_dpo/q_t": 0.36962372064590454,
"grad_norm": 22.292245864868164,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.45513978600502014,
"logits/rejected": -0.4522014260292053,
"logps/chosen": -370.46917724609375,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -750.0982666015625,
"loss": 0.9618,
"margin_dpo/margin_mean": 338.02001953125,
"margin_dpo/margin_std": 297.86004638671875,
"step": 400
},
{
"KL/chosen_KL_mean": -357.10784912109375,
"KL/mean": -451.53125,
"KL/rejected_KL_mean": -545.95458984375,
"KL/std": 285.63330078125,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.0016490190755575895,
"fcm_dpo/delta": 0.09153245389461517,
"fcm_dpo/margin": 188.84674072265625,
"fcm_dpo/q_t": 0.4310322701931,
"grad_norm": 23.559465408325195,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -0.45114535093307495,
"logits/rejected": -0.44320282340049744,
"logps/chosen": -414.2708740234375,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -629.7471313476562,
"loss": 1.1949,
"margin_dpo/margin_mean": 188.84677124023438,
"margin_dpo/margin_std": 386.6992492675781,
"step": 401
},
{
"KL/chosen_KL_mean": -268.1985778808594,
"KL/mean": -443.2376708984375,
"KL/rejected_KL_mean": -618.2767333984375,
"KL/std": 305.66064453125,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.0016179950907826424,
"fcm_dpo/delta": -0.1764608919620514,
"fcm_dpo/margin": 350.0781555175781,
"fcm_dpo/q_t": 0.36862170696258545,
"grad_norm": 20.347381591796875,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -0.4648516774177551,
"logits/rejected": -0.4514453411102295,
"logps/chosen": -318.93896484375,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -699.32275390625,
"loss": 0.9548,
"margin_dpo/margin_mean": 350.078125,
"margin_dpo/margin_std": 312.98248291015625,
"step": 402
},
{
"KL/chosen_KL_mean": -298.02740478515625,
"KL/mean": -428.8231201171875,
"KL/rejected_KL_mean": -559.6187744140625,
"KL/std": 291.15869140625,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.0015987367369234562,
"fcm_dpo/delta": -0.019017325714230537,
"fcm_dpo/margin": 261.5914306640625,
"fcm_dpo/q_t": 0.40307024121284485,
"grad_norm": 25.700403213500977,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -0.44661349058151245,
"logits/rejected": -0.4446351230144501,
"logps/chosen": -345.1747131347656,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -636.8854370117188,
"loss": 1.0672,
"margin_dpo/margin_mean": 261.5914306640625,
"margin_dpo/margin_std": 303.60894775390625,
"step": 403
},
{
"KL/chosen_KL_mean": -324.199951171875,
"KL/mean": -455.5159606933594,
"KL/rejected_KL_mean": -586.8319091796875,
"KL/std": 273.1387634277344,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.0015905757900327444,
"fcm_dpo/delta": -0.018594812601804733,
"fcm_dpo/margin": 262.6319885253906,
"fcm_dpo/q_t": 0.4038143754005432,
"grad_norm": 27.593461990356445,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -0.46002256870269775,
"logits/rejected": -0.45228320360183716,
"logps/chosen": -372.0752258300781,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -663.9869384765625,
"loss": 1.0834,
"margin_dpo/margin_mean": 262.6319885253906,
"margin_dpo/margin_std": 347.2093505859375,
"step": 404
},
{
"KL/chosen_KL_mean": -376.49700927734375,
"KL/mean": -483.60076904296875,
"KL/rejected_KL_mean": -590.7044677734375,
"KL/std": 307.7935791015625,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.001616810099221766,
"fcm_dpo/delta": 0.054005078971385956,
"fcm_dpo/margin": 214.20745849609375,
"fcm_dpo/q_t": 0.4231037497520447,
"grad_norm": 28.185096740722656,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -0.4508972465991974,
"logits/rejected": -0.44256657361984253,
"logps/chosen": -441.659912109375,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -677.8912353515625,
"loss": 1.1654,
"margin_dpo/margin_mean": 214.20745849609375,
"margin_dpo/margin_std": 385.24530029296875,
"step": 405
},
{
"KL/chosen_KL_mean": -331.07080078125,
"KL/mean": -467.66162109375,
"KL/rejected_KL_mean": -604.2523803710938,
"KL/std": 297.1915588378906,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.0016056987224146724,
"fcm_dpo/delta": -0.04058470577001572,
"fcm_dpo/margin": 273.1816101074219,
"fcm_dpo/q_t": 0.39960601925849915,
"grad_norm": 24.00153160095215,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -0.4905382990837097,
"logits/rejected": -0.49678927659988403,
"logps/chosen": -380.8116149902344,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -696.3310546875,
"loss": 1.0601,
"margin_dpo/margin_mean": 273.1816101074219,
"margin_dpo/margin_std": 328.4947814941406,
"step": 406
},
{
"KL/chosen_KL_mean": -363.4420166015625,
"KL/mean": -452.39483642578125,
"KL/rejected_KL_mean": -541.34765625,
"KL/std": 244.7425537109375,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.0016330704092979431,
"fcm_dpo/delta": 0.1112029105424881,
"fcm_dpo/margin": 177.90560913085938,
"fcm_dpo/q_t": 0.4322769343852997,
"grad_norm": 26.514976501464844,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -0.470073401927948,
"logits/rejected": -0.4409082531929016,
"logps/chosen": -419.7727355957031,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -618.8597412109375,
"loss": 1.1879,
"margin_dpo/margin_mean": 177.9055938720703,
"margin_dpo/margin_std": 325.5947265625,
"step": 407
},
{
"KL/chosen_KL_mean": -365.21722412109375,
"KL/mean": -465.7977294921875,
"KL/rejected_KL_mean": -566.3782958984375,
"KL/std": 237.32412719726562,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.0016487594693899155,
"fcm_dpo/delta": 0.07069391012191772,
"fcm_dpo/margin": 201.16098022460938,
"fcm_dpo/q_t": 0.42220157384872437,
"grad_norm": 21.884017944335938,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -0.4787837266921997,
"logits/rejected": -0.4536997079849243,
"logps/chosen": -435.0065612792969,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -656.4752197265625,
"loss": 1.1279,
"margin_dpo/margin_mean": 201.16098022460938,
"margin_dpo/margin_std": 266.10650634765625,
"step": 408
},
{
"KL/chosen_KL_mean": -356.888671875,
"KL/mean": -453.8036804199219,
"KL/rejected_KL_mean": -550.71875,
"KL/std": 255.46417236328125,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.0016783413011580706,
"fcm_dpo/delta": 0.07688158750534058,
"fcm_dpo/margin": 193.83004760742188,
"fcm_dpo/q_t": 0.42450258135795593,
"grad_norm": 29.449851989746094,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -0.45866113901138306,
"logits/rejected": -0.44195905327796936,
"logps/chosen": -424.20611572265625,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -635.623046875,
"loss": 1.1479,
"margin_dpo/margin_mean": 193.83004760742188,
"margin_dpo/margin_std": 298.97589111328125,
"step": 409
},
{
"KL/chosen_KL_mean": -321.3678283691406,
"KL/mean": -441.85791015625,
"KL/rejected_KL_mean": -562.3480224609375,
"KL/std": 258.2183837890625,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.001676855026744306,
"fcm_dpo/delta": -0.004324676468968391,
"fcm_dpo/margin": 240.9801483154297,
"fcm_dpo/q_t": 0.4049345850944519,
"grad_norm": 23.625701904296875,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -0.46683716773986816,
"logits/rejected": -0.4663783311843872,
"logps/chosen": -372.83319091796875,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -645.5469970703125,
"loss": 1.0765,
"margin_dpo/margin_mean": 240.98016357421875,
"margin_dpo/margin_std": 284.5562744140625,
"step": 410
},
{
"KL/chosen_KL_mean": -347.264404296875,
"KL/mean": -463.2464599609375,
"KL/rejected_KL_mean": -579.228515625,
"KL/std": 278.7031555175781,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.001669498858973384,
"fcm_dpo/delta": 0.012138403952121735,
"fcm_dpo/margin": 231.9641876220703,
"fcm_dpo/q_t": 0.41212886571884155,
"grad_norm": 23.38148307800293,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -0.4481458365917206,
"logits/rejected": -0.45459499955177307,
"logps/chosen": -399.5716552734375,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -659.9235229492188,
"loss": 1.1047,
"margin_dpo/margin_mean": 231.9641876220703,
"margin_dpo/margin_std": 314.99127197265625,
"step": 411
},
{
"KL/chosen_KL_mean": -357.5395202636719,
"KL/mean": -476.832763671875,
"KL/rejected_KL_mean": -596.1259765625,
"KL/std": 277.5762939453125,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.0016845406498759985,
"fcm_dpo/delta": -0.00200699083507061,
"fcm_dpo/margin": 238.58645629882812,
"fcm_dpo/q_t": 0.40782129764556885,
"grad_norm": 41.79743194580078,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -0.49470025300979614,
"logits/rejected": -0.5206660628318787,
"logps/chosen": -410.68365478515625,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -696.186767578125,
"loss": 1.0941,
"margin_dpo/margin_mean": 238.58645629882812,
"margin_dpo/margin_std": 326.08343505859375,
"step": 412
},
{
"KL/chosen_KL_mean": -381.70831298828125,
"KL/mean": -499.70184326171875,
"KL/rejected_KL_mean": -617.6954345703125,
"KL/std": 265.35125732421875,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.001688747899606824,
"fcm_dpo/delta": 0.0012981817126274109,
"fcm_dpo/margin": 235.98709106445312,
"fcm_dpo/q_t": 0.406361848115921,
"grad_norm": 25.019241333007812,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -0.47015732526779175,
"logits/rejected": -0.47621041536331177,
"logps/chosen": -443.290283203125,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -717.1688232421875,
"loss": 1.0904,
"margin_dpo/margin_mean": 235.9871063232422,
"margin_dpo/margin_std": 305.47210693359375,
"step": 413
},
{
"KL/chosen_KL_mean": -334.1068115234375,
"KL/mean": -468.125732421875,
"KL/rejected_KL_mean": -602.1447143554688,
"KL/std": 256.7291259765625,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.001681898720562458,
"fcm_dpo/delta": -0.053896043449640274,
"fcm_dpo/margin": 268.03790283203125,
"fcm_dpo/q_t": 0.3947378396987915,
"grad_norm": 29.387535095214844,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -0.4133932590484619,
"logits/rejected": -0.40770968794822693,
"logps/chosen": -380.73828125,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -689.791259765625,
"loss": 1.0452,
"margin_dpo/margin_mean": 268.03790283203125,
"margin_dpo/margin_std": 288.237548828125,
"step": 414
},
{
"KL/chosen_KL_mean": -390.490966796875,
"KL/mean": -491.0526123046875,
"KL/rejected_KL_mean": -591.6142578125,
"KL/std": 267.60589599609375,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.0016747894696891308,
"fcm_dpo/delta": 0.06530318409204483,
"fcm_dpo/margin": 201.123291015625,
"fcm_dpo/q_t": 0.4213160574436188,
"grad_norm": 26.970125198364258,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -0.448618084192276,
"logits/rejected": -0.4439224600791931,
"logps/chosen": -469.1092529296875,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -692.091796875,
"loss": 1.1369,
"margin_dpo/margin_mean": 201.123291015625,
"margin_dpo/margin_std": 294.55072021484375,
"step": 415
},
{
"KL/chosen_KL_mean": -348.8500061035156,
"KL/mean": -504.3551330566406,
"KL/rejected_KL_mean": -659.8602294921875,
"KL/std": 290.89202880859375,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.001661974936723709,
"fcm_dpo/delta": -0.1230802834033966,
"fcm_dpo/margin": 311.01019287109375,
"fcm_dpo/q_t": 0.37868714332580566,
"grad_norm": 31.66852569580078,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -0.4952217638492584,
"logits/rejected": -0.5007729530334473,
"logps/chosen": -407.129150390625,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -750.428955078125,
"loss": 0.9841,
"margin_dpo/margin_mean": 311.01019287109375,
"margin_dpo/margin_std": 273.85443115234375,
"step": 416
},
{
"KL/chosen_KL_mean": -348.1707763671875,
"KL/mean": -476.07635498046875,
"KL/rejected_KL_mean": -603.98193359375,
"KL/std": 262.30242919921875,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.001643989235162735,
"fcm_dpo/delta": -0.02149224281311035,
"fcm_dpo/margin": 255.8111572265625,
"fcm_dpo/q_t": 0.40100353956222534,
"grad_norm": 24.26694107055664,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -0.43751639127731323,
"logits/rejected": -0.4203481674194336,
"logps/chosen": -398.3695068359375,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -672.1337280273438,
"loss": 1.0534,
"margin_dpo/margin_mean": 255.8111572265625,
"margin_dpo/margin_std": 262.398193359375,
"step": 417
},
{
"KL/chosen_KL_mean": -375.5062255859375,
"KL/mean": -509.50982666015625,
"KL/rejected_KL_mean": -643.513427734375,
"KL/std": 293.5723571777344,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.0016407601069658995,
"fcm_dpo/delta": -0.042371716350317,
"fcm_dpo/margin": 268.0071716308594,
"fcm_dpo/q_t": 0.4010557532310486,
"grad_norm": 27.076887130737305,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -0.5030707120895386,
"logits/rejected": -0.48085683584213257,
"logps/chosen": -457.4847106933594,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -748.2048950195312,
"loss": 1.0719,
"margin_dpo/margin_mean": 268.0072021484375,
"margin_dpo/margin_std": 345.320068359375,
"step": 418
},
{
"KL/chosen_KL_mean": -347.1816101074219,
"KL/mean": -493.19476318359375,
"KL/rejected_KL_mean": -639.2078857421875,
"KL/std": 260.90594482421875,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.0016074271406978369,
"fcm_dpo/delta": -0.07283525168895721,
"fcm_dpo/margin": 292.02630615234375,
"fcm_dpo/q_t": 0.3896368443965912,
"grad_norm": 30.91160011291504,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -0.49828916788101196,
"logits/rejected": -0.49863529205322266,
"logps/chosen": -400.1302490234375,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -730.791015625,
"loss": 1.0255,
"margin_dpo/margin_mean": 292.02630615234375,
"margin_dpo/margin_std": 299.172119140625,
"step": 419
},
{
"KL/chosen_KL_mean": -439.3621520996094,
"KL/mean": -528.0501098632812,
"KL/rejected_KL_mean": -616.7380981445312,
"KL/std": 280.856689453125,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.001630987972021103,
"fcm_dpo/delta": 0.11383231729269028,
"fcm_dpo/margin": 177.37596130371094,
"fcm_dpo/q_t": 0.43548548221588135,
"grad_norm": 60.948829650878906,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -0.5088773369789124,
"logits/rejected": -0.47673243284225464,
"logps/chosen": -517.132080078125,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -686.0579833984375,
"loss": 1.2097,
"margin_dpo/margin_mean": 177.3759765625,
"margin_dpo/margin_std": 384.9363098144531,
"step": 420
},
{
"KL/chosen_KL_mean": -349.59552001953125,
"KL/mean": -450.66204833984375,
"KL/rejected_KL_mean": -551.728515625,
"KL/std": 265.665283203125,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.0016611446626484394,
"fcm_dpo/delta": 0.06588370352983475,
"fcm_dpo/margin": 202.13302612304688,
"fcm_dpo/q_t": 0.4204084873199463,
"grad_norm": 26.5180606842041,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -0.519219160079956,
"logits/rejected": -0.5066835880279541,
"logps/chosen": -403.36138916015625,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -641.010009765625,
"loss": 1.1353,
"margin_dpo/margin_mean": 202.13302612304688,
"margin_dpo/margin_std": 287.11669921875,
"step": 421
},
{
"KL/chosen_KL_mean": -403.3852233886719,
"KL/mean": -521.2222900390625,
"KL/rejected_KL_mean": -639.059326171875,
"KL/std": 274.61279296875,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.001660653855651617,
"fcm_dpo/delta": 0.008965391665697098,
"fcm_dpo/margin": 235.6741485595703,
"fcm_dpo/q_t": 0.4084932208061218,
"grad_norm": 31.94446563720703,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -0.5387804508209229,
"logits/rejected": -0.5378388166427612,
"logps/chosen": -472.01898193359375,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -726.9228515625,
"loss": 1.1001,
"margin_dpo/margin_mean": 235.6741485595703,
"margin_dpo/margin_std": 322.131103515625,
"step": 422
},
{
"KL/chosen_KL_mean": -375.15228271484375,
"KL/mean": -501.4312744140625,
"KL/rejected_KL_mean": -627.710205078125,
"KL/std": 263.3236083984375,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.0016572801396250725,
"fcm_dpo/delta": -0.019391080364584923,
"fcm_dpo/margin": 252.5579833984375,
"fcm_dpo/q_t": 0.4022686183452606,
"grad_norm": 31.633256912231445,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -0.5140703916549683,
"logits/rejected": -0.519053041934967,
"logps/chosen": -430.142578125,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -714.0167846679688,
"loss": 1.0628,
"margin_dpo/margin_mean": 252.5579833984375,
"margin_dpo/margin_std": 284.78033447265625,
"step": 423
},
{
"KL/chosen_KL_mean": -334.7992858886719,
"KL/mean": -459.7820739746094,
"KL/rejected_KL_mean": -584.7648315429688,
"KL/std": 269.6551513671875,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.0016480376943945885,
"fcm_dpo/delta": -0.012738510966300964,
"fcm_dpo/margin": 249.96554565429688,
"fcm_dpo/q_t": 0.4058811068534851,
"grad_norm": 32.633731842041016,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -0.5273067951202393,
"logits/rejected": -0.5043067932128906,
"logps/chosen": -390.81121826171875,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -651.2437744140625,
"loss": 1.0844,
"margin_dpo/margin_mean": 249.96554565429688,
"margin_dpo/margin_std": 328.8323974609375,
"step": 424
},
{
"KL/chosen_KL_mean": -362.98529052734375,
"KL/mean": -461.9312744140625,
"KL/rejected_KL_mean": -560.877197265625,
"KL/std": 245.79722595214844,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.0016736630350351334,
"fcm_dpo/delta": 0.07090499997138977,
"fcm_dpo/margin": 197.89198303222656,
"fcm_dpo/q_t": 0.4216083884239197,
"grad_norm": 25.201358795166016,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -0.50770503282547,
"logits/rejected": -0.5075915455818176,
"logps/chosen": -409.8542785644531,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -656.802734375,
"loss": 1.1421,
"margin_dpo/margin_mean": 197.89198303222656,
"margin_dpo/margin_std": 296.4322509765625,
"step": 425
},
{
"KL/chosen_KL_mean": -336.8538818359375,
"KL/mean": -458.18670654296875,
"KL/rejected_KL_mean": -579.51953125,
"KL/std": 251.29937744140625,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.0016819476149976254,
"fcm_dpo/delta": -0.008926056325435638,
"fcm_dpo/margin": 242.66563415527344,
"fcm_dpo/q_t": 0.40451914072036743,
"grad_norm": 25.522932052612305,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -0.5204522609710693,
"logits/rejected": -0.4901271462440491,
"logps/chosen": -413.43743896484375,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -660.7861328125,
"loss": 1.0736,
"margin_dpo/margin_mean": 242.66561889648438,
"margin_dpo/margin_std": 277.174560546875,
"step": 426
},
{
"KL/chosen_KL_mean": -315.9246520996094,
"KL/mean": -404.6652526855469,
"KL/rejected_KL_mean": -493.4059143066406,
"KL/std": 217.07647705078125,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.0016970799770206213,
"fcm_dpo/delta": 0.10195031762123108,
"fcm_dpo/margin": 177.48121643066406,
"fcm_dpo/q_t": 0.42938411235809326,
"grad_norm": 23.957731246948242,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -0.5153839588165283,
"logits/rejected": -0.49676990509033203,
"logps/chosen": -380.7785339355469,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -571.971923828125,
"loss": 1.1509,
"margin_dpo/margin_mean": 177.48121643066406,
"margin_dpo/margin_std": 249.63307189941406,
"step": 427
},
{
"KL/chosen_KL_mean": -394.0665283203125,
"KL/mean": -510.107421875,
"KL/rejected_KL_mean": -626.1483154296875,
"KL/std": 281.40484619140625,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.0017051963368430734,
"fcm_dpo/delta": 0.0043541621416807175,
"fcm_dpo/margin": 232.08180236816406,
"fcm_dpo/q_t": 0.40895557403564453,
"grad_norm": 30.00721549987793,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -0.5060927271842957,
"logits/rejected": -0.5087655782699585,
"logps/chosen": -456.70318603515625,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -729.43017578125,
"loss": 1.1043,
"margin_dpo/margin_mean": 232.081787109375,
"margin_dpo/margin_std": 331.1031494140625,
"step": 428
},
{
"KL/chosen_KL_mean": -389.79180908203125,
"KL/mean": -480.70843505859375,
"KL/rejected_KL_mean": -571.6251220703125,
"KL/std": 241.70196533203125,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.0017423585522919893,
"fcm_dpo/delta": 0.08442431688308716,
"fcm_dpo/margin": 181.8333282470703,
"fcm_dpo/q_t": 0.42665740847587585,
"grad_norm": 35.86955642700195,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -0.5312508940696716,
"logits/rejected": -0.5112993717193604,
"logps/chosen": -471.02581787109375,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -663.4200439453125,
"loss": 1.163,
"margin_dpo/margin_mean": 181.83334350585938,
"margin_dpo/margin_std": 292.9608154296875,
"step": 429
},
{
"KL/chosen_KL_mean": -317.3845520019531,
"KL/mean": -419.97271728515625,
"KL/rejected_KL_mean": -522.560791015625,
"KL/std": 234.30323791503906,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.001752232201397419,
"fcm_dpo/delta": 0.04184893146157265,
"fcm_dpo/margin": 205.17626953125,
"fcm_dpo/q_t": 0.4163983464241028,
"grad_norm": 30.209016799926758,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.5189450979232788,
"logits/rejected": -0.5163074731826782,
"logps/chosen": -378.30487060546875,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -626.983642578125,
"loss": 1.1061,
"margin_dpo/margin_mean": 205.17623901367188,
"margin_dpo/margin_std": 255.99505615234375,
"step": 430
},
{
"KL/chosen_KL_mean": -305.75433349609375,
"KL/mean": -447.6547546386719,
"KL/rejected_KL_mean": -589.55517578125,
"KL/std": 262.2629089355469,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.0017262442270293832,
"fcm_dpo/delta": -0.094680055975914,
"fcm_dpo/margin": 283.80078125,
"fcm_dpo/q_t": 0.3853592872619629,
"grad_norm": 27.11848258972168,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -0.4816255271434784,
"logits/rejected": -0.48415422439575195,
"logps/chosen": -363.10308837890625,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -682.3953857421875,
"loss": 1.0109,
"margin_dpo/margin_mean": 283.80078125,
"margin_dpo/margin_std": 278.22015380859375,
"step": 431
},
{
"KL/chosen_KL_mean": -311.2400817871094,
"KL/mean": -443.8277893066406,
"KL/rejected_KL_mean": -576.41552734375,
"KL/std": 255.052490234375,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.0017163490410894156,
"fcm_dpo/delta": -0.05826277658343315,
"fcm_dpo/margin": 265.17547607421875,
"fcm_dpo/q_t": 0.39427268505096436,
"grad_norm": 25.935993194580078,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -0.48976317048072815,
"logits/rejected": -0.4787473678588867,
"logps/chosen": -382.31488037109375,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -674.9950561523438,
"loss": 1.0351,
"margin_dpo/margin_mean": 265.1754455566406,
"margin_dpo/margin_std": 264.5519104003906,
"step": 432
},
{
"KL/chosen_KL_mean": -400.88958740234375,
"KL/mean": -498.4219970703125,
"KL/rejected_KL_mean": -595.9544067382812,
"KL/std": 252.05137634277344,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.0017208922654390335,
"fcm_dpo/delta": 0.06628023833036423,
"fcm_dpo/margin": 195.0648193359375,
"fcm_dpo/q_t": 0.42144525051116943,
"grad_norm": 28.61791229248047,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -0.5252653360366821,
"logits/rejected": -0.514911413192749,
"logps/chosen": -459.16278076171875,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -691.9052734375,
"loss": 1.1435,
"margin_dpo/margin_mean": 195.0648193359375,
"margin_dpo/margin_std": 301.6573486328125,
"step": 433
},
{
"KL/chosen_KL_mean": -365.26397705078125,
"KL/mean": -477.8414306640625,
"KL/rejected_KL_mean": -590.4189453125,
"KL/std": 274.9288024902344,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.0017205406911671162,
"fcm_dpo/delta": 0.012956922873854637,
"fcm_dpo/margin": 225.1549530029297,
"fcm_dpo/q_t": 0.4162324070930481,
"grad_norm": 27.521728515625,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -0.5110592842102051,
"logits/rejected": -0.5141887664794922,
"logps/chosen": -427.2376708984375,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -668.91748046875,
"loss": 1.1143,
"margin_dpo/margin_mean": 225.1549530029297,
"margin_dpo/margin_std": 354.01708984375,
"step": 434
},
{
"KL/chosen_KL_mean": -334.04791259765625,
"KL/mean": -459.4658203125,
"KL/rejected_KL_mean": -584.8836669921875,
"KL/std": 268.1673583984375,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.0017172786174342036,
"fcm_dpo/delta": -0.03218632563948631,
"fcm_dpo/margin": 250.83578491210938,
"fcm_dpo/q_t": 0.40059614181518555,
"grad_norm": 23.964658737182617,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -0.5041155219078064,
"logits/rejected": -0.4924761950969696,
"logps/chosen": -385.54998779296875,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -672.4505615234375,
"loss": 1.0677,
"margin_dpo/margin_mean": 250.83578491210938,
"margin_dpo/margin_std": 302.60870361328125,
"step": 435
},
{
"KL/chosen_KL_mean": -356.885986328125,
"KL/mean": -461.7406005859375,
"KL/rejected_KL_mean": -566.5952758789062,
"KL/std": 239.44638061523438,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.0017226835479959846,
"fcm_dpo/delta": 0.040182169526815414,
"fcm_dpo/margin": 209.70928955078125,
"fcm_dpo/q_t": 0.4159030616283417,
"grad_norm": 49.06702423095703,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -0.5320940613746643,
"logits/rejected": -0.5051020979881287,
"logps/chosen": -428.2896728515625,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -649.322998046875,
"loss": 1.1129,
"margin_dpo/margin_mean": 209.70928955078125,
"margin_dpo/margin_std": 280.531982421875,
"step": 436
},
{
"KL/chosen_KL_mean": -356.98883056640625,
"KL/mean": -465.893310546875,
"KL/rejected_KL_mean": -574.7978515625,
"KL/std": 229.29736328125,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.0017339956248179078,
"fcm_dpo/delta": 0.023197388276457787,
"fcm_dpo/margin": 217.80899047851562,
"fcm_dpo/q_t": 0.4120755195617676,
"grad_norm": 27.109731674194336,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -0.537100613117218,
"logits/rejected": -0.5201215147972107,
"logps/chosen": -421.73309326171875,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -656.8413696289062,
"loss": 1.1006,
"margin_dpo/margin_mean": 217.80899047851562,
"margin_dpo/margin_std": 282.6852111816406,
"step": 437
},
{
"KL/chosen_KL_mean": -356.2583923339844,
"KL/mean": -477.8333740234375,
"KL/rejected_KL_mean": -599.4083251953125,
"KL/std": 254.6542510986328,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.0017272335244342685,
"fcm_dpo/delta": -0.02110411413013935,
"fcm_dpo/margin": 243.14990234375,
"fcm_dpo/q_t": 0.401958167552948,
"grad_norm": 36.0264892578125,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -0.47867268323898315,
"logits/rejected": -0.4559587240219116,
"logps/chosen": -415.2770690917969,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -682.485107421875,
"loss": 1.0646,
"margin_dpo/margin_mean": 243.14990234375,
"margin_dpo/margin_std": 278.13189697265625,
"step": 438
},
{
"KL/chosen_KL_mean": -355.92974853515625,
"KL/mean": -459.84600830078125,
"KL/rejected_KL_mean": -563.76220703125,
"KL/std": 248.40817260742188,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.0017257456202059984,
"fcm_dpo/delta": 0.041571587324142456,
"fcm_dpo/margin": 207.83248901367188,
"fcm_dpo/q_t": 0.4176570177078247,
"grad_norm": 27.175893783569336,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -0.49970486760139465,
"logits/rejected": -0.5002373456954956,
"logps/chosen": -409.7138366699219,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -647.7476806640625,
"loss": 1.1219,
"margin_dpo/margin_mean": 207.83248901367188,
"margin_dpo/margin_std": 278.52850341796875,
"step": 439
},
{
"KL/chosen_KL_mean": -379.3209533691406,
"KL/mean": -483.71697998046875,
"KL/rejected_KL_mean": -588.113037109375,
"KL/std": 282.49066162109375,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.0017588550690561533,
"fcm_dpo/delta": 0.0337531603872776,
"fcm_dpo/margin": 208.7920684814453,
"fcm_dpo/q_t": 0.4173220992088318,
"grad_norm": 26.570308685302734,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -0.5298205614089966,
"logits/rejected": -0.5196566581726074,
"logps/chosen": -457.8876953125,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -684.6107788085938,
"loss": 1.1246,
"margin_dpo/margin_mean": 208.79208374023438,
"margin_dpo/margin_std": 317.7086181640625,
"step": 440
},
{
"KL/chosen_KL_mean": -393.6994323730469,
"KL/mean": -507.71136474609375,
"KL/rejected_KL_mean": -621.7232666015625,
"KL/std": 289.2242431640625,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.0017611184157431126,
"fcm_dpo/delta": -0.0017841961234807968,
"fcm_dpo/margin": 228.02386474609375,
"fcm_dpo/q_t": 0.40944066643714905,
"grad_norm": 30.262908935546875,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -0.4817603826522827,
"logits/rejected": -0.4669860601425171,
"logps/chosen": -454.52386474609375,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -718.194091796875,
"loss": 1.1168,
"margin_dpo/margin_mean": 228.02386474609375,
"margin_dpo/margin_std": 352.9437255859375,
"step": 441
},
{
"KL/chosen_KL_mean": -332.66845703125,
"KL/mean": -464.35418701171875,
"KL/rejected_KL_mean": -596.0399169921875,
"KL/std": 273.4464416503906,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.0017507702577859163,
"fcm_dpo/delta": -0.06448774039745331,
"fcm_dpo/margin": 263.3714599609375,
"fcm_dpo/q_t": 0.39469367265701294,
"grad_norm": 22.49814796447754,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -0.4911942481994629,
"logits/rejected": -0.4789756238460541,
"logps/chosen": -379.6796875,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -672.5791625976562,
"loss": 1.0467,
"margin_dpo/margin_mean": 263.3714599609375,
"margin_dpo/margin_std": 305.1097717285156,
"step": 442
},
{
"KL/chosen_KL_mean": -378.4925842285156,
"KL/mean": -465.4745788574219,
"KL/rejected_KL_mean": -552.45654296875,
"KL/std": 265.3682861328125,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.0017513078637421131,
"fcm_dpo/delta": 0.09834998100996017,
"fcm_dpo/margin": 173.96397399902344,
"fcm_dpo/q_t": 0.4313589036464691,
"grad_norm": 29.68878746032715,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -0.4677377939224243,
"logits/rejected": -0.4488041400909424,
"logps/chosen": -449.765625,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -639.136474609375,
"loss": 1.1875,
"margin_dpo/margin_mean": 173.96395874023438,
"margin_dpo/margin_std": 334.8567810058594,
"step": 443
},
{
"KL/chosen_KL_mean": -357.6883850097656,
"KL/mean": -494.31622314453125,
"KL/rejected_KL_mean": -630.944091796875,
"KL/std": 298.2752685546875,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.0017422267701476812,
"fcm_dpo/delta": -0.08027061074972153,
"fcm_dpo/margin": 273.25567626953125,
"fcm_dpo/q_t": 0.39155757427215576,
"grad_norm": 29.274503707885742,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -0.45265936851501465,
"logits/rejected": -0.4536314010620117,
"logps/chosen": -414.902099609375,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -728.198974609375,
"loss": 1.0395,
"margin_dpo/margin_mean": 273.25567626953125,
"margin_dpo/margin_std": 326.13409423828125,
"step": 444
},
{
"KL/chosen_KL_mean": -323.1199035644531,
"KL/mean": -449.0135803222656,
"KL/rejected_KL_mean": -574.9072265625,
"KL/std": 250.39508056640625,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.001730577670969069,
"fcm_dpo/delta": -0.037361592054367065,
"fcm_dpo/margin": 251.78732299804688,
"fcm_dpo/q_t": 0.40040573477745056,
"grad_norm": 30.908275604248047,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -0.46896958351135254,
"logits/rejected": -0.4585559368133545,
"logps/chosen": -390.4197082519531,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -667.5899658203125,
"loss": 1.0632,
"margin_dpo/margin_mean": 251.78732299804688,
"margin_dpo/margin_std": 303.667724609375,
"step": 445
},
{
"KL/chosen_KL_mean": -299.8826904296875,
"KL/mean": -439.4361877441406,
"KL/rejected_KL_mean": -578.9896850585938,
"KL/std": 266.3847961425781,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.0017157963011413813,
"fcm_dpo/delta": -0.08306018263101578,
"fcm_dpo/margin": 279.10699462890625,
"fcm_dpo/q_t": 0.38841933012008667,
"grad_norm": 24.125913619995117,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -0.46759241819381714,
"logits/rejected": -0.4777601361274719,
"logps/chosen": -358.98114013671875,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -680.25390625,
"loss": 1.018,
"margin_dpo/margin_mean": 279.10699462890625,
"margin_dpo/margin_std": 273.501220703125,
"step": 446
},
{
"KL/chosen_KL_mean": -321.5812683105469,
"KL/mean": -465.3448486328125,
"KL/rejected_KL_mean": -609.1083984375,
"KL/std": 309.42169189453125,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.0016721412539482117,
"fcm_dpo/delta": -0.08531050384044647,
"fcm_dpo/margin": 287.5271301269531,
"fcm_dpo/q_t": 0.39167481660842896,
"grad_norm": 30.128538131713867,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -0.5374941229820251,
"logits/rejected": -0.542584240436554,
"logps/chosen": -377.6566162109375,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -707.8031616210938,
"loss": 1.0343,
"margin_dpo/margin_mean": 287.52716064453125,
"margin_dpo/margin_std": 343.70501708984375,
"step": 447
},
{
"KL/chosen_KL_mean": -364.1661071777344,
"KL/mean": -509.53765869140625,
"KL/rejected_KL_mean": -654.9091796875,
"KL/std": 271.6998291015625,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.0016555753536522388,
"fcm_dpo/delta": -0.08545278757810593,
"fcm_dpo/margin": 290.74310302734375,
"fcm_dpo/q_t": 0.38933056592941284,
"grad_norm": 40.963924407958984,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -0.47368836402893066,
"logits/rejected": -0.4778289794921875,
"logps/chosen": -424.1699523925781,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -757.173828125,
"loss": 1.0235,
"margin_dpo/margin_mean": 290.74310302734375,
"margin_dpo/margin_std": 306.6837463378906,
"step": 448
},
{
"KL/chosen_KL_mean": -390.13323974609375,
"KL/mean": -518.724853515625,
"KL/rejected_KL_mean": -647.3164672851562,
"KL/std": 310.909912109375,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.001634610234759748,
"fcm_dpo/delta": -0.021336160600185394,
"fcm_dpo/margin": 257.1832275390625,
"fcm_dpo/q_t": 0.40584796667099,
"grad_norm": 25.04412078857422,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -0.5128796100616455,
"logits/rejected": -0.5053173899650574,
"logps/chosen": -457.65985107421875,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -735.9133911132812,
"loss": 1.0971,
"margin_dpo/margin_mean": 257.1832275390625,
"margin_dpo/margin_std": 382.7791748046875,
"step": 449
},
{
"KL/chosen_KL_mean": -325.1204833984375,
"KL/mean": -481.97296142578125,
"KL/rejected_KL_mean": -638.825439453125,
"KL/std": 312.12579345703125,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.0016077951295301318,
"fcm_dpo/delta": -0.10993964970111847,
"fcm_dpo/margin": 313.70501708984375,
"fcm_dpo/q_t": 0.3819790780544281,
"grad_norm": 31.933448791503906,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -0.5362136363983154,
"logits/rejected": -0.5440558195114136,
"logps/chosen": -382.22857666015625,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -741.5804443359375,
"loss": 1.0019,
"margin_dpo/margin_mean": 313.7049865722656,
"margin_dpo/margin_std": 309.9493408203125,
"step": 450
},
{
"KL/chosen_KL_mean": -407.0479431152344,
"KL/mean": -521.1527099609375,
"KL/rejected_KL_mean": -635.2574462890625,
"KL/std": 311.9068603515625,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.0016091731376945972,
"fcm_dpo/delta": 0.03394605964422226,
"fcm_dpo/margin": 228.20953369140625,
"fcm_dpo/q_t": 0.41256624460220337,
"grad_norm": 28.686208724975586,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -0.4843297302722931,
"logits/rejected": -0.4612458348274231,
"logps/chosen": -465.51678466796875,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -708.1868896484375,
"loss": 1.1327,
"margin_dpo/margin_mean": 228.2095184326172,
"margin_dpo/margin_std": 363.1060791015625,
"step": 451
},
{
"KL/chosen_KL_mean": -299.3313293457031,
"KL/mean": -445.92633056640625,
"KL/rejected_KL_mean": -592.5213012695312,
"KL/std": 276.49298095703125,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.0015910858055576682,
"fcm_dpo/delta": -0.0698796957731247,
"fcm_dpo/margin": 293.19000244140625,
"fcm_dpo/q_t": 0.39026927947998047,
"grad_norm": 20.935033798217773,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -0.4869954288005829,
"logits/rejected": -0.5046002864837646,
"logps/chosen": -349.7218933105469,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -690.292724609375,
"loss": 1.0204,
"margin_dpo/margin_mean": 293.1899719238281,
"margin_dpo/margin_std": 282.45556640625,
"step": 452
},
{
"KL/chosen_KL_mean": -366.5586242675781,
"KL/mean": -491.7527160644531,
"KL/rejected_KL_mean": -616.94677734375,
"KL/std": 276.7699279785156,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.0015832162462174892,
"fcm_dpo/delta": 0.0035196300595998764,
"fcm_dpo/margin": 250.38816833496094,
"fcm_dpo/q_t": 0.4097445607185364,
"grad_norm": 32.47494888305664,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -0.5162127614021301,
"logits/rejected": -0.4957190752029419,
"logps/chosen": -424.27349853515625,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -699.1542358398438,
"loss": 1.0915,
"margin_dpo/margin_mean": 250.38815307617188,
"margin_dpo/margin_std": 329.19891357421875,
"step": 453
},
{
"KL/chosen_KL_mean": -425.7057189941406,
"KL/mean": -568.9153442382812,
"KL/rejected_KL_mean": -712.125,
"KL/std": 310.9266357421875,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.0015724084805697203,
"fcm_dpo/delta": -0.052992694079875946,
"fcm_dpo/margin": 286.4192810058594,
"fcm_dpo/q_t": 0.39700382947921753,
"grad_norm": 29.691364288330078,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -0.5279435515403748,
"logits/rejected": -0.5177085995674133,
"logps/chosen": -486.6513671875,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -797.0758056640625,
"loss": 1.0566,
"margin_dpo/margin_mean": 286.4192810058594,
"margin_dpo/margin_std": 355.6832275390625,
"step": 454
},
{
"KL/chosen_KL_mean": -374.63739013671875,
"KL/mean": -549.975341796875,
"KL/rejected_KL_mean": -725.3132934570312,
"KL/std": 342.216064453125,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.0015454981476068497,
"fcm_dpo/delta": -0.14988133311271667,
"fcm_dpo/margin": 350.6759033203125,
"fcm_dpo/q_t": 0.3785054683685303,
"grad_norm": 39.72151565551758,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -0.5236682295799255,
"logits/rejected": -0.5419257283210754,
"logps/chosen": -419.5240783691406,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -840.61474609375,
"loss": 1.0004,
"margin_dpo/margin_mean": 350.6759033203125,
"margin_dpo/margin_std": 388.170166015625,
"step": 455
},
{
"KL/chosen_KL_mean": -396.84381103515625,
"KL/mean": -554.3494262695312,
"KL/rejected_KL_mean": -711.85498046875,
"KL/std": 319.39385986328125,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.001508980873040855,
"fcm_dpo/delta": -0.07905428111553192,
"fcm_dpo/margin": 315.0111389160156,
"fcm_dpo/q_t": 0.38942915201187134,
"grad_norm": 20.266353607177734,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -0.5147756338119507,
"logits/rejected": -0.5317715406417847,
"logps/chosen": -453.880615234375,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -817.0728149414062,
"loss": 1.0175,
"margin_dpo/margin_mean": 315.01116943359375,
"margin_dpo/margin_std": 312.59521484375,
"step": 456
},
{
"KL/chosen_KL_mean": -372.20489501953125,
"KL/mean": -528.918212890625,
"KL/rejected_KL_mean": -685.631591796875,
"KL/std": 302.023193359375,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.001482290681451559,
"fcm_dpo/delta": -0.06809078902006149,
"fcm_dpo/margin": 313.4267272949219,
"fcm_dpo/q_t": 0.3901534080505371,
"grad_norm": 25.995811462402344,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -0.5356336832046509,
"logits/rejected": -0.5320440530776978,
"logps/chosen": -426.44744873046875,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -770.7412109375,
"loss": 1.0214,
"margin_dpo/margin_mean": 313.4267272949219,
"margin_dpo/margin_std": 297.42291259765625,
"step": 457
},
{
"KL/chosen_KL_mean": -381.38238525390625,
"KL/mean": -524.291748046875,
"KL/rejected_KL_mean": -667.201171875,
"KL/std": 277.86822509765625,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.001474408432841301,
"fcm_dpo/delta": -0.022421175613999367,
"fcm_dpo/margin": 285.8187255859375,
"fcm_dpo/q_t": 0.4017055928707123,
"grad_norm": 24.2961483001709,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -0.5444470047950745,
"logits/rejected": -0.5421079397201538,
"logps/chosen": -436.7912902832031,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -764.8843994140625,
"loss": 1.06,
"margin_dpo/margin_mean": 285.8187255859375,
"margin_dpo/margin_std": 318.3499450683594,
"step": 458
},
{
"KL/chosen_KL_mean": -400.17242431640625,
"KL/mean": -548.4536743164062,
"KL/rejected_KL_mean": -696.73486328125,
"KL/std": 310.79791259765625,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.0014622649177908897,
"fcm_dpo/delta": -0.035495489835739136,
"fcm_dpo/margin": 296.5623779296875,
"fcm_dpo/q_t": 0.399467408657074,
"grad_norm": 21.809309005737305,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -0.5215349793434143,
"logits/rejected": -0.5344473719596863,
"logps/chosen": -446.72991943359375,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -782.9033813476562,
"loss": 1.0613,
"margin_dpo/margin_mean": 296.5624084472656,
"margin_dpo/margin_std": 346.4207458496094,
"step": 459
},
{
"KL/chosen_KL_mean": -457.33087158203125,
"KL/mean": -610.467041015625,
"KL/rejected_KL_mean": -763.6031494140625,
"KL/std": 300.7550354003906,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.0014558184193447232,
"fcm_dpo/delta": -0.047990500926971436,
"fcm_dpo/margin": 306.27227783203125,
"fcm_dpo/q_t": 0.3961849808692932,
"grad_norm": 37.35396957397461,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -0.5847633481025696,
"logits/rejected": -0.6056011915206909,
"logps/chosen": -508.96575927734375,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -867.7225341796875,
"loss": 1.0468,
"margin_dpo/margin_mean": 306.2723083496094,
"margin_dpo/margin_std": 339.9352722167969,
"step": 460
},
{
"KL/chosen_KL_mean": -445.0636901855469,
"KL/mean": -572.2376708984375,
"KL/rejected_KL_mean": -699.4116821289062,
"KL/std": 290.94036865234375,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.0014602546580135822,
"fcm_dpo/delta": 0.029491884633898735,
"fcm_dpo/margin": 254.34796142578125,
"fcm_dpo/q_t": 0.4137762188911438,
"grad_norm": 22.10498046875,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -0.5995633006095886,
"logits/rejected": -0.6073707342147827,
"logps/chosen": -500.24566650390625,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -785.8885498046875,
"loss": 1.1051,
"margin_dpo/margin_mean": 254.34796142578125,
"margin_dpo/margin_std": 332.9735107421875,
"step": 461
},
{
"KL/chosen_KL_mean": -459.8350830078125,
"KL/mean": -553.7359619140625,
"KL/rejected_KL_mean": -647.6368408203125,
"KL/std": 295.6435852050781,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.0014856795314699411,
"fcm_dpo/delta": 0.12450301647186279,
"fcm_dpo/margin": 187.80177307128906,
"fcm_dpo/q_t": 0.4358844757080078,
"grad_norm": 43.47697067260742,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -0.6230775117874146,
"logits/rejected": -0.616161584854126,
"logps/chosen": -529.7631225585938,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -726.4779052734375,
"loss": 1.1844,
"margin_dpo/margin_mean": 187.8017578125,
"margin_dpo/margin_std": 329.5667419433594,
"step": 462
},
{
"KL/chosen_KL_mean": -415.004150390625,
"KL/mean": -553.5238037109375,
"KL/rejected_KL_mean": -692.04345703125,
"KL/std": 322.8453674316406,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.001497290562838316,
"fcm_dpo/delta": -0.015637099742889404,
"fcm_dpo/margin": 277.03924560546875,
"fcm_dpo/q_t": 0.40619271993637085,
"grad_norm": 33.27143478393555,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -0.6098886728286743,
"logits/rejected": -0.6173523664474487,
"logps/chosen": -470.27850341796875,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -781.068359375,
"loss": 1.0932,
"margin_dpo/margin_mean": 277.03924560546875,
"margin_dpo/margin_std": 387.570068359375,
"step": 463
},
{
"KL/chosen_KL_mean": -402.9927978515625,
"KL/mean": -604.0712890625,
"KL/rejected_KL_mean": -805.1497802734375,
"KL/std": 352.389892578125,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.0014413490425795317,
"fcm_dpo/delta": -0.1922934651374817,
"fcm_dpo/margin": 402.1569519042969,
"fcm_dpo/q_t": 0.3662247657775879,
"grad_norm": 29.33611488342285,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -0.5622963309288025,
"logits/rejected": -0.6052860021591187,
"logps/chosen": -453.9051208496094,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -907.63916015625,
"loss": 0.9519,
"margin_dpo/margin_mean": 402.156982421875,
"margin_dpo/margin_std": 368.00384521484375,
"step": 464
},
{
"KL/chosen_KL_mean": -372.560791015625,
"KL/mean": -539.6180419921875,
"KL/rejected_KL_mean": -706.67529296875,
"KL/std": 334.50079345703125,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.0014234642731025815,
"fcm_dpo/delta": -0.07937896996736526,
"fcm_dpo/margin": 334.114501953125,
"fcm_dpo/q_t": 0.3893335163593292,
"grad_norm": 27.498571395874023,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -0.5741344094276428,
"logits/rejected": -0.5896367430686951,
"logps/chosen": -432.6776428222656,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -820.621337890625,
"loss": 1.022,
"margin_dpo/margin_mean": 334.114501953125,
"margin_dpo/margin_std": 343.14959716796875,
"step": 465
},
{
"KL/chosen_KL_mean": -411.8160400390625,
"KL/mean": -545.8191528320312,
"KL/rejected_KL_mean": -679.822265625,
"KL/std": 299.4898681640625,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.0014133648946881294,
"fcm_dpo/delta": 0.021980978548526764,
"fcm_dpo/margin": 268.0061950683594,
"fcm_dpo/q_t": 0.4110508859157562,
"grad_norm": 29.728809356689453,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -0.5952455997467041,
"logits/rejected": -0.5951662063598633,
"logps/chosen": -464.73699951171875,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -770.1376953125,
"loss": 1.0968,
"margin_dpo/margin_mean": 268.00616455078125,
"margin_dpo/margin_std": 334.8175354003906,
"step": 466
},
{
"KL/chosen_KL_mean": -508.6435546875,
"KL/mean": -648.5333251953125,
"KL/rejected_KL_mean": -788.423095703125,
"KL/std": 349.01898193359375,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.0014083052519708872,
"fcm_dpo/delta": 0.005007736384868622,
"fcm_dpo/margin": 279.7795715332031,
"fcm_dpo/q_t": 0.4113559424877167,
"grad_norm": 33.90370559692383,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -0.6591016054153442,
"logits/rejected": -0.6488548517227173,
"logps/chosen": -587.359375,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -891.2833251953125,
"loss": 1.1234,
"margin_dpo/margin_mean": 279.779541015625,
"margin_dpo/margin_std": 433.38726806640625,
"step": 467
},
{
"KL/chosen_KL_mean": -386.95941162109375,
"KL/mean": -549.1633911132812,
"KL/rejected_KL_mean": -711.3673706054688,
"KL/std": 344.0048522949219,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.0014025418786332011,
"fcm_dpo/delta": -0.05807999521493912,
"fcm_dpo/margin": 324.407958984375,
"fcm_dpo/q_t": 0.39575815200805664,
"grad_norm": 39.35798645019531,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -0.632330596446991,
"logits/rejected": -0.6243791580200195,
"logps/chosen": -456.894775390625,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -812.3961791992188,
"loss": 1.0463,
"margin_dpo/margin_mean": 324.407958984375,
"margin_dpo/margin_std": 376.43096923828125,
"step": 468
},
{
"KL/chosen_KL_mean": -436.9048767089844,
"KL/mean": -599.1432495117188,
"KL/rejected_KL_mean": -761.3817138671875,
"KL/std": 320.456787109375,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.0013855324359610677,
"fcm_dpo/delta": -0.0526653453707695,
"fcm_dpo/margin": 324.476806640625,
"fcm_dpo/q_t": 0.3954392969608307,
"grad_norm": 27.19671058654785,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -0.6334969997406006,
"logits/rejected": -0.6272901296615601,
"logps/chosen": -505.0295715332031,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -866.1680908203125,
"loss": 1.0435,
"margin_dpo/margin_mean": 324.476806640625,
"margin_dpo/margin_std": 351.4849853515625,
"step": 469
},
{
"KL/chosen_KL_mean": -332.91339111328125,
"KL/mean": -487.83050537109375,
"KL/rejected_KL_mean": -642.7476806640625,
"KL/std": 311.11309814453125,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.0013843157794326544,
"fcm_dpo/delta": -0.030201081186532974,
"fcm_dpo/margin": 309.83428955078125,
"fcm_dpo/q_t": 0.39850544929504395,
"grad_norm": 25.313594818115234,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -0.6387758851051331,
"logits/rejected": -0.6436039209365845,
"logps/chosen": -376.7052917480469,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -725.4505004882812,
"loss": 1.0555,
"margin_dpo/margin_mean": 309.83428955078125,
"margin_dpo/margin_std": 340.318115234375,
"step": 470
},
{
"KL/chosen_KL_mean": -412.908447265625,
"KL/mean": -547.954833984375,
"KL/rejected_KL_mean": -683.0012817382812,
"KL/std": 324.60595703125,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.0013804736081510782,
"fcm_dpo/delta": 0.027975287288427353,
"fcm_dpo/margin": 270.0928039550781,
"fcm_dpo/q_t": 0.41345933079719543,
"grad_norm": 24.690170288085938,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -0.6649577617645264,
"logits/rejected": -0.6651647090911865,
"logps/chosen": -476.24798583984375,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -766.6117553710938,
"loss": 1.0987,
"margin_dpo/margin_mean": 270.0928039550781,
"margin_dpo/margin_std": 332.7794189453125,
"step": 471
},
{
"KL/chosen_KL_mean": -438.15020751953125,
"KL/mean": -597.1299438476562,
"KL/rejected_KL_mean": -756.1097412109375,
"KL/std": 392.02294921875,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.001379038905724883,
"fcm_dpo/delta": -0.04026825353503227,
"fcm_dpo/margin": 317.9595031738281,
"fcm_dpo/q_t": 0.40268874168395996,
"grad_norm": 25.000944137573242,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -0.7143632173538208,
"logits/rejected": -0.6818025708198547,
"logps/chosen": -521.8162841796875,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -873.3189086914062,
"loss": 1.0767,
"margin_dpo/margin_mean": 317.95953369140625,
"margin_dpo/margin_std": 434.4294738769531,
"step": 472
},
{
"KL/chosen_KL_mean": -489.4718322753906,
"KL/mean": -592.4940185546875,
"KL/rejected_KL_mean": -695.5161743164062,
"KL/std": 372.8263244628906,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0014005075208842754,
"fcm_dpo/delta": 0.11475996673107147,
"fcm_dpo/margin": 206.04432678222656,
"fcm_dpo/q_t": 0.43647685647010803,
"grad_norm": 41.77243423461914,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -0.6815752983093262,
"logits/rejected": -0.6562888622283936,
"logps/chosen": -552.9688110351562,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -776.6627197265625,
"loss": 1.2097,
"margin_dpo/margin_mean": 206.04432678222656,
"margin_dpo/margin_std": 453.6682434082031,
"step": 473
},
{
"KL/chosen_KL_mean": -412.56927490234375,
"KL/mean": -540.0201416015625,
"KL/rejected_KL_mean": -667.470947265625,
"KL/std": 328.03204345703125,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.001424451475031674,
"fcm_dpo/delta": 0.03749527409672737,
"fcm_dpo/margin": 254.90162658691406,
"fcm_dpo/q_t": 0.41584312915802,
"grad_norm": 30.2032527923584,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -0.6723858118057251,
"logits/rejected": -0.6669915914535522,
"logps/chosen": -465.18121337890625,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -757.5513305664062,
"loss": 1.1276,
"margin_dpo/margin_mean": 254.90164184570312,
"margin_dpo/margin_std": 379.5084228515625,
"step": 474
},
{
"KL/chosen_KL_mean": -334.7874755859375,
"KL/mean": -509.8450622558594,
"KL/rejected_KL_mean": -684.9027099609375,
"KL/std": 321.20361328125,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.0013988735154271126,
"fcm_dpo/delta": -0.09445011615753174,
"fcm_dpo/margin": 350.11517333984375,
"fcm_dpo/q_t": 0.3866940140724182,
"grad_norm": 22.809551239013672,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -0.7078989744186401,
"logits/rejected": -0.7233259677886963,
"logps/chosen": -377.2826843261719,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -774.9656372070312,
"loss": 1.0106,
"margin_dpo/margin_mean": 350.1152038574219,
"margin_dpo/margin_std": 354.17852783203125,
"step": 475
},
{
"KL/chosen_KL_mean": -391.2220458984375,
"KL/mean": -534.6827392578125,
"KL/rejected_KL_mean": -678.1434326171875,
"KL/std": 323.22772216796875,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.001390953897498548,
"fcm_dpo/delta": 0.0008967118337750435,
"fcm_dpo/margin": 286.92138671875,
"fcm_dpo/q_t": 0.4077247977256775,
"grad_norm": 29.844114303588867,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -0.7228225469589233,
"logits/rejected": -0.7230457663536072,
"logps/chosen": -434.17138671875,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -751.8536376953125,
"loss": 1.0791,
"margin_dpo/margin_mean": 286.92138671875,
"margin_dpo/margin_std": 342.6464538574219,
"step": 476
},
{
"KL/chosen_KL_mean": -419.1278991699219,
"KL/mean": -544.8071899414062,
"KL/rejected_KL_mean": -670.4865112304688,
"KL/std": 317.6999816894531,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.0014057450462132692,
"fcm_dpo/delta": 0.0482083223760128,
"fcm_dpo/margin": 251.35861206054688,
"fcm_dpo/q_t": 0.4184876084327698,
"grad_norm": 27.383092880249023,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -0.7424951791763306,
"logits/rejected": -0.70060133934021,
"logps/chosen": -489.9005126953125,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -746.6239013671875,
"loss": 1.1313,
"margin_dpo/margin_mean": 251.35861206054688,
"margin_dpo/margin_std": 380.20428466796875,
"step": 477
},
{
"KL/chosen_KL_mean": -347.18548583984375,
"KL/mean": -512.42431640625,
"KL/rejected_KL_mean": -677.6632080078125,
"KL/std": 301.6866455078125,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.001397792948409915,
"fcm_dpo/delta": -0.06495002657175064,
"fcm_dpo/margin": 330.4776611328125,
"fcm_dpo/q_t": 0.3927929997444153,
"grad_norm": 28.49869155883789,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.6548566818237305,
"logits/rejected": -0.6655555367469788,
"logps/chosen": -388.6259765625,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -763.025146484375,
"loss": 1.0329,
"margin_dpo/margin_mean": 330.4776611328125,
"margin_dpo/margin_std": 345.8775634765625,
"step": 478
},
{
"KL/chosen_KL_mean": -487.4216613769531,
"KL/mean": -634.49609375,
"KL/rejected_KL_mean": -781.5704345703125,
"KL/std": 378.451416015625,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.0013954101596027613,
"fcm_dpo/delta": -0.011742450296878815,
"fcm_dpo/margin": 294.1488037109375,
"fcm_dpo/q_t": 0.4071451425552368,
"grad_norm": 25.941673278808594,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -0.7130151987075806,
"logits/rejected": -0.7304632663726807,
"logps/chosen": -541.32958984375,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -876.686767578125,
"loss": 1.0994,
"margin_dpo/margin_mean": 294.1488037109375,
"margin_dpo/margin_std": 420.4874572753906,
"step": 479
},
{
"KL/chosen_KL_mean": -545.10693359375,
"KL/mean": -653.86328125,
"KL/rejected_KL_mean": -762.61962890625,
"KL/std": 339.1335754394531,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.0014152206713333726,
"fcm_dpo/delta": 0.09325779229402542,
"fcm_dpo/margin": 217.51266479492188,
"fcm_dpo/q_t": 0.42927083373069763,
"grad_norm": 57.129112243652344,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -0.7444524765014648,
"logits/rejected": -0.7359182834625244,
"logps/chosen": -603.7896118164062,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -845.5521240234375,
"loss": 1.1837,
"margin_dpo/margin_mean": 217.5126953125,
"margin_dpo/margin_std": 391.7486572265625,
"step": 480
},
{
"KL/chosen_KL_mean": -469.24285888671875,
"KL/mean": -651.1468505859375,
"KL/rejected_KL_mean": -833.05078125,
"KL/std": 393.1416320800781,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.0013865388464182615,
"fcm_dpo/delta": -0.110165536403656,
"fcm_dpo/margin": 363.80792236328125,
"fcm_dpo/q_t": 0.38523510098457336,
"grad_norm": 45.171932220458984,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -0.7980456352233887,
"logits/rejected": -0.8220420479774475,
"logps/chosen": -524.2071533203125,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -925.47119140625,
"loss": 1.0202,
"margin_dpo/margin_mean": 363.80792236328125,
"margin_dpo/margin_std": 419.04107666015625,
"step": 481
},
{
"KL/chosen_KL_mean": -532.6513671875,
"KL/mean": -629.147705078125,
"KL/rejected_KL_mean": -725.64404296875,
"KL/std": 367.68963623046875,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.0014093199279159307,
"fcm_dpo/delta": 0.13144272565841675,
"fcm_dpo/margin": 192.99264526367188,
"fcm_dpo/q_t": 0.4362224340438843,
"grad_norm": 43.460269927978516,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -0.7118823528289795,
"logits/rejected": -0.7040268182754517,
"logps/chosen": -600.204833984375,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -813.2335815429688,
"loss": 1.2274,
"margin_dpo/margin_mean": 192.99264526367188,
"margin_dpo/margin_std": 455.07916259765625,
"step": 482
},
{
"KL/chosen_KL_mean": -494.7724304199219,
"KL/mean": -688.48193359375,
"KL/rejected_KL_mean": -882.1915283203125,
"KL/std": 397.38714599609375,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.0013840030878782272,
"fcm_dpo/delta": -0.14435875415802002,
"fcm_dpo/margin": 387.4190979003906,
"fcm_dpo/q_t": 0.3799830973148346,
"grad_norm": 40.94780349731445,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -0.7805616855621338,
"logits/rejected": -0.7874066233634949,
"logps/chosen": -558.102294921875,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -977.978515625,
"loss": 1.0135,
"margin_dpo/margin_mean": 387.4190673828125,
"margin_dpo/margin_std": 465.99285888671875,
"step": 483
},
{
"KL/chosen_KL_mean": -447.64361572265625,
"KL/mean": -612.8146362304688,
"KL/rejected_KL_mean": -777.985595703125,
"KL/std": 382.26495361328125,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.0013740049907937646,
"fcm_dpo/delta": -0.056898586452007294,
"fcm_dpo/margin": 330.342041015625,
"fcm_dpo/q_t": 0.39715200662612915,
"grad_norm": 36.74729537963867,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -0.7907060384750366,
"logits/rejected": -0.8072522282600403,
"logps/chosen": -506.78173828125,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -862.3570556640625,
"loss": 1.0552,
"margin_dpo/margin_mean": 330.342041015625,
"margin_dpo/margin_std": 402.3882141113281,
"step": 484
},
{
"KL/chosen_KL_mean": -473.51666259765625,
"KL/mean": -629.485107421875,
"KL/rejected_KL_mean": -785.4535522460938,
"KL/std": 376.95941162109375,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.0013583770487457514,
"fcm_dpo/delta": -0.024785015732049942,
"fcm_dpo/margin": 311.9368896484375,
"fcm_dpo/q_t": 0.4046768844127655,
"grad_norm": 34.864871978759766,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -0.7671700716018677,
"logits/rejected": -0.785019040107727,
"logps/chosen": -532.3662109375,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -888.817626953125,
"loss": 1.0916,
"margin_dpo/margin_mean": 311.9368896484375,
"margin_dpo/margin_std": 449.0213317871094,
"step": 485
},
{
"KL/chosen_KL_mean": -506.8376770019531,
"KL/mean": -694.2506103515625,
"KL/rejected_KL_mean": -881.6634521484375,
"KL/std": 416.7193603515625,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.0013335715048015118,
"fcm_dpo/delta": -0.10512037575244904,
"fcm_dpo/margin": 374.8258056640625,
"fcm_dpo/q_t": 0.3889124393463135,
"grad_norm": 39.17012023925781,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -0.8197872638702393,
"logits/rejected": -0.8387131094932556,
"logps/chosen": -562.0973510742188,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -973.8028564453125,
"loss": 1.0538,
"margin_dpo/margin_mean": 374.8258056640625,
"margin_dpo/margin_std": 518.553466796875,
"step": 486
},
{
"KL/chosen_KL_mean": -560.105224609375,
"KL/mean": -713.3970947265625,
"KL/rejected_KL_mean": -866.6890258789062,
"KL/std": 408.19464111328125,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.0013294760137796402,
"fcm_dpo/delta": -0.008133085444569588,
"fcm_dpo/margin": 306.58380126953125,
"fcm_dpo/q_t": 0.4039134085178375,
"grad_norm": 61.269317626953125,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -0.8197336196899414,
"logits/rejected": -0.8259349465370178,
"logps/chosen": -613.1685791015625,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -959.1078491210938,
"loss": 1.0925,
"margin_dpo/margin_mean": 306.58380126953125,
"margin_dpo/margin_std": 416.7705078125,
"step": 487
},
{
"KL/chosen_KL_mean": -450.8982238769531,
"KL/mean": -607.0410766601562,
"KL/rejected_KL_mean": -763.1839599609375,
"KL/std": 365.67449951171875,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.0013271074276417494,
"fcm_dpo/delta": -0.015382722020149231,
"fcm_dpo/margin": 312.2857360839844,
"fcm_dpo/q_t": 0.4043211042881012,
"grad_norm": 38.323486328125,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -0.7659053802490234,
"logits/rejected": -0.7713078260421753,
"logps/chosen": -503.1263732910156,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -847.1905517578125,
"loss": 1.0717,
"margin_dpo/margin_mean": 312.2857666015625,
"margin_dpo/margin_std": 371.44061279296875,
"step": 488
},
{
"KL/chosen_KL_mean": -421.9764709472656,
"KL/mean": -567.7413330078125,
"KL/rejected_KL_mean": -713.5062255859375,
"KL/std": 350.5052185058594,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.0013246666640043259,
"fcm_dpo/delta": 0.014329822733998299,
"fcm_dpo/margin": 291.52972412109375,
"fcm_dpo/q_t": 0.4118216931819916,
"grad_norm": 31.000072479248047,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -0.8222206234931946,
"logits/rejected": -0.8239779472351074,
"logps/chosen": -477.9660949707031,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -792.904296875,
"loss": 1.0936,
"margin_dpo/margin_mean": 291.52972412109375,
"margin_dpo/margin_std": 376.18890380859375,
"step": 489
},
{
"KL/chosen_KL_mean": -505.61871337890625,
"KL/mean": -669.4989013671875,
"KL/rejected_KL_mean": -833.3790893554688,
"KL/std": 459.050048828125,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.0013201385736465454,
"fcm_dpo/delta": -0.03418659418821335,
"fcm_dpo/margin": 327.7603759765625,
"fcm_dpo/q_t": 0.4046947658061981,
"grad_norm": 35.84400177001953,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -0.8307057619094849,
"logits/rejected": -0.8715535402297974,
"logps/chosen": -557.985107421875,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -943.7881469726562,
"loss": 1.1084,
"margin_dpo/margin_mean": 327.7603759765625,
"margin_dpo/margin_std": 522.5004272460938,
"step": 490
},
{
"KL/chosen_KL_mean": -479.70025634765625,
"KL/mean": -585.4370727539062,
"KL/rejected_KL_mean": -691.173828125,
"KL/std": 367.56024169921875,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.0013417748268693686,
"fcm_dpo/delta": 0.11926029622554779,
"fcm_dpo/margin": 211.4735870361328,
"fcm_dpo/q_t": 0.43593573570251465,
"grad_norm": 30.89192008972168,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -0.7931652069091797,
"logits/rejected": -0.7898507714271545,
"logps/chosen": -539.8165283203125,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -764.4466552734375,
"loss": 1.1864,
"margin_dpo/margin_mean": 211.47357177734375,
"margin_dpo/margin_std": 388.8064270019531,
"step": 491
},
{
"KL/chosen_KL_mean": -498.99969482421875,
"KL/mean": -605.6187744140625,
"KL/rejected_KL_mean": -712.2379150390625,
"KL/std": 378.2779541015625,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.0013660395052284002,
"fcm_dpo/delta": 0.11212383210659027,
"fcm_dpo/margin": 213.23822021484375,
"fcm_dpo/q_t": 0.4349971413612366,
"grad_norm": 34.141265869140625,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -0.8712701797485352,
"logits/rejected": -0.8622636795043945,
"logps/chosen": -552.993896484375,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -784.8974609375,
"loss": 1.2029,
"margin_dpo/margin_mean": 213.23822021484375,
"margin_dpo/margin_std": 443.1600341796875,
"step": 492
},
{
"KL/chosen_KL_mean": -544.2478637695312,
"KL/mean": -646.387451171875,
"KL/rejected_KL_mean": -748.5269775390625,
"KL/std": 407.29376220703125,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.0013930967543274164,
"fcm_dpo/delta": 0.11872616410255432,
"fcm_dpo/margin": 204.27920532226562,
"fcm_dpo/q_t": 0.43394380807876587,
"grad_norm": 40.129520416259766,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -0.9206933379173279,
"logits/rejected": -0.8987519145011902,
"logps/chosen": -619.7451171875,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -835.8500366210938,
"loss": 1.1997,
"margin_dpo/margin_mean": 204.27920532226562,
"margin_dpo/margin_std": 401.0374755859375,
"step": 493
},
{
"KL/chosen_KL_mean": -415.2734375,
"KL/mean": -600.3134765625,
"KL/rejected_KL_mean": -785.353515625,
"KL/std": 375.8345642089844,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.001385992392897606,
"fcm_dpo/delta": -0.11901067197322845,
"fcm_dpo/margin": 370.080078125,
"fcm_dpo/q_t": 0.38057684898376465,
"grad_norm": 46.4956169128418,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -0.7722222805023193,
"logits/rejected": -0.8065148591995239,
"logps/chosen": -456.6326904296875,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -871.4448852539062,
"loss": 0.9966,
"margin_dpo/margin_mean": 370.0801086425781,
"margin_dpo/margin_std": 362.3285827636719,
"step": 494
},
{
"KL/chosen_KL_mean": -454.084228515625,
"KL/mean": -611.7135009765625,
"KL/rejected_KL_mean": -769.3427124023438,
"KL/std": 360.3024597167969,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.0013737970730289817,
"fcm_dpo/delta": -0.03465069457888603,
"fcm_dpo/margin": 315.25848388671875,
"fcm_dpo/q_t": 0.40194958448410034,
"grad_norm": 42.17742919921875,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -0.8575547933578491,
"logits/rejected": -0.8662198781967163,
"logps/chosen": -517.6192626953125,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -860.76708984375,
"loss": 1.0748,
"margin_dpo/margin_mean": 315.25848388671875,
"margin_dpo/margin_std": 420.1734619140625,
"step": 495
},
{
"KL/chosen_KL_mean": -543.9061279296875,
"KL/mean": -646.899658203125,
"KL/rejected_KL_mean": -749.89306640625,
"KL/std": 301.260498046875,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.0013906970852985978,
"fcm_dpo/delta": 0.11696639657020569,
"fcm_dpo/margin": 205.9869842529297,
"fcm_dpo/q_t": 0.43300455808639526,
"grad_norm": 38.68362808227539,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -0.8373208045959473,
"logits/rejected": -0.8159177303314209,
"logps/chosen": -616.498046875,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -834.222412109375,
"loss": 1.1732,
"margin_dpo/margin_mean": 205.9869842529297,
"margin_dpo/margin_std": 331.5146484375,
"step": 496
},
{
"KL/chosen_KL_mean": -514.3320922851562,
"KL/mean": -602.7001953125,
"KL/rejected_KL_mean": -691.068359375,
"KL/std": 345.37384033203125,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.0014234920963644981,
"fcm_dpo/delta": 0.15240351855754852,
"fcm_dpo/margin": 176.73617553710938,
"fcm_dpo/q_t": 0.44170081615448,
"grad_norm": 47.40353012084961,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -0.7741077542304993,
"logits/rejected": -0.7563482522964478,
"logps/chosen": -572.9260864257812,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -767.356689453125,
"loss": 1.2084,
"margin_dpo/margin_mean": 176.73619079589844,
"margin_dpo/margin_std": 327.4029541015625,
"step": 497
},
{
"KL/chosen_KL_mean": -505.73297119140625,
"KL/mean": -641.7274169921875,
"KL/rejected_KL_mean": -777.7219848632812,
"KL/std": 385.7830810546875,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.0014440558152273297,
"fcm_dpo/delta": 0.007523723877966404,
"fcm_dpo/margin": 271.9889831542969,
"fcm_dpo/q_t": 0.4101245403289795,
"grad_norm": 38.1798210144043,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -0.8347723484039307,
"logits/rejected": -0.8251057267189026,
"logps/chosen": -576.9386596679688,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -861.6800537109375,
"loss": 1.1289,
"margin_dpo/margin_mean": 271.98895263671875,
"margin_dpo/margin_std": 450.01959228515625,
"step": 498
},
{
"KL/chosen_KL_mean": -444.7691345214844,
"KL/mean": -620.37109375,
"KL/rejected_KL_mean": -795.97314453125,
"KL/std": 380.4271240234375,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.00142248859629035,
"fcm_dpo/delta": -0.10488341003656387,
"fcm_dpo/margin": 351.2040710449219,
"fcm_dpo/q_t": 0.3861340880393982,
"grad_norm": 28.65833854675293,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -0.7777875065803528,
"logits/rejected": -0.8081082105636597,
"logps/chosen": -496.0243225097656,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -897.0518798828125,
"loss": 1.0342,
"margin_dpo/margin_mean": 351.20404052734375,
"margin_dpo/margin_std": 435.44036865234375,
"step": 499
},
{
"KL/chosen_KL_mean": -509.4615478515625,
"KL/mean": -653.1812744140625,
"KL/rejected_KL_mean": -796.90087890625,
"KL/std": 342.46307373046875,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.0014125257730484009,
"fcm_dpo/delta": -0.006309333257377148,
"fcm_dpo/margin": 287.4393310546875,
"fcm_dpo/q_t": 0.4078175127506256,
"grad_norm": 32.10559844970703,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -0.756510853767395,
"logits/rejected": -0.7809162735939026,
"logps/chosen": -566.489013671875,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -890.8350830078125,
"loss": 1.1072,
"margin_dpo/margin_mean": 287.4393310546875,
"margin_dpo/margin_std": 430.96832275390625,
"step": 500
},
{
"KL/chosen_KL_mean": -445.49664306640625,
"KL/mean": -589.9784545898438,
"KL/rejected_KL_mean": -734.460205078125,
"KL/std": 344.7445068359375,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.00141524663195014,
"fcm_dpo/delta": -0.00943204015493393,
"fcm_dpo/margin": 288.9635314941406,
"fcm_dpo/q_t": 0.4075087308883667,
"grad_norm": 35.74781799316406,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -0.7470359206199646,
"logits/rejected": -0.7275895476341248,
"logps/chosen": -499.856201171875,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -814.616943359375,
"loss": 1.1037,
"margin_dpo/margin_mean": 288.96356201171875,
"margin_dpo/margin_std": 428.626708984375,
"step": 501
},
{
"KL/chosen_KL_mean": -402.1201171875,
"KL/mean": -555.928955078125,
"KL/rejected_KL_mean": -709.73779296875,
"KL/std": 308.30999755859375,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.0014068656601011753,
"fcm_dpo/delta": -0.034274935722351074,
"fcm_dpo/margin": 307.6176452636719,
"fcm_dpo/q_t": 0.3976163864135742,
"grad_norm": 46.3470573425293,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -0.7059692144393921,
"logits/rejected": -0.6937296390533447,
"logps/chosen": -469.7206115722656,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -792.6865234375,
"loss": 1.0425,
"margin_dpo/margin_mean": 307.6176452636719,
"margin_dpo/margin_std": 298.7598876953125,
"step": 502
},
{
"KL/chosen_KL_mean": -417.1700439453125,
"KL/mean": -571.521240234375,
"KL/rejected_KL_mean": -725.8723754882812,
"KL/std": 324.25213623046875,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.001392916077747941,
"fcm_dpo/delta": -0.03143535181879997,
"fcm_dpo/margin": 308.7023620605469,
"fcm_dpo/q_t": 0.4012370705604553,
"grad_norm": 24.095443725585938,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -0.6967535614967346,
"logits/rejected": -0.6839867830276489,
"logps/chosen": -472.2484130859375,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -808.3778076171875,
"loss": 1.0592,
"margin_dpo/margin_mean": 308.7023620605469,
"margin_dpo/margin_std": 360.79278564453125,
"step": 503
},
{
"KL/chosen_KL_mean": -498.46856689453125,
"KL/mean": -675.3897705078125,
"KL/rejected_KL_mean": -852.3109130859375,
"KL/std": 413.42071533203125,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.0013741697184741497,
"fcm_dpo/delta": -0.09064232558012009,
"fcm_dpo/margin": 353.84228515625,
"fcm_dpo/q_t": 0.3909027874469757,
"grad_norm": 31.768949508666992,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -0.7702962160110474,
"logits/rejected": -0.7931787967681885,
"logps/chosen": -558.434326171875,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -956.072998046875,
"loss": 1.0515,
"margin_dpo/margin_mean": 353.84228515625,
"margin_dpo/margin_std": 460.11822509765625,
"step": 504
},
{
"KL/chosen_KL_mean": -557.1580810546875,
"KL/mean": -650.727783203125,
"KL/rejected_KL_mean": -744.2975463867188,
"KL/std": 392.6376037597656,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.0013989988947287202,
"fcm_dpo/delta": 0.14166060090065002,
"fcm_dpo/margin": 187.13946533203125,
"fcm_dpo/q_t": 0.4404255449771881,
"grad_norm": 34.36621856689453,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -0.7928653955459595,
"logits/rejected": -0.7725697159767151,
"logps/chosen": -633.31298828125,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -832.8829345703125,
"loss": 1.2135,
"margin_dpo/margin_mean": 187.13946533203125,
"margin_dpo/margin_std": 389.143310546875,
"step": 505
},
{
"KL/chosen_KL_mean": -444.1624450683594,
"KL/mean": -609.744140625,
"KL/rejected_KL_mean": -775.3258666992188,
"KL/std": 375.8392333984375,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.0013877790188416839,
"fcm_dpo/delta": -0.06312032043933868,
"fcm_dpo/margin": 331.1634521484375,
"fcm_dpo/q_t": 0.3947676420211792,
"grad_norm": 24.712621688842773,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -0.7426855564117432,
"logits/rejected": -0.7506879568099976,
"logps/chosen": -493.1229248046875,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -853.7409057617188,
"loss": 1.0503,
"margin_dpo/margin_mean": 331.1634521484375,
"margin_dpo/margin_std": 399.01446533203125,
"step": 506
},
{
"KL/chosen_KL_mean": -518.7902221679688,
"KL/mean": -660.9124755859375,
"KL/rejected_KL_mean": -803.0347900390625,
"KL/std": 432.55389404296875,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.0013897776370868087,
"fcm_dpo/delta": 0.005165234208106995,
"fcm_dpo/margin": 284.24456787109375,
"fcm_dpo/q_t": 0.4138604402542114,
"grad_norm": 36.502052307128906,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -0.7783904075622559,
"logits/rejected": -0.7834802269935608,
"logps/chosen": -585.5916748046875,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -898.4076538085938,
"loss": 1.1169,
"margin_dpo/margin_mean": 284.2445373535156,
"margin_dpo/margin_std": 456.73846435546875,
"step": 507
},
{
"KL/chosen_KL_mean": -553.865966796875,
"KL/mean": -671.4600830078125,
"KL/rejected_KL_mean": -789.0540161132812,
"KL/std": 411.0580749511719,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.0014068866148591042,
"fcm_dpo/delta": 0.07140786945819855,
"fcm_dpo/margin": 235.18801879882812,
"fcm_dpo/q_t": 0.4281538724899292,
"grad_norm": 52.258079528808594,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -0.7359493970870972,
"logits/rejected": -0.7305607795715332,
"logps/chosen": -625.16943359375,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -884.6815795898438,
"loss": 1.1927,
"margin_dpo/margin_mean": 235.18801879882812,
"margin_dpo/margin_std": 488.09149169921875,
"step": 508
},
{
"KL/chosen_KL_mean": -454.72467041015625,
"KL/mean": -576.01513671875,
"KL/rejected_KL_mean": -697.3055419921875,
"KL/std": 319.4141845703125,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.0014271587133407593,
"fcm_dpo/delta": 0.05532916262745857,
"fcm_dpo/margin": 242.58090209960938,
"fcm_dpo/q_t": 0.42185845971107483,
"grad_norm": 53.33353805541992,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -0.7149254083633423,
"logits/rejected": -0.7081091403961182,
"logps/chosen": -518.5435791015625,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -780.56201171875,
"loss": 1.1447,
"margin_dpo/margin_mean": 242.58090209960938,
"margin_dpo/margin_std": 395.3203125,
"step": 509
},
{
"KL/chosen_KL_mean": -523.7684326171875,
"KL/mean": -695.6725463867188,
"KL/rejected_KL_mean": -867.57666015625,
"KL/std": 365.83953857421875,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.0014092667261138558,
"fcm_dpo/delta": -0.08875143527984619,
"fcm_dpo/margin": 343.80816650390625,
"fcm_dpo/q_t": 0.3868986666202545,
"grad_norm": 34.31364059448242,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -0.6882549524307251,
"logits/rejected": -0.7085413932800293,
"logps/chosen": -575.6468505859375,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -970.341796875,
"loss": 1.0208,
"margin_dpo/margin_mean": 343.80816650390625,
"margin_dpo/margin_std": 363.1875,
"step": 510
},
{
"KL/chosen_KL_mean": -496.804443359375,
"KL/mean": -639.4493408203125,
"KL/rejected_KL_mean": -782.0941162109375,
"KL/std": 395.576416015625,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.0014029962476342916,
"fcm_dpo/delta": -0.0002991212531924248,
"fcm_dpo/margin": 285.2897644042969,
"fcm_dpo/q_t": 0.40967386960983276,
"grad_norm": 41.153446197509766,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -0.7291253805160522,
"logits/rejected": -0.7314298152923584,
"logps/chosen": -557.0425415039062,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -874.950927734375,
"loss": 1.1254,
"margin_dpo/margin_mean": 285.2897644042969,
"margin_dpo/margin_std": 475.3314208984375,
"step": 511
},
{
"KL/chosen_KL_mean": -421.3726806640625,
"KL/mean": -607.905517578125,
"KL/rejected_KL_mean": -794.4383544921875,
"KL/std": 373.86981201171875,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.001376358326524496,
"fcm_dpo/delta": -0.11963961273431778,
"fcm_dpo/margin": 373.065673828125,
"fcm_dpo/q_t": 0.3809126019477844,
"grad_norm": 37.31480026245117,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -0.7321999073028564,
"logits/rejected": -0.746883749961853,
"logps/chosen": -476.2781677246094,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -876.314208984375,
"loss": 0.9998,
"margin_dpo/margin_mean": 373.065673828125,
"margin_dpo/margin_std": 378.77947998046875,
"step": 512
},
{
"KL/chosen_KL_mean": -525.3746337890625,
"KL/mean": -649.6375732421875,
"KL/rejected_KL_mean": -773.9005737304688,
"KL/std": 345.8955078125,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.001371821272186935,
"fcm_dpo/delta": 0.060696233063936234,
"fcm_dpo/margin": 248.52593994140625,
"fcm_dpo/q_t": 0.4223766028881073,
"grad_norm": 43.97354507446289,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -0.78472501039505,
"logits/rejected": -0.7688409090042114,
"logps/chosen": -590.2910766601562,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -849.9630126953125,
"loss": 1.1507,
"margin_dpo/margin_mean": 248.52593994140625,
"margin_dpo/margin_std": 403.25299072265625,
"step": 513
},
{
"KL/chosen_KL_mean": -545.6099243164062,
"KL/mean": -675.9661865234375,
"KL/rejected_KL_mean": -806.322509765625,
"KL/std": 379.65130615234375,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.0013956009643152356,
"fcm_dpo/delta": 0.0373426228761673,
"fcm_dpo/margin": 260.7126159667969,
"fcm_dpo/q_t": 0.4192820191383362,
"grad_norm": 38.2194709777832,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -0.7396078705787659,
"logits/rejected": -0.7294738292694092,
"logps/chosen": -619.8394775390625,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -885.26806640625,
"loss": 1.1391,
"margin_dpo/margin_mean": 260.7126159667969,
"margin_dpo/margin_std": 434.5355224609375,
"step": 514
},
{
"KL/chosen_KL_mean": -449.01666259765625,
"KL/mean": -563.1497802734375,
"KL/rejected_KL_mean": -677.2828979492188,
"KL/std": 333.6114196777344,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.0014119221596047282,
"fcm_dpo/delta": 0.08030429482460022,
"fcm_dpo/margin": 228.26622009277344,
"fcm_dpo/q_t": 0.4252746105194092,
"grad_norm": 49.32374954223633,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -0.7190433740615845,
"logits/rejected": -0.7307946681976318,
"logps/chosen": -499.4182434082031,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -764.380615234375,
"loss": 1.159,
"margin_dpo/margin_mean": 228.26620483398438,
"margin_dpo/margin_std": 386.7580261230469,
"step": 515
},
{
"KL/chosen_KL_mean": -490.2201843261719,
"KL/mean": -656.5306396484375,
"KL/rejected_KL_mean": -822.841064453125,
"KL/std": 385.257568359375,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.001406691619195044,
"fcm_dpo/delta": -0.07118304073810577,
"fcm_dpo/margin": 332.62091064453125,
"fcm_dpo/q_t": 0.3913043141365051,
"grad_norm": 38.687232971191406,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -0.7688533067703247,
"logits/rejected": -0.7967864871025085,
"logps/chosen": -554.8697509765625,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -934.5634765625,
"loss": 1.0369,
"margin_dpo/margin_mean": 332.62091064453125,
"margin_dpo/margin_std": 372.27069091796875,
"step": 516
},
{
"KL/chosen_KL_mean": -515.1275024414062,
"KL/mean": -695.8507080078125,
"KL/rejected_KL_mean": -876.5738525390625,
"KL/std": 418.8572082519531,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.0013741871807724237,
"fcm_dpo/delta": -0.1020156666636467,
"fcm_dpo/margin": 361.4462890625,
"fcm_dpo/q_t": 0.3897242248058319,
"grad_norm": 26.913551330566406,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -0.7710027694702148,
"logits/rejected": -0.7722501754760742,
"logps/chosen": -576.0410766601562,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -965.656982421875,
"loss": 1.0368,
"margin_dpo/margin_mean": 361.4462890625,
"margin_dpo/margin_std": 460.18988037109375,
"step": 517
},
{
"KL/chosen_KL_mean": -483.17401123046875,
"KL/mean": -631.18896484375,
"KL/rejected_KL_mean": -779.2039794921875,
"KL/std": 357.8949279785156,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.0013638620730489492,
"fcm_dpo/delta": -0.00417877733707428,
"fcm_dpo/margin": 296.0299072265625,
"fcm_dpo/q_t": 0.4065293073654175,
"grad_norm": 49.986297607421875,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -0.7640267610549927,
"logits/rejected": -0.7799776792526245,
"logps/chosen": -540.6298828125,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -864.5166015625,
"loss": 1.0928,
"margin_dpo/margin_mean": 296.0299072265625,
"margin_dpo/margin_std": 397.06072998046875,
"step": 518
},
{
"KL/chosen_KL_mean": -480.3702697753906,
"KL/mean": -592.9776000976562,
"KL/rejected_KL_mean": -705.5848388671875,
"KL/std": 326.01904296875,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.0013907022075727582,
"fcm_dpo/delta": 0.08918791264295578,
"fcm_dpo/margin": 225.21456909179688,
"fcm_dpo/q_t": 0.4279451370239258,
"grad_norm": 30.363061904907227,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -0.7486674189567566,
"logits/rejected": -0.732239305973053,
"logps/chosen": -554.43359375,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -810.029052734375,
"loss": 1.1746,
"margin_dpo/margin_mean": 225.21456909179688,
"margin_dpo/margin_std": 407.32513427734375,
"step": 519
},
{
"KL/chosen_KL_mean": -511.0517578125,
"KL/mean": -647.2855834960938,
"KL/rejected_KL_mean": -783.5194091796875,
"KL/std": 358.12750244140625,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.0013931184075772762,
"fcm_dpo/delta": 0.021092122420668602,
"fcm_dpo/margin": 272.4676208496094,
"fcm_dpo/q_t": 0.4135820269584656,
"grad_norm": 35.9310302734375,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -0.7427826523780823,
"logits/rejected": -0.7525472640991211,
"logps/chosen": -581.3515625,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -883.500732421875,
"loss": 1.1106,
"margin_dpo/margin_mean": 272.4676208496094,
"margin_dpo/margin_std": 379.3016357421875,
"step": 520
},
{
"KL/chosen_KL_mean": -464.24853515625,
"KL/mean": -612.560791015625,
"KL/rejected_KL_mean": -760.8731079101562,
"KL/std": 374.5102844238281,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.0014033226761966944,
"fcm_dpo/delta": -0.017528019845485687,
"fcm_dpo/margin": 296.62457275390625,
"fcm_dpo/q_t": 0.404031366109848,
"grad_norm": 31.091215133666992,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -0.774972677230835,
"logits/rejected": -0.7772347331047058,
"logps/chosen": -522.3914794921875,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -844.1536865234375,
"loss": 1.0795,
"margin_dpo/margin_mean": 296.62457275390625,
"margin_dpo/margin_std": 372.91729736328125,
"step": 521
},
{
"KL/chosen_KL_mean": -514.0162353515625,
"KL/mean": -652.778564453125,
"KL/rejected_KL_mean": -791.5409545898438,
"KL/std": 397.7562255859375,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.0013970948057249188,
"fcm_dpo/delta": 0.01276197750121355,
"fcm_dpo/margin": 277.5247497558594,
"fcm_dpo/q_t": 0.4113382399082184,
"grad_norm": 36.46541976928711,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -0.7818618416786194,
"logits/rejected": -0.7891424894332886,
"logps/chosen": -560.5638427734375,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -857.5548095703125,
"loss": 1.1174,
"margin_dpo/margin_mean": 277.5247497558594,
"margin_dpo/margin_std": 421.7791748046875,
"step": 522
},
{
"KL/chosen_KL_mean": -539.9659423828125,
"KL/mean": -685.8133544921875,
"KL/rejected_KL_mean": -831.66064453125,
"KL/std": 432.6346435546875,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.0013963168021291494,
"fcm_dpo/delta": -0.00760743347927928,
"fcm_dpo/margin": 291.6947326660156,
"fcm_dpo/q_t": 0.4086528420448303,
"grad_norm": 36.53788757324219,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -0.8017454147338867,
"logits/rejected": -0.8047584295272827,
"logps/chosen": -601.735595703125,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -915.4220581054688,
"loss": 1.1164,
"margin_dpo/margin_mean": 291.6947326660156,
"margin_dpo/margin_std": 465.686279296875,
"step": 523
},
{
"KL/chosen_KL_mean": -532.7937622070312,
"KL/mean": -692.2679443359375,
"KL/rejected_KL_mean": -851.7421264648438,
"KL/std": 435.285400390625,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.0013786728959530592,
"fcm_dpo/delta": -0.04249449074268341,
"fcm_dpo/margin": 318.9483337402344,
"fcm_dpo/q_t": 0.398825079202652,
"grad_norm": 43.52704620361328,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -0.8273106813430786,
"logits/rejected": -0.8241423964500427,
"logps/chosen": -610.8658447265625,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -933.0441284179688,
"loss": 1.0843,
"margin_dpo/margin_mean": 318.9483337402344,
"margin_dpo/margin_std": 438.8890380859375,
"step": 524
},
{
"KL/chosen_KL_mean": -513.194580078125,
"KL/mean": -706.9784545898438,
"KL/rejected_KL_mean": -900.7623291015625,
"KL/std": 436.58209228515625,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.0013523332308977842,
"fcm_dpo/delta": -0.13166292011737823,
"fcm_dpo/margin": 387.5677490234375,
"fcm_dpo/q_t": 0.3833525776863098,
"grad_norm": 43.69297790527344,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -0.7880626916885376,
"logits/rejected": -0.8217567205429077,
"logps/chosen": -564.0224609375,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -1000.8153076171875,
"loss": 1.0179,
"margin_dpo/margin_mean": 387.5677490234375,
"margin_dpo/margin_std": 468.5898742675781,
"step": 525
},
{
"KL/chosen_KL_mean": -560.8143310546875,
"KL/mean": -753.6343994140625,
"KL/rejected_KL_mean": -946.4544067382812,
"KL/std": 424.5549621582031,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.001329383347183466,
"fcm_dpo/delta": -0.11859972029924393,
"fcm_dpo/margin": 385.64013671875,
"fcm_dpo/q_t": 0.3826919198036194,
"grad_norm": 35.98128128051758,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -0.8110041618347168,
"logits/rejected": -0.8155307769775391,
"logps/chosen": -623.9815673828125,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -1032.7637939453125,
"loss": 1.029,
"margin_dpo/margin_mean": 385.64013671875,
"margin_dpo/margin_std": 466.6395263671875,
"step": 526
},
{
"KL/chosen_KL_mean": -580.950439453125,
"KL/mean": -715.412109375,
"KL/rejected_KL_mean": -849.8738403320312,
"KL/std": 442.2706298828125,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.0013231317279860377,
"fcm_dpo/delta": 0.04583089426159859,
"fcm_dpo/margin": 268.92340087890625,
"fcm_dpo/q_t": 0.4169968366622925,
"grad_norm": 36.08964538574219,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -0.9514946937561035,
"logits/rejected": -0.9313217997550964,
"logps/chosen": -652.0791015625,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -928.2164306640625,
"loss": 1.1218,
"margin_dpo/margin_mean": 268.92340087890625,
"margin_dpo/margin_std": 378.7649230957031,
"step": 527
},
{
"KL/chosen_KL_mean": -611.4620361328125,
"KL/mean": -767.3988037109375,
"KL/rejected_KL_mean": -923.3355102539062,
"KL/std": 427.5599365234375,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.0013234864454716444,
"fcm_dpo/delta": -0.013384838588535786,
"fcm_dpo/margin": 311.8734130859375,
"fcm_dpo/q_t": 0.40960851311683655,
"grad_norm": 42.53367614746094,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -0.9003564119338989,
"logits/rejected": -0.8869940042495728,
"logps/chosen": -679.551513671875,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -1017.24560546875,
"loss": 1.1211,
"margin_dpo/margin_mean": 311.8734130859375,
"margin_dpo/margin_std": 515.3593139648438,
"step": 528
},
{
"KL/chosen_KL_mean": -696.36767578125,
"KL/mean": -806.6844482421875,
"KL/rejected_KL_mean": -917.001220703125,
"KL/std": 397.0294494628906,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.001338808098807931,
"fcm_dpo/delta": 0.10776834189891815,
"fcm_dpo/margin": 220.63357543945312,
"fcm_dpo/q_t": 0.43243926763534546,
"grad_norm": 42.70530700683594,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -0.9018471240997314,
"logits/rejected": -0.9128869771957397,
"logps/chosen": -751.942626953125,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -1006.2103271484375,
"loss": 1.2233,
"margin_dpo/margin_mean": 220.63357543945312,
"margin_dpo/margin_std": 502.29888916015625,
"step": 529
},
{
"KL/chosen_KL_mean": -592.1580810546875,
"KL/mean": -767.647705078125,
"KL/rejected_KL_mean": -943.1373291015625,
"KL/std": 471.09625244140625,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.0013396573485806584,
"fcm_dpo/delta": -0.07361838221549988,
"fcm_dpo/margin": 350.97930908203125,
"fcm_dpo/q_t": 0.3977614939212799,
"grad_norm": 49.951263427734375,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -0.8480117321014404,
"logits/rejected": -0.846229076385498,
"logps/chosen": -639.7594604492188,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -1030.421875,
"loss": 1.1034,
"margin_dpo/margin_mean": 350.97930908203125,
"margin_dpo/margin_std": 581.3236694335938,
"step": 530
},
{
"KL/chosen_KL_mean": -643.9351806640625,
"KL/mean": -796.6851806640625,
"KL/rejected_KL_mean": -949.4352416992188,
"KL/std": 499.3472900390625,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.0013343794271349907,
"fcm_dpo/delta": -0.008019059896469116,
"fcm_dpo/margin": 305.5000305175781,
"fcm_dpo/q_t": 0.40879255533218384,
"grad_norm": 36.798309326171875,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -0.9161735773086548,
"logits/rejected": -0.9046072959899902,
"logps/chosen": -699.1812744140625,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -1020.041259765625,
"loss": 1.1112,
"margin_dpo/margin_mean": 305.5000305175781,
"margin_dpo/margin_std": 474.65972900390625,
"step": 531
},
{
"KL/chosen_KL_mean": -626.92822265625,
"KL/mean": -798.6822509765625,
"KL/rejected_KL_mean": -970.436279296875,
"KL/std": 478.4112548828125,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.0013115124311298132,
"fcm_dpo/delta": -0.054340578615665436,
"fcm_dpo/margin": 343.50811767578125,
"fcm_dpo/q_t": 0.399822860956192,
"grad_norm": 50.558799743652344,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -0.8500086069107056,
"logits/rejected": -0.8299944400787354,
"logps/chosen": -697.2142333984375,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -1057.027587890625,
"loss": 1.1119,
"margin_dpo/margin_mean": 343.50811767578125,
"margin_dpo/margin_std": 558.3848876953125,
"step": 532
},
{
"KL/chosen_KL_mean": -598.7301025390625,
"KL/mean": -717.7310791015625,
"KL/rejected_KL_mean": -836.7320556640625,
"KL/std": 411.3270263671875,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.0013432963751256466,
"fcm_dpo/delta": 0.08121401071548462,
"fcm_dpo/margin": 238.00205993652344,
"fcm_dpo/q_t": 0.4286707639694214,
"grad_norm": 53.88816833496094,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -0.8658875226974487,
"logits/rejected": -0.8455657958984375,
"logps/chosen": -663.585205078125,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -913.3201904296875,
"loss": 1.1979,
"margin_dpo/margin_mean": 238.0020751953125,
"margin_dpo/margin_std": 496.88311767578125,
"step": 533
},
{
"KL/chosen_KL_mean": -611.169189453125,
"KL/mean": -767.7733154296875,
"KL/rejected_KL_mean": -924.37744140625,
"KL/std": 438.1583557128906,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.0013270191848278046,
"fcm_dpo/delta": -0.01723591610789299,
"fcm_dpo/margin": 313.2082824707031,
"fcm_dpo/q_t": 0.4059499502182007,
"grad_norm": 34.78287124633789,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -0.843059778213501,
"logits/rejected": -0.8394489288330078,
"logps/chosen": -671.28857421875,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -1002.9208984375,
"loss": 1.0967,
"margin_dpo/margin_mean": 313.2082824707031,
"margin_dpo/margin_std": 444.6209716796875,
"step": 534
},
{
"KL/chosen_KL_mean": -533.5681762695312,
"KL/mean": -717.9885864257812,
"KL/rejected_KL_mean": -902.408935546875,
"KL/std": 436.5372314453125,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.001325107179582119,
"fcm_dpo/delta": -0.09366312623023987,
"fcm_dpo/margin": 368.8407897949219,
"fcm_dpo/q_t": 0.39030542969703674,
"grad_norm": 29.257259368896484,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -0.8032083511352539,
"logits/rejected": -0.8018609285354614,
"logps/chosen": -587.8984375,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -998.716552734375,
"loss": 1.0469,
"margin_dpo/margin_mean": 368.8408203125,
"margin_dpo/margin_std": 462.23028564453125,
"step": 535
},
{
"KL/chosen_KL_mean": -478.49005126953125,
"KL/mean": -697.0470581054688,
"KL/rejected_KL_mean": -915.6041259765625,
"KL/std": 505.50726318359375,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.0012766944710165262,
"fcm_dpo/delta": -0.1674920618534088,
"fcm_dpo/margin": 437.11407470703125,
"fcm_dpo/q_t": 0.38029739260673523,
"grad_norm": 31.428855895996094,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -0.7796632051467896,
"logits/rejected": -0.8067115545272827,
"logps/chosen": -525.570556640625,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -1004.701904296875,
"loss": 1.0216,
"margin_dpo/margin_mean": 437.11407470703125,
"margin_dpo/margin_std": 601.05322265625,
"step": 536
},
{
"KL/chosen_KL_mean": -491.5775146484375,
"KL/mean": -637.9295654296875,
"KL/rejected_KL_mean": -784.2815551757812,
"KL/std": 396.8555908203125,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.0012780611868947744,
"fcm_dpo/delta": 0.026200681924819946,
"fcm_dpo/margin": 292.7039794921875,
"fcm_dpo/q_t": 0.4136512577533722,
"grad_norm": 37.28866195678711,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -0.8498191833496094,
"logits/rejected": -0.8356490135192871,
"logps/chosen": -549.3250122070312,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -854.719970703125,
"loss": 1.136,
"margin_dpo/margin_mean": 292.7039794921875,
"margin_dpo/margin_std": 474.5274353027344,
"step": 537
},
{
"KL/chosen_KL_mean": -613.0432739257812,
"KL/mean": -787.3213500976562,
"KL/rejected_KL_mean": -961.5994262695312,
"KL/std": 445.82562255859375,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.001262559788301587,
"fcm_dpo/delta": -0.04207906872034073,
"fcm_dpo/margin": 348.55621337890625,
"fcm_dpo/q_t": 0.4003813862800598,
"grad_norm": 28.12194061279297,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -0.8495669364929199,
"logits/rejected": -0.8379867076873779,
"logps/chosen": -679.459228515625,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -1045.8275146484375,
"loss": 1.0696,
"margin_dpo/margin_mean": 348.55621337890625,
"margin_dpo/margin_std": 457.8056335449219,
"step": 538
},
{
"KL/chosen_KL_mean": -530.21875,
"KL/mean": -680.2120361328125,
"KL/rejected_KL_mean": -830.205322265625,
"KL/std": 353.58636474609375,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.0012676361948251724,
"fcm_dpo/delta": 0.0204551313072443,
"fcm_dpo/margin": 299.98663330078125,
"fcm_dpo/q_t": 0.4122438430786133,
"grad_norm": 25.66387176513672,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -0.8782637119293213,
"logits/rejected": -0.8664537668228149,
"logps/chosen": -588.7115478515625,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -922.059326171875,
"loss": 1.1019,
"margin_dpo/margin_mean": 299.98663330078125,
"margin_dpo/margin_std": 396.3056335449219,
"step": 539
},
{
"KL/chosen_KL_mean": -508.0380859375,
"KL/mean": -721.2171630859375,
"KL/rejected_KL_mean": -934.3963623046875,
"KL/std": 451.8458251953125,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.00124639505520463,
"fcm_dpo/delta": -0.13855527341365814,
"fcm_dpo/margin": 426.35821533203125,
"fcm_dpo/q_t": 0.3821779489517212,
"grad_norm": 36.2562370300293,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -0.8005784749984741,
"logits/rejected": -0.8170750141143799,
"logps/chosen": -571.5206298828125,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -1050.8262939453125,
"loss": 1.0202,
"margin_dpo/margin_mean": 426.35821533203125,
"margin_dpo/margin_std": 517.6613159179688,
"step": 540
},
{
"KL/chosen_KL_mean": -607.1495361328125,
"KL/mean": -720.770263671875,
"KL/rejected_KL_mean": -834.390869140625,
"KL/std": 394.2872619628906,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.0012464348692446947,
"fcm_dpo/delta": 0.12000001966953278,
"fcm_dpo/margin": 227.2414093017578,
"fcm_dpo/q_t": 0.4358953833580017,
"grad_norm": 39.796974182128906,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -0.8924692273139954,
"logits/rejected": -0.8837276697158813,
"logps/chosen": -685.4298706054688,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -937.66455078125,
"loss": 1.2091,
"margin_dpo/margin_mean": 227.24142456054688,
"margin_dpo/margin_std": 467.9242248535156,
"step": 541
},
{
"KL/chosen_KL_mean": -537.729248046875,
"KL/mean": -746.4808959960938,
"KL/rejected_KL_mean": -955.2325439453125,
"KL/std": 462.728759765625,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.001229890389367938,
"fcm_dpo/delta": -0.1220366433262825,
"fcm_dpo/margin": 417.50335693359375,
"fcm_dpo/q_t": 0.3854454755783081,
"grad_norm": 39.909820556640625,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -0.8912371397018433,
"logits/rejected": -0.9178431034088135,
"logps/chosen": -595.2142333984375,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -1051.7076416015625,
"loss": 1.0364,
"margin_dpo/margin_mean": 417.5033264160156,
"margin_dpo/margin_std": 530.0671997070312,
"step": 542
},
{
"KL/chosen_KL_mean": -606.9930419921875,
"KL/mean": -751.1796264648438,
"KL/rejected_KL_mean": -895.3662109375,
"KL/std": 553.8678588867188,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.0012381270062178373,
"fcm_dpo/delta": 0.04455637186765671,
"fcm_dpo/margin": 288.3731994628906,
"fcm_dpo/q_t": 0.43025535345077515,
"grad_norm": 39.136451721191406,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -0.8726138472557068,
"logits/rejected": -0.8934823274612427,
"logps/chosen": -667.6105346679688,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -1000.9652099609375,
"loss": 1.1918,
"margin_dpo/margin_mean": 288.3731994628906,
"margin_dpo/margin_std": 641.470703125,
"step": 543
},
{
"KL/chosen_KL_mean": -598.482177734375,
"KL/mean": -781.916748046875,
"KL/rejected_KL_mean": -965.3511962890625,
"KL/std": 443.2710266113281,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.0012327476870268583,
"fcm_dpo/delta": -0.054760560393333435,
"fcm_dpo/margin": 366.86907958984375,
"fcm_dpo/q_t": 0.397747278213501,
"grad_norm": 33.70466232299805,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -0.8226778507232666,
"logits/rejected": -0.8345876932144165,
"logps/chosen": -658.12451171875,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -1066.305908203125,
"loss": 1.0675,
"margin_dpo/margin_mean": 366.86907958984375,
"margin_dpo/margin_std": 492.79803466796875,
"step": 544
},
{
"KL/chosen_KL_mean": -577.6429443359375,
"KL/mean": -770.2427978515625,
"KL/rejected_KL_mean": -962.8426513671875,
"KL/std": 431.33111572265625,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.0012169883120805025,
"fcm_dpo/delta": -0.07215145230293274,
"fcm_dpo/margin": 385.19970703125,
"fcm_dpo/q_t": 0.3939034640789032,
"grad_norm": 37.47612380981445,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -0.8058052062988281,
"logits/rejected": -0.8308186531066895,
"logps/chosen": -645.29150390625,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -1058.7506103515625,
"loss": 1.0537,
"margin_dpo/margin_mean": 385.19970703125,
"margin_dpo/margin_std": 491.25616455078125,
"step": 545
},
{
"KL/chosen_KL_mean": -527.8880004882812,
"KL/mean": -677.0577392578125,
"KL/rejected_KL_mean": -826.2274169921875,
"KL/std": 387.1895751953125,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.00121834478341043,
"fcm_dpo/delta": 0.037912800908088684,
"fcm_dpo/margin": 298.3394470214844,
"fcm_dpo/q_t": 0.41466593742370605,
"grad_norm": 26.89870262145996,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -0.877615213394165,
"logits/rejected": -0.8663809299468994,
"logps/chosen": -578.6322021484375,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -908.0936279296875,
"loss": 1.1145,
"margin_dpo/margin_mean": 298.33941650390625,
"margin_dpo/margin_std": 405.95281982421875,
"step": 546
},
{
"KL/chosen_KL_mean": -543.017333984375,
"KL/mean": -709.8662109375,
"KL/rejected_KL_mean": -876.715087890625,
"KL/std": 433.509521484375,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.0012220775242894888,
"fcm_dpo/delta": -0.00818348303437233,
"fcm_dpo/margin": 333.69775390625,
"fcm_dpo/q_t": 0.4070436358451843,
"grad_norm": 49.96892166137695,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -0.8075991272926331,
"logits/rejected": -0.812584400177002,
"logps/chosen": -616.705078125,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -967.4764404296875,
"loss": 1.0976,
"margin_dpo/margin_mean": 333.69775390625,
"margin_dpo/margin_std": 473.7314758300781,
"step": 547
},
{
"KL/chosen_KL_mean": -549.0206298828125,
"KL/mean": -712.397705078125,
"KL/rejected_KL_mean": -875.7747802734375,
"KL/std": 449.38262939453125,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.0012183936778455973,
"fcm_dpo/delta": 0.0019378187134861946,
"fcm_dpo/margin": 326.75421142578125,
"fcm_dpo/q_t": 0.41230571269989014,
"grad_norm": 25.078372955322266,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -0.8877569437026978,
"logits/rejected": -0.8838699460029602,
"logps/chosen": -614.2669677734375,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -969.8928833007812,
"loss": 1.1042,
"margin_dpo/margin_mean": 326.75421142578125,
"margin_dpo/margin_std": 487.56951904296875,
"step": 548
},
{
"KL/chosen_KL_mean": -589.3922729492188,
"KL/mean": -711.4036254882812,
"KL/rejected_KL_mean": -833.4149780273438,
"KL/std": 379.79595947265625,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.0012427914189174771,
"fcm_dpo/delta": 0.09908977150917053,
"fcm_dpo/margin": 244.022705078125,
"fcm_dpo/q_t": 0.4320880174636841,
"grad_norm": 40.19181442260742,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -0.8845070600509644,
"logits/rejected": -0.8731452226638794,
"logps/chosen": -638.6046142578125,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -907.3253173828125,
"loss": 1.181,
"margin_dpo/margin_mean": 244.02273559570312,
"margin_dpo/margin_std": 453.96075439453125,
"step": 549
},
{
"KL/chosen_KL_mean": -575.8084716796875,
"KL/mean": -719.7702026367188,
"KL/rejected_KL_mean": -863.7319946289062,
"KL/std": 412.17730712890625,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.0012514062691479921,
"fcm_dpo/delta": 0.04118921235203743,
"fcm_dpo/margin": 287.9236145019531,
"fcm_dpo/q_t": 0.41750797629356384,
"grad_norm": 40.92441940307617,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -0.8064329624176025,
"logits/rejected": -0.8260148763656616,
"logps/chosen": -632.6154174804688,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -958.8577880859375,
"loss": 1.1311,
"margin_dpo/margin_mean": 287.923583984375,
"margin_dpo/margin_std": 450.417236328125,
"step": 550
},
{
"KL/chosen_KL_mean": -510.1761169433594,
"KL/mean": -744.1221923828125,
"KL/rejected_KL_mean": -978.068359375,
"KL/std": 450.1910400390625,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.001218787394464016,
"fcm_dpo/delta": -0.18103429675102234,
"fcm_dpo/margin": 467.89215087890625,
"fcm_dpo/q_t": 0.3692789375782013,
"grad_norm": 53.24525833129883,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -0.7796275615692139,
"logits/rejected": -0.8252531886100769,
"logps/chosen": -569.282470703125,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -1089.7412109375,
"loss": 0.9655,
"margin_dpo/margin_mean": 467.89215087890625,
"margin_dpo/margin_std": 460.2682800292969,
"step": 551
},
{
"KL/chosen_KL_mean": -502.0711975097656,
"KL/mean": -760.1432495117188,
"KL/rejected_KL_mean": -1018.2152099609375,
"KL/std": 528.348876953125,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.001160782529041171,
"fcm_dpo/delta": -0.21574066579341888,
"fcm_dpo/margin": 516.14404296875,
"fcm_dpo/q_t": 0.36910372972488403,
"grad_norm": 41.11149597167969,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -0.859273374080658,
"logits/rejected": -0.8876142501831055,
"logps/chosen": -564.42578125,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -1122.77734375,
"loss": 0.9781,
"margin_dpo/margin_mean": 516.14404296875,
"margin_dpo/margin_std": 595.195068359375,
"step": 552
},
{
"KL/chosen_KL_mean": -559.717529296875,
"KL/mean": -693.601806640625,
"KL/rejected_KL_mean": -827.4859619140625,
"KL/std": 416.41571044921875,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.0011716227745637298,
"fcm_dpo/delta": 0.08916747570037842,
"fcm_dpo/margin": 267.76837158203125,
"fcm_dpo/q_t": 0.43077874183654785,
"grad_norm": 29.42908477783203,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -0.8399643898010254,
"logits/rejected": -0.8348550200462341,
"logps/chosen": -627.9763793945312,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -925.5831298828125,
"loss": 1.1687,
"margin_dpo/margin_mean": 267.7684020996094,
"margin_dpo/margin_std": 478.6548767089844,
"step": 553
},
{
"KL/chosen_KL_mean": -586.1414794921875,
"KL/mean": -774.9417724609375,
"KL/rejected_KL_mean": -963.7420654296875,
"KL/std": 468.91375732421875,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.0011788744013756514,
"fcm_dpo/delta": -0.04776027426123619,
"fcm_dpo/margin": 377.6005859375,
"fcm_dpo/q_t": 0.4016305208206177,
"grad_norm": 67.11528778076172,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -0.8197340369224548,
"logits/rejected": -0.7931340932846069,
"logps/chosen": -654.089111328125,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -1053.5247802734375,
"loss": 1.1021,
"margin_dpo/margin_mean": 377.6005859375,
"margin_dpo/margin_std": 588.55615234375,
"step": 554
},
{
"KL/chosen_KL_mean": -597.59375,
"KL/mean": -807.906005859375,
"KL/rejected_KL_mean": -1018.2183227539062,
"KL/std": 496.7644348144531,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.0011544254375621676,
"fcm_dpo/delta": -0.0898696631193161,
"fcm_dpo/margin": 420.6246032714844,
"fcm_dpo/q_t": 0.3962913751602173,
"grad_norm": 28.15028953552246,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -0.897038459777832,
"logits/rejected": -0.9390174150466919,
"logps/chosen": -650.9241943359375,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -1126.69775390625,
"loss": 1.0808,
"margin_dpo/margin_mean": 420.6246032714844,
"margin_dpo/margin_std": 651.3489990234375,
"step": 555
},
{
"KL/chosen_KL_mean": -573.87255859375,
"KL/mean": -702.5924072265625,
"KL/rejected_KL_mean": -831.312255859375,
"KL/std": 383.98846435546875,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.001165332505479455,
"fcm_dpo/delta": 0.10310876369476318,
"fcm_dpo/margin": 257.43975830078125,
"fcm_dpo/q_t": 0.4312754273414612,
"grad_norm": 32.8856086730957,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -0.8695886731147766,
"logits/rejected": -0.8604377508163452,
"logps/chosen": -632.5169677734375,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -932.6526489257812,
"loss": 1.1598,
"margin_dpo/margin_mean": 257.43975830078125,
"margin_dpo/margin_std": 401.28082275390625,
"step": 556
},
{
"KL/chosen_KL_mean": -607.11181640625,
"KL/mean": -808.7969970703125,
"KL/rejected_KL_mean": -1010.482177734375,
"KL/std": 527.2576904296875,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.0011563836596906185,
"fcm_dpo/delta": -0.07004015892744064,
"fcm_dpo/margin": 403.37042236328125,
"fcm_dpo/q_t": 0.3975653350353241,
"grad_norm": 53.122276306152344,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -0.895065426826477,
"logits/rejected": -0.9235693216323853,
"logps/chosen": -674.9524536132812,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -1120.421875,
"loss": 1.0825,
"margin_dpo/margin_mean": 403.37042236328125,
"margin_dpo/margin_std": 600.696044921875,
"step": 557
},
{
"KL/chosen_KL_mean": -559.891845703125,
"KL/mean": -821.7356567382812,
"KL/rejected_KL_mean": -1083.579345703125,
"KL/std": 499.6360168457031,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.0011204106267541647,
"fcm_dpo/delta": -0.19866636395454407,
"fcm_dpo/margin": 523.6875,
"fcm_dpo/q_t": 0.3648674488067627,
"grad_norm": 35.1843376159668,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -0.9096598625183105,
"logits/rejected": -0.92439866065979,
"logps/chosen": -622.2601318359375,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -1185.740478515625,
"loss": 0.9752,
"margin_dpo/margin_mean": 523.6875,
"margin_dpo/margin_std": 566.728515625,
"step": 558
},
{
"KL/chosen_KL_mean": -622.984130859375,
"KL/mean": -833.32373046875,
"KL/rejected_KL_mean": -1043.6634521484375,
"KL/std": 492.6358642578125,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.001100032590329647,
"fcm_dpo/delta": -0.0657808855175972,
"fcm_dpo/margin": 420.67926025390625,
"fcm_dpo/q_t": 0.39390745759010315,
"grad_norm": 35.2745475769043,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -1.016822338104248,
"logits/rejected": -1.0179095268249512,
"logps/chosen": -683.7364501953125,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -1137.105712890625,
"loss": 1.0476,
"margin_dpo/margin_mean": 420.6792907714844,
"margin_dpo/margin_std": 506.8309326171875,
"step": 559
},
{
"KL/chosen_KL_mean": -593.2108154296875,
"KL/mean": -745.4317626953125,
"KL/rejected_KL_mean": -897.6527099609375,
"KL/std": 441.3681945800781,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.001101277768611908,
"fcm_dpo/delta": 0.0667891800403595,
"fcm_dpo/margin": 304.4419250488281,
"fcm_dpo/q_t": 0.4226078391075134,
"grad_norm": 28.90320587158203,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -0.8108816146850586,
"logits/rejected": -0.7898497581481934,
"logps/chosen": -651.314697265625,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -977.6439208984375,
"loss": 1.146,
"margin_dpo/margin_mean": 304.4419250488281,
"margin_dpo/margin_std": 474.00811767578125,
"step": 560
},
{
"KL/chosen_KL_mean": -665.6651611328125,
"KL/mean": -807.514404296875,
"KL/rejected_KL_mean": -949.363525390625,
"KL/std": 453.2119140625,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.0011282196501269937,
"fcm_dpo/delta": 0.08202661573886871,
"fcm_dpo/margin": 283.6983642578125,
"fcm_dpo/q_t": 0.4258641004562378,
"grad_norm": 38.57765579223633,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -0.9530788660049438,
"logits/rejected": -0.969383716583252,
"logps/chosen": -716.5780639648438,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -1048.43212890625,
"loss": 1.1834,
"margin_dpo/margin_mean": 283.6983947753906,
"margin_dpo/margin_std": 554.9716796875,
"step": 561
},
{
"KL/chosen_KL_mean": -642.4440307617188,
"KL/mean": -829.6681518554688,
"KL/rejected_KL_mean": -1016.8922119140625,
"KL/std": 508.0831298828125,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.0011184532195329666,
"fcm_dpo/delta": -0.020561814308166504,
"fcm_dpo/margin": 374.4482727050781,
"fcm_dpo/q_t": 0.4057563841342926,
"grad_norm": 35.47517776489258,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -0.8892195224761963,
"logits/rejected": -0.89787757396698,
"logps/chosen": -701.908447265625,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -1113.4349365234375,
"loss": 1.0976,
"margin_dpo/margin_mean": 374.4482421875,
"margin_dpo/margin_std": 549.26806640625,
"step": 562
},
{
"KL/chosen_KL_mean": -701.1400756835938,
"KL/mean": -842.6939697265625,
"KL/rejected_KL_mean": -984.247802734375,
"KL/std": 542.4169921875,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.00114821782335639,
"fcm_dpo/delta": 0.07472698390483856,
"fcm_dpo/margin": 283.1077575683594,
"fcm_dpo/q_t": 0.4208974540233612,
"grad_norm": 55.25715255737305,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -0.901836633682251,
"logits/rejected": -0.9079591035842896,
"logps/chosen": -761.748291015625,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -1078.8155517578125,
"loss": 1.2174,
"margin_dpo/margin_mean": 283.1077575683594,
"margin_dpo/margin_std": 628.2413330078125,
"step": 563
},
{
"KL/chosen_KL_mean": -627.564453125,
"KL/mean": -802.5059814453125,
"KL/rejected_KL_mean": -977.4475708007812,
"KL/std": 454.1446533203125,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.001143982750363648,
"fcm_dpo/delta": -0.0003933161497116089,
"fcm_dpo/margin": 349.8831481933594,
"fcm_dpo/q_t": 0.4085201025009155,
"grad_norm": 34.40708541870117,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -0.9562081098556519,
"logits/rejected": -0.9461531639099121,
"logps/chosen": -701.832763671875,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -1070.685791015625,
"loss": 1.0896,
"margin_dpo/margin_mean": 349.8831481933594,
"margin_dpo/margin_std": 450.19439697265625,
"step": 564
},
{
"KL/chosen_KL_mean": -659.664794921875,
"KL/mean": -855.8405151367188,
"KL/rejected_KL_mean": -1052.0162353515625,
"KL/std": 532.7481079101562,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.0011324284132570028,
"fcm_dpo/delta": -0.046417634934186935,
"fcm_dpo/margin": 392.35150146484375,
"fcm_dpo/q_t": 0.40353649854660034,
"grad_norm": 34.46331024169922,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -0.9307489395141602,
"logits/rejected": -0.9408478736877441,
"logps/chosen": -728.6846923828125,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -1137.795166015625,
"loss": 1.1014,
"margin_dpo/margin_mean": 392.35150146484375,
"margin_dpo/margin_std": 624.1721801757812,
"step": 565
},
{
"KL/chosen_KL_mean": -628.304931640625,
"KL/mean": -842.6285400390625,
"KL/rejected_KL_mean": -1056.9521484375,
"KL/std": 567.1864624023438,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.0011231814278289676,
"fcm_dpo/delta": -0.08581139892339706,
"fcm_dpo/margin": 428.6472473144531,
"fcm_dpo/q_t": 0.39444002509117126,
"grad_norm": 38.54751205444336,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -0.9179797768592834,
"logits/rejected": -0.9470702409744263,
"logps/chosen": -694.8502197265625,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -1160.821533203125,
"loss": 1.0874,
"margin_dpo/margin_mean": 428.64727783203125,
"margin_dpo/margin_std": 652.5841064453125,
"step": 566
},
{
"KL/chosen_KL_mean": -615.6129760742188,
"KL/mean": -772.991943359375,
"KL/rejected_KL_mean": -930.370849609375,
"KL/std": 386.2548828125,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.0011194807011634111,
"fcm_dpo/delta": 0.049378592520952225,
"fcm_dpo/margin": 314.75799560546875,
"fcm_dpo/q_t": 0.4178185760974884,
"grad_norm": 45.093955993652344,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -0.9833869934082031,
"logits/rejected": -0.9983804225921631,
"logps/chosen": -668.4712524414062,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -1015.7418212890625,
"loss": 1.1226,
"margin_dpo/margin_mean": 314.75799560546875,
"margin_dpo/margin_std": 445.0365905761719,
"step": 567
},
{
"KL/chosen_KL_mean": -607.3502197265625,
"KL/mean": -825.4920654296875,
"KL/rejected_KL_mean": -1043.6339111328125,
"KL/std": 465.9037780761719,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.0011085269507020712,
"fcm_dpo/delta": -0.08783602714538574,
"fcm_dpo/margin": 436.28375244140625,
"fcm_dpo/q_t": 0.38784247636795044,
"grad_norm": 32.346004486083984,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -0.8905673027038574,
"logits/rejected": -0.9232343435287476,
"logps/chosen": -652.5426025390625,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -1132.726318359375,
"loss": 1.0244,
"margin_dpo/margin_mean": 436.2837219238281,
"margin_dpo/margin_std": 477.65777587890625,
"step": 568
},
{
"KL/chosen_KL_mean": -678.6146850585938,
"KL/mean": -856.0438232421875,
"KL/rejected_KL_mean": -1033.472900390625,
"KL/std": 577.4537353515625,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.0011109868064522743,
"fcm_dpo/delta": 0.005078420042991638,
"fcm_dpo/margin": 354.8581237792969,
"fcm_dpo/q_t": 0.4105854630470276,
"grad_norm": 57.44594955444336,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -0.9483610391616821,
"logits/rejected": -0.9578467011451721,
"logps/chosen": -742.3352661132812,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -1112.576171875,
"loss": 1.1429,
"margin_dpo/margin_mean": 354.8581237792969,
"margin_dpo/margin_std": 614.8703002929688,
"step": 569
},
{
"KL/chosen_KL_mean": -638.6702880859375,
"KL/mean": -804.5684814453125,
"KL/rejected_KL_mean": -970.466552734375,
"KL/std": 463.08331298828125,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.0011088717728853226,
"fcm_dpo/delta": 0.03331791236996651,
"fcm_dpo/margin": 331.7962646484375,
"fcm_dpo/q_t": 0.41667434573173523,
"grad_norm": 29.472156524658203,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -1.0172860622406006,
"logits/rejected": -1.0165293216705322,
"logps/chosen": -700.2848510742188,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -1052.6083984375,
"loss": 1.1261,
"margin_dpo/margin_mean": 331.7962646484375,
"margin_dpo/margin_std": 504.40728759765625,
"step": 570
},
{
"KL/chosen_KL_mean": -635.20947265625,
"KL/mean": -841.0794067382812,
"KL/rejected_KL_mean": -1046.949462890625,
"KL/std": 483.367431640625,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.0010986864799633622,
"fcm_dpo/delta": -0.05524833872914314,
"fcm_dpo/margin": 411.7398376464844,
"fcm_dpo/q_t": 0.3991958796977997,
"grad_norm": 33.479488372802734,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -0.9390131235122681,
"logits/rejected": -0.9548516273498535,
"logps/chosen": -688.2635498046875,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -1138.2861328125,
"loss": 1.0845,
"margin_dpo/margin_mean": 411.7398681640625,
"margin_dpo/margin_std": 602.9391479492188,
"step": 571
},
{
"KL/chosen_KL_mean": -681.075439453125,
"KL/mean": -884.181884765625,
"KL/rejected_KL_mean": -1087.2882080078125,
"KL/std": 541.6312255859375,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.0010886520612984896,
"fcm_dpo/delta": -0.04448999464511871,
"fcm_dpo/margin": 406.21282958984375,
"fcm_dpo/q_t": 0.40421679615974426,
"grad_norm": 34.047908782958984,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -0.9380755424499512,
"logits/rejected": -0.9658199548721313,
"logps/chosen": -729.5347290039062,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -1170.84521484375,
"loss": 1.1046,
"margin_dpo/margin_mean": 406.21282958984375,
"margin_dpo/margin_std": 657.9480590820312,
"step": 572
},
{
"KL/chosen_KL_mean": -669.11083984375,
"KL/mean": -855.2178955078125,
"KL/rejected_KL_mean": -1041.324951171875,
"KL/std": 513.5198974609375,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.001089778495952487,
"fcm_dpo/delta": -0.0058921948075294495,
"fcm_dpo/margin": 372.2140808105469,
"fcm_dpo/q_t": 0.40893957018852234,
"grad_norm": 29.2487850189209,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -0.9847406148910522,
"logits/rejected": -0.9911180734634399,
"logps/chosen": -720.7334594726562,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -1126.64990234375,
"loss": 1.0893,
"margin_dpo/margin_mean": 372.2140808105469,
"margin_dpo/margin_std": 510.26092529296875,
"step": 573
},
{
"KL/chosen_KL_mean": -611.013916015625,
"KL/mean": -807.5015869140625,
"KL/rejected_KL_mean": -1003.9892578125,
"KL/std": 465.5011901855469,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.0010860915062949061,
"fcm_dpo/delta": -0.028046101331710815,
"fcm_dpo/margin": 392.9753112792969,
"fcm_dpo/q_t": 0.4033794403076172,
"grad_norm": 43.88747787475586,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -0.9889034032821655,
"logits/rejected": -1.0169628858566284,
"logps/chosen": -662.0584106445312,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -1096.795654296875,
"loss": 1.0841,
"margin_dpo/margin_mean": 392.97528076171875,
"margin_dpo/margin_std": 539.8909912109375,
"step": 574
},
{
"KL/chosen_KL_mean": -683.813232421875,
"KL/mean": -864.3245849609375,
"KL/rejected_KL_mean": -1044.8359375,
"KL/std": 464.4603271484375,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.0010744205210357904,
"fcm_dpo/delta": 0.0111289843916893,
"fcm_dpo/margin": 361.0226745605469,
"fcm_dpo/q_t": 0.41186636686325073,
"grad_norm": 33.57511901855469,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -0.8667001724243164,
"logits/rejected": -0.8740522265434265,
"logps/chosen": -755.6033935546875,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -1140.22216796875,
"loss": 1.1383,
"margin_dpo/margin_mean": 361.0226745605469,
"margin_dpo/margin_std": 593.4249267578125,
"step": 575
},
{
"KL/chosen_KL_mean": -668.3992919921875,
"KL/mean": -807.94384765625,
"KL/rejected_KL_mean": -947.4883422851562,
"KL/std": 447.33709716796875,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.0010960063664242625,
"fcm_dpo/delta": 0.09715408086776733,
"fcm_dpo/margin": 279.0890808105469,
"fcm_dpo/q_t": 0.430780291557312,
"grad_norm": 42.60184097290039,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -0.979659914970398,
"logits/rejected": -1.0043472051620483,
"logps/chosen": -722.6622314453125,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -1048.24267578125,
"loss": 1.1827,
"margin_dpo/margin_mean": 279.0890808105469,
"margin_dpo/margin_std": 521.076416015625,
"step": 576
},
{
"KL/chosen_KL_mean": -573.982666015625,
"KL/mean": -754.96923828125,
"KL/rejected_KL_mean": -935.955810546875,
"KL/std": 477.551025390625,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.0011036575306206942,
"fcm_dpo/delta": 0.0004506707191467285,
"fcm_dpo/margin": 361.97320556640625,
"fcm_dpo/q_t": 0.4099828004837036,
"grad_norm": 30.019325256347656,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -0.9628755450248718,
"logits/rejected": -0.957021951675415,
"logps/chosen": -635.892333984375,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -1020.0265502929688,
"loss": 1.1096,
"margin_dpo/margin_mean": 361.97320556640625,
"margin_dpo/margin_std": 514.8892211914062,
"step": 577
},
{
"KL/chosen_KL_mean": -574.492431640625,
"KL/mean": -777.5902099609375,
"KL/rejected_KL_mean": -980.6880493164062,
"KL/std": 478.97015380859375,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.001092255231924355,
"fcm_dpo/delta": -0.04655010625720024,
"fcm_dpo/margin": 406.195556640625,
"fcm_dpo/q_t": 0.3978912830352783,
"grad_norm": 43.33347702026367,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -0.8652215003967285,
"logits/rejected": -0.8970048427581787,
"logps/chosen": -623.7561645507812,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -1064.124267578125,
"loss": 1.0536,
"margin_dpo/margin_mean": 406.1955261230469,
"margin_dpo/margin_std": 466.63641357421875,
"step": 578
},
{
"KL/chosen_KL_mean": -647.93896484375,
"KL/mean": -775.7979125976562,
"KL/rejected_KL_mean": -903.6568603515625,
"KL/std": 477.6488037109375,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.0011125376913696527,
"fcm_dpo/delta": 0.11905627697706223,
"fcm_dpo/margin": 255.71783447265625,
"fcm_dpo/q_t": 0.43392279744148254,
"grad_norm": 68.53885650634766,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -0.9105473756790161,
"logits/rejected": -0.9102990031242371,
"logps/chosen": -706.8248291015625,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -998.4444580078125,
"loss": 1.2003,
"margin_dpo/margin_mean": 255.7178192138672,
"margin_dpo/margin_std": 512.1516723632812,
"step": 579
},
{
"KL/chosen_KL_mean": -494.50726318359375,
"KL/mean": -724.7994384765625,
"KL/rejected_KL_mean": -955.0916137695312,
"KL/std": 545.931640625,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.0010995608754456043,
"fcm_dpo/delta": -0.11274391412734985,
"fcm_dpo/margin": 460.5844421386719,
"fcm_dpo/q_t": 0.3904712200164795,
"grad_norm": 38.69807052612305,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -0.8189841508865356,
"logits/rejected": -0.8483502268791199,
"logps/chosen": -543.2140502929688,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -1036.85009765625,
"loss": 1.043,
"margin_dpo/margin_mean": 460.5844421386719,
"margin_dpo/margin_std": 635.3092041015625,
"step": 580
},
{
"KL/chosen_KL_mean": -643.0278930664062,
"KL/mean": -802.6070556640625,
"KL/rejected_KL_mean": -962.1861572265625,
"KL/std": 479.75579833984375,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.001105679664760828,
"fcm_dpo/delta": 0.048796892166137695,
"fcm_dpo/margin": 319.1583557128906,
"fcm_dpo/q_t": 0.4213637113571167,
"grad_norm": 30.005510330200195,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -0.9855085611343384,
"logits/rejected": -0.9959384202957153,
"logps/chosen": -705.250244140625,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -1053.921875,
"loss": 1.1458,
"margin_dpo/margin_mean": 319.15838623046875,
"margin_dpo/margin_std": 545.41748046875,
"step": 581
},
{
"KL/chosen_KL_mean": -564.5281982421875,
"KL/mean": -749.2686157226562,
"KL/rejected_KL_mean": -934.009033203125,
"KL/std": 440.322021484375,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.0011159980203956366,
"fcm_dpo/delta": -0.014601297676563263,
"fcm_dpo/margin": 369.480712890625,
"fcm_dpo/q_t": 0.4044383764266968,
"grad_norm": 39.41720962524414,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -0.955707848072052,
"logits/rejected": -0.9739981889724731,
"logps/chosen": -622.7568969726562,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -1044.07861328125,
"loss": 1.093,
"margin_dpo/margin_mean": 369.480712890625,
"margin_dpo/margin_std": 481.0858154296875,
"step": 582
},
{
"KL/chosen_KL_mean": -668.479248046875,
"KL/mean": -783.5263671875,
"KL/rejected_KL_mean": -898.5736083984375,
"KL/std": 474.0484619140625,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.0011011306196451187,
"fcm_dpo/delta": -0.015219791792333126,
"fcm_dpo/margin": 230.09426879882812,
"fcm_dpo/q_t": 0.44363144040107727,
"grad_norm": 43.34550857543945,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -0.9127052426338196,
"logits/rejected": -0.9093770980834961,
"logps/chosen": -725.3421630859375,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -986.9774780273438,
"loss": 1.25,
"margin_dpo/margin_mean": 230.09423828125,
"margin_dpo/margin_std": 576.1717529296875,
"step": 583
},
{
"KL/chosen_KL_mean": -553.2639770507812,
"KL/mean": -785.3903198242188,
"KL/rejected_KL_mean": -1017.5167236328125,
"KL/std": 505.50018310546875,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.0010839344467967749,
"fcm_dpo/delta": -0.10864884406328201,
"fcm_dpo/margin": 464.25274658203125,
"fcm_dpo/q_t": 0.38627010583877563,
"grad_norm": 28.635910034179688,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -0.924347996711731,
"logits/rejected": -0.9356412887573242,
"logps/chosen": -610.1646728515625,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -1115.15283203125,
"loss": 1.0195,
"margin_dpo/margin_mean": 464.25274658203125,
"margin_dpo/margin_std": 526.7293090820312,
"step": 584
},
{
"KL/chosen_KL_mean": -675.3355712890625,
"KL/mean": -866.5830078125,
"KL/rejected_KL_mean": -1057.83056640625,
"KL/std": 549.9703369140625,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.0010719253914430737,
"fcm_dpo/delta": -0.010730547830462456,
"fcm_dpo/margin": 382.4949645996094,
"fcm_dpo/q_t": 0.4081985652446747,
"grad_norm": 31.741275787353516,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -0.964898943901062,
"logits/rejected": -0.9843629002571106,
"logps/chosen": -736.3095703125,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -1142.0,
"loss": 1.1303,
"margin_dpo/margin_mean": 382.4949645996094,
"margin_dpo/margin_std": 656.9327392578125,
"step": 585
},
{
"KL/chosen_KL_mean": -676.3819580078125,
"KL/mean": -811.5240478515625,
"KL/rejected_KL_mean": -946.6661376953125,
"KL/std": 476.75653076171875,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.0010906466050073504,
"fcm_dpo/delta": 0.10854475200176239,
"fcm_dpo/margin": 270.2841796875,
"fcm_dpo/q_t": 0.43295344710350037,
"grad_norm": 32.98674774169922,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -0.9334938526153564,
"logits/rejected": -0.9494249820709229,
"logps/chosen": -733.237548828125,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -1038.46875,
"loss": 1.1953,
"margin_dpo/margin_mean": 270.2841796875,
"margin_dpo/margin_std": 539.8758544921875,
"step": 586
},
{
"KL/chosen_KL_mean": -490.77911376953125,
"KL/mean": -676.01806640625,
"KL/rejected_KL_mean": -861.257080078125,
"KL/std": 499.88653564453125,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.0011002285173162818,
"fcm_dpo/delta": -0.008045274764299393,
"fcm_dpo/margin": 370.47796630859375,
"fcm_dpo/q_t": 0.40609079599380493,
"grad_norm": 28.055667877197266,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -0.9820740222930908,
"logits/rejected": -1.0075342655181885,
"logps/chosen": -535.470703125,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -943.8809204101562,
"loss": 1.0937,
"margin_dpo/margin_mean": 370.47796630859375,
"margin_dpo/margin_std": 512.2026977539062,
"step": 587
},
{
"KL/chosen_KL_mean": -632.4581298828125,
"KL/mean": -804.46728515625,
"KL/rejected_KL_mean": -976.4764404296875,
"KL/std": 447.35491943359375,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.0010953000746667385,
"fcm_dpo/delta": 0.02369820699095726,
"fcm_dpo/margin": 344.018310546875,
"fcm_dpo/q_t": 0.4127538204193115,
"grad_norm": 31.495437622070312,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -0.9647231101989746,
"logits/rejected": -0.9858247637748718,
"logps/chosen": -682.7531127929688,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -1083.846435546875,
"loss": 1.1092,
"margin_dpo/margin_mean": 344.018310546875,
"margin_dpo/margin_std": 468.7295227050781,
"step": 588
},
{
"KL/chosen_KL_mean": -627.8831787109375,
"KL/mean": -838.1376953125,
"KL/rejected_KL_mean": -1048.392333984375,
"KL/std": 478.62567138671875,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.0010934111196547747,
"fcm_dpo/delta": -0.06262395530939102,
"fcm_dpo/margin": 420.50909423828125,
"fcm_dpo/q_t": 0.3944852352142334,
"grad_norm": 33.55838394165039,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -0.92295241355896,
"logits/rejected": -0.9320027828216553,
"logps/chosen": -687.8131103515625,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -1160.047607421875,
"loss": 1.0472,
"margin_dpo/margin_mean": 420.50909423828125,
"margin_dpo/margin_std": 499.3512878417969,
"step": 589
},
{
"KL/chosen_KL_mean": -557.1012573242188,
"KL/mean": -763.3665771484375,
"KL/rejected_KL_mean": -969.6317749023438,
"KL/std": 488.5511474609375,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.0010802859906107187,
"fcm_dpo/delta": -0.04780504107475281,
"fcm_dpo/margin": 412.5305480957031,
"fcm_dpo/q_t": 0.39945223927497864,
"grad_norm": 30.184764862060547,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -0.905895471572876,
"logits/rejected": -0.9255493879318237,
"logps/chosen": -612.9110717773438,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -1075.694580078125,
"loss": 1.0646,
"margin_dpo/margin_mean": 412.5305480957031,
"margin_dpo/margin_std": 534.6390380859375,
"step": 590
},
{
"KL/chosen_KL_mean": -611.14794921875,
"KL/mean": -795.2865600585938,
"KL/rejected_KL_mean": -979.4251708984375,
"KL/std": 462.18768310546875,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.0010774084366858006,
"fcm_dpo/delta": 0.0033245673403143883,
"fcm_dpo/margin": 368.2772216796875,
"fcm_dpo/q_t": 0.4083741009235382,
"grad_norm": 42.21658706665039,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -0.9570738673210144,
"logits/rejected": -0.9459799528121948,
"logps/chosen": -667.3885498046875,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -1063.21142578125,
"loss": 1.1034,
"margin_dpo/margin_mean": 368.2772216796875,
"margin_dpo/margin_std": 524.36181640625,
"step": 591
},
{
"KL/chosen_KL_mean": -624.6557006835938,
"KL/mean": -784.8875732421875,
"KL/rejected_KL_mean": -945.1195068359375,
"KL/std": 459.7552490234375,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.0010868444805964828,
"fcm_dpo/delta": 0.05359634757041931,
"fcm_dpo/margin": 320.46380615234375,
"fcm_dpo/q_t": 0.42158329486846924,
"grad_norm": 32.69625473022461,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -0.9273663759231567,
"logits/rejected": -0.9230743646621704,
"logps/chosen": -672.595947265625,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -1020.8524169921875,
"loss": 1.18,
"margin_dpo/margin_mean": 320.46380615234375,
"margin_dpo/margin_std": 626.68798828125,
"step": 592
},
{
"KL/chosen_KL_mean": -588.4630737304688,
"KL/mean": -720.7249755859375,
"KL/rejected_KL_mean": -852.9869995117188,
"KL/std": 485.0901794433594,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.0011051694164052606,
"fcm_dpo/delta": 0.11104539781808853,
"fcm_dpo/margin": 264.52392578125,
"fcm_dpo/q_t": 0.4346621036529541,
"grad_norm": 51.25044250488281,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -0.8214367628097534,
"logits/rejected": -0.8108229637145996,
"logps/chosen": -637.15380859375,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -913.89501953125,
"loss": 1.2031,
"margin_dpo/margin_mean": 264.52392578125,
"margin_dpo/margin_std": 547.176513671875,
"step": 593
},
{
"KL/chosen_KL_mean": -561.3931884765625,
"KL/mean": -733.5811767578125,
"KL/rejected_KL_mean": -905.7691650390625,
"KL/std": 492.49456787109375,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.001125779584981501,
"fcm_dpo/delta": 0.011429011821746826,
"fcm_dpo/margin": 344.37591552734375,
"fcm_dpo/q_t": 0.41177088022232056,
"grad_norm": 41.104610443115234,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -0.9229486584663391,
"logits/rejected": -0.912185788154602,
"logps/chosen": -616.3280639648438,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -991.8687744140625,
"loss": 1.1215,
"margin_dpo/margin_mean": 344.37591552734375,
"margin_dpo/margin_std": 514.7716674804688,
"step": 594
},
{
"KL/chosen_KL_mean": -565.7555541992188,
"KL/mean": -749.776123046875,
"KL/rejected_KL_mean": -933.7967529296875,
"KL/std": 474.48052978515625,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.0011134643573313951,
"fcm_dpo/delta": -0.010329552926123142,
"fcm_dpo/margin": 368.0412292480469,
"fcm_dpo/q_t": 0.40979164838790894,
"grad_norm": 36.07206344604492,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -0.9135459065437317,
"logits/rejected": -0.9275071620941162,
"logps/chosen": -615.1759643554688,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -1014.424072265625,
"loss": 1.0946,
"margin_dpo/margin_mean": 368.0412292480469,
"margin_dpo/margin_std": 528.9100341796875,
"step": 595
},
{
"KL/chosen_KL_mean": -627.2183837890625,
"KL/mean": -771.135498046875,
"KL/rejected_KL_mean": -915.052490234375,
"KL/std": 470.12677001953125,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.0011297892779111862,
"fcm_dpo/delta": 0.07716254889965057,
"fcm_dpo/margin": 287.83416748046875,
"fcm_dpo/q_t": 0.42858174443244934,
"grad_norm": 44.115386962890625,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -0.9362180233001709,
"logits/rejected": -0.9097167253494263,
"logps/chosen": -687.0100708007812,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -995.1436767578125,
"loss": 1.1769,
"margin_dpo/margin_mean": 287.83416748046875,
"margin_dpo/margin_std": 555.0716552734375,
"step": 596
},
{
"KL/chosen_KL_mean": -596.168701171875,
"KL/mean": -822.4722900390625,
"KL/rejected_KL_mean": -1048.77587890625,
"KL/std": 528.4112548828125,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.0011037664953619242,
"fcm_dpo/delta": -0.1077902540564537,
"fcm_dpo/margin": 452.60723876953125,
"fcm_dpo/q_t": 0.38863760232925415,
"grad_norm": 33.63905334472656,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -0.8626628518104553,
"logits/rejected": -0.9061563611030579,
"logps/chosen": -653.429443359375,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -1149.4697265625,
"loss": 1.0451,
"margin_dpo/margin_mean": 452.60723876953125,
"margin_dpo/margin_std": 572.915771484375,
"step": 597
},
{
"KL/chosen_KL_mean": -605.6923828125,
"KL/mean": -782.9190673828125,
"KL/rejected_KL_mean": -960.1458740234375,
"KL/std": 463.83331298828125,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.0011007774155586958,
"fcm_dpo/delta": 0.008908487856388092,
"fcm_dpo/margin": 354.4535827636719,
"fcm_dpo/q_t": 0.4098372757434845,
"grad_norm": 47.247886657714844,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -0.9317420721054077,
"logits/rejected": -0.9370558261871338,
"logps/chosen": -658.2108154296875,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -1049.5897216796875,
"loss": 1.1114,
"margin_dpo/margin_mean": 354.4535827636719,
"margin_dpo/margin_std": 501.76495361328125,
"step": 598
},
{
"KL/chosen_KL_mean": -596.0595703125,
"KL/mean": -746.37451171875,
"KL/rejected_KL_mean": -896.6893920898438,
"KL/std": 440.1773681640625,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.0011121116112917662,
"fcm_dpo/delta": 0.06703872978687286,
"fcm_dpo/margin": 300.6299133300781,
"fcm_dpo/q_t": 0.4199250042438507,
"grad_norm": 27.749969482421875,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -0.8937709331512451,
"logits/rejected": -0.899101972579956,
"logps/chosen": -645.8621826171875,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -979.6679077148438,
"loss": 1.1383,
"margin_dpo/margin_mean": 300.6298828125,
"margin_dpo/margin_std": 414.0669250488281,
"step": 599
},
{
"KL/chosen_KL_mean": -645.6766967773438,
"KL/mean": -785.5516357421875,
"KL/rejected_KL_mean": -925.4264526367188,
"KL/std": 447.0907897949219,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.0011385063407942653,
"fcm_dpo/delta": 0.08424638211727142,
"fcm_dpo/margin": 279.7497863769531,
"fcm_dpo/q_t": 0.42901042103767395,
"grad_norm": 34.18629837036133,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -0.9779192209243774,
"logits/rejected": -0.9724453687667847,
"logps/chosen": -712.111572265625,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -1010.8829345703125,
"loss": 1.177,
"margin_dpo/margin_mean": 279.74981689453125,
"margin_dpo/margin_std": 529.72802734375,
"step": 600
},
{
"KL/chosen_KL_mean": -664.976806640625,
"KL/mean": -846.01318359375,
"KL/rejected_KL_mean": -1027.0494384765625,
"KL/std": 486.04486083984375,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.0011400362709537148,
"fcm_dpo/delta": -0.01342380978167057,
"fcm_dpo/margin": 362.07269287109375,
"fcm_dpo/q_t": 0.40398281812667847,
"grad_norm": 33.51792907714844,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -0.9247018098831177,
"logits/rejected": -0.9378587603569031,
"logps/chosen": -724.1104736328125,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -1121.740478515625,
"loss": 1.0822,
"margin_dpo/margin_mean": 362.07269287109375,
"margin_dpo/margin_std": 467.5895690917969,
"step": 601
},
{
"KL/chosen_KL_mean": -456.95404052734375,
"KL/mean": -683.4141845703125,
"KL/rejected_KL_mean": -909.8743896484375,
"KL/std": 451.41448974609375,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.0011287988163530827,
"fcm_dpo/delta": -0.11732130497694016,
"fcm_dpo/margin": 452.92034912109375,
"fcm_dpo/q_t": 0.3834123909473419,
"grad_norm": 54.98399353027344,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -0.9682570695877075,
"logits/rejected": -0.9984170198440552,
"logps/chosen": -505.54754638671875,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -997.54296875,
"loss": 1.0084,
"margin_dpo/margin_mean": 452.92034912109375,
"margin_dpo/margin_std": 480.7337646484375,
"step": 602
},
{
"KL/chosen_KL_mean": -533.0413818359375,
"KL/mean": -735.8339233398438,
"KL/rejected_KL_mean": -938.6265258789062,
"KL/std": 480.28125,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.0011060773395001888,
"fcm_dpo/delta": -0.05091490596532822,
"fcm_dpo/margin": 405.5850830078125,
"fcm_dpo/q_t": 0.400734007358551,
"grad_norm": 38.465694427490234,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -0.9273467063903809,
"logits/rejected": -0.9231489896774292,
"logps/chosen": -603.4559936523438,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -1038.9521484375,
"loss": 1.0805,
"margin_dpo/margin_mean": 405.5850830078125,
"margin_dpo/margin_std": 583.6143798828125,
"step": 603
},
{
"KL/chosen_KL_mean": -568.7514038085938,
"KL/mean": -793.9749755859375,
"KL/rejected_KL_mean": -1019.198486328125,
"KL/std": 485.3682861328125,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.0010875534499064088,
"fcm_dpo/delta": -0.09458193182945251,
"fcm_dpo/margin": 450.44708251953125,
"fcm_dpo/q_t": 0.3903900384902954,
"grad_norm": 36.59914016723633,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -0.905841052532196,
"logits/rejected": -0.9343771934509277,
"logps/chosen": -615.20947265625,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -1111.052978515625,
"loss": 1.0455,
"margin_dpo/margin_mean": 450.44708251953125,
"margin_dpo/margin_std": 581.6656494140625,
"step": 604
},
{
"KL/chosen_KL_mean": -586.8192749023438,
"KL/mean": -758.067626953125,
"KL/rejected_KL_mean": -929.31591796875,
"KL/std": 455.83184814453125,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.0010850478429347277,
"fcm_dpo/delta": 0.029418762773275375,
"fcm_dpo/margin": 342.49664306640625,
"fcm_dpo/q_t": 0.41720741987228394,
"grad_norm": 39.87853240966797,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -0.8960555791854858,
"logits/rejected": -0.910882294178009,
"logps/chosen": -653.068603515625,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -1031.620849609375,
"loss": 1.1268,
"margin_dpo/margin_mean": 342.49664306640625,
"margin_dpo/margin_std": 546.2432861328125,
"step": 605
},
{
"KL/chosen_KL_mean": -597.602783203125,
"KL/mean": -785.0491333007812,
"KL/rejected_KL_mean": -972.4955444335938,
"KL/std": 471.9700012207031,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.0010922958608716726,
"fcm_dpo/delta": -0.01021907851099968,
"fcm_dpo/margin": 374.89276123046875,
"fcm_dpo/q_t": 0.40733814239501953,
"grad_norm": 32.22350311279297,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -0.9396861791610718,
"logits/rejected": -0.9551488161087036,
"logps/chosen": -652.421875,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -1070.866943359375,
"loss": 1.0912,
"margin_dpo/margin_mean": 374.8927917480469,
"margin_dpo/margin_std": 513.25732421875,
"step": 606
},
{
"KL/chosen_KL_mean": -603.7939453125,
"KL/mean": -754.0906982421875,
"KL/rejected_KL_mean": -904.387451171875,
"KL/std": 452.41448974609375,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.0011022321414202452,
"fcm_dpo/delta": 0.07058005034923553,
"fcm_dpo/margin": 300.59344482421875,
"fcm_dpo/q_t": 0.42460712790489197,
"grad_norm": 27.491031646728516,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -0.8635603785514832,
"logits/rejected": -0.8497999310493469,
"logps/chosen": -661.8780517578125,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -984.16455078125,
"loss": 1.151,
"margin_dpo/margin_mean": 300.5934143066406,
"margin_dpo/margin_std": 491.64447021484375,
"step": 607
},
{
"KL/chosen_KL_mean": -552.9517211914062,
"KL/mean": -731.7744750976562,
"KL/rejected_KL_mean": -910.5972900390625,
"KL/std": 413.10968017578125,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.0011053085327148438,
"fcm_dpo/delta": 0.0048129381611943245,
"fcm_dpo/margin": 357.64556884765625,
"fcm_dpo/q_t": 0.40791454911231995,
"grad_norm": 29.21363639831543,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -0.9508018493652344,
"logits/rejected": -0.9595135450363159,
"logps/chosen": -610.402587890625,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -1005.3706665039062,
"loss": 1.081,
"margin_dpo/margin_mean": 357.64556884765625,
"margin_dpo/margin_std": 424.187255859375,
"step": 608
},
{
"KL/chosen_KL_mean": -582.7489013671875,
"KL/mean": -807.50830078125,
"KL/rejected_KL_mean": -1032.267822265625,
"KL/std": 577.1176147460938,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.001089900266379118,
"fcm_dpo/delta": -0.09446786344051361,
"fcm_dpo/margin": 449.5188293457031,
"fcm_dpo/q_t": 0.3957676887512207,
"grad_norm": 31.981115341186523,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -1.0054192543029785,
"logits/rejected": -1.0256993770599365,
"logps/chosen": -641.5543212890625,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -1121.083740234375,
"loss": 1.0637,
"margin_dpo/margin_mean": 449.5188293457031,
"margin_dpo/margin_std": 674.6402587890625,
"step": 609
},
{
"KL/chosen_KL_mean": -577.7659912109375,
"KL/mean": -720.6400146484375,
"KL/rejected_KL_mean": -863.5140380859375,
"KL/std": 434.3399658203125,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.0010983757674694061,
"fcm_dpo/delta": 0.08884203433990479,
"fcm_dpo/margin": 285.748046875,
"fcm_dpo/q_t": 0.42750847339630127,
"grad_norm": 40.23204040527344,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -0.9262121319770813,
"logits/rejected": -0.904991626739502,
"logps/chosen": -643.4610595703125,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -946.91943359375,
"loss": 1.1661,
"margin_dpo/margin_mean": 285.7480773925781,
"margin_dpo/margin_std": 495.6571044921875,
"step": 610
},
{
"KL/chosen_KL_mean": -595.0731811523438,
"KL/mean": -832.9767456054688,
"KL/rejected_KL_mean": -1070.88037109375,
"KL/std": 559.13427734375,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.0010830024257302284,
"fcm_dpo/delta": -0.12153010070323944,
"fcm_dpo/margin": 475.8071594238281,
"fcm_dpo/q_t": 0.38699936866760254,
"grad_norm": 34.51700973510742,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -0.9600176215171814,
"logits/rejected": -1.0014249086380005,
"logps/chosen": -647.672607421875,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -1157.2113037109375,
"loss": 1.0378,
"margin_dpo/margin_mean": 475.8071594238281,
"margin_dpo/margin_std": 629.8643188476562,
"step": 611
},
{
"KL/chosen_KL_mean": -638.7127075195312,
"KL/mean": -837.2454223632812,
"KL/rejected_KL_mean": -1035.778076171875,
"KL/std": 478.2944030761719,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.0010748032946139574,
"fcm_dpo/delta": -0.028072573244571686,
"fcm_dpo/margin": 397.0654296875,
"fcm_dpo/q_t": 0.4017370641231537,
"grad_norm": 29.581735610961914,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -0.9273912310600281,
"logits/rejected": -0.9367831945419312,
"logps/chosen": -698.0364379882812,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -1124.0904541015625,
"loss": 1.072,
"margin_dpo/margin_mean": 397.0653991699219,
"margin_dpo/margin_std": 500.4986572265625,
"step": 612
},
{
"KL/chosen_KL_mean": -573.99462890625,
"KL/mean": -799.8018798828125,
"KL/rejected_KL_mean": -1025.609130859375,
"KL/std": 517.7454833984375,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.00105574750341475,
"fcm_dpo/delta": -0.0807756707072258,
"fcm_dpo/margin": 451.61456298828125,
"fcm_dpo/q_t": 0.3911280035972595,
"grad_norm": 31.94742774963379,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -0.9322279691696167,
"logits/rejected": -0.983695387840271,
"logps/chosen": -633.724609375,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -1130.7166748046875,
"loss": 1.0376,
"margin_dpo/margin_mean": 451.6145324707031,
"margin_dpo/margin_std": 535.5646362304688,
"step": 613
},
{
"KL/chosen_KL_mean": -613.453369140625,
"KL/mean": -825.698974609375,
"KL/rejected_KL_mean": -1037.944580078125,
"KL/std": 496.1231689453125,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.0010477991309016943,
"fcm_dpo/delta": -0.04686400666832924,
"fcm_dpo/margin": 424.4912414550781,
"fcm_dpo/q_t": 0.39960259199142456,
"grad_norm": 43.36309814453125,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -0.8932386636734009,
"logits/rejected": -0.9586418271064758,
"logps/chosen": -666.392333984375,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -1142.6240234375,
"loss": 1.0767,
"margin_dpo/margin_mean": 424.4912414550781,
"margin_dpo/margin_std": 584.0821533203125,
"step": 614
},
{
"KL/chosen_KL_mean": -608.7545166015625,
"KL/mean": -797.580078125,
"KL/rejected_KL_mean": -986.405517578125,
"KL/std": 550.36865234375,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.001047707162797451,
"fcm_dpo/delta": 0.0041780248284339905,
"fcm_dpo/margin": 377.65106201171875,
"fcm_dpo/q_t": 0.41096800565719604,
"grad_norm": 25.20032501220703,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -0.9183000326156616,
"logits/rejected": -0.9365602731704712,
"logps/chosen": -674.5718383789062,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -1081.5830078125,
"loss": 1.1209,
"margin_dpo/margin_mean": 377.65106201171875,
"margin_dpo/margin_std": 608.4381713867188,
"step": 615
},
{
"KL/chosen_KL_mean": -699.6544189453125,
"KL/mean": -851.59375,
"KL/rejected_KL_mean": -1003.532958984375,
"KL/std": 449.58392333984375,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.001057529472745955,
"fcm_dpo/delta": 0.08115516602993011,
"fcm_dpo/margin": 303.8785400390625,
"fcm_dpo/q_t": 0.42706164717674255,
"grad_norm": 28.30909538269043,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -0.9700459241867065,
"logits/rejected": -0.956931471824646,
"logps/chosen": -764.7872924804688,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -1078.2335205078125,
"loss": 1.1614,
"margin_dpo/margin_mean": 303.8785400390625,
"margin_dpo/margin_std": 516.1566772460938,
"step": 616
},
{
"KL/chosen_KL_mean": -680.9326171875,
"KL/mean": -784.277587890625,
"KL/rejected_KL_mean": -887.62255859375,
"KL/std": 428.1174621582031,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.0010916695464402437,
"fcm_dpo/delta": 0.1779545247554779,
"fcm_dpo/margin": 206.68988037109375,
"fcm_dpo/q_t": 0.44872158765792847,
"grad_norm": 50.34171676635742,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -0.8789099454879761,
"logits/rejected": -0.8473949432373047,
"logps/chosen": -743.9381713867188,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -951.856689453125,
"loss": 1.2433,
"margin_dpo/margin_mean": 206.6898956298828,
"margin_dpo/margin_std": 494.8637390136719,
"step": 617
},
{
"KL/chosen_KL_mean": -682.5164184570312,
"KL/mean": -914.16064453125,
"KL/rejected_KL_mean": -1145.804931640625,
"KL/std": 565.1162719726562,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.0010877924505621195,
"fcm_dpo/delta": -0.10951869189739227,
"fcm_dpo/margin": 463.28851318359375,
"fcm_dpo/q_t": 0.39071911573410034,
"grad_norm": 43.230098724365234,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -0.9272968769073486,
"logits/rejected": -0.9583449363708496,
"logps/chosen": -749.6177978515625,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -1237.9583740234375,
"loss": 1.0792,
"margin_dpo/margin_mean": 463.2885437011719,
"margin_dpo/margin_std": 716.175048828125,
"step": 618
},
{
"KL/chosen_KL_mean": -674.689697265625,
"KL/mean": -854.6988525390625,
"KL/rejected_KL_mean": -1034.7080078125,
"KL/std": 527.9037475585938,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.0010771185625344515,
"fcm_dpo/delta": 0.012712216936051846,
"fcm_dpo/margin": 360.01837158203125,
"fcm_dpo/q_t": 0.4185749888420105,
"grad_norm": 38.901329040527344,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -0.9141817092895508,
"logits/rejected": -0.9271951913833618,
"logps/chosen": -730.6679077148438,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -1127.8935546875,
"loss": 1.1562,
"margin_dpo/margin_mean": 360.01837158203125,
"margin_dpo/margin_std": 684.8106079101562,
"step": 619
},
{
"KL/chosen_KL_mean": -612.038330078125,
"KL/mean": -780.5623779296875,
"KL/rejected_KL_mean": -949.08642578125,
"KL/std": 464.1794128417969,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.0010800679447129369,
"fcm_dpo/delta": 0.03716272860765457,
"fcm_dpo/margin": 337.04815673828125,
"fcm_dpo/q_t": 0.4180569648742676,
"grad_norm": 30.291711807250977,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -0.9410725831985474,
"logits/rejected": -0.9461079835891724,
"logps/chosen": -671.8358154296875,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -1027.4971923828125,
"loss": 1.1314,
"margin_dpo/margin_mean": 337.04815673828125,
"margin_dpo/margin_std": 529.4715576171875,
"step": 620
},
{
"KL/chosen_KL_mean": -583.8446655273438,
"KL/mean": -841.3868408203125,
"KL/rejected_KL_mean": -1098.928955078125,
"KL/std": 564.9022827148438,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.0010678130201995373,
"fcm_dpo/delta": -0.15856234729290009,
"fcm_dpo/margin": 515.0842895507812,
"fcm_dpo/q_t": 0.3761498034000397,
"grad_norm": 56.53019714355469,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -0.9774070382118225,
"logits/rejected": -1.0240281820297241,
"logps/chosen": -637.7784423828125,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -1187.298583984375,
"loss": 1.0088,
"margin_dpo/margin_mean": 515.0842895507812,
"margin_dpo/margin_std": 613.7774658203125,
"step": 621
},
{
"KL/chosen_KL_mean": -600.4814453125,
"KL/mean": -777.9830322265625,
"KL/rejected_KL_mean": -955.4846801757812,
"KL/std": 432.13836669921875,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.0010518047492951155,
"fcm_dpo/delta": 0.02732599899172783,
"fcm_dpo/margin": 355.00323486328125,
"fcm_dpo/q_t": 0.41628405451774597,
"grad_norm": 30.599348068237305,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -0.8828511238098145,
"logits/rejected": -0.8755066394805908,
"logps/chosen": -660.767333984375,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -1041.00341796875,
"loss": 1.1217,
"margin_dpo/margin_mean": 355.00323486328125,
"margin_dpo/margin_std": 544.4783935546875,
"step": 622
},
{
"KL/chosen_KL_mean": -659.3543701171875,
"KL/mean": -799.5592041015625,
"KL/rejected_KL_mean": -939.7640380859375,
"KL/std": 451.401123046875,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.0010745935142040253,
"fcm_dpo/delta": 0.10171453654766083,
"fcm_dpo/margin": 280.4097900390625,
"fcm_dpo/q_t": 0.43300747871398926,
"grad_norm": 38.104915618896484,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -0.9893115758895874,
"logits/rejected": -0.9900562763214111,
"logps/chosen": -723.5112915039062,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -1024.84716796875,
"loss": 1.1854,
"margin_dpo/margin_mean": 280.4097900390625,
"margin_dpo/margin_std": 543.878173828125,
"step": 623
},
{
"KL/chosen_KL_mean": -652.4624633789062,
"KL/mean": -863.5037841796875,
"KL/rejected_KL_mean": -1074.544921875,
"KL/std": 486.3994445800781,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.0010720172431319952,
"fcm_dpo/delta": -0.05493501201272011,
"fcm_dpo/margin": 422.08258056640625,
"fcm_dpo/q_t": 0.3948482275009155,
"grad_norm": 35.71148681640625,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -1.014406442642212,
"logits/rejected": -1.0207338333129883,
"logps/chosen": -724.381103515625,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -1171.677001953125,
"loss": 1.0571,
"margin_dpo/margin_mean": 422.0825500488281,
"margin_dpo/margin_std": 520.140380859375,
"step": 624
},
{
"KL/chosen_KL_mean": -627.500244140625,
"KL/mean": -859.1607055664062,
"KL/rejected_KL_mean": -1090.8212890625,
"KL/std": 523.5878295898438,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.0010553135070949793,
"fcm_dpo/delta": -0.09341917186975479,
"fcm_dpo/margin": 463.32098388671875,
"fcm_dpo/q_t": 0.3881823420524597,
"grad_norm": 56.8843994140625,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -0.9432724714279175,
"logits/rejected": -0.9646409749984741,
"logps/chosen": -685.84228515625,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -1176.91162109375,
"loss": 1.0156,
"margin_dpo/margin_mean": 463.32098388671875,
"margin_dpo/margin_std": 491.7691955566406,
"step": 625
},
{
"KL/chosen_KL_mean": -746.43310546875,
"KL/mean": -880.2539672851562,
"KL/rejected_KL_mean": -1014.0748291015625,
"KL/std": 577.3873901367188,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.0010682584252208471,
"fcm_dpo/delta": 0.11725667864084244,
"fcm_dpo/margin": 267.6417236328125,
"fcm_dpo/q_t": 0.43391209840774536,
"grad_norm": 35.362770080566406,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -0.9193681478500366,
"logits/rejected": -0.9120655059814453,
"logps/chosen": -821.5457153320312,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -1113.2635498046875,
"loss": 1.2164,
"margin_dpo/margin_mean": 267.6416931152344,
"margin_dpo/margin_std": 581.1749267578125,
"step": 626
},
{
"KL/chosen_KL_mean": -532.278564453125,
"KL/mean": -807.0740966796875,
"KL/rejected_KL_mean": -1081.86962890625,
"KL/std": 631.8243408203125,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.0010512054432183504,
"fcm_dpo/delta": -0.18833430111408234,
"fcm_dpo/margin": 549.591064453125,
"fcm_dpo/q_t": 0.37993723154067993,
"grad_norm": 28.75301742553711,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -0.8789898753166199,
"logits/rejected": -0.9454772472381592,
"logps/chosen": -580.021728515625,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -1188.6240234375,
"loss": 1.0129,
"margin_dpo/margin_mean": 549.591064453125,
"margin_dpo/margin_std": 725.5487060546875,
"step": 627
},
{
"KL/chosen_KL_mean": -685.268798828125,
"KL/mean": -895.5054931640625,
"KL/rejected_KL_mean": -1105.7421875,
"KL/std": 521.978515625,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.0010267079342156649,
"fcm_dpo/delta": -0.033246103674173355,
"fcm_dpo/margin": 420.47332763671875,
"fcm_dpo/q_t": 0.40259799361228943,
"grad_norm": 31.96737289428711,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -0.993385910987854,
"logits/rejected": -1.0080006122589111,
"logps/chosen": -745.4517822265625,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -1207.296875,
"loss": 1.0827,
"margin_dpo/margin_mean": 420.47332763671875,
"margin_dpo/margin_std": 580.8540649414062,
"step": 628
},
{
"KL/chosen_KL_mean": -672.7838134765625,
"KL/mean": -860.986572265625,
"KL/rejected_KL_mean": -1049.1893310546875,
"KL/std": 503.16357421875,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.0010284529998898506,
"fcm_dpo/delta": 0.0134055744856596,
"fcm_dpo/margin": 376.40557861328125,
"fcm_dpo/q_t": 0.41000691056251526,
"grad_norm": 39.398704528808594,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -1.0148801803588867,
"logits/rejected": -1.0346441268920898,
"logps/chosen": -736.997314453125,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -1140.843017578125,
"loss": 1.0958,
"margin_dpo/margin_mean": 376.40557861328125,
"margin_dpo/margin_std": 488.128173828125,
"step": 629
},
{
"KL/chosen_KL_mean": -619.2073974609375,
"KL/mean": -723.8460693359375,
"KL/rejected_KL_mean": -828.4847412109375,
"KL/std": 491.6463928222656,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.0010610285680741072,
"fcm_dpo/delta": 0.18122641742229462,
"fcm_dpo/margin": 209.27731323242188,
"fcm_dpo/q_t": 0.4533764719963074,
"grad_norm": 50.54798889160156,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -0.9331322312355042,
"logits/rejected": -0.9073890447616577,
"logps/chosen": -678.4984130859375,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -912.0830078125,
"loss": 1.2625,
"margin_dpo/margin_mean": 209.2772979736328,
"margin_dpo/margin_std": 571.025634765625,
"step": 630
},
{
"KL/chosen_KL_mean": -727.552978515625,
"KL/mean": -893.5679931640625,
"KL/rejected_KL_mean": -1059.5828857421875,
"KL/std": 512.9461059570312,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.001076185842975974,
"fcm_dpo/delta": 0.04413389414548874,
"fcm_dpo/margin": 332.0299072265625,
"fcm_dpo/q_t": 0.4186234474182129,
"grad_norm": 44.696895599365234,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -1.0083227157592773,
"logits/rejected": -1.012683391571045,
"logps/chosen": -787.006591796875,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -1140.534423828125,
"loss": 1.1549,
"margin_dpo/margin_mean": 332.0299072265625,
"margin_dpo/margin_std": 589.2799072265625,
"step": 631
},
{
"KL/chosen_KL_mean": -637.191162109375,
"KL/mean": -817.064697265625,
"KL/rejected_KL_mean": -996.9381713867188,
"KL/std": 446.905029296875,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.00107399164699018,
"fcm_dpo/delta": 0.013784755021333694,
"fcm_dpo/margin": 359.7469787597656,
"fcm_dpo/q_t": 0.4118819236755371,
"grad_norm": 36.90241241455078,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -0.9714510440826416,
"logits/rejected": -0.9745583534240723,
"logps/chosen": -698.542724609375,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -1083.098388671875,
"loss": 1.1202,
"margin_dpo/margin_mean": 359.7469787597656,
"margin_dpo/margin_std": 549.3491821289062,
"step": 632
},
{
"KL/chosen_KL_mean": -677.3255615234375,
"KL/mean": -810.3518676757812,
"KL/rejected_KL_mean": -943.378173828125,
"KL/std": 452.0666198730469,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.001101176138035953,
"fcm_dpo/delta": 0.1097092255949974,
"fcm_dpo/margin": 266.0525817871094,
"fcm_dpo/q_t": 0.4321713149547577,
"grad_norm": 35.17088317871094,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -1.0107479095458984,
"logits/rejected": -1.019244909286499,
"logps/chosen": -734.603759765625,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -1034.962158203125,
"loss": 1.1926,
"margin_dpo/margin_mean": 266.0526123046875,
"margin_dpo/margin_std": 517.71240234375,
"step": 633
},
{
"KL/chosen_KL_mean": -777.34423828125,
"KL/mean": -895.5999755859375,
"KL/rejected_KL_mean": -1013.855712890625,
"KL/std": 547.462890625,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.0011245100758969784,
"fcm_dpo/delta": 0.13790102303028107,
"fcm_dpo/margin": 236.51150512695312,
"fcm_dpo/q_t": 0.44136252999305725,
"grad_norm": 34.355796813964844,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -0.9018381834030151,
"logits/rejected": -0.8940595388412476,
"logps/chosen": -843.9631958007812,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -1120.9814453125,
"loss": 1.2218,
"margin_dpo/margin_mean": 236.51150512695312,
"margin_dpo/margin_std": 533.4472045898438,
"step": 634
},
{
"KL/chosen_KL_mean": -675.8604736328125,
"KL/mean": -839.7138671875,
"KL/rejected_KL_mean": -1003.567138671875,
"KL/std": 590.4039306640625,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.001137568149715662,
"fcm_dpo/delta": 0.028254780918359756,
"fcm_dpo/margin": 327.7066955566406,
"fcm_dpo/q_t": 0.41930854320526123,
"grad_norm": 34.57207107543945,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -0.9981896877288818,
"logits/rejected": -0.9815343618392944,
"logps/chosen": -749.8160400390625,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -1086.067626953125,
"loss": 1.1431,
"margin_dpo/margin_mean": 327.7066650390625,
"margin_dpo/margin_std": 579.1690063476562,
"step": 635
},
{
"KL/chosen_KL_mean": -641.6556396484375,
"KL/mean": -837.667724609375,
"KL/rejected_KL_mean": -1033.679931640625,
"KL/std": 567.2379150390625,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.0011395248584449291,
"fcm_dpo/delta": -0.049328967928886414,
"fcm_dpo/margin": 392.0242004394531,
"fcm_dpo/q_t": 0.40262505412101746,
"grad_norm": 27.586217880249023,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -0.9401626586914062,
"logits/rejected": -0.9458721876144409,
"logps/chosen": -701.2845458984375,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -1115.65869140625,
"loss": 1.0791,
"margin_dpo/margin_mean": 392.0242004394531,
"margin_dpo/margin_std": 573.56640625,
"step": 636
},
{
"KL/chosen_KL_mean": -650.4024658203125,
"KL/mean": -858.9671630859375,
"KL/rejected_KL_mean": -1067.5318603515625,
"KL/std": 553.26611328125,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.0011137896217405796,
"fcm_dpo/delta": -0.06838998943567276,
"fcm_dpo/margin": 417.1293029785156,
"fcm_dpo/q_t": 0.3966093063354492,
"grad_norm": 26.735336303710938,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -1.0075198411941528,
"logits/rejected": -1.0491002798080444,
"logps/chosen": -700.05517578125,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -1165.93701171875,
"loss": 1.0642,
"margin_dpo/margin_mean": 417.1293029785156,
"margin_dpo/margin_std": 567.79345703125,
"step": 637
},
{
"KL/chosen_KL_mean": -648.9947509765625,
"KL/mean": -814.8811645507812,
"KL/rejected_KL_mean": -980.767578125,
"KL/std": 530.7965087890625,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.001130024204030633,
"fcm_dpo/delta": 0.023703396320343018,
"fcm_dpo/margin": 331.7728271484375,
"fcm_dpo/q_t": 0.4090859293937683,
"grad_norm": 37.624141693115234,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -0.8736113905906677,
"logits/rejected": -0.860742449760437,
"logps/chosen": -707.1513671875,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -1060.069091796875,
"loss": 1.1595,
"margin_dpo/margin_mean": 331.7728271484375,
"margin_dpo/margin_std": 607.9287109375,
"step": 638
},
{
"KL/chosen_KL_mean": -850.44189453125,
"KL/mean": -925.354736328125,
"KL/rejected_KL_mean": -1000.2677001953125,
"KL/std": 479.6988525390625,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.0011593474773690104,
"fcm_dpo/delta": 0.23026630282402039,
"fcm_dpo/margin": 149.82571411132812,
"fcm_dpo/q_t": 0.46169108152389526,
"grad_norm": 100.88923645019531,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -0.8598443269729614,
"logits/rejected": -0.8015762567520142,
"logps/chosen": -922.76513671875,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -1074.5426025390625,
"loss": 1.3238,
"margin_dpo/margin_mean": 149.8257293701172,
"margin_dpo/margin_std": 569.2753295898438,
"step": 639
},
{
"KL/chosen_KL_mean": -669.903076171875,
"KL/mean": -923.6558837890625,
"KL/rejected_KL_mean": -1177.40869140625,
"KL/std": 576.5742797851562,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.0011469844030216336,
"fcm_dpo/delta": -0.1929648518562317,
"fcm_dpo/margin": 507.5057067871094,
"fcm_dpo/q_t": 0.3761150538921356,
"grad_norm": 36.260929107666016,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -0.8822273015975952,
"logits/rejected": -0.9096044898033142,
"logps/chosen": -726.037353515625,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -1286.0087890625,
"loss": 1.0178,
"margin_dpo/margin_mean": 507.50567626953125,
"margin_dpo/margin_std": 679.51318359375,
"step": 640
},
{
"KL/chosen_KL_mean": -733.76953125,
"KL/mean": -893.8585205078125,
"KL/rejected_KL_mean": -1053.947509765625,
"KL/std": 490.9017333984375,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.0011330367997288704,
"fcm_dpo/delta": 0.038639139384031296,
"fcm_dpo/margin": 320.1780700683594,
"fcm_dpo/q_t": 0.4209401607513428,
"grad_norm": 39.58655548095703,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -0.9786254167556763,
"logits/rejected": -0.9939931035041809,
"logps/chosen": -798.7664184570312,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -1140.93994140625,
"loss": 1.1682,
"margin_dpo/margin_mean": 320.1781005859375,
"margin_dpo/margin_std": 625.5211181640625,
"step": 641
},
{
"KL/chosen_KL_mean": -726.5020751953125,
"KL/mean": -920.57958984375,
"KL/rejected_KL_mean": -1114.6572265625,
"KL/std": 639.9869384765625,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.0011348032858222723,
"fcm_dpo/delta": -0.04255044087767601,
"fcm_dpo/margin": 388.155029296875,
"fcm_dpo/q_t": 0.41343218088150024,
"grad_norm": 42.09096908569336,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -0.9162915945053101,
"logits/rejected": -0.9479303359985352,
"logps/chosen": -792.1912841796875,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -1224.899169921875,
"loss": 1.1468,
"margin_dpo/margin_mean": 388.1550598144531,
"margin_dpo/margin_std": 787.761474609375,
"step": 642
},
{
"KL/chosen_KL_mean": -623.6024780273438,
"KL/mean": -785.3634033203125,
"KL/rejected_KL_mean": -947.1244506835938,
"KL/std": 469.08172607421875,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.0011254868004471064,
"fcm_dpo/delta": 0.0364561453461647,
"fcm_dpo/margin": 323.52197265625,
"fcm_dpo/q_t": 0.41738998889923096,
"grad_norm": 43.73933029174805,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -0.9865999221801758,
"logits/rejected": -0.9877755045890808,
"logps/chosen": -675.552490234375,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -1034.5927734375,
"loss": 1.1271,
"margin_dpo/margin_mean": 323.52197265625,
"margin_dpo/margin_std": 478.4233703613281,
"step": 643
},
{
"KL/chosen_KL_mean": -632.0460205078125,
"KL/mean": -788.1431884765625,
"KL/rejected_KL_mean": -944.2403564453125,
"KL/std": 557.23095703125,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.0011425323318690062,
"fcm_dpo/delta": 0.044929929077625275,
"fcm_dpo/margin": 312.19439697265625,
"fcm_dpo/q_t": 0.42244064807891846,
"grad_norm": 49.204463958740234,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -0.9041519165039062,
"logits/rejected": -0.9101707339286804,
"logps/chosen": -691.0636596679688,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -1031.3770751953125,
"loss": 1.1822,
"margin_dpo/margin_mean": 312.19439697265625,
"margin_dpo/margin_std": 634.4713745117188,
"step": 644
},
{
"KL/chosen_KL_mean": -692.5667724609375,
"KL/mean": -776.70458984375,
"KL/rejected_KL_mean": -860.8424072265625,
"KL/std": 455.5659484863281,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.001176076359115541,
"fcm_dpo/delta": 0.2067035287618637,
"fcm_dpo/margin": 168.275634765625,
"fcm_dpo/q_t": 0.458232045173645,
"grad_norm": 80.0379638671875,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -0.9043595194816589,
"logits/rejected": -0.8900693655014038,
"logps/chosen": -748.4427490234375,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -958.6231689453125,
"loss": 1.3254,
"margin_dpo/margin_mean": 168.27561950683594,
"margin_dpo/margin_std": 608.6322021484375,
"step": 645
},
{
"KL/chosen_KL_mean": -624.6588134765625,
"KL/mean": -764.05859375,
"KL/rejected_KL_mean": -903.4583740234375,
"KL/std": 412.8096618652344,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.0012013267260044813,
"fcm_dpo/delta": 0.06725561618804932,
"fcm_dpo/margin": 278.79949951171875,
"fcm_dpo/q_t": 0.4242590665817261,
"grad_norm": 43.91450881958008,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -0.9505150318145752,
"logits/rejected": -0.9369181990623474,
"logps/chosen": -685.9346313476562,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -980.964111328125,
"loss": 1.1552,
"margin_dpo/margin_mean": 278.79949951171875,
"margin_dpo/margin_std": 466.7037353515625,
"step": 646
},
{
"KL/chosen_KL_mean": -572.581298828125,
"KL/mean": -745.3472900390625,
"KL/rejected_KL_mean": -918.1133422851562,
"KL/std": 485.7171936035156,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.001207266584970057,
"fcm_dpo/delta": -0.017894629389047623,
"fcm_dpo/margin": 345.5320129394531,
"fcm_dpo/q_t": 0.40421944856643677,
"grad_norm": 36.778465270996094,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -0.8633547425270081,
"logits/rejected": -0.891248345375061,
"logps/chosen": -627.4337158203125,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -1011.6327514648438,
"loss": 1.0833,
"margin_dpo/margin_mean": 345.53204345703125,
"margin_dpo/margin_std": 453.4792785644531,
"step": 647
},
{
"KL/chosen_KL_mean": -597.9881591796875,
"KL/mean": -854.043212890625,
"KL/rejected_KL_mean": -1110.098388671875,
"KL/std": 566.09912109375,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.0011682284530252218,
"fcm_dpo/delta": -0.2110145092010498,
"fcm_dpo/margin": 512.1102905273438,
"fcm_dpo/q_t": 0.37206360697746277,
"grad_norm": 33.85206985473633,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -0.9995071887969971,
"logits/rejected": -1.050903558731079,
"logps/chosen": -652.1595458984375,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -1208.8111572265625,
"loss": 0.9965,
"margin_dpo/margin_mean": 512.1102294921875,
"margin_dpo/margin_std": 664.9869384765625,
"step": 648
},
{
"KL/chosen_KL_mean": -615.007080078125,
"KL/mean": -742.46533203125,
"KL/rejected_KL_mean": -869.923583984375,
"KL/std": 460.30889892578125,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.0011777379550039768,
"fcm_dpo/delta": 0.10234013199806213,
"fcm_dpo/margin": 254.91647338867188,
"fcm_dpo/q_t": 0.4372375011444092,
"grad_norm": 29.467697143554688,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -1.0217046737670898,
"logits/rejected": -1.0160858631134033,
"logps/chosen": -677.4874267578125,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -950.000732421875,
"loss": 1.2132,
"margin_dpo/margin_mean": 254.91644287109375,
"margin_dpo/margin_std": 566.233642578125,
"step": 649
},
{
"KL/chosen_KL_mean": -667.095947265625,
"KL/mean": -870.520263671875,
"KL/rejected_KL_mean": -1073.9447021484375,
"KL/std": 554.6080322265625,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.001163251232355833,
"fcm_dpo/delta": -0.07730758190155029,
"fcm_dpo/margin": 406.8487548828125,
"fcm_dpo/q_t": 0.3968607783317566,
"grad_norm": 42.1073112487793,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -0.948840320110321,
"logits/rejected": -0.9683371186256409,
"logps/chosen": -723.188720703125,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -1172.20947265625,
"loss": 1.0906,
"margin_dpo/margin_mean": 406.8487548828125,
"margin_dpo/margin_std": 639.1219482421875,
"step": 650
},
{
"KL/chosen_KL_mean": -465.42462158203125,
"KL/mean": -709.3658447265625,
"KL/rejected_KL_mean": -953.3070678710938,
"KL/std": 522.031005859375,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.0011323931394144893,
"fcm_dpo/delta": -0.16175681352615356,
"fcm_dpo/margin": 487.8824768066406,
"fcm_dpo/q_t": 0.3747558891773224,
"grad_norm": 32.2318000793457,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -0.9470324516296387,
"logits/rejected": -0.9983922243118286,
"logps/chosen": -508.8500671386719,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -1053.264892578125,
"loss": 0.9846,
"margin_dpo/margin_mean": 487.88250732421875,
"margin_dpo/margin_std": 518.0916137695312,
"step": 651
},
{
"KL/chosen_KL_mean": -579.7500610351562,
"KL/mean": -770.1712646484375,
"KL/rejected_KL_mean": -960.592529296875,
"KL/std": 537.4967041015625,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.0011219610460102558,
"fcm_dpo/delta": -0.028511468321084976,
"fcm_dpo/margin": 380.8424377441406,
"fcm_dpo/q_t": 0.4063616394996643,
"grad_norm": 38.36195373535156,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -0.9568244218826294,
"logits/rejected": -0.9792181253433228,
"logps/chosen": -642.326904296875,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -1072.3603515625,
"loss": 1.1056,
"margin_dpo/margin_mean": 380.8424072265625,
"margin_dpo/margin_std": 606.596435546875,
"step": 652
},
{
"KL/chosen_KL_mean": -694.0017700195312,
"KL/mean": -866.1856689453125,
"KL/rejected_KL_mean": -1038.3695068359375,
"KL/std": 565.44384765625,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.0011236886493861675,
"fcm_dpo/delta": 0.013402625918388367,
"fcm_dpo/margin": 344.36773681640625,
"fcm_dpo/q_t": 0.41541993618011475,
"grad_norm": 35.408912658691406,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -1.0132495164871216,
"logits/rejected": -1.0341593027114868,
"logps/chosen": -755.11474609375,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -1141.619140625,
"loss": 1.1366,
"margin_dpo/margin_mean": 344.36773681640625,
"margin_dpo/margin_std": 593.7517700195312,
"step": 653
},
{
"KL/chosen_KL_mean": -636.8229370117188,
"KL/mean": -798.19287109375,
"KL/rejected_KL_mean": -959.562744140625,
"KL/std": 455.2934265136719,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.0011275492142885923,
"fcm_dpo/delta": 0.03746868297457695,
"fcm_dpo/margin": 322.73980712890625,
"fcm_dpo/q_t": 0.4194484353065491,
"grad_norm": 34.15508270263672,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -0.9693796634674072,
"logits/rejected": -0.9811614751815796,
"logps/chosen": -698.5510864257812,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -1058.3365478515625,
"loss": 1.1369,
"margin_dpo/margin_mean": 322.73980712890625,
"margin_dpo/margin_std": 528.53564453125,
"step": 654
},
{
"KL/chosen_KL_mean": -576.0261840820312,
"KL/mean": -755.9908447265625,
"KL/rejected_KL_mean": -935.9555053710938,
"KL/std": 469.98699951171875,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.0011292758863419294,
"fcm_dpo/delta": -0.006732765585184097,
"fcm_dpo/margin": 359.9292297363281,
"fcm_dpo/q_t": 0.4099391996860504,
"grad_norm": 34.95536804199219,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -0.9169396162033081,
"logits/rejected": -0.9512918591499329,
"logps/chosen": -625.60302734375,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -1034.247314453125,
"loss": 1.1086,
"margin_dpo/margin_mean": 359.9292297363281,
"margin_dpo/margin_std": 552.3767700195312,
"step": 655
},
{
"KL/chosen_KL_mean": -630.7530517578125,
"KL/mean": -918.9047241210938,
"KL/rejected_KL_mean": -1207.0562744140625,
"KL/std": 602.3056640625,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.0010918871266767383,
"fcm_dpo/delta": -0.2440999299287796,
"fcm_dpo/margin": 576.30322265625,
"fcm_dpo/q_t": 0.361335813999176,
"grad_norm": 84.34712982177734,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -0.9227770566940308,
"logits/rejected": -0.9797782897949219,
"logps/chosen": -683.302490234375,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -1320.73095703125,
"loss": 0.948,
"margin_dpo/margin_mean": 576.30322265625,
"margin_dpo/margin_std": 611.642822265625,
"step": 656
},
{
"KL/chosen_KL_mean": -592.7017822265625,
"KL/mean": -825.6055297851562,
"KL/rejected_KL_mean": -1058.50927734375,
"KL/std": 583.1033325195312,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.0010563830146566033,
"fcm_dpo/delta": -0.0972730815410614,
"fcm_dpo/margin": 465.8076171875,
"fcm_dpo/q_t": 0.3898601531982422,
"grad_norm": 42.103797912597656,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -0.9886813163757324,
"logits/rejected": -1.0220025777816772,
"logps/chosen": -639.4022827148438,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -1156.4241943359375,
"loss": 1.045,
"margin_dpo/margin_mean": 465.8076171875,
"margin_dpo/margin_std": 609.584228515625,
"step": 657
},
{
"KL/chosen_KL_mean": -648.281982421875,
"KL/mean": -832.0382690429688,
"KL/rejected_KL_mean": -1015.79443359375,
"KL/std": 454.24188232421875,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.001051408820785582,
"fcm_dpo/delta": 0.013769730925559998,
"fcm_dpo/margin": 367.512451171875,
"fcm_dpo/q_t": 0.4112740457057953,
"grad_norm": 31.373876571655273,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -0.9426242709159851,
"logits/rejected": -0.9675799012184143,
"logps/chosen": -709.240234375,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -1111.73388671875,
"loss": 1.1061,
"margin_dpo/margin_mean": 367.512451171875,
"margin_dpo/margin_std": 511.381103515625,
"step": 658
},
{
"KL/chosen_KL_mean": -570.4702758789062,
"KL/mean": -759.959716796875,
"KL/rejected_KL_mean": -949.4490966796875,
"KL/std": 473.38311767578125,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.0010536739137023687,
"fcm_dpo/delta": 0.0004968568682670593,
"fcm_dpo/margin": 378.97882080078125,
"fcm_dpo/q_t": 0.4098956882953644,
"grad_norm": 42.95841979980469,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -0.878132700920105,
"logits/rejected": -0.8574471473693848,
"logps/chosen": -647.2132568359375,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -1036.9200439453125,
"loss": 1.0935,
"margin_dpo/margin_mean": 378.97882080078125,
"margin_dpo/margin_std": 513.1124267578125,
"step": 659
},
{
"KL/chosen_KL_mean": -608.8428344726562,
"KL/mean": -835.480224609375,
"KL/rejected_KL_mean": -1062.117431640625,
"KL/std": 522.89111328125,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.001046313438564539,
"fcm_dpo/delta": -0.07789793610572815,
"fcm_dpo/margin": 453.274658203125,
"fcm_dpo/q_t": 0.39197492599487305,
"grad_norm": 43.292850494384766,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -0.9468445777893066,
"logits/rejected": -0.9635279178619385,
"logps/chosen": -667.8907470703125,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -1138.07763671875,
"loss": 1.0454,
"margin_dpo/margin_mean": 453.274658203125,
"margin_dpo/margin_std": 558.73583984375,
"step": 660
},
{
"KL/chosen_KL_mean": -549.8631591796875,
"KL/mean": -752.0217895507812,
"KL/rejected_KL_mean": -954.1804809570312,
"KL/std": 563.0736083984375,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.0010425238870084286,
"fcm_dpo/delta": -0.022867929190397263,
"fcm_dpo/margin": 404.3173522949219,
"fcm_dpo/q_t": 0.40591317415237427,
"grad_norm": 44.050010681152344,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -0.9941369295120239,
"logits/rejected": -1.0034549236297607,
"logps/chosen": -600.537109375,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -1040.1861572265625,
"loss": 1.0828,
"margin_dpo/margin_mean": 404.3173522949219,
"margin_dpo/margin_std": 560.7882080078125,
"step": 661
},
{
"KL/chosen_KL_mean": -640.8428955078125,
"KL/mean": -805.0003662109375,
"KL/rejected_KL_mean": -969.1578369140625,
"KL/std": 488.666015625,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.0010448182001709938,
"fcm_dpo/delta": 0.05901217460632324,
"fcm_dpo/margin": 328.3148498535156,
"fcm_dpo/q_t": 0.4229253828525543,
"grad_norm": 29.6393985748291,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -0.8987467288970947,
"logits/rejected": -0.8944799900054932,
"logps/chosen": -710.10400390625,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -1058.2137451171875,
"loss": 1.164,
"margin_dpo/margin_mean": 328.3148498535156,
"margin_dpo/margin_std": 602.18505859375,
"step": 662
},
{
"KL/chosen_KL_mean": -611.591796875,
"KL/mean": -805.6707763671875,
"KL/rejected_KL_mean": -999.7498168945312,
"KL/std": 552.761474609375,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.00104548130184412,
"fcm_dpo/delta": -0.006134232506155968,
"fcm_dpo/margin": 388.15789794921875,
"fcm_dpo/q_t": 0.4104207158088684,
"grad_norm": 25.260534286499023,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -0.9808641076087952,
"logits/rejected": -1.0001773834228516,
"logps/chosen": -676.4707641601562,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -1113.6751708984375,
"loss": 1.1198,
"margin_dpo/margin_mean": 388.15789794921875,
"margin_dpo/margin_std": 645.2306518554688,
"step": 663
},
{
"KL/chosen_KL_mean": -636.8848876953125,
"KL/mean": -862.3253173828125,
"KL/rejected_KL_mean": -1087.765869140625,
"KL/std": 568.6349487304688,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.001032976433634758,
"fcm_dpo/delta": -0.06939011812210083,
"fcm_dpo/margin": 450.8810119628906,
"fcm_dpo/q_t": 0.39612704515457153,
"grad_norm": 26.127405166625977,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -0.9494335651397705,
"logits/rejected": -0.9770565032958984,
"logps/chosen": -697.7733154296875,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -1193.28759765625,
"loss": 1.0633,
"margin_dpo/margin_mean": 450.8809814453125,
"margin_dpo/margin_std": 610.881103515625,
"step": 664
},
{
"KL/chosen_KL_mean": -567.0701904296875,
"KL/mean": -746.9297485351562,
"KL/rejected_KL_mean": -926.7893676757812,
"KL/std": 455.4164733886719,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.0010364458430558443,
"fcm_dpo/delta": 0.028231685981154442,
"fcm_dpo/margin": 359.71923828125,
"fcm_dpo/q_t": 0.4139707386493683,
"grad_norm": 37.42238235473633,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -1.0132501125335693,
"logits/rejected": -1.0314836502075195,
"logps/chosen": -627.63427734375,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -1011.5982055664062,
"loss": 1.1014,
"margin_dpo/margin_mean": 359.71923828125,
"margin_dpo/margin_std": 458.2942199707031,
"step": 665
},
{
"KL/chosen_KL_mean": -619.1126708984375,
"KL/mean": -813.4324951171875,
"KL/rejected_KL_mean": -1007.7523193359375,
"KL/std": 487.46881103515625,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.00103611103259027,
"fcm_dpo/delta": -0.0028592273592948914,
"fcm_dpo/margin": 388.6396484375,
"fcm_dpo/q_t": 0.40877360105514526,
"grad_norm": 33.26556396484375,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -0.8844137191772461,
"logits/rejected": -0.8921805024147034,
"logps/chosen": -683.5326538085938,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -1103.64404296875,
"loss": 1.0977,
"margin_dpo/margin_mean": 388.6396484375,
"margin_dpo/margin_std": 539.1551513671875,
"step": 666
},
{
"KL/chosen_KL_mean": -677.048828125,
"KL/mean": -896.513671875,
"KL/rejected_KL_mean": -1115.978515625,
"KL/std": 515.116455078125,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.00102834845893085,
"fcm_dpo/delta": -0.05389273539185524,
"fcm_dpo/margin": 438.9295654296875,
"fcm_dpo/q_t": 0.3974354863166809,
"grad_norm": 49.83727264404297,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -0.8834275007247925,
"logits/rejected": -0.8831124305725098,
"logps/chosen": -746.3258666992188,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -1203.81396484375,
"loss": 1.0624,
"margin_dpo/margin_mean": 438.9295349121094,
"margin_dpo/margin_std": 566.962890625,
"step": 667
},
{
"KL/chosen_KL_mean": -733.402587890625,
"KL/mean": -894.012939453125,
"KL/rejected_KL_mean": -1054.623291015625,
"KL/std": 560.261962890625,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.0010424241190776229,
"fcm_dpo/delta": 0.06649725884199142,
"fcm_dpo/margin": 321.22064208984375,
"fcm_dpo/q_t": 0.42711547017097473,
"grad_norm": 41.80741500854492,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -0.9494549036026001,
"logits/rejected": -0.9580808877944946,
"logps/chosen": -806.006591796875,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -1158.3623046875,
"loss": 1.1829,
"margin_dpo/margin_mean": 321.22064208984375,
"margin_dpo/margin_std": 639.4466552734375,
"step": 668
},
{
"KL/chosen_KL_mean": -553.5703735351562,
"KL/mean": -774.89501953125,
"KL/rejected_KL_mean": -996.2195434570312,
"KL/std": 520.9459228515625,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.0010317617561668158,
"fcm_dpo/delta": -0.059400614351034164,
"fcm_dpo/margin": 442.649169921875,
"fcm_dpo/q_t": 0.3953366279602051,
"grad_norm": 26.183456420898438,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -0.950742244720459,
"logits/rejected": -0.9570592641830444,
"logps/chosen": -599.686767578125,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -1074.1439208984375,
"loss": 1.0634,
"margin_dpo/margin_mean": 442.649169921875,
"margin_dpo/margin_std": 582.044921875,
"step": 669
},
{
"KL/chosen_KL_mean": -542.683837890625,
"KL/mean": -772.5099487304688,
"KL/rejected_KL_mean": -1002.3360595703125,
"KL/std": 490.7078552246094,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.001021178555674851,
"fcm_dpo/delta": -0.07283845543861389,
"fcm_dpo/margin": 459.6522521972656,
"fcm_dpo/q_t": 0.3936052620410919,
"grad_norm": 26.939546585083008,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -0.9101635217666626,
"logits/rejected": -0.9309284687042236,
"logps/chosen": -605.0296020507812,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -1099.276611328125,
"loss": 1.0577,
"margin_dpo/margin_mean": 459.6522521972656,
"margin_dpo/margin_std": 595.2643432617188,
"step": 670
},
{
"KL/chosen_KL_mean": -657.3043823242188,
"KL/mean": -844.4332275390625,
"KL/rejected_KL_mean": -1031.56201171875,
"KL/std": 473.44189453125,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.0010192393092438579,
"fcm_dpo/delta": 0.019037535414099693,
"fcm_dpo/margin": 374.25762939453125,
"fcm_dpo/q_t": 0.41170093417167664,
"grad_norm": 26.634918212890625,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -0.9962849617004395,
"logits/rejected": -1.0246731042861938,
"logps/chosen": -705.3045043945312,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -1115.38134765625,
"loss": 1.1151,
"margin_dpo/margin_mean": 374.25762939453125,
"margin_dpo/margin_std": 549.691162109375,
"step": 671
},
{
"KL/chosen_KL_mean": -719.322998046875,
"KL/mean": -906.4647216796875,
"KL/rejected_KL_mean": -1093.6064453125,
"KL/std": 566.9267578125,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.0010189152089878917,
"fcm_dpo/delta": 0.01937013305723667,
"fcm_dpo/margin": 374.28350830078125,
"fcm_dpo/q_t": 0.41550666093826294,
"grad_norm": 35.86020278930664,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -1.0255699157714844,
"logits/rejected": -1.027112364768982,
"logps/chosen": -777.90625,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -1186.74658203125,
"loss": 1.1382,
"margin_dpo/margin_mean": 374.2834777832031,
"margin_dpo/margin_std": 629.8716430664062,
"step": 672
},
{
"KL/chosen_KL_mean": -668.7772216796875,
"KL/mean": -847.4249267578125,
"KL/rejected_KL_mean": -1026.072509765625,
"KL/std": 507.55853271484375,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.0010265372693538666,
"fcm_dpo/delta": 0.034482501447200775,
"fcm_dpo/margin": 357.29541015625,
"fcm_dpo/q_t": 0.4174554646015167,
"grad_norm": 31.460580825805664,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -0.9649622440338135,
"logits/rejected": -0.968742847442627,
"logps/chosen": -715.5003662109375,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -1111.3687744140625,
"loss": 1.127,
"margin_dpo/margin_mean": 357.29541015625,
"margin_dpo/margin_std": 567.166015625,
"step": 673
},
{
"KL/chosen_KL_mean": -557.0751953125,
"KL/mean": -779.7994384765625,
"KL/rejected_KL_mean": -1002.5236206054688,
"KL/std": 495.4407958984375,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.001021510106511414,
"fcm_dpo/delta": -0.05762124061584473,
"fcm_dpo/margin": 445.4484558105469,
"fcm_dpo/q_t": 0.39889925718307495,
"grad_norm": 22.074878692626953,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -0.8819149136543274,
"logits/rejected": -0.8991641998291016,
"logps/chosen": -602.5206909179688,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -1072.569580078125,
"loss": 1.0619,
"margin_dpo/margin_mean": 445.4484558105469,
"margin_dpo/margin_std": 588.2993774414062,
"step": 674
},
{
"KL/chosen_KL_mean": -620.5293579101562,
"KL/mean": -859.3121337890625,
"KL/rejected_KL_mean": -1098.0947265625,
"KL/std": 547.046875,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.000996412243694067,
"fcm_dpo/delta": -0.08150520920753479,
"fcm_dpo/margin": 477.5653381347656,
"fcm_dpo/q_t": 0.3942088484764099,
"grad_norm": 24.40850257873535,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -0.9287329316139221,
"logits/rejected": -0.9418089389801025,
"logps/chosen": -664.7056884765625,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -1172.186767578125,
"loss": 1.0481,
"margin_dpo/margin_mean": 477.56536865234375,
"margin_dpo/margin_std": 582.3136596679688,
"step": 675
},
{
"KL/chosen_KL_mean": -674.1378784179688,
"KL/mean": -893.676025390625,
"KL/rejected_KL_mean": -1113.2142333984375,
"KL/std": 516.1575927734375,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.000995003618299961,
"fcm_dpo/delta": -0.038550153374671936,
"fcm_dpo/margin": 439.076416015625,
"fcm_dpo/q_t": 0.3991606831550598,
"grad_norm": 26.64472007751465,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -0.8965151906013489,
"logits/rejected": -0.9073841571807861,
"logps/chosen": -745.536376953125,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -1201.572998046875,
"loss": 1.0554,
"margin_dpo/margin_mean": 439.076416015625,
"margin_dpo/margin_std": 500.6483154296875,
"step": 676
},
{
"KL/chosen_KL_mean": -677.4783935546875,
"KL/mean": -880.2694702148438,
"KL/rejected_KL_mean": -1083.060546875,
"KL/std": 518.0269775390625,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.0009909621439874172,
"fcm_dpo/delta": -0.00202114786952734,
"fcm_dpo/margin": 405.5820617675781,
"fcm_dpo/q_t": 0.4127524495124817,
"grad_norm": 39.86590576171875,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -0.9709941744804382,
"logits/rejected": -0.962664008140564,
"logps/chosen": -734.005859375,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -1161.287109375,
"loss": 1.107,
"margin_dpo/margin_mean": 405.5820617675781,
"margin_dpo/margin_std": 623.3728637695312,
"step": 677
},
{
"KL/chosen_KL_mean": -572.25390625,
"KL/mean": -817.163818359375,
"KL/rejected_KL_mean": -1062.07373046875,
"KL/std": 575.556884765625,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.0009832927025854588,
"fcm_dpo/delta": -0.0857529491186142,
"fcm_dpo/margin": 489.81982421875,
"fcm_dpo/q_t": 0.39209699630737305,
"grad_norm": 40.62564468383789,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -0.9188889265060425,
"logits/rejected": -0.9523910880088806,
"logps/chosen": -618.388427734375,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -1142.678466796875,
"loss": 1.0514,
"margin_dpo/margin_mean": 489.81982421875,
"margin_dpo/margin_std": 646.03662109375,
"step": 678
},
{
"KL/chosen_KL_mean": -651.777099609375,
"KL/mean": -824.8280029296875,
"KL/rejected_KL_mean": -997.8789672851562,
"KL/std": 468.39227294921875,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.0009836409008130431,
"fcm_dpo/delta": 0.06168115884065628,
"fcm_dpo/margin": 346.101806640625,
"fcm_dpo/q_t": 0.4226842224597931,
"grad_norm": 32.5433349609375,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -0.9497741460800171,
"logits/rejected": -0.9401757121086121,
"logps/chosen": -702.072021484375,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -1074.47705078125,
"loss": 1.1381,
"margin_dpo/margin_mean": 346.1018371582031,
"margin_dpo/margin_std": 531.5057983398438,
"step": 679
},
{
"KL/chosen_KL_mean": -660.9338989257812,
"KL/mean": -887.5235595703125,
"KL/rejected_KL_mean": -1114.11328125,
"KL/std": 608.47216796875,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.0009757577208802104,
"fcm_dpo/delta": -0.04485994204878807,
"fcm_dpo/margin": 453.1793212890625,
"fcm_dpo/q_t": 0.39889073371887207,
"grad_norm": 42.46123504638672,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -0.9346154928207397,
"logits/rejected": -0.9627900123596191,
"logps/chosen": -737.849609375,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -1226.498046875,
"loss": 1.0907,
"margin_dpo/margin_mean": 453.1793518066406,
"margin_dpo/margin_std": 672.1544799804688,
"step": 680
},
{
"KL/chosen_KL_mean": -638.86279296875,
"KL/mean": -822.6510009765625,
"KL/rejected_KL_mean": -1006.4391479492188,
"KL/std": 489.5135498046875,
"epoch": 1.0,
"fcm_dpo/beta": 0.000990185420960188,
"fcm_dpo/delta": 0.03625159710645676,
"fcm_dpo/margin": 367.5763244628906,
"fcm_dpo/q_t": 0.41684556007385254,
"grad_norm": 29.843454360961914,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -0.9915759563446045,
"logits/rejected": -1.0077660083770752,
"logps/chosen": -699.820068359375,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -1094.9970703125,
"loss": 1.1249,
"margin_dpo/margin_mean": 367.5762939453125,
"margin_dpo/margin_std": 529.50439453125,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 1.0968060302664355,
"train_runtime": 1742.1859,
"train_samples_per_second": 25.025,
"train_steps_per_second": 0.391
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink