W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
02bf1f76cb73516ac4cce846aeaa15a266e2be8f
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC 02bf1f76cb 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8
Source: Original Platform
2026-05-12 14:46:33 +08:00

15707 lines
579 KiB
JSON
Raw Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.5267562866211,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.22103118896484,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": 0.016141891479492188,
"KL/mean": 0.009041428565979004,
"KL/rejected_KL_mean": 0.001941680908203125,
"KL/std": 0.2510850429534912,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014200329780578613,
"fcm_dpo/q_t": 0.4996456503868103,
"grad_norm": 70.8984375,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.4816073179244995,
"logits/rejected": -0.44214364886283875,
"logps/chosen": -60.9654541015625,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.670654296875,
"loss": 1.3852,
"margin_dpo/margin_mean": 0.01420101523399353,
"margin_dpo/margin_std": 0.37771791219711304,
"step": 3
},
{
"KL/chosen_KL_mean": -0.038990020751953125,
"KL/mean": -0.02172142267227173,
"KL/rejected_KL_mean": -0.00445556640625,
"KL/std": 0.22829246520996094,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.034536004066467285,
"fcm_dpo/q_t": 0.500863790512085,
"grad_norm": 72.12220764160156,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.46893513202667236,
"logits/rejected": -0.44124215841293335,
"logps/chosen": -56.80670166015625,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.65156555175781,
"loss": 1.3901,
"margin_dpo/margin_mean": -0.03453657031059265,
"margin_dpo/margin_std": 0.3669918179512024,
"step": 4
},
{
"KL/chosen_KL_mean": 0.02498626708984375,
"KL/mean": 0.023038864135742188,
"KL/rejected_KL_mean": 0.021091461181640625,
"KL/std": 0.25224569439888,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0038943886756896973,
"fcm_dpo/q_t": 0.4999024569988251,
"grad_norm": 90.13763427734375,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.5147161483764648,
"logits/rejected": -0.47098612785339355,
"logps/chosen": -53.834388732910156,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.12809753417969,
"loss": 1.3862,
"margin_dpo/margin_mean": 0.0038947463035583496,
"margin_dpo/margin_std": 0.3366306722164154,
"step": 5
},
{
"KL/chosen_KL_mean": -0.008636474609375,
"KL/mean": -0.022702336311340332,
"KL/rejected_KL_mean": -0.036769866943359375,
"KL/std": 0.2681416869163513,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.028134524822235107,
"fcm_dpo/q_t": 0.49929672479629517,
"grad_norm": 91.89168548583984,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.5082045793533325,
"logits/rejected": -0.4662151038646698,
"logps/chosen": -63.01612091064453,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.68211364746094,
"loss": 1.3838,
"margin_dpo/margin_mean": 0.02813410758972168,
"margin_dpo/margin_std": 0.35160428285598755,
"step": 6
},
{
"KL/chosen_KL_mean": -0.013416290283203125,
"KL/mean": 0.02394537627696991,
"KL/rejected_KL_mean": 0.061305999755859375,
"KL/std": 0.2824476361274719,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.07472285628318787,
"fcm_dpo/q_t": 0.5018666982650757,
"grad_norm": 82.56436157226562,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.5052725076675415,
"logits/rejected": -0.47143226861953735,
"logps/chosen": -57.788238525390625,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.85928344726562,
"loss": 1.3943,
"margin_dpo/margin_mean": -0.07472258806228638,
"margin_dpo/margin_std": 0.43447619676589966,
"step": 7
},
{
"KL/chosen_KL_mean": 0.012325286865234375,
"KL/mean": 0.027680113911628723,
"KL/rejected_KL_mean": 0.04303741455078125,
"KL/std": 0.3071158230304718,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.030712276697158813,
"fcm_dpo/q_t": 0.5007674694061279,
"grad_norm": 78.390869140625,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.49178242683410645,
"logits/rejected": -0.4640960395336151,
"logps/chosen": -58.703712463378906,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.26838684082031,
"loss": 1.3898,
"margin_dpo/margin_mean": -0.03071233630180359,
"margin_dpo/margin_std": 0.37911269068717957,
"step": 8
},
{
"KL/chosen_KL_mean": 0.020702362060546875,
"KL/mean": 0.0013383924961090088,
"KL/rejected_KL_mean": -0.01802825927734375,
"KL/std": 0.2934737205505371,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03873598575592041,
"fcm_dpo/q_t": 0.49903160333633423,
"grad_norm": 85.00302124023438,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.5062781572341919,
"logits/rejected": -0.4602203369140625,
"logps/chosen": -69.84614562988281,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.62069702148438,
"loss": 1.3828,
"margin_dpo/margin_mean": 0.03873533010482788,
"margin_dpo/margin_std": 0.39692068099975586,
"step": 9
},
{
"KL/chosen_KL_mean": 0.013669967651367188,
"KL/mean": -0.007347121834754944,
"KL/rejected_KL_mean": -0.028362274169921875,
"KL/std": 0.2732405662536621,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.042034536600112915,
"fcm_dpo/q_t": 0.49894949793815613,
"grad_norm": 70.61676025390625,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.5029403567314148,
"logits/rejected": -0.46021097898483276,
"logps/chosen": -48.344017028808594,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.40042114257812,
"loss": 1.3825,
"margin_dpo/margin_mean": 0.04203528165817261,
"margin_dpo/margin_std": 0.3880940079689026,
"step": 10
},
{
"KL/chosen_KL_mean": 0.007905960083007812,
"KL/mean": -0.01761634647846222,
"KL/rejected_KL_mean": -0.0431365966796875,
"KL/std": 0.2690313160419464,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05104079842567444,
"fcm_dpo/q_t": 0.4987240731716156,
"grad_norm": 68.2398681640625,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.4520139694213867,
"logits/rejected": -0.4266074299812317,
"logps/chosen": -53.008949279785156,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.82351684570312,
"loss": 1.3816,
"margin_dpo/margin_mean": 0.05104094743728638,
"margin_dpo/margin_std": 0.3854028582572937,
"step": 11
},
{
"KL/chosen_KL_mean": 0.0002536773681640625,
"KL/mean": -0.010517977178096771,
"KL/rejected_KL_mean": -0.0212860107421875,
"KL/std": 0.2928847372531891,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.021543309092521667,
"fcm_dpo/q_t": 0.49946165084838867,
"grad_norm": 89.65792846679688,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.5362281799316406,
"logits/rejected": -0.49995797872543335,
"logps/chosen": -61.80518341064453,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.87956237792969,
"loss": 1.3846,
"margin_dpo/margin_mean": 0.021542653441429138,
"margin_dpo/margin_std": 0.40630000829696655,
"step": 12
},
{
"KL/chosen_KL_mean": 0.0028934478759765625,
"KL/mean": -0.017609775066375732,
"KL/rejected_KL_mean": -0.03810882568359375,
"KL/std": 0.3167712390422821,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04100680351257324,
"fcm_dpo/q_t": 0.4989779591560364,
"grad_norm": 79.11839294433594,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.4737343192100525,
"logits/rejected": -0.44504889845848083,
"logps/chosen": -64.25746154785156,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.24119567871094,
"loss": 1.3827,
"margin_dpo/margin_mean": 0.0410064160823822,
"margin_dpo/margin_std": 0.4179210662841797,
"step": 13
},
{
"KL/chosen_KL_mean": -0.03240966796875,
"KL/mean": -0.003829583525657654,
"KL/rejected_KL_mean": 0.024745941162109375,
"KL/std": 0.2639964520931244,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.05715218186378479,
"fcm_dpo/q_t": 0.5014278888702393,
"grad_norm": 85.87844848632812,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.4652435779571533,
"logits/rejected": -0.4254474639892578,
"logps/chosen": -58.14262008666992,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.0223388671875,
"loss": 1.3924,
"margin_dpo/margin_mean": -0.0571514368057251,
"margin_dpo/margin_std": 0.3682215213775635,
"step": 14
},
{
"KL/chosen_KL_mean": -0.03320121765136719,
"KL/mean": -0.0465509295463562,
"KL/rejected_KL_mean": -0.059902191162109375,
"KL/std": 0.23382139205932617,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.026701807975769043,
"fcm_dpo/q_t": 0.4993324279785156,
"grad_norm": 64.0005874633789,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.5041216015815735,
"logits/rejected": -0.48607927560806274,
"logps/chosen": -57.0001106262207,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.86854553222656,
"loss": 1.3839,
"margin_dpo/margin_mean": 0.026701688766479492,
"margin_dpo/margin_std": 0.33812767267227173,
"step": 15
},
{
"KL/chosen_KL_mean": -0.0052700042724609375,
"KL/mean": -0.040506258606910706,
"KL/rejected_KL_mean": -0.07574081420898438,
"KL/std": 0.2736942172050476,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07047191262245178,
"fcm_dpo/q_t": 0.4982394576072693,
"grad_norm": 83.81981658935547,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.5340527296066284,
"logits/rejected": -0.4940548539161682,
"logps/chosen": -61.74516296386719,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.4452133178711,
"loss": 1.3796,
"margin_dpo/margin_mean": 0.07047182321548462,
"margin_dpo/margin_std": 0.3839455842971802,
"step": 16
},
{
"KL/chosen_KL_mean": -0.008800506591796875,
"KL/mean": -0.020151451230049133,
"KL/rejected_KL_mean": -0.031505584716796875,
"KL/std": 0.2639004588127136,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02270624041557312,
"fcm_dpo/q_t": 0.49943220615386963,
"grad_norm": 78.86737060546875,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.504452109336853,
"logits/rejected": -0.46770912408828735,
"logps/chosen": -67.71913146972656,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.41015625,
"loss": 1.3843,
"margin_dpo/margin_mean": 0.022706151008605957,
"margin_dpo/margin_std": 0.3559093475341797,
"step": 17
},
{
"KL/chosen_KL_mean": -0.010873794555664062,
"KL/mean": -0.05961640179157257,
"KL/rejected_KL_mean": -0.10836029052734375,
"KL/std": 0.30446290969848633,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09748539328575134,
"fcm_dpo/q_t": 0.497562050819397,
"grad_norm": 81.79119110107422,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.5100164413452148,
"logits/rejected": -0.4564880430698395,
"logps/chosen": -47.7503662109375,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.58064270019531,
"loss": 1.377,
"margin_dpo/margin_mean": 0.09748479723930359,
"margin_dpo/margin_std": 0.4179475009441376,
"step": 18
},
{
"KL/chosen_KL_mean": 0.0426177978515625,
"KL/mean": -0.00010053813457489014,
"KL/rejected_KL_mean": -0.042816162109375,
"KL/std": 0.2286176085472107,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08543136715888977,
"fcm_dpo/q_t": 0.497865229845047,
"grad_norm": 72.6661376953125,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.49207890033721924,
"logits/rejected": -0.44229188561439514,
"logps/chosen": -70.1627426147461,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.80040740966797,
"loss": 1.378,
"margin_dpo/margin_mean": 0.08543169498443604,
"margin_dpo/margin_std": 0.3262811303138733,
"step": 19
},
{
"KL/chosen_KL_mean": 0.0006504058837890625,
"KL/mean": -0.034426525235176086,
"KL/rejected_KL_mean": -0.06949996948242188,
"KL/std": 0.27714747190475464,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07015708088874817,
"fcm_dpo/q_t": 0.4982466399669647,
"grad_norm": 73.6260986328125,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5626459121704102,
"logits/rejected": -0.5075259804725647,
"logps/chosen": -50.802589416503906,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.89285278320312,
"loss": 1.3797,
"margin_dpo/margin_mean": 0.07015728950500488,
"margin_dpo/margin_std": 0.38508284091949463,
"step": 20
},
{
"KL/chosen_KL_mean": 0.022371292114257812,
"KL/mean": -0.05759835243225098,
"KL/rejected_KL_mean": -0.13756561279296875,
"KL/std": 0.3261793851852417,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15993809700012207,
"fcm_dpo/q_t": 0.4960044026374817,
"grad_norm": 77.39993286132812,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.5166388750076294,
"logits/rejected": -0.4944336712360382,
"logps/chosen": -50.040645599365234,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -78.00636291503906,
"loss": 1.3708,
"margin_dpo/margin_mean": 0.15993818640708923,
"margin_dpo/margin_std": 0.40345868468284607,
"step": 21
},
{
"KL/chosen_KL_mean": 0.03532218933105469,
"KL/mean": -0.06052166223526001,
"KL/rejected_KL_mean": -0.15636444091796875,
"KL/std": 0.314807265996933,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.191686749458313,
"fcm_dpo/q_t": 0.49521100521087646,
"grad_norm": 84.29412841796875,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.4891554117202759,
"logits/rejected": -0.44577914476394653,
"logps/chosen": -59.022315979003906,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.66104125976562,
"loss": 1.3678,
"margin_dpo/margin_mean": 0.1916864514350891,
"margin_dpo/margin_std": 0.4764353930950165,
"step": 22
},
{
"KL/chosen_KL_mean": 0.03306007385253906,
"KL/mean": -0.06440095603466034,
"KL/rejected_KL_mean": -0.16185760498046875,
"KL/std": 0.31806907057762146,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19491460919380188,
"fcm_dpo/q_t": 0.49513041973114014,
"grad_norm": 80.66082000732422,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.4961227774620056,
"logits/rejected": -0.47408023476600647,
"logps/chosen": -60.04463577270508,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.30140686035156,
"loss": 1.3674,
"margin_dpo/margin_mean": 0.19491499662399292,
"margin_dpo/margin_std": 0.43538808822631836,
"step": 23
},
{
"KL/chosen_KL_mean": 0.03735542297363281,
"KL/mean": -0.08538687229156494,
"KL/rejected_KL_mean": -0.2081298828125,
"KL/std": 0.27013736963272095,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2454851269721985,
"fcm_dpo/q_t": 0.49386560916900635,
"grad_norm": 80.96056365966797,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5057855844497681,
"logits/rejected": -0.4892767071723938,
"logps/chosen": -44.25367736816406,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.33334350585938,
"loss": 1.3622,
"margin_dpo/margin_mean": 0.2454843521118164,
"margin_dpo/margin_std": 0.3252241015434265,
"step": 24
},
{
"KL/chosen_KL_mean": 0.0063114166259765625,
"KL/mean": -0.09504103660583496,
"KL/rejected_KL_mean": -0.19639205932617188,
"KL/std": 0.31363558769226074,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20270639657974243,
"fcm_dpo/q_t": 0.4949347674846649,
"grad_norm": 73.89991760253906,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.5002730488777161,
"logits/rejected": -0.47041672468185425,
"logps/chosen": -52.530738830566406,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.53858184814453,
"loss": 1.3665,
"margin_dpo/margin_mean": 0.20270657539367676,
"margin_dpo/margin_std": 0.37986764311790466,
"step": 25
},
{
"KL/chosen_KL_mean": 0.09214401245117188,
"KL/mean": -0.10877400636672974,
"KL/rejected_KL_mean": -0.3096961975097656,
"KL/std": 0.4387274384498596,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4018453359603882,
"fcm_dpo/q_t": 0.4899684488773346,
"grad_norm": 87.35763549804688,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5315044522285461,
"logits/rejected": -0.4998011589050293,
"logps/chosen": -53.830665588378906,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.6694107055664,
"loss": 1.3474,
"margin_dpo/margin_mean": 0.40184569358825684,
"margin_dpo/margin_std": 0.5766628980636597,
"step": 26
},
{
"KL/chosen_KL_mean": 0.14463043212890625,
"KL/mean": -0.12463589012622833,
"KL/rejected_KL_mean": -0.3939056396484375,
"KL/std": 0.4663229286670685,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5385316610336304,
"fcm_dpo/q_t": 0.48655465245246887,
"grad_norm": 93.67314147949219,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.5397769212722778,
"logits/rejected": -0.503648579120636,
"logps/chosen": -42.75389862060547,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.11810302734375,
"loss": 1.334,
"margin_dpo/margin_mean": 0.5385308265686035,
"margin_dpo/margin_std": 0.5469115972518921,
"step": 27
},
{
"KL/chosen_KL_mean": -0.03975486755371094,
"KL/mean": -0.16874754428863525,
"KL/rejected_KL_mean": -0.29773712158203125,
"KL/std": 0.41560396552085876,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2579805254936218,
"fcm_dpo/q_t": 0.49355655908584595,
"grad_norm": 75.45986938476562,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.5163806080818176,
"logits/rejected": -0.4621371328830719,
"logps/chosen": -60.596256256103516,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.6988525390625,
"loss": 1.3616,
"margin_dpo/margin_mean": 0.25798097252845764,
"margin_dpo/margin_std": 0.6161943674087524,
"step": 28
},
{
"KL/chosen_KL_mean": 0.11548995971679688,
"KL/mean": -0.14322738349437714,
"KL/rejected_KL_mean": -0.4019432067871094,
"KL/std": 0.47610336542129517,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5174299478530884,
"fcm_dpo/q_t": 0.48708075284957886,
"grad_norm": 90.54450225830078,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.5383598208427429,
"logits/rejected": -0.49088796973228455,
"logps/chosen": -57.69229507446289,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.79629516601562,
"loss": 1.3361,
"margin_dpo/margin_mean": 0.5174306631088257,
"margin_dpo/margin_std": 0.5744448900222778,
"step": 29
},
{
"KL/chosen_KL_mean": 0.1258563995361328,
"KL/mean": -0.19550739228725433,
"KL/rejected_KL_mean": -0.5168724060058594,
"KL/std": 0.5710160136222839,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6427308320999146,
"fcm_dpo/q_t": 0.48395419120788574,
"grad_norm": 87.73825073242188,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.5053558349609375,
"logits/rejected": -0.4750453233718872,
"logps/chosen": -52.451515197753906,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -99.00607299804688,
"loss": 1.324,
"margin_dpo/margin_mean": 0.6427305340766907,
"margin_dpo/margin_std": 0.6174843311309814,
"step": 30
},
{
"KL/chosen_KL_mean": 0.0767364501953125,
"KL/mean": -0.1584608256816864,
"KL/rejected_KL_mean": -0.3936576843261719,
"KL/std": 0.5441341400146484,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.47039252519607544,
"fcm_dpo/q_t": 0.48826301097869873,
"grad_norm": 68.57320404052734,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.5012216567993164,
"logits/rejected": -0.456045925617218,
"logps/chosen": -63.730186462402344,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.28765869140625,
"loss": 1.341,
"margin_dpo/margin_mean": 0.4703920781612396,
"margin_dpo/margin_std": 0.7006645202636719,
"step": 31
},
{
"KL/chosen_KL_mean": 0.20220184326171875,
"KL/mean": -0.18391788005828857,
"KL/rejected_KL_mean": -0.570037841796875,
"KL/std": 0.7176538705825806,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7722392082214355,
"fcm_dpo/q_t": 0.4807533621788025,
"grad_norm": 83.21366119384766,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.5237495303153992,
"logits/rejected": -0.48302900791168213,
"logps/chosen": -62.537322998046875,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.88754272460938,
"loss": 1.3123,
"margin_dpo/margin_mean": 0.7722393274307251,
"margin_dpo/margin_std": 0.842420220375061,
"step": 32
},
{
"KL/chosen_KL_mean": 0.12135887145996094,
"KL/mean": -0.17980101704597473,
"KL/rejected_KL_mean": -0.4809608459472656,
"KL/std": 0.5358744859695435,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6023146510124207,
"fcm_dpo/q_t": 0.48496127128601074,
"grad_norm": 72.2737045288086,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.5086983442306519,
"logits/rejected": -0.4830739498138428,
"logps/chosen": -53.13961410522461,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.36609649658203,
"loss": 1.3279,
"margin_dpo/margin_mean": 0.6023147106170654,
"margin_dpo/margin_std": 0.5953254699707031,
"step": 33
},
{
"KL/chosen_KL_mean": 0.08457756042480469,
"KL/mean": -0.30423083901405334,
"KL/rejected_KL_mean": -0.6930427551269531,
"KL/std": 0.7175294160842896,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.77761310338974,
"fcm_dpo/q_t": 0.4806172847747803,
"grad_norm": 77.8739013671875,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.5117413401603699,
"logits/rejected": -0.4942744970321655,
"logps/chosen": -50.732749938964844,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.61488342285156,
"loss": 1.312,
"margin_dpo/margin_mean": 0.7776126861572266,
"margin_dpo/margin_std": 0.8780879974365234,
"step": 34
},
{
"KL/chosen_KL_mean": 0.09637641906738281,
"KL/mean": -0.5095885992050171,
"KL/rejected_KL_mean": -1.1155509948730469,
"KL/std": 0.9993947744369507,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2119297981262207,
"fcm_dpo/q_t": 0.46987059712409973,
"grad_norm": 82.90363311767578,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.5279215574264526,
"logits/rejected": -0.49103760719299316,
"logps/chosen": -50.92811584472656,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.93997955322266,
"loss": 1.2717,
"margin_dpo/margin_mean": 1.2119297981262207,
"margin_dpo/margin_std": 1.0788286924362183,
"step": 35
},
{
"KL/chosen_KL_mean": 0.040618896484375,
"KL/mean": -0.5388714671134949,
"KL/rejected_KL_mean": -1.1183624267578125,
"KL/std": 1.0968472957611084,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1589829921722412,
"fcm_dpo/q_t": 0.4711868166923523,
"grad_norm": 72.5723648071289,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5531824827194214,
"logits/rejected": -0.5167204737663269,
"logps/chosen": -51.95087432861328,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.15898895263672,
"loss": 1.2777,
"margin_dpo/margin_mean": 1.158982753753662,
"margin_dpo/margin_std": 1.2281889915466309,
"step": 36
},
{
"KL/chosen_KL_mean": -0.025007247924804688,
"KL/mean": -0.5262736082077026,
"KL/rejected_KL_mean": -1.0275382995605469,
"KL/std": 1.0382901430130005,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.002532720565796,
"fcm_dpo/q_t": 0.4751211404800415,
"grad_norm": 61.644954681396484,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.5020361542701721,
"logits/rejected": -0.45762014389038086,
"logps/chosen": -62.832115173339844,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.922607421875,
"loss": 1.2928,
"margin_dpo/margin_mean": 1.0025322437286377,
"margin_dpo/margin_std": 1.2644333839416504,
"step": 37
},
{
"KL/chosen_KL_mean": 0.15131187438964844,
"KL/mean": -0.5383007526397705,
"KL/rejected_KL_mean": -1.2279129028320312,
"KL/std": 1.3978111743927002,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3792251348495483,
"fcm_dpo/q_t": 0.4659920334815979,
"grad_norm": 69.25381469726562,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5107109546661377,
"logits/rejected": -0.4779038727283478,
"logps/chosen": -48.23920822143555,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.14035034179688,
"loss": 1.2602,
"margin_dpo/margin_mean": 1.379225730895996,
"margin_dpo/margin_std": 1.6747769117355347,
"step": 38
},
{
"KL/chosen_KL_mean": 0.03895378112792969,
"KL/mean": -0.7721768617630005,
"KL/rejected_KL_mean": -1.5833053588867188,
"KL/std": 1.307983636856079,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6222527027130127,
"fcm_dpo/q_t": 0.45977842807769775,
"grad_norm": 74.82653045654297,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.5557675361633301,
"logits/rejected": -0.5158565044403076,
"logps/chosen": -50.711517333984375,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.15281677246094,
"loss": 1.2361,
"margin_dpo/margin_mean": 1.6222525835037231,
"margin_dpo/margin_std": 1.4611896276474,
"step": 39
},
{
"KL/chosen_KL_mean": 0.18808746337890625,
"KL/mean": -0.5939724445343018,
"KL/rejected_KL_mean": -1.37603759765625,
"KL/std": 1.4154611825942993,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5641274452209473,
"fcm_dpo/q_t": 0.46141234040260315,
"grad_norm": 60.164329528808594,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5338292121887207,
"logits/rejected": -0.50484299659729,
"logps/chosen": -57.79698181152344,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.67610931396484,
"loss": 1.2436,
"margin_dpo/margin_mean": 1.5641283988952637,
"margin_dpo/margin_std": 1.7055679559707642,
"step": 40
},
{
"KL/chosen_KL_mean": 0.035152435302734375,
"KL/mean": -0.9116830825805664,
"KL/rejected_KL_mean": -1.8585128784179688,
"KL/std": 1.8412256240844727,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.2878764867782593,
"fcm_dpo/delta": 0.09748303890228271,
"fcm_dpo/margin": 1.893665075302124,
"fcm_dpo/q_t": 0.4534370005130768,
"grad_norm": 148.9955596923828,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.5401488542556763,
"logits/rejected": -0.5035785436630249,
"logps/chosen": -62.66066360473633,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.88204193115234,
"loss": 1.0448,
"margin_dpo/margin_mean": 1.8936657905578613,
"margin_dpo/margin_std": 2.0037331581115723,
"step": 41
},
{
"KL/chosen_KL_mean": 0.2072906494140625,
"KL/mean": -1.0888673067092896,
"KL/rejected_KL_mean": -2.3850173950195312,
"KL/std": 2.1535332202911377,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": -0.38896632194519043,
"fcm_dpo/margin": 2.592313289642334,
"fcm_dpo/q_t": 0.3977593779563904,
"grad_norm": 0.964080810546875,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.527120053768158,
"logits/rejected": -0.4803665280342102,
"logps/chosen": -58.75913619995117,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.29339599609375,
"loss": 1.3839,
"margin_dpo/margin_mean": 2.592312812805176,
"margin_dpo/margin_std": 2.3912642002105713,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5225181579589844,
"KL/mean": -0.7178996205329895,
"KL/rejected_KL_mean": -1.9583206176757812,
"KL/std": 1.8281306028366089,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.480837345123291,
"fcm_dpo/q_t": 0.4994150698184967,
"grad_norm": 0.8368467688560486,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.5465719699859619,
"logits/rejected": -0.5219491720199585,
"logps/chosen": -53.63347625732422,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.43850708007812,
"loss": 1.384,
"margin_dpo/margin_mean": 2.480837345123291,
"margin_dpo/margin_std": 1.950972557067871,
"step": 43
},
{
"KL/chosen_KL_mean": 0.2370586395263672,
"KL/mean": -1.170802116394043,
"KL/rejected_KL_mean": -2.578662872314453,
"KL/std": 2.2450320720672607,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.815722703933716,
"fcm_dpo/q_t": 0.4993360936641693,
"grad_norm": 0.9514136910438538,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.4892653822898865,
"logits/rejected": -0.468932181596756,
"logps/chosen": -49.84143829345703,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.3624267578125,
"loss": 1.3836,
"margin_dpo/margin_mean": 2.8157224655151367,
"margin_dpo/margin_std": 2.159979820251465,
"step": 44
},
{
"KL/chosen_KL_mean": 0.1477680206298828,
"KL/mean": -0.9873858690261841,
"KL/rejected_KL_mean": -2.122539520263672,
"KL/std": 2.0420613288879395,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.2703118324279785,
"fcm_dpo/q_t": 0.4994646906852722,
"grad_norm": 0.7598158717155457,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.5002434253692627,
"logits/rejected": -0.4881095290184021,
"logps/chosen": -48.26715850830078,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.05896759033203,
"loss": 1.3842,
"margin_dpo/margin_mean": 2.2703113555908203,
"margin_dpo/margin_std": 2.5159013271331787,
"step": 45
},
{
"KL/chosen_KL_mean": 0.1900959014892578,
"KL/mean": -1.2448352575302124,
"KL/rejected_KL_mean": -2.679767608642578,
"KL/std": 2.5515077114105225,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.869863510131836,
"fcm_dpo/q_t": 0.4993233382701874,
"grad_norm": 0.9118256568908691,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.5423866510391235,
"logits/rejected": -0.4921818673610687,
"logps/chosen": -55.809329986572266,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.33235168457031,
"loss": 1.3836,
"margin_dpo/margin_mean": 2.869863510131836,
"margin_dpo/margin_std": 3.1087698936462402,
"step": 46
},
{
"KL/chosen_KL_mean": 0.38346290588378906,
"KL/mean": -1.0860923528671265,
"KL/rejected_KL_mean": -2.5556488037109375,
"KL/std": 2.5621590614318848,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.9391136169433594,
"fcm_dpo/q_t": 0.4993070363998413,
"grad_norm": 0.8042023777961731,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.5681043267250061,
"logits/rejected": -0.515215277671814,
"logps/chosen": -57.54261779785156,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.23485565185547,
"loss": 1.3835,
"margin_dpo/margin_mean": 2.9391140937805176,
"margin_dpo/margin_std": 2.639211654663086,
"step": 47
},
{
"KL/chosen_KL_mean": 0.08052635192871094,
"KL/mean": -1.415499210357666,
"KL/rejected_KL_mean": -2.9115257263183594,
"KL/std": 2.53715181350708,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.992050886154175,
"fcm_dpo/q_t": 0.49929454922676086,
"grad_norm": 0.9025480151176453,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.5785880088806152,
"logits/rejected": -0.5198041200637817,
"logps/chosen": -57.107547760009766,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -90.92813110351562,
"loss": 1.3835,
"margin_dpo/margin_mean": 2.992050886154175,
"margin_dpo/margin_std": 2.7764430046081543,
"step": 48
},
{
"KL/chosen_KL_mean": 0.3786048889160156,
"KL/mean": -1.4605863094329834,
"KL/rejected_KL_mean": -3.299774169921875,
"KL/std": 3.252890110015869,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.6783740520477295,
"fcm_dpo/q_t": 0.4991326928138733,
"grad_norm": 0.8797588348388672,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.5455284118652344,
"logits/rejected": -0.48684394359588623,
"logps/chosen": -61.306663513183594,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.06724548339844,
"loss": 1.3828,
"margin_dpo/margin_mean": 3.6783740520477295,
"margin_dpo/margin_std": 3.7025225162506104,
"step": 49
},
{
"KL/chosen_KL_mean": -0.12292289733886719,
"KL/mean": -2.0747485160827637,
"KL/rejected_KL_mean": -4.026569366455078,
"KL/std": 3.336162567138672,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.9036431312561035,
"fcm_dpo/q_t": 0.4990795850753784,
"grad_norm": 0.8429669141769409,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5466989278793335,
"logits/rejected": -0.5109710693359375,
"logps/chosen": -58.84706115722656,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.38471984863281,
"loss": 1.3826,
"margin_dpo/margin_mean": 3.9036436080932617,
"margin_dpo/margin_std": 3.8791186809539795,
"step": 50
},
{
"KL/chosen_KL_mean": -0.27999114990234375,
"KL/mean": -2.2037620544433594,
"KL/rejected_KL_mean": -4.127536773681641,
"KL/std": 3.985077142715454,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.847546100616455,
"fcm_dpo/q_t": 0.49909284710884094,
"grad_norm": 0.7773189544677734,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.5368514060974121,
"logits/rejected": -0.5046820044517517,
"logps/chosen": -61.65365982055664,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.1295394897461,
"loss": 1.3827,
"margin_dpo/margin_mean": 3.847546100616455,
"margin_dpo/margin_std": 4.900827407836914,
"step": 51
},
{
"KL/chosen_KL_mean": 0.37728309631347656,
"KL/mean": -2.584096908569336,
"KL/rejected_KL_mean": -5.545478820800781,
"KL/std": 4.599925994873047,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.922760963439941,
"fcm_dpo/q_t": 0.4986035227775574,
"grad_norm": 0.9805387258529663,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.5656962394714355,
"logits/rejected": -0.5110952258110046,
"logps/chosen": -51.96007537841797,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.51939392089844,
"loss": 1.3807,
"margin_dpo/margin_mean": 5.922760963439941,
"margin_dpo/margin_std": 4.9100751876831055,
"step": 52
},
{
"KL/chosen_KL_mean": -0.1439056396484375,
"KL/mean": -3.184206485748291,
"KL/rejected_KL_mean": -6.224506378173828,
"KL/std": 5.271965980529785,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.08060884475708,
"fcm_dpo/q_t": 0.49856632947921753,
"grad_norm": 1.0281976461410522,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.6287667155265808,
"logits/rejected": -0.6076550483703613,
"logps/chosen": -53.45855712890625,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.00810241699219,
"loss": 1.3806,
"margin_dpo/margin_mean": 6.080608367919922,
"margin_dpo/margin_std": 6.02375602722168,
"step": 53
},
{
"KL/chosen_KL_mean": -0.37529754638671875,
"KL/mean": -3.0028505325317383,
"KL/rejected_KL_mean": -5.630397796630859,
"KL/std": 4.981309413909912,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.255102157592773,
"fcm_dpo/q_t": 0.4987609386444092,
"grad_norm": 0.8609330654144287,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.5812989473342896,
"logits/rejected": -0.5273776054382324,
"logps/chosen": -51.06395721435547,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.34579467773438,
"loss": 1.3814,
"margin_dpo/margin_mean": 5.255101203918457,
"margin_dpo/margin_std": 5.078667163848877,
"step": 54
},
{
"KL/chosen_KL_mean": -0.8801307678222656,
"KL/mean": -4.046672344207764,
"KL/rejected_KL_mean": -7.213207244873047,
"KL/std": 6.092576026916504,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.333078384399414,
"fcm_dpo/q_t": 0.4985068142414093,
"grad_norm": 0.9243965744972229,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6232470273971558,
"logits/rejected": -0.5590590238571167,
"logps/chosen": -63.495365142822266,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.20671081542969,
"loss": 1.3803,
"margin_dpo/margin_mean": 6.333077907562256,
"margin_dpo/margin_std": 7.390386581420898,
"step": 55
},
{
"KL/chosen_KL_mean": -0.6633682250976562,
"KL/mean": -3.713230609893799,
"KL/rejected_KL_mean": -6.763092041015625,
"KL/std": 5.995309829711914,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.099724769592285,
"fcm_dpo/q_t": 0.4985618591308594,
"grad_norm": 0.8911805152893066,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.6286439895629883,
"logits/rejected": -0.5872344970703125,
"logps/chosen": -58.59609603881836,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -100.93753051757812,
"loss": 1.3806,
"margin_dpo/margin_mean": 6.099723815917969,
"margin_dpo/margin_std": 7.246601104736328,
"step": 56
},
{
"KL/chosen_KL_mean": -0.7514553070068359,
"KL/mean": -4.159761428833008,
"KL/rejected_KL_mean": -7.568061828613281,
"KL/std": 5.89918327331543,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.816606521606445,
"fcm_dpo/q_t": 0.498392790555954,
"grad_norm": 0.9736577868461609,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.5877238512039185,
"logits/rejected": -0.5602452754974365,
"logps/chosen": -71.24674224853516,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -103.13351440429688,
"loss": 1.3799,
"margin_dpo/margin_mean": 6.816606521606445,
"margin_dpo/margin_std": 6.500908851623535,
"step": 57
},
{
"KL/chosen_KL_mean": -1.0264396667480469,
"KL/mean": -4.852134704589844,
"KL/rejected_KL_mean": -8.67782974243164,
"KL/std": 6.786162376403809,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.651383399963379,
"fcm_dpo/q_t": 0.498196005821228,
"grad_norm": 0.9956030249595642,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.6043895483016968,
"logits/rejected": -0.5297507047653198,
"logps/chosen": -63.15937805175781,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -93.29512023925781,
"loss": 1.3791,
"margin_dpo/margin_mean": 7.651384353637695,
"margin_dpo/margin_std": 8.097295761108398,
"step": 58
},
{
"KL/chosen_KL_mean": -1.4949493408203125,
"KL/mean": -5.701706886291504,
"KL/rejected_KL_mean": -9.908466339111328,
"KL/std": 7.460696697235107,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.413518905639648,
"fcm_dpo/q_t": 0.4980163276195526,
"grad_norm": 1.1153010129928589,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.6216074228286743,
"logits/rejected": -0.5813175439834595,
"logps/chosen": -53.42747497558594,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.79367065429688,
"loss": 1.3784,
"margin_dpo/margin_mean": 8.413517951965332,
"margin_dpo/margin_std": 8.59628677368164,
"step": 59
},
{
"KL/chosen_KL_mean": -2.799711227416992,
"KL/mean": -6.168231964111328,
"KL/rejected_KL_mean": -9.536746978759766,
"KL/std": 6.918834686279297,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.737033843994141,
"fcm_dpo/q_t": 0.4984115660190582,
"grad_norm": 0.9285561442375183,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.606009840965271,
"logits/rejected": -0.5456796884536743,
"logps/chosen": -63.74189758300781,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.93014526367188,
"loss": 1.38,
"margin_dpo/margin_mean": 6.737033367156982,
"margin_dpo/margin_std": 7.816346645355225,
"step": 60
},
{
"KL/chosen_KL_mean": -1.6795711517333984,
"KL/mean": -5.893689155578613,
"KL/rejected_KL_mean": -10.107810974121094,
"KL/std": 9.398534774780273,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.428237915039062,
"fcm_dpo/q_t": 0.498012900352478,
"grad_norm": 1.0661332607269287,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.6031610369682312,
"logits/rejected": -0.5691741704940796,
"logps/chosen": -62.31309509277344,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -99.96031188964844,
"loss": 1.3784,
"margin_dpo/margin_mean": 8.428237915039062,
"margin_dpo/margin_std": 11.791951179504395,
"step": 61
},
{
"KL/chosen_KL_mean": -1.8402385711669922,
"KL/mean": -5.0417022705078125,
"KL/rejected_KL_mean": -8.243167877197266,
"KL/std": 7.013890266418457,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.402925491333008,
"fcm_dpo/q_t": 0.4984903633594513,
"grad_norm": 0.8285877704620361,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.6119799613952637,
"logits/rejected": -0.577934980392456,
"logps/chosen": -57.99101257324219,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -83.80936431884766,
"loss": 1.3803,
"margin_dpo/margin_mean": 6.402926445007324,
"margin_dpo/margin_std": 8.621729850769043,
"step": 62
},
{
"KL/chosen_KL_mean": -3.1350955963134766,
"KL/mean": -7.4803924560546875,
"KL/rejected_KL_mean": -11.825679779052734,
"KL/std": 8.981889724731445,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.690584182739258,
"fcm_dpo/q_t": 0.4979510009288788,
"grad_norm": 1.0361840724945068,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.6032196283340454,
"logits/rejected": -0.5593255758285522,
"logps/chosen": -76.28248596191406,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -109.43574523925781,
"loss": 1.3781,
"margin_dpo/margin_mean": 8.690584182739258,
"margin_dpo/margin_std": 9.405391693115234,
"step": 63
},
{
"KL/chosen_KL_mean": -1.344888687133789,
"KL/mean": -6.571680068969727,
"KL/rejected_KL_mean": -11.798473358154297,
"KL/std": 9.811971664428711,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.45357894897461,
"fcm_dpo/q_t": 0.49753537774086,
"grad_norm": 1.1841435432434082,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.5892411470413208,
"logits/rejected": -0.5572826862335205,
"logps/chosen": -55.34349060058594,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -105.32867431640625,
"loss": 1.3765,
"margin_dpo/margin_mean": 10.45357894897461,
"margin_dpo/margin_std": 12.130199432373047,
"step": 64
},
{
"KL/chosen_KL_mean": -3.9116477966308594,
"KL/mean": -9.302034378051758,
"KL/rejected_KL_mean": -14.692424774169922,
"KL/std": 11.291559219360352,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.780777931213379,
"fcm_dpo/q_t": 0.49745821952819824,
"grad_norm": 1.2158167362213135,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6837905049324036,
"logits/rejected": -0.6725139021873474,
"logps/chosen": -68.74765014648438,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -124.63888549804688,
"loss": 1.3762,
"margin_dpo/margin_mean": 10.780776977539062,
"margin_dpo/margin_std": 12.423912048339844,
"step": 65
},
{
"KL/chosen_KL_mean": -3.748075485229492,
"KL/mean": -8.604604721069336,
"KL/rejected_KL_mean": -13.461128234863281,
"KL/std": 10.147451400756836,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.713046073913574,
"fcm_dpo/q_t": 0.4977099895477295,
"grad_norm": 1.069097638130188,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.6540935039520264,
"logits/rejected": -0.6227428913116455,
"logps/chosen": -55.19160079956055,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -89.09742736816406,
"loss": 1.3772,
"margin_dpo/margin_mean": 9.713046073913574,
"margin_dpo/margin_std": 12.457484245300293,
"step": 66
},
{
"KL/chosen_KL_mean": -3.5582504272460938,
"KL/mean": -8.677906036376953,
"KL/rejected_KL_mean": -13.797565460205078,
"KL/std": 10.710579872131348,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.239309310913086,
"fcm_dpo/q_t": 0.4975859522819519,
"grad_norm": 1.0920296907424927,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.6097283363342285,
"logits/rejected": -0.5669122934341431,
"logps/chosen": -62.89905548095703,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -86.58485412597656,
"loss": 1.3767,
"margin_dpo/margin_mean": 10.239309310913086,
"margin_dpo/margin_std": 13.077437400817871,
"step": 67
},
{
"KL/chosen_KL_mean": -4.973361968994141,
"KL/mean": -9.859228134155273,
"KL/rejected_KL_mean": -14.745094299316406,
"KL/std": 9.751331329345703,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.771727561950684,
"fcm_dpo/q_t": 0.4976961016654968,
"grad_norm": 0.9640390276908875,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6477208137512207,
"logits/rejected": -0.5843826532363892,
"logps/chosen": -70.17919921875,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -91.95234680175781,
"loss": 1.3771,
"margin_dpo/margin_mean": 9.771727561950684,
"margin_dpo/margin_std": 11.175668716430664,
"step": 68
},
{
"KL/chosen_KL_mean": -6.378795623779297,
"KL/mean": -13.339784622192383,
"KL/rejected_KL_mean": -20.300777435302734,
"KL/std": 13.335014343261719,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 13.921974182128906,
"fcm_dpo/q_t": 0.49671778082847595,
"grad_norm": 1.3528519868850708,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.6241370439529419,
"logits/rejected": -0.6030097007751465,
"logps/chosen": -66.19803619384766,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -123.68963623046875,
"loss": 1.3733,
"margin_dpo/margin_mean": 13.921974182128906,
"margin_dpo/margin_std": 15.130611419677734,
"step": 69
},
{
"KL/chosen_KL_mean": -9.457656860351562,
"KL/mean": -16.863452911376953,
"KL/rejected_KL_mean": -24.269248962402344,
"KL/std": 17.1834716796875,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 14.811593055725098,
"fcm_dpo/q_t": 0.4965081214904785,
"grad_norm": 1.351995825767517,
"learning_rate": 5e-07,
"logits/chosen": -0.6134965419769287,
"logits/rejected": -0.5804483890533447,
"logps/chosen": -71.38829803466797,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -115.33003234863281,
"loss": 1.3724,
"margin_dpo/margin_mean": 14.811592102050781,
"margin_dpo/margin_std": 18.324480056762695,
"step": 70
},
{
"KL/chosen_KL_mean": -10.057798385620117,
"KL/mean": -19.228286743164062,
"KL/rejected_KL_mean": -28.398780822753906,
"KL/std": 19.04326820373535,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 18.340984344482422,
"fcm_dpo/q_t": 0.4956763982772827,
"grad_norm": 1.5885552167892456,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.6829918622970581,
"logits/rejected": -0.650745153427124,
"logps/chosen": -71.80813598632812,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -125.73539733886719,
"loss": 1.3692,
"margin_dpo/margin_mean": 18.340984344482422,
"margin_dpo/margin_std": 21.016489028930664,
"step": 71
},
{
"KL/chosen_KL_mean": -11.907201766967773,
"KL/mean": -21.156211853027344,
"KL/rejected_KL_mean": -30.405223846435547,
"KL/std": 20.45846939086914,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 18.498023986816406,
"fcm_dpo/q_t": 0.4956393539905548,
"grad_norm": 1.4867459535598755,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.6599299907684326,
"logits/rejected": -0.6240644454956055,
"logps/chosen": -77.96061706542969,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -125.69221496582031,
"loss": 1.3691,
"margin_dpo/margin_mean": 18.498022079467773,
"margin_dpo/margin_std": 25.244348526000977,
"step": 72
},
{
"KL/chosen_KL_mean": -14.035341262817383,
"KL/mean": -23.60753059387207,
"KL/rejected_KL_mean": -33.17972183227539,
"KL/std": 26.00308609008789,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 19.144380569458008,
"fcm_dpo/q_t": 0.4954878091812134,
"grad_norm": 1.645006775856018,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.7174981832504272,
"logits/rejected": -0.681929349899292,
"logps/chosen": -80.29161071777344,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -123.63585662841797,
"loss": 1.3686,
"margin_dpo/margin_mean": 19.144378662109375,
"margin_dpo/margin_std": 33.420066833496094,
"step": 73
},
{
"KL/chosen_KL_mean": -14.948610305786133,
"KL/mean": -26.0163631439209,
"KL/rejected_KL_mean": -37.0841178894043,
"KL/std": 29.37421989440918,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 22.13550567626953,
"fcm_dpo/q_t": 0.49478527903556824,
"grad_norm": 1.8224822282791138,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.6496589183807373,
"logits/rejected": -0.6474369764328003,
"logps/chosen": -68.37348937988281,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -133.0310516357422,
"loss": 1.3659,
"margin_dpo/margin_mean": 22.13550567626953,
"margin_dpo/margin_std": 37.34225845336914,
"step": 74
},
{
"KL/chosen_KL_mean": -15.307416915893555,
"KL/mean": -31.02456283569336,
"KL/rejected_KL_mean": -46.74169158935547,
"KL/std": 30.637062072753906,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 31.434282302856445,
"fcm_dpo/q_t": 0.4925920069217682,
"grad_norm": 2.1851608753204346,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6600038409233093,
"logits/rejected": -0.6581864953041077,
"logps/chosen": -67.16908264160156,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -157.99566650390625,
"loss": 1.3572,
"margin_dpo/margin_mean": 31.434280395507812,
"margin_dpo/margin_std": 35.90174102783203,
"step": 75
},
{
"KL/chosen_KL_mean": -18.540132522583008,
"KL/mean": -28.030113220214844,
"KL/rejected_KL_mean": -37.52008819580078,
"KL/std": 25.205242156982422,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 18.979961395263672,
"fcm_dpo/q_t": 0.49552568793296814,
"grad_norm": 1.614672064781189,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.6978548765182495,
"logits/rejected": -0.6618174910545349,
"logps/chosen": -71.80616760253906,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -115.7367172241211,
"loss": 1.3687,
"margin_dpo/margin_mean": 18.979961395263672,
"margin_dpo/margin_std": 28.169784545898438,
"step": 76
},
{
"KL/chosen_KL_mean": -17.505142211914062,
"KL/mean": -33.80626678466797,
"KL/rejected_KL_mean": -50.107398986816406,
"KL/std": 34.56831359863281,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 32.602256774902344,
"fcm_dpo/q_t": 0.4923163056373596,
"grad_norm": 2.1774914264678955,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.6647493839263916,
"logits/rejected": -0.631820797920227,
"logps/chosen": -75.60182189941406,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -143.88101196289062,
"loss": 1.3561,
"margin_dpo/margin_mean": 32.602256774902344,
"margin_dpo/margin_std": 40.01980972290039,
"step": 77
},
{
"KL/chosen_KL_mean": -18.282596588134766,
"KL/mean": -30.055492401123047,
"KL/rejected_KL_mean": -41.82838821411133,
"KL/std": 29.950658798217773,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 23.545791625976562,
"fcm_dpo/q_t": 0.4944510757923126,
"grad_norm": 1.7334191799163818,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.631340503692627,
"logits/rejected": -0.6118667721748352,
"logps/chosen": -73.89637756347656,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -126.76275634765625,
"loss": 1.3645,
"margin_dpo/margin_mean": 23.545791625976562,
"margin_dpo/margin_std": 35.435997009277344,
"step": 78
},
{
"KL/chosen_KL_mean": -19.754838943481445,
"KL/mean": -32.51957702636719,
"KL/rejected_KL_mean": -45.28432083129883,
"KL/std": 32.64012908935547,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 25.52947998046875,
"fcm_dpo/q_t": 0.4939836859703064,
"grad_norm": 1.8259940147399902,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6151328682899475,
"logits/rejected": -0.5880134105682373,
"logps/chosen": -75.205322265625,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -132.931884765625,
"loss": 1.3627,
"margin_dpo/margin_mean": 25.529478073120117,
"margin_dpo/margin_std": 39.56281280517578,
"step": 79
},
{
"KL/chosen_KL_mean": -24.592361450195312,
"KL/mean": -38.224281311035156,
"KL/rejected_KL_mean": -51.856201171875,
"KL/std": 38.630069732666016,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 27.263832092285156,
"fcm_dpo/q_t": 0.49357831478118896,
"grad_norm": 2.117938995361328,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.7042999267578125,
"logits/rejected": -0.672730565071106,
"logps/chosen": -83.11164855957031,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -139.40370178222656,
"loss": 1.3613,
"margin_dpo/margin_mean": 27.263832092285156,
"margin_dpo/margin_std": 45.90215301513672,
"step": 80
},
{
"KL/chosen_KL_mean": -25.093727111816406,
"KL/mean": -43.66114044189453,
"KL/rejected_KL_mean": -62.228553771972656,
"KL/std": 46.15410232543945,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 37.13482666015625,
"fcm_dpo/q_t": 0.4912540912628174,
"grad_norm": 2.5071187019348145,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.6825768947601318,
"logits/rejected": -0.6628377437591553,
"logps/chosen": -91.54259490966797,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -191.89126586914062,
"loss": 1.3523,
"margin_dpo/margin_mean": 37.13482666015625,
"margin_dpo/margin_std": 55.59607696533203,
"step": 81
},
{
"KL/chosen_KL_mean": -28.598669052124023,
"KL/mean": -43.321502685546875,
"KL/rejected_KL_mean": -58.04433822631836,
"KL/std": 42.634342193603516,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 29.445667266845703,
"fcm_dpo/q_t": 0.4930664896965027,
"grad_norm": 2.0818185806274414,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.5801957249641418,
"logits/rejected": -0.5773532390594482,
"logps/chosen": -80.8310546875,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -148.78759765625,
"loss": 1.3596,
"margin_dpo/margin_mean": 29.445667266845703,
"margin_dpo/margin_std": 59.75962829589844,
"step": 82
},
{
"KL/chosen_KL_mean": -30.05629539489746,
"KL/mean": -49.223304748535156,
"KL/rejected_KL_mean": -68.39031982421875,
"KL/std": 45.316627502441406,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 38.33401870727539,
"fcm_dpo/q_t": 0.4909701645374298,
"grad_norm": 2.6023781299591064,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.614409327507019,
"logits/rejected": -0.5958919525146484,
"logps/chosen": -85.88368225097656,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -172.10623168945312,
"loss": 1.3511,
"margin_dpo/margin_mean": 38.334014892578125,
"margin_dpo/margin_std": 54.87122344970703,
"step": 83
},
{
"KL/chosen_KL_mean": -32.64569091796875,
"KL/mean": -46.82188415527344,
"KL/rejected_KL_mean": -60.99808120727539,
"KL/std": 39.93206787109375,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 28.352384567260742,
"fcm_dpo/q_t": 0.49331820011138916,
"grad_norm": 2.06538462638855,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.645265519618988,
"logits/rejected": -0.6129859685897827,
"logps/chosen": -99.82185363769531,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -148.29669189453125,
"loss": 1.3601,
"margin_dpo/margin_mean": 28.352384567260742,
"margin_dpo/margin_std": 39.42462158203125,
"step": 84
},
{
"KL/chosen_KL_mean": -32.759803771972656,
"KL/mean": -47.751686096191406,
"KL/rejected_KL_mean": -62.743568420410156,
"KL/std": 39.7208251953125,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 29.9837646484375,
"fcm_dpo/q_t": 0.4929351806640625,
"grad_norm": 2.0294606685638428,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.5920377969741821,
"logits/rejected": -0.5608456134796143,
"logps/chosen": -91.16642761230469,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -141.3823699951172,
"loss": 1.3588,
"margin_dpo/margin_mean": 29.9837646484375,
"margin_dpo/margin_std": 49.01397705078125,
"step": 85
},
{
"KL/chosen_KL_mean": -44.688865661621094,
"KL/mean": -60.836769104003906,
"KL/rejected_KL_mean": -76.98466491699219,
"KL/std": 61.91584396362305,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 32.29580307006836,
"fcm_dpo/q_t": 0.492409884929657,
"grad_norm": 2.3101718425750732,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.6010197401046753,
"logits/rejected": -0.5972357392311096,
"logps/chosen": -100.82633209228516,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -165.1063232421875,
"loss": 1.3578,
"margin_dpo/margin_mean": 32.29580307006836,
"margin_dpo/margin_std": 87.33891296386719,
"step": 86
},
{
"KL/chosen_KL_mean": -44.94416046142578,
"KL/mean": -66.77362823486328,
"KL/rejected_KL_mean": -88.60308837890625,
"KL/std": 63.363311767578125,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 43.65892028808594,
"fcm_dpo/q_t": 0.48971858620643616,
"grad_norm": 2.6735432147979736,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.6408615112304688,
"logits/rejected": -0.6341418027877808,
"logps/chosen": -100.58026123046875,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -184.0706787109375,
"loss": 1.3471,
"margin_dpo/margin_mean": 43.6589241027832,
"margin_dpo/margin_std": 80.54689025878906,
"step": 87
},
{
"KL/chosen_KL_mean": -58.59263229370117,
"KL/mean": -75.60669708251953,
"KL/rejected_KL_mean": -92.62075805664062,
"KL/std": 71.91948699951172,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 34.02812957763672,
"fcm_dpo/q_t": 0.49200400710105896,
"grad_norm": 2.6304991245269775,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.5753469467163086,
"logits/rejected": -0.57887864112854,
"logps/chosen": -132.26377868652344,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -199.32925415039062,
"loss": 1.3562,
"margin_dpo/margin_mean": 34.02812576293945,
"margin_dpo/margin_std": 82.46296691894531,
"step": 88
},
{
"KL/chosen_KL_mean": -44.48945617675781,
"KL/mean": -63.188812255859375,
"KL/rejected_KL_mean": -81.88816833496094,
"KL/std": 67.53860473632812,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 37.39871597290039,
"fcm_dpo/q_t": 0.4911901354789734,
"grad_norm": 2.4624993801116943,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.6243025660514832,
"logits/rejected": -0.6062852144241333,
"logps/chosen": -105.11437225341797,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -163.97171020507812,
"loss": 1.3528,
"margin_dpo/margin_mean": 37.398719787597656,
"margin_dpo/margin_std": 81.66424560546875,
"step": 89
},
{
"KL/chosen_KL_mean": -48.97517395019531,
"KL/mean": -81.48391723632812,
"KL/rejected_KL_mean": -113.99266052246094,
"KL/std": 89.38653564453125,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 65.01748657226562,
"fcm_dpo/q_t": 0.48473870754241943,
"grad_norm": 3.8177247047424316,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.5124300122261047,
"logits/rejected": -0.5386440753936768,
"logps/chosen": -102.26048278808594,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -225.53738403320312,
"loss": 1.3289,
"margin_dpo/margin_mean": 65.01748657226562,
"margin_dpo/margin_std": 113.12422943115234,
"step": 90
},
{
"KL/chosen_KL_mean": -57.929931640625,
"KL/mean": -77.27233123779297,
"KL/rejected_KL_mean": -96.6147232055664,
"KL/std": 69.36103057861328,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 38.684791564941406,
"fcm_dpo/q_t": 0.4909099340438843,
"grad_norm": 2.578143358230591,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.5383755564689636,
"logits/rejected": -0.5206432342529297,
"logps/chosen": -119.73289489746094,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -184.48867797851562,
"loss": 1.3522,
"margin_dpo/margin_mean": 38.68478775024414,
"margin_dpo/margin_std": 95.96929931640625,
"step": 91
},
{
"KL/chosen_KL_mean": -53.14446258544922,
"KL/mean": -75.70481872558594,
"KL/rejected_KL_mean": -98.26518249511719,
"KL/std": 75.28404235839844,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 45.120723724365234,
"fcm_dpo/q_t": 0.4893913269042969,
"grad_norm": 3.023728370666504,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.5423229932785034,
"logits/rejected": -0.5309740304946899,
"logps/chosen": -104.78523254394531,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -176.1463623046875,
"loss": 1.3461,
"margin_dpo/margin_mean": 45.1207275390625,
"margin_dpo/margin_std": 90.38905334472656,
"step": 92
},
{
"KL/chosen_KL_mean": -59.912742614746094,
"KL/mean": -81.39234924316406,
"KL/rejected_KL_mean": -102.8719482421875,
"KL/std": 67.60503387451172,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 42.959205627441406,
"fcm_dpo/q_t": 0.4898951053619385,
"grad_norm": 2.951615810394287,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.5452234745025635,
"logits/rejected": -0.524033784866333,
"logps/chosen": -112.44198608398438,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -180.03269958496094,
"loss": 1.3476,
"margin_dpo/margin_mean": 42.959205627441406,
"margin_dpo/margin_std": 79.7041015625,
"step": 93
},
{
"KL/chosen_KL_mean": -66.89620208740234,
"KL/mean": -95.25006103515625,
"KL/rejected_KL_mean": -123.60391235351562,
"KL/std": 84.54957580566406,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 56.70771408081055,
"fcm_dpo/q_t": 0.4866675138473511,
"grad_norm": 3.3346071243286133,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.5058275461196899,
"logits/rejected": -0.5132472515106201,
"logps/chosen": -128.11880493164062,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -223.20294189453125,
"loss": 1.3356,
"margin_dpo/margin_mean": 56.70771026611328,
"margin_dpo/margin_std": 94.90916442871094,
"step": 94
},
{
"KL/chosen_KL_mean": -66.25115203857422,
"KL/mean": -89.31552124023438,
"KL/rejected_KL_mean": -112.37989044189453,
"KL/std": 95.1351089477539,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 46.12873840332031,
"fcm_dpo/q_t": 0.4891487956047058,
"grad_norm": 3.2070772647857666,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.47669315338134766,
"logits/rejected": -0.4623689651489258,
"logps/chosen": -118.77479553222656,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -188.26023864746094,
"loss": 1.3457,
"margin_dpo/margin_mean": 46.12873840332031,
"margin_dpo/margin_std": 102.57861328125,
"step": 95
},
{
"KL/chosen_KL_mean": -75.35250854492188,
"KL/mean": -108.11980438232422,
"KL/rejected_KL_mean": -140.88711547851562,
"KL/std": 99.36506652832031,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 65.53459930419922,
"fcm_dpo/q_t": 0.4846518039703369,
"grad_norm": 3.4605512619018555,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.4755854606628418,
"logits/rejected": -0.46957653760910034,
"logps/chosen": -137.50949096679688,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -237.48312377929688,
"loss": 1.3288,
"margin_dpo/margin_mean": 65.53459167480469,
"margin_dpo/margin_std": 118.84884643554688,
"step": 96
},
{
"KL/chosen_KL_mean": -70.3343734741211,
"KL/mean": -93.92042541503906,
"KL/rejected_KL_mean": -117.50647735595703,
"KL/std": 90.63258361816406,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 47.1721076965332,
"fcm_dpo/q_t": 0.4889163076877594,
"grad_norm": 3.436333417892456,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.4467729330062866,
"logits/rejected": -0.43124696612358093,
"logps/chosen": -124.98074340820312,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -194.47122192382812,
"loss": 1.3443,
"margin_dpo/margin_mean": 47.17211151123047,
"margin_dpo/margin_std": 92.33546447753906,
"step": 97
},
{
"KL/chosen_KL_mean": -95.86998748779297,
"KL/mean": -124.71914672851562,
"KL/rejected_KL_mean": -153.5682830810547,
"KL/std": 112.18405151367188,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 57.69831848144531,
"fcm_dpo/q_t": 0.486421674489975,
"grad_norm": 4.174478530883789,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.4279845356941223,
"logits/rejected": -0.43450891971588135,
"logps/chosen": -161.1286163330078,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -250.0957794189453,
"loss": 1.3368,
"margin_dpo/margin_mean": 57.69831085205078,
"margin_dpo/margin_std": 136.08026123046875,
"step": 98
},
{
"KL/chosen_KL_mean": -70.87637329101562,
"KL/mean": -108.80039978027344,
"KL/rejected_KL_mean": -146.72442626953125,
"KL/std": 105.301513671875,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 75.84803771972656,
"fcm_dpo/q_t": 0.48224809765815735,
"grad_norm": 4.009462833404541,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.3845648467540741,
"logits/rejected": -0.39073482155799866,
"logps/chosen": -116.51486206054688,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -233.162353515625,
"loss": 1.3202,
"margin_dpo/margin_mean": 75.84803771972656,
"margin_dpo/margin_std": 136.87684631347656,
"step": 99
},
{
"KL/chosen_KL_mean": -88.81651306152344,
"KL/mean": -101.03938293457031,
"KL/rejected_KL_mean": -113.26225280761719,
"KL/std": 101.6782455444336,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 24.445741653442383,
"fcm_dpo/q_t": 0.49424445629119873,
"grad_norm": 3.8610727787017822,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.3569236397743225,
"logits/rejected": -0.35417813062667847,
"logps/chosen": -146.41049194335938,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -187.3224639892578,
"loss": 1.3669,
"margin_dpo/margin_mean": 24.44573974609375,
"margin_dpo/margin_std": 121.69652557373047,
"step": 100
},
{
"KL/chosen_KL_mean": -103.06278991699219,
"KL/mean": -123.61421203613281,
"KL/rejected_KL_mean": -144.16561889648438,
"KL/std": 117.26971435546875,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 41.10283660888672,
"fcm_dpo/q_t": 0.49035578966140747,
"grad_norm": 3.403747081756592,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.3375687599182129,
"logits/rejected": -0.3361296057701111,
"logps/chosen": -164.71163940429688,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -227.35531616210938,
"loss": 1.352,
"margin_dpo/margin_mean": 41.10283660888672,
"margin_dpo/margin_std": 134.45703125,
"step": 101
},
{
"KL/chosen_KL_mean": -104.34712982177734,
"KL/mean": -118.05978393554688,
"KL/rejected_KL_mean": -131.77243041992188,
"KL/std": 111.49862670898438,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 27.42530059814453,
"fcm_dpo/q_t": 0.4935542345046997,
"grad_norm": 3.6488194465637207,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.36390089988708496,
"logits/rejected": -0.3599966764450073,
"logps/chosen": -168.42601013183594,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -199.95950317382812,
"loss": 1.3651,
"margin_dpo/margin_mean": 27.425302505493164,
"margin_dpo/margin_std": 138.6975555419922,
"step": 102
},
{
"KL/chosen_KL_mean": -99.00223541259766,
"KL/mean": -133.520263671875,
"KL/rejected_KL_mean": -168.03829956054688,
"KL/std": 128.6646728515625,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 69.03605651855469,
"fcm_dpo/q_t": 0.4838695824146271,
"grad_norm": 3.918245553970337,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.3378918170928955,
"logits/rejected": -0.33644479513168335,
"logps/chosen": -160.301513671875,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -261.61102294921875,
"loss": 1.3284,
"margin_dpo/margin_mean": 69.03605651855469,
"margin_dpo/margin_std": 166.41140747070312,
"step": 103
},
{
"KL/chosen_KL_mean": -91.2227783203125,
"KL/mean": -124.46497344970703,
"KL/rejected_KL_mean": -157.70718383789062,
"KL/std": 121.44420623779297,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 66.48440551757812,
"fcm_dpo/q_t": 0.4844798445701599,
"grad_norm": 4.455353736877441,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.3243046998977661,
"logits/rejected": -0.33145880699157715,
"logps/chosen": -145.59555053710938,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -247.2718963623047,
"loss": 1.33,
"margin_dpo/margin_mean": 66.4843978881836,
"margin_dpo/margin_std": 155.1224365234375,
"step": 104
},
{
"KL/chosen_KL_mean": -89.49755859375,
"KL/mean": -140.19944763183594,
"KL/rejected_KL_mean": -190.9013214111328,
"KL/std": 112.19564819335938,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 101.40377807617188,
"fcm_dpo/q_t": 0.47620758414268494,
"grad_norm": 4.004859924316406,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.2886583209037781,
"logits/rejected": -0.2872416079044342,
"logps/chosen": -144.13650512695312,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -288.8748474121094,
"loss": 1.297,
"margin_dpo/margin_mean": 101.40377807617188,
"margin_dpo/margin_std": 133.38304138183594,
"step": 105
},
{
"KL/chosen_KL_mean": -103.17796325683594,
"KL/mean": -136.15133666992188,
"KL/rejected_KL_mean": -169.12469482421875,
"KL/std": 121.09173583984375,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 65.94671630859375,
"fcm_dpo/q_t": 0.4845297336578369,
"grad_norm": 3.938218593597412,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.2749488949775696,
"logits/rejected": -0.29554685950279236,
"logps/chosen": -158.0108642578125,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -254.34930419921875,
"loss": 1.3289,
"margin_dpo/margin_mean": 65.94671630859375,
"margin_dpo/margin_std": 128.7685546875,
"step": 106
},
{
"KL/chosen_KL_mean": -125.75239562988281,
"KL/mean": -159.4122314453125,
"KL/rejected_KL_mean": -193.07208251953125,
"KL/std": 142.40440368652344,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 67.31967163085938,
"fcm_dpo/q_t": 0.48424452543258667,
"grad_norm": 4.5661797523498535,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.2691690921783447,
"logits/rejected": -0.27037227153778076,
"logps/chosen": -195.460205078125,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -287.81158447265625,
"loss": 1.3291,
"margin_dpo/margin_mean": 67.31967163085938,
"margin_dpo/margin_std": 150.08763122558594,
"step": 107
},
{
"KL/chosen_KL_mean": -99.49694061279297,
"KL/mean": -149.0161895751953,
"KL/rejected_KL_mean": -198.53543090820312,
"KL/std": 152.70819091796875,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.03849029541016,
"fcm_dpo/q_t": 0.4770011305809021,
"grad_norm": 4.919802665710449,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.2482558786869049,
"logits/rejected": -0.2509337365627289,
"logps/chosen": -155.5068359375,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -294.3314514160156,
"loss": 1.3035,
"margin_dpo/margin_mean": 99.03849029541016,
"margin_dpo/margin_std": 194.90606689453125,
"step": 108
},
{
"KL/chosen_KL_mean": -109.02256774902344,
"KL/mean": -155.30252075195312,
"KL/rejected_KL_mean": -201.5824737548828,
"KL/std": 154.06240844726562,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 92.55990600585938,
"fcm_dpo/q_t": 0.4782617688179016,
"grad_norm": 4.247311592102051,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.23186370730400085,
"logits/rejected": -0.22922776639461517,
"logps/chosen": -171.90805053710938,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -300.2681884765625,
"loss": 1.3106,
"margin_dpo/margin_mean": 92.55990600585938,
"margin_dpo/margin_std": 207.37530517578125,
"step": 109
},
{
"KL/chosen_KL_mean": -112.6722640991211,
"KL/mean": -150.91159057617188,
"KL/rejected_KL_mean": -189.1509552001953,
"KL/std": 145.97552490234375,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 76.47869873046875,
"fcm_dpo/q_t": 0.48217493295669556,
"grad_norm": 4.57998514175415,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.19486288726329803,
"logits/rejected": -0.20499581098556519,
"logps/chosen": -171.4259490966797,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -268.9009704589844,
"loss": 1.3238,
"margin_dpo/margin_mean": 76.47869873046875,
"margin_dpo/margin_std": 185.61337280273438,
"step": 110
},
{
"KL/chosen_KL_mean": -127.63728332519531,
"KL/mean": -176.09266662597656,
"KL/rejected_KL_mean": -224.5480499267578,
"KL/std": 172.54864501953125,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 96.91078186035156,
"fcm_dpo/q_t": 0.4774002432823181,
"grad_norm": 4.575905799865723,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.2067311704158783,
"logits/rejected": -0.20800232887268066,
"logps/chosen": -196.26138305664062,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -322.9769287109375,
"loss": 1.3096,
"margin_dpo/margin_mean": 96.91078186035156,
"margin_dpo/margin_std": 237.3184356689453,
"step": 111
},
{
"KL/chosen_KL_mean": -111.8941650390625,
"KL/mean": -135.28634643554688,
"KL/rejected_KL_mean": -158.6785430908203,
"KL/std": 107.61951446533203,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 46.78435516357422,
"fcm_dpo/q_t": 0.48899713158607483,
"grad_norm": 5.0745391845703125,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.13949084281921387,
"logits/rejected": -0.1618117392063141,
"logps/chosen": -162.143798828125,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -223.45297241210938,
"loss": 1.3476,
"margin_dpo/margin_mean": 46.78435516357422,
"margin_dpo/margin_std": 145.43516540527344,
"step": 112
},
{
"KL/chosen_KL_mean": -146.45416259765625,
"KL/mean": -168.53370666503906,
"KL/rejected_KL_mean": -190.61326599121094,
"KL/std": 156.10946655273438,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 44.15910339355469,
"fcm_dpo/q_t": 0.48973506689071655,
"grad_norm": 4.978002071380615,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.18881914019584656,
"logits/rejected": -0.1964729130268097,
"logps/chosen": -213.1671142578125,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -268.58197021484375,
"loss": 1.3508,
"margin_dpo/margin_mean": 44.15910339355469,
"margin_dpo/margin_std": 160.76222229003906,
"step": 113
},
{
"KL/chosen_KL_mean": -121.00819396972656,
"KL/mean": -166.08595275878906,
"KL/rejected_KL_mean": -211.16371154785156,
"KL/std": 159.60922241210938,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 90.155517578125,
"fcm_dpo/q_t": 0.47918766736984253,
"grad_norm": 4.788021087646484,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.1772385984659195,
"logits/rejected": -0.18084228038787842,
"logps/chosen": -178.79327392578125,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -298.27337646484375,
"loss": 1.3122,
"margin_dpo/margin_mean": 90.155517578125,
"margin_dpo/margin_std": 198.69287109375,
"step": 114
},
{
"KL/chosen_KL_mean": -145.8248748779297,
"KL/mean": -179.474365234375,
"KL/rejected_KL_mean": -213.12384033203125,
"KL/std": 172.7808837890625,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 67.29896545410156,
"fcm_dpo/q_t": 0.48439520597457886,
"grad_norm": 4.369140148162842,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.23590882122516632,
"logits/rejected": -0.24995499849319458,
"logps/chosen": -211.40750122070312,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -311.6893615722656,
"loss": 1.3361,
"margin_dpo/margin_mean": 67.29896545410156,
"margin_dpo/margin_std": 233.44024658203125,
"step": 115
},
{
"KL/chosen_KL_mean": -99.08972930908203,
"KL/mean": -135.49847412109375,
"KL/rejected_KL_mean": -171.9072265625,
"KL/std": 123.32942199707031,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 72.8175048828125,
"fcm_dpo/q_t": 0.48297327756881714,
"grad_norm": 5.752792835235596,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.23723204433918,
"logits/rejected": -0.25562626123428345,
"logps/chosen": -150.49005126953125,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -252.4290771484375,
"loss": 1.3235,
"margin_dpo/margin_mean": 72.81749725341797,
"margin_dpo/margin_std": 141.2083282470703,
"step": 116
},
{
"KL/chosen_KL_mean": -142.8017120361328,
"KL/mean": -156.0880126953125,
"KL/rejected_KL_mean": -169.37429809570312,
"KL/std": 135.4499053955078,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 26.572589874267578,
"fcm_dpo/q_t": 0.49375978112220764,
"grad_norm": 4.810681343078613,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.19404463469982147,
"logits/rejected": -0.19508536159992218,
"logps/chosen": -212.10012817382812,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -235.95828247070312,
"loss": 1.3678,
"margin_dpo/margin_mean": 26.572589874267578,
"margin_dpo/margin_std": 167.6946258544922,
"step": 117
},
{
"KL/chosen_KL_mean": -102.44888305664062,
"KL/mean": -137.05758666992188,
"KL/rejected_KL_mean": -171.66629028320312,
"KL/std": 128.6864776611328,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 69.2174072265625,
"fcm_dpo/q_t": 0.48380762338638306,
"grad_norm": 5.621657371520996,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.3070230484008789,
"logits/rejected": -0.3028327226638794,
"logps/chosen": -158.08985900878906,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -247.33534240722656,
"loss": 1.327,
"margin_dpo/margin_mean": 69.2174072265625,
"margin_dpo/margin_std": 148.856201171875,
"step": 118
},
{
"KL/chosen_KL_mean": -149.1088104248047,
"KL/mean": -178.7309112548828,
"KL/rejected_KL_mean": -208.35299682617188,
"KL/std": 138.42144775390625,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 59.24419403076172,
"fcm_dpo/q_t": 0.4861356019973755,
"grad_norm": 4.433512210845947,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.297086238861084,
"logits/rejected": -0.3139578104019165,
"logps/chosen": -222.61900329589844,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -311.3302917480469,
"loss": 1.3383,
"margin_dpo/margin_mean": 59.24420166015625,
"margin_dpo/margin_std": 175.93429565429688,
"step": 119
},
{
"KL/chosen_KL_mean": -147.74859619140625,
"KL/mean": -195.46881103515625,
"KL/rejected_KL_mean": -243.18905639648438,
"KL/std": 166.3782958984375,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 95.4404525756836,
"fcm_dpo/q_t": 0.4778479337692261,
"grad_norm": 8.700891494750977,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.293715238571167,
"logits/rejected": -0.30091625452041626,
"logps/chosen": -224.5294189453125,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -351.2127685546875,
"loss": 1.3067,
"margin_dpo/margin_mean": 95.4404525756836,
"margin_dpo/margin_std": 195.20413208007812,
"step": 120
},
{
"KL/chosen_KL_mean": -136.00462341308594,
"KL/mean": -193.11111450195312,
"KL/rejected_KL_mean": -250.2176055908203,
"KL/std": 168.69393920898438,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 114.21296691894531,
"fcm_dpo/q_t": 0.473272442817688,
"grad_norm": 6.882490634918213,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.2620672583580017,
"logits/rejected": -0.2657386064529419,
"logps/chosen": -197.79452514648438,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -360.212158203125,
"loss": 1.29,
"margin_dpo/margin_mean": 114.21297454833984,
"margin_dpo/margin_std": 193.61260986328125,
"step": 121
},
{
"KL/chosen_KL_mean": -107.97517395019531,
"KL/mean": -183.01693725585938,
"KL/rejected_KL_mean": -258.0587158203125,
"KL/std": 169.63442993164062,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 150.0835418701172,
"fcm_dpo/q_t": 0.46513625979423523,
"grad_norm": 10.18962287902832,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.19436302781105042,
"logits/rejected": -0.20805394649505615,
"logps/chosen": -154.87738037109375,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -364.7729187011719,
"loss": 1.2581,
"margin_dpo/margin_mean": 150.0835418701172,
"margin_dpo/margin_std": 193.8829345703125,
"step": 122
},
{
"KL/chosen_KL_mean": -145.32444763183594,
"KL/mean": -189.00643920898438,
"KL/rejected_KL_mean": -232.6884307861328,
"KL/std": 162.26889038085938,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 87.36398315429688,
"fcm_dpo/q_t": 0.47969871759414673,
"grad_norm": 4.467272758483887,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.15873804688453674,
"logits/rejected": -0.1691892147064209,
"logps/chosen": -206.6630859375,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -320.46380615234375,
"loss": 1.3167,
"margin_dpo/margin_mean": 87.36398315429688,
"margin_dpo/margin_std": 223.19760131835938,
"step": 123
},
{
"KL/chosen_KL_mean": -179.08816528320312,
"KL/mean": -245.78233337402344,
"KL/rejected_KL_mean": -312.47650146484375,
"KL/std": 236.51095581054688,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 133.3883056640625,
"fcm_dpo/q_t": 0.4694249927997589,
"grad_norm": 5.300126552581787,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.1333819478750229,
"logits/rejected": -0.14498867094516754,
"logps/chosen": -250.5364990234375,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -430.05706787109375,
"loss": 1.2852,
"margin_dpo/margin_mean": 133.38832092285156,
"margin_dpo/margin_std": 305.14990234375,
"step": 124
},
{
"KL/chosen_KL_mean": -135.3007049560547,
"KL/mean": -188.19273376464844,
"KL/rejected_KL_mean": -241.08474731445312,
"KL/std": 185.8060302734375,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 105.78404998779297,
"fcm_dpo/q_t": 0.47543567419052124,
"grad_norm": 5.876065254211426,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.11746616661548615,
"logits/rejected": -0.1338275969028473,
"logps/chosen": -185.43765258789062,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -325.0733642578125,
"loss": 1.3011,
"margin_dpo/margin_mean": 105.7840576171875,
"margin_dpo/margin_std": 230.14541625976562,
"step": 125
},
{
"KL/chosen_KL_mean": -139.29977416992188,
"KL/mean": -200.52383422851562,
"KL/rejected_KL_mean": -261.7478942871094,
"KL/std": 185.92526245117188,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 122.4481201171875,
"fcm_dpo/q_t": 0.47186100482940674,
"grad_norm": 4.977867126464844,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.1627553403377533,
"logits/rejected": -0.17375735938549042,
"logps/chosen": -194.96685791015625,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -359.8776550292969,
"loss": 1.2868,
"margin_dpo/margin_mean": 122.4481201171875,
"margin_dpo/margin_std": 236.15780639648438,
"step": 126
},
{
"KL/chosen_KL_mean": -138.76910400390625,
"KL/mean": -169.734130859375,
"KL/rejected_KL_mean": -200.6991729736328,
"KL/std": 156.78909301757812,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 61.930057525634766,
"fcm_dpo/q_t": 0.4853692650794983,
"grad_norm": 5.053523540496826,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.20304161310195923,
"logits/rejected": -0.21361683309078217,
"logps/chosen": -195.3237762451172,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -277.49493408203125,
"loss": 1.3373,
"margin_dpo/margin_mean": 61.9300537109375,
"margin_dpo/margin_std": 196.28656005859375,
"step": 127
},
{
"KL/chosen_KL_mean": -137.2354736328125,
"KL/mean": -177.0436553955078,
"KL/rejected_KL_mean": -216.85183715820312,
"KL/std": 186.32858276367188,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 79.61640167236328,
"fcm_dpo/q_t": 0.4815317690372467,
"grad_norm": 4.554657936096191,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.18499845266342163,
"logits/rejected": -0.2032622992992401,
"logps/chosen": -195.35641479492188,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -293.2908020019531,
"loss": 1.3258,
"margin_dpo/margin_mean": 79.61640167236328,
"margin_dpo/margin_std": 241.3561248779297,
"step": 128
},
{
"KL/chosen_KL_mean": -175.23907470703125,
"KL/mean": -220.06051635742188,
"KL/rejected_KL_mean": -264.8819580078125,
"KL/std": 237.86366271972656,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.64288330078125,
"fcm_dpo/q_t": 0.4801744818687439,
"grad_norm": 4.864995002746582,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.187529057264328,
"logits/rejected": -0.1925836205482483,
"logps/chosen": -242.1554412841797,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -361.524169921875,
"loss": 1.3266,
"margin_dpo/margin_mean": 89.64288330078125,
"margin_dpo/margin_std": 326.5260009765625,
"step": 129
},
{
"KL/chosen_KL_mean": -115.89335632324219,
"KL/mean": -175.7540283203125,
"KL/rejected_KL_mean": -235.61468505859375,
"KL/std": 157.26568603515625,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 119.72134399414062,
"fcm_dpo/q_t": 0.47216594219207764,
"grad_norm": 4.814020156860352,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.17016103863716125,
"logits/rejected": -0.17204934358596802,
"logps/chosen": -160.56021118164062,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -318.3963623046875,
"loss": 1.2868,
"margin_dpo/margin_mean": 119.72134399414062,
"margin_dpo/margin_std": 210.9378204345703,
"step": 130
},
{
"KL/chosen_KL_mean": -119.94183349609375,
"KL/mean": -184.79937744140625,
"KL/rejected_KL_mean": -249.65692138671875,
"KL/std": 192.86123657226562,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 129.715087890625,
"fcm_dpo/q_t": 0.4696594178676605,
"grad_norm": 7.062121868133545,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.15699008107185364,
"logits/rejected": -0.16018380224704742,
"logps/chosen": -164.86642456054688,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -338.1009521484375,
"loss": 1.2773,
"margin_dpo/margin_mean": 129.715087890625,
"margin_dpo/margin_std": 207.52944946289062,
"step": 131
},
{
"KL/chosen_KL_mean": -152.28724670410156,
"KL/mean": -203.29061889648438,
"KL/rejected_KL_mean": -254.29400634765625,
"KL/std": 203.4266357421875,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.00675964355469,
"fcm_dpo/q_t": 0.47675275802612305,
"grad_norm": 6.158812522888184,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.16013693809509277,
"logits/rejected": -0.18753768503665924,
"logps/chosen": -211.288330078125,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -342.1861572265625,
"loss": 1.3077,
"margin_dpo/margin_mean": 102.00675964355469,
"margin_dpo/margin_std": 263.32513427734375,
"step": 132
},
{
"KL/chosen_KL_mean": -176.15794372558594,
"KL/mean": -220.2735137939453,
"KL/rejected_KL_mean": -264.3890686035156,
"KL/std": 171.11268615722656,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 88.23113250732422,
"fcm_dpo/q_t": 0.47959333658218384,
"grad_norm": 5.555715084075928,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.16391275823116302,
"logits/rejected": -0.17706236243247986,
"logps/chosen": -242.762451171875,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -360.7226257324219,
"loss": 1.3163,
"margin_dpo/margin_mean": 88.23113250732422,
"margin_dpo/margin_std": 228.1553955078125,
"step": 133
},
{
"KL/chosen_KL_mean": -138.08108520507812,
"KL/mean": -183.7069091796875,
"KL/rejected_KL_mean": -229.33270263671875,
"KL/std": 146.15443420410156,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 91.2516098022461,
"fcm_dpo/q_t": 0.4787331819534302,
"grad_norm": 5.492349624633789,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.14529156684875488,
"logits/rejected": -0.16231930255889893,
"logps/chosen": -190.15036010742188,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -316.9872131347656,
"loss": 1.3092,
"margin_dpo/margin_mean": 91.2516098022461,
"margin_dpo/margin_std": 175.44699096679688,
"step": 134
},
{
"KL/chosen_KL_mean": -141.79519653320312,
"KL/mean": -230.31271362304688,
"KL/rejected_KL_mean": -318.8302001953125,
"KL/std": 236.4542694091797,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 177.03501892089844,
"fcm_dpo/q_t": 0.45972955226898193,
"grad_norm": 10.549337387084961,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.12863895297050476,
"logits/rejected": -0.1378084421157837,
"logps/chosen": -192.14907836914062,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -434.8099670410156,
"loss": 1.2474,
"margin_dpo/margin_mean": 177.03500366210938,
"margin_dpo/margin_std": 301.78912353515625,
"step": 135
},
{
"KL/chosen_KL_mean": -178.12527465820312,
"KL/mean": -235.43304443359375,
"KL/rejected_KL_mean": -292.7408142089844,
"KL/std": 245.4122772216797,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 114.61553192138672,
"fcm_dpo/q_t": 0.47485947608947754,
"grad_norm": 5.174078941345215,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.10895825922489166,
"logits/rejected": -0.12058396637439728,
"logps/chosen": -243.1977996826172,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -389.06207275390625,
"loss": 1.3052,
"margin_dpo/margin_mean": 114.61553192138672,
"margin_dpo/margin_std": 335.3597106933594,
"step": 136
},
{
"KL/chosen_KL_mean": -150.92208862304688,
"KL/mean": -239.39349365234375,
"KL/rejected_KL_mean": -327.86492919921875,
"KL/std": 264.5162353515625,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 176.94281005859375,
"fcm_dpo/q_t": 0.46023792028427124,
"grad_norm": 6.715754508972168,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.08718404918909073,
"logits/rejected": -0.09329766035079956,
"logps/chosen": -199.68121337890625,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -441.72869873046875,
"loss": 1.2512,
"margin_dpo/margin_mean": 176.94281005859375,
"margin_dpo/margin_std": 338.74969482421875,
"step": 137
},
{
"KL/chosen_KL_mean": -174.56582641601562,
"KL/mean": -233.0918426513672,
"KL/rejected_KL_mean": -291.61785888671875,
"KL/std": 202.8188934326172,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 117.0520248413086,
"fcm_dpo/q_t": 0.47279053926467896,
"grad_norm": 6.762569904327393,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.09231037646532059,
"logits/rejected": -0.10022014379501343,
"logps/chosen": -235.08547973632812,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -384.8148193359375,
"loss": 1.2889,
"margin_dpo/margin_mean": 117.0520248413086,
"margin_dpo/margin_std": 211.6190185546875,
"step": 138
},
{
"KL/chosen_KL_mean": -153.19482421875,
"KL/mean": -214.05612182617188,
"KL/rejected_KL_mean": -274.9173889160156,
"KL/std": 174.04698181152344,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 121.72257232666016,
"fcm_dpo/q_t": 0.4718576669692993,
"grad_norm": 5.579147815704346,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.053176525980234146,
"logits/rejected": -0.06008898839354515,
"logps/chosen": -200.08621215820312,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -354.6453857421875,
"loss": 1.2875,
"margin_dpo/margin_mean": 121.72257232666016,
"margin_dpo/margin_std": 241.1826171875,
"step": 139
},
{
"KL/chosen_KL_mean": -195.38796997070312,
"KL/mean": -242.62347412109375,
"KL/rejected_KL_mean": -289.8590087890625,
"KL/std": 201.94166564941406,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 94.47102355957031,
"fcm_dpo/q_t": 0.47789740562438965,
"grad_norm": 7.205073833465576,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.008332394063472748,
"logits/rejected": -0.023142114281654358,
"logps/chosen": -254.36268615722656,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -373.14312744140625,
"loss": 1.3137,
"margin_dpo/margin_mean": 94.47102355957031,
"margin_dpo/margin_std": 255.87286376953125,
"step": 140
},
{
"KL/chosen_KL_mean": -251.51254272460938,
"KL/mean": -293.2801513671875,
"KL/rejected_KL_mean": -335.04779052734375,
"KL/std": 324.080078125,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 83.5352554321289,
"fcm_dpo/q_t": 0.4800110161304474,
"grad_norm": 14.238146781921387,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.020058374851942062,
"logits/rejected": -0.05175706744194031,
"logps/chosen": -326.58819580078125,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -433.24005126953125,
"loss": 1.3458,
"margin_dpo/margin_mean": 83.53524780273438,
"margin_dpo/margin_std": 406.5972900390625,
"step": 141
},
{
"KL/chosen_KL_mean": -205.25633239746094,
"KL/mean": -271.9860534667969,
"KL/rejected_KL_mean": -338.71575927734375,
"KL/std": 258.38970947265625,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 133.45944213867188,
"fcm_dpo/q_t": 0.4691423177719116,
"grad_norm": 6.6939826011657715,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.04343932867050171,
"logits/rejected": -0.06279127299785614,
"logps/chosen": -263.28424072265625,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -433.2979736328125,
"loss": 1.2852,
"margin_dpo/margin_mean": 133.45944213867188,
"margin_dpo/margin_std": 297.07684326171875,
"step": 142
},
{
"KL/chosen_KL_mean": -197.44586181640625,
"KL/mean": -235.8623046875,
"KL/rejected_KL_mean": -274.2787780761719,
"KL/std": 196.98611450195312,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 76.83290100097656,
"fcm_dpo/q_t": 0.4824400544166565,
"grad_norm": 6.687804222106934,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.008544637821614742,
"logits/rejected": -0.0213165283203125,
"logps/chosen": -255.0423126220703,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -353.2783203125,
"loss": 1.3288,
"margin_dpo/margin_mean": 76.83290100097656,
"margin_dpo/margin_std": 242.87838745117188,
"step": 143
},
{
"KL/chosen_KL_mean": -199.11557006835938,
"KL/mean": -241.8614959716797,
"KL/rejected_KL_mean": -284.607421875,
"KL/std": 204.11404418945312,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 85.49183654785156,
"fcm_dpo/q_t": 0.48005104064941406,
"grad_norm": 6.8599467277526855,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.04758778586983681,
"logits/rejected": -0.08662058413028717,
"logps/chosen": -259.0219421386719,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -366.607666015625,
"loss": 1.319,
"margin_dpo/margin_mean": 85.49183654785156,
"margin_dpo/margin_std": 223.2879638671875,
"step": 144
},
{
"KL/chosen_KL_mean": -174.992919921875,
"KL/mean": -220.36129760742188,
"KL/rejected_KL_mean": -265.72967529296875,
"KL/std": 170.8227996826172,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 90.73674774169922,
"fcm_dpo/q_t": 0.47885861992836,
"grad_norm": 8.647239685058594,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.11733024567365646,
"logits/rejected": -0.14884145557880402,
"logps/chosen": -231.59359741210938,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -343.59600830078125,
"loss": 1.3128,
"margin_dpo/margin_mean": 90.73674011230469,
"margin_dpo/margin_std": 214.95761108398438,
"step": 145
},
{
"KL/chosen_KL_mean": -224.58987426757812,
"KL/mean": -252.029541015625,
"KL/rejected_KL_mean": -279.46923828125,
"KL/std": 185.42648315429688,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 54.87935256958008,
"fcm_dpo/q_t": 0.48712092638015747,
"grad_norm": 10.154541969299316,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.047742657363414764,
"logits/rejected": -0.07091985642910004,
"logps/chosen": -290.59033203125,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -361.1719970703125,
"loss": 1.3475,
"margin_dpo/margin_mean": 54.87934875488281,
"margin_dpo/margin_std": 230.03529357910156,
"step": 146
},
{
"KL/chosen_KL_mean": -175.193359375,
"KL/mean": -218.81103515625,
"KL/rejected_KL_mean": -262.4287109375,
"KL/std": 185.93280029296875,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 87.23533630371094,
"fcm_dpo/q_t": 0.4797496497631073,
"grad_norm": 5.8443684577941895,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.08670485019683838,
"logits/rejected": -0.11605459451675415,
"logps/chosen": -228.59884643554688,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -333.8193359375,
"loss": 1.3179,
"margin_dpo/margin_mean": 87.23533630371094,
"margin_dpo/margin_std": 234.81607055664062,
"step": 147
},
{
"KL/chosen_KL_mean": -186.26583862304688,
"KL/mean": -237.65237426757812,
"KL/rejected_KL_mean": -289.0389404296875,
"KL/std": 223.15664672851562,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.77307891845703,
"fcm_dpo/q_t": 0.476764976978302,
"grad_norm": 6.361453056335449,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.14481475949287415,
"logits/rejected": -0.15246915817260742,
"logps/chosen": -251.20291137695312,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -392.13275146484375,
"loss": 1.3073,
"margin_dpo/margin_mean": 102.77307891845703,
"margin_dpo/margin_std": 248.232666015625,
"step": 148
},
{
"KL/chosen_KL_mean": -163.42788696289062,
"KL/mean": -227.01951599121094,
"KL/rejected_KL_mean": -290.61114501953125,
"KL/std": 197.52664184570312,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 127.1832275390625,
"fcm_dpo/q_t": 0.4706050753593445,
"grad_norm": 9.70356273651123,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.1462225615978241,
"logits/rejected": -0.16033855080604553,
"logps/chosen": -221.90164184570312,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -389.9258728027344,
"loss": 1.2842,
"margin_dpo/margin_mean": 127.1832275390625,
"margin_dpo/margin_std": 244.41700744628906,
"step": 149
},
{
"KL/chosen_KL_mean": -142.97955322265625,
"KL/mean": -210.34365844726562,
"KL/rejected_KL_mean": -277.707763671875,
"KL/std": 207.92286682128906,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 134.72821044921875,
"fcm_dpo/q_t": 0.469161719083786,
"grad_norm": 6.090360641479492,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.12846021354198456,
"logits/rejected": -0.1382494419813156,
"logps/chosen": -188.68536376953125,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -361.05535888671875,
"loss": 1.2781,
"margin_dpo/margin_mean": 134.72821044921875,
"margin_dpo/margin_std": 255.6205596923828,
"step": 150
},
{
"KL/chosen_KL_mean": -200.44656372070312,
"KL/mean": -258.9799499511719,
"KL/rejected_KL_mean": -317.5133056640625,
"KL/std": 248.0785369873047,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 117.06673431396484,
"fcm_dpo/q_t": 0.47344115376472473,
"grad_norm": 6.703395843505859,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.14405345916748047,
"logits/rejected": -0.16287846863269806,
"logps/chosen": -271.01739501953125,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -417.97711181640625,
"loss": 1.2977,
"margin_dpo/margin_mean": 117.06673431396484,
"margin_dpo/margin_std": 292.07391357421875,
"step": 151
},
{
"KL/chosen_KL_mean": -173.80703735351562,
"KL/mean": -253.65969848632812,
"KL/rejected_KL_mean": -333.5123596191406,
"KL/std": 224.35980224609375,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 159.705322265625,
"fcm_dpo/q_t": 0.4632638990879059,
"grad_norm": 9.056612968444824,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.09410399198532104,
"logits/rejected": -0.10684510320425034,
"logps/chosen": -233.97142028808594,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -439.65283203125,
"loss": 1.2584,
"margin_dpo/margin_mean": 159.705322265625,
"margin_dpo/margin_std": 275.8470458984375,
"step": 152
},
{
"KL/chosen_KL_mean": -167.72113037109375,
"KL/mean": -232.05372619628906,
"KL/rejected_KL_mean": -296.38629150390625,
"KL/std": 238.9656219482422,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 128.6651611328125,
"fcm_dpo/q_t": 0.47082751989364624,
"grad_norm": 7.3872175216674805,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.09062840044498444,
"logits/rejected": -0.11342403292655945,
"logps/chosen": -224.03640747070312,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -382.0421142578125,
"loss": 1.29,
"margin_dpo/margin_mean": 128.6651611328125,
"margin_dpo/margin_std": 311.77520751953125,
"step": 153
},
{
"KL/chosen_KL_mean": -209.06495666503906,
"KL/mean": -261.43896484375,
"KL/rejected_KL_mean": -313.81298828125,
"KL/std": 230.873291015625,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.7480239868164,
"fcm_dpo/q_t": 0.47572967410087585,
"grad_norm": 6.2794013023376465,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.12751531600952148,
"logits/rejected": -0.14344094693660736,
"logps/chosen": -271.8075256347656,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -418.05718994140625,
"loss": 1.3126,
"margin_dpo/margin_mean": 104.74801635742188,
"margin_dpo/margin_std": 321.2839660644531,
"step": 154
},
{
"KL/chosen_KL_mean": -190.59161376953125,
"KL/mean": -236.80764770507812,
"KL/rejected_KL_mean": -283.023681640625,
"KL/std": 196.6082763671875,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 92.43206787109375,
"fcm_dpo/q_t": 0.4784696102142334,
"grad_norm": 6.7807159423828125,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.12184707820415497,
"logits/rejected": -0.1378769725561142,
"logps/chosen": -251.2447967529297,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -360.515869140625,
"loss": 1.3159,
"margin_dpo/margin_mean": 92.43206787109375,
"margin_dpo/margin_std": 260.202880859375,
"step": 155
},
{
"KL/chosen_KL_mean": -253.88046264648438,
"KL/mean": -270.5169677734375,
"KL/rejected_KL_mean": -287.15350341796875,
"KL/std": 232.72618103027344,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 33.27302551269531,
"fcm_dpo/q_t": 0.49217426776885986,
"grad_norm": 8.957015037536621,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.08439283818006516,
"logits/rejected": -0.12552709877490997,
"logps/chosen": -323.3723449707031,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -364.3227844238281,
"loss": 1.3753,
"margin_dpo/margin_mean": 33.27302551269531,
"margin_dpo/margin_std": 293.46331787109375,
"step": 156
},
{
"KL/chosen_KL_mean": -210.72842407226562,
"KL/mean": -293.7404479980469,
"KL/rejected_KL_mean": -376.75250244140625,
"KL/std": 239.9976043701172,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 166.02407836914062,
"fcm_dpo/q_t": 0.46170127391815186,
"grad_norm": 8.381214141845703,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.029853641986846924,
"logits/rejected": -0.06525542587041855,
"logps/chosen": -272.09686279296875,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -484.39886474609375,
"loss": 1.2578,
"margin_dpo/margin_mean": 166.02407836914062,
"margin_dpo/margin_std": 295.1898498535156,
"step": 157
},
{
"KL/chosen_KL_mean": -221.72073364257812,
"KL/mean": -324.26605224609375,
"KL/rejected_KL_mean": -426.8114013671875,
"KL/std": 309.63800048828125,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 205.0906524658203,
"fcm_dpo/q_t": 0.4540192782878876,
"grad_norm": 9.694887161254883,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.0386735275387764,
"logits/rejected": -0.06256888806819916,
"logps/chosen": -279.3336486816406,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -540.506103515625,
"loss": 1.2334,
"margin_dpo/margin_mean": 205.09066772460938,
"margin_dpo/margin_std": 380.8212890625,
"step": 158
},
{
"KL/chosen_KL_mean": -299.9361267089844,
"KL/mean": -334.1776123046875,
"KL/rejected_KL_mean": -368.41912841796875,
"KL/std": 297.918212890625,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 68.48296356201172,
"fcm_dpo/q_t": 0.48391008377075195,
"grad_norm": 11.454606056213379,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": 0.013804029673337936,
"logits/rejected": -0.0027273818850517273,
"logps/chosen": -381.4964599609375,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -457.31781005859375,
"loss": 1.3469,
"margin_dpo/margin_mean": 68.48297119140625,
"margin_dpo/margin_std": 327.9689636230469,
"step": 159
},
{
"KL/chosen_KL_mean": -264.8038330078125,
"KL/mean": -340.09210205078125,
"KL/rejected_KL_mean": -415.38037109375,
"KL/std": 267.2978515625,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 150.57656860351562,
"fcm_dpo/q_t": 0.4657573103904724,
"grad_norm": 8.678204536437988,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.02935023419559002,
"logits/rejected": -0.05578979104757309,
"logps/chosen": -330.53472900390625,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -512.5982055664062,
"loss": 1.275,
"margin_dpo/margin_mean": 150.57656860351562,
"margin_dpo/margin_std": 334.5924072265625,
"step": 160
},
{
"KL/chosen_KL_mean": -215.1328125,
"KL/mean": -277.35003662109375,
"KL/rejected_KL_mean": -339.5672607421875,
"KL/std": 214.84197998046875,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 124.43446350097656,
"fcm_dpo/q_t": 0.47127845883369446,
"grad_norm": 8.156302452087402,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.02828654646873474,
"logits/rejected": -0.06187158077955246,
"logps/chosen": -267.56927490234375,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -422.99822998046875,
"loss": 1.2874,
"margin_dpo/margin_mean": 124.43446350097656,
"margin_dpo/margin_std": 254.58370971679688,
"step": 161
},
{
"KL/chosen_KL_mean": -217.1569366455078,
"KL/mean": -270.75323486328125,
"KL/rejected_KL_mean": -324.34954833984375,
"KL/std": 228.0889129638672,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 107.19261169433594,
"fcm_dpo/q_t": 0.47551965713500977,
"grad_norm": 9.41181755065918,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.13572916388511658,
"logits/rejected": -0.15741822123527527,
"logps/chosen": -279.76751708984375,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -413.7401428222656,
"loss": 1.3073,
"margin_dpo/margin_mean": 107.1926040649414,
"margin_dpo/margin_std": 286.09466552734375,
"step": 162
},
{
"KL/chosen_KL_mean": -229.8784637451172,
"KL/mean": -275.2326354980469,
"KL/rejected_KL_mean": -320.5867919921875,
"KL/std": 224.8272705078125,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 90.70832824707031,
"fcm_dpo/q_t": 0.47908735275268555,
"grad_norm": 7.177928447723389,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.07068107277154922,
"logits/rejected": -0.08296166360378265,
"logps/chosen": -284.90008544921875,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -396.0050048828125,
"loss": 1.3193,
"margin_dpo/margin_mean": 90.70832824707031,
"margin_dpo/margin_std": 275.54254150390625,
"step": 163
},
{
"KL/chosen_KL_mean": -202.0023956298828,
"KL/mean": -269.85015869140625,
"KL/rejected_KL_mean": -337.6978759765625,
"KL/std": 206.05075073242188,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 135.6955108642578,
"fcm_dpo/q_t": 0.4684979319572449,
"grad_norm": 8.586596488952637,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.0649288222193718,
"logits/rejected": -0.08924803882837296,
"logps/chosen": -257.64306640625,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -417.362548828125,
"loss": 1.2751,
"margin_dpo/margin_mean": 135.6955108642578,
"margin_dpo/margin_std": 236.37542724609375,
"step": 164
},
{
"KL/chosen_KL_mean": -217.8095703125,
"KL/mean": -258.1341552734375,
"KL/rejected_KL_mean": -298.45867919921875,
"KL/std": 203.58636474609375,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 80.64910888671875,
"fcm_dpo/q_t": 0.4814724326133728,
"grad_norm": 6.596712112426758,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.09720340371131897,
"logits/rejected": -0.09911538660526276,
"logps/chosen": -279.1202697753906,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -372.1292724609375,
"loss": 1.3307,
"margin_dpo/margin_mean": 80.64909362792969,
"margin_dpo/margin_std": 288.8643798828125,
"step": 165
},
{
"KL/chosen_KL_mean": -188.13812255859375,
"KL/mean": -287.0330505371094,
"KL/rejected_KL_mean": -385.927978515625,
"KL/std": 297.14111328125,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 197.7898406982422,
"fcm_dpo/q_t": 0.45629557967185974,
"grad_norm": 8.050973892211914,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.1186860129237175,
"logits/rejected": -0.127618670463562,
"logps/chosen": -239.12173461914062,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -484.0231018066406,
"loss": 1.2411,
"margin_dpo/margin_mean": 197.78982543945312,
"margin_dpo/margin_std": 393.486572265625,
"step": 166
},
{
"KL/chosen_KL_mean": -192.49603271484375,
"KL/mean": -283.8441162109375,
"KL/rejected_KL_mean": -375.19219970703125,
"KL/std": 246.08155822753906,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 182.6961669921875,
"fcm_dpo/q_t": 0.4583081901073456,
"grad_norm": 11.262971878051758,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.13433954119682312,
"logits/rejected": -0.15051549673080444,
"logps/chosen": -242.92013549804688,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -471.22259521484375,
"loss": 1.2432,
"margin_dpo/margin_mean": 182.69615173339844,
"margin_dpo/margin_std": 304.00164794921875,
"step": 167
},
{
"KL/chosen_KL_mean": -209.07247924804688,
"KL/mean": -267.29833984375,
"KL/rejected_KL_mean": -325.524169921875,
"KL/std": 204.67034912109375,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 116.45173645019531,
"fcm_dpo/q_t": 0.4730868935585022,
"grad_norm": 8.38344669342041,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.07969611883163452,
"logits/rejected": -0.08738286793231964,
"logps/chosen": -258.5353088378906,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -400.832763671875,
"loss": 1.2933,
"margin_dpo/margin_mean": 116.45173645019531,
"margin_dpo/margin_std": 250.1861572265625,
"step": 168
},
{
"KL/chosen_KL_mean": -235.8634033203125,
"KL/mean": -288.0467834472656,
"KL/rejected_KL_mean": -340.2301330566406,
"KL/std": 220.77529907226562,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 104.36674499511719,
"fcm_dpo/q_t": 0.4762327969074249,
"grad_norm": 8.089078903198242,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.09106568992137909,
"logits/rejected": -0.091008760035038,
"logps/chosen": -295.6668395996094,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -423.5758972167969,
"loss": 1.3126,
"margin_dpo/margin_mean": 104.36674499511719,
"margin_dpo/margin_std": 308.4825439453125,
"step": 169
},
{
"KL/chosen_KL_mean": -209.93124389648438,
"KL/mean": -269.19219970703125,
"KL/rejected_KL_mean": -328.453125,
"KL/std": 228.42990112304688,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 118.52188110351562,
"fcm_dpo/q_t": 0.4724269509315491,
"grad_norm": 9.027761459350586,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.14047937095165253,
"logits/rejected": -0.1535281240940094,
"logps/chosen": -259.40301513671875,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -404.3704833984375,
"loss": 1.2953,
"margin_dpo/margin_mean": 118.52188110351562,
"margin_dpo/margin_std": 277.45550537109375,
"step": 170
},
{
"KL/chosen_KL_mean": -336.32196044921875,
"KL/mean": -386.96771240234375,
"KL/rejected_KL_mean": -437.61346435546875,
"KL/std": 330.1646728515625,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 101.29150390625,
"fcm_dpo/q_t": 0.47753575444221497,
"grad_norm": 14.035201072692871,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.09937100857496262,
"logits/rejected": -0.11626763641834259,
"logps/chosen": -420.8212890625,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -546.99560546875,
"loss": 1.3297,
"margin_dpo/margin_mean": 101.29150390625,
"margin_dpo/margin_std": 415.2909851074219,
"step": 171
},
{
"KL/chosen_KL_mean": -284.5125427246094,
"KL/mean": -329.1851806640625,
"KL/rejected_KL_mean": -373.85784912109375,
"KL/std": 302.93798828125,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 89.34530639648438,
"fcm_dpo/q_t": 0.478992223739624,
"grad_norm": 11.372665405273438,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.11791045218706131,
"logits/rejected": -0.13465595245361328,
"logps/chosen": -353.16644287109375,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -459.29449462890625,
"loss": 1.3256,
"margin_dpo/margin_mean": 89.34529876708984,
"margin_dpo/margin_std": 311.55474853515625,
"step": 172
},
{
"KL/chosen_KL_mean": -248.29794311523438,
"KL/mean": -298.60186767578125,
"KL/rejected_KL_mean": -348.9057922363281,
"KL/std": 261.31658935546875,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 100.60782623291016,
"fcm_dpo/q_t": 0.4770353138446808,
"grad_norm": 8.505526542663574,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.14444559812545776,
"logits/rejected": -0.15536616742610931,
"logps/chosen": -311.34881591796875,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -427.5897216796875,
"loss": 1.3094,
"margin_dpo/margin_mean": 100.60781860351562,
"margin_dpo/margin_std": 267.5063171386719,
"step": 173
},
{
"KL/chosen_KL_mean": -225.7945556640625,
"KL/mean": -311.4512023925781,
"KL/rejected_KL_mean": -397.10784912109375,
"KL/std": 295.2328186035156,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.0009431193466298282,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 171.3133087158203,
"fcm_dpo/q_t": 0.46146026253700256,
"grad_norm": 9.906888008117676,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.1199919730424881,
"logits/rejected": -0.13182440400123596,
"logps/chosen": -279.15753173828125,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -499.0190734863281,
"loss": 1.2638,
"margin_dpo/margin_mean": 171.31329345703125,
"margin_dpo/margin_std": 387.0136413574219,
"step": 174
},
{
"KL/chosen_KL_mean": -183.84222412109375,
"KL/mean": -282.45843505859375,
"KL/rejected_KL_mean": -381.07464599609375,
"KL/std": 243.96095275878906,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.002304242691025138,
"fcm_dpo/delta": -0.08028450608253479,
"fcm_dpo/margin": 197.232421875,
"fcm_dpo/q_t": 0.40342098474502563,
"grad_norm": 22.565650939941406,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.1468554139137268,
"logits/rejected": -0.1525323987007141,
"logps/chosen": -229.260009765625,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -470.5804443359375,
"loss": 1.1443,
"margin_dpo/margin_mean": 197.232421875,
"margin_dpo/margin_std": 280.858642578125,
"step": 175
},
{
"KL/chosen_KL_mean": -198.94644165039062,
"KL/mean": -293.33734130859375,
"KL/rejected_KL_mean": -387.728271484375,
"KL/std": 279.7598876953125,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 188.78184509277344,
"fcm_dpo/q_t": 0.4877324104309082,
"grad_norm": 3.078137159347534,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.15219050645828247,
"logits/rejected": -0.16848905384540558,
"logps/chosen": -249.39926147460938,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -483.2872314453125,
"loss": 1.3396,
"margin_dpo/margin_mean": 188.7818603515625,
"margin_dpo/margin_std": 341.77484130859375,
"step": 176
},
{
"KL/chosen_KL_mean": -251.27587890625,
"KL/mean": -338.42828369140625,
"KL/rejected_KL_mean": -425.5806884765625,
"KL/std": 328.7501220703125,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 174.30484008789062,
"fcm_dpo/q_t": 0.4886671304702759,
"grad_norm": 3.2213590145111084,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.15985408425331116,
"logits/rejected": -0.1774776577949524,
"logps/chosen": -312.4923400878906,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -521.4744873046875,
"loss": 1.3435,
"margin_dpo/margin_mean": 174.30482482910156,
"margin_dpo/margin_std": 355.32843017578125,
"step": 177
},
{
"KL/chosen_KL_mean": -255.82669067382812,
"KL/mean": -358.01220703125,
"KL/rejected_KL_mean": -460.19775390625,
"KL/std": 332.4327392578125,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 204.37107849121094,
"fcm_dpo/q_t": 0.4866999387741089,
"grad_norm": 4.488763332366943,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.07843378186225891,
"logits/rejected": -0.09438927471637726,
"logps/chosen": -314.09149169921875,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -565.5631103515625,
"loss": 1.3357,
"margin_dpo/margin_mean": 204.37109375,
"margin_dpo/margin_std": 342.59222412109375,
"step": 178
},
{
"KL/chosen_KL_mean": -269.11553955078125,
"KL/mean": -328.9101867675781,
"KL/rejected_KL_mean": -388.704833984375,
"KL/std": 307.3699951171875,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 119.58927917480469,
"fcm_dpo/q_t": 0.4922333359718323,
"grad_norm": 2.9506919384002686,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.13306602835655212,
"logits/rejected": -0.15629252791404724,
"logps/chosen": -330.17388916015625,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -479.232666015625,
"loss": 1.3581,
"margin_dpo/margin_mean": 119.58926391601562,
"margin_dpo/margin_std": 401.2672119140625,
"step": 179
},
{
"KL/chosen_KL_mean": -231.9340362548828,
"KL/mean": -326.68878173828125,
"KL/rejected_KL_mean": -421.44354248046875,
"KL/std": 240.43899536132812,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 189.5094757080078,
"fcm_dpo/q_t": 0.4876583218574524,
"grad_norm": 2.660843849182129,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.1463892161846161,
"logits/rejected": -0.16165244579315186,
"logps/chosen": -286.2767639160156,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -519.6553955078125,
"loss": 1.3391,
"margin_dpo/margin_mean": 189.50949096679688,
"margin_dpo/margin_std": 316.0293273925781,
"step": 180
},
{
"KL/chosen_KL_mean": -196.69769287109375,
"KL/mean": -247.92457580566406,
"KL/rejected_KL_mean": -299.15142822265625,
"KL/std": 217.79359436035156,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 102.4537353515625,
"fcm_dpo/q_t": 0.49332791566848755,
"grad_norm": 2.223689556121826,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.14982840418815613,
"logits/rejected": -0.1486806869506836,
"logps/chosen": -251.69815063476562,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -360.8076171875,
"loss": 1.3612,
"margin_dpo/margin_mean": 102.4537353515625,
"margin_dpo/margin_std": 290.69586181640625,
"step": 181
},
{
"KL/chosen_KL_mean": -208.18597412109375,
"KL/mean": -319.1571044921875,
"KL/rejected_KL_mean": -430.12823486328125,
"KL/std": 292.2494812011719,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 221.94229125976562,
"fcm_dpo/q_t": 0.4855644702911377,
"grad_norm": 3.2457919120788574,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.12511727213859558,
"logits/rejected": -0.1399608701467514,
"logps/chosen": -249.2938232421875,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -519.6498413085938,
"loss": 1.3315,
"margin_dpo/margin_mean": 221.94229125976562,
"margin_dpo/margin_std": 369.4161376953125,
"step": 182
},
{
"KL/chosen_KL_mean": -293.37567138671875,
"KL/mean": -343.2914123535156,
"KL/rejected_KL_mean": -393.2071533203125,
"KL/std": 252.21807861328125,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 99.83150482177734,
"fcm_dpo/q_t": 0.49350279569625854,
"grad_norm": 2.8942596912384033,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.12328383326530457,
"logits/rejected": -0.1357804387807846,
"logps/chosen": -350.90020751953125,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -469.182861328125,
"loss": 1.3619,
"margin_dpo/margin_mean": 99.83151245117188,
"margin_dpo/margin_std": 281.327880859375,
"step": 183
},
{
"KL/chosen_KL_mean": -286.9473571777344,
"KL/mean": -329.740966796875,
"KL/rejected_KL_mean": -372.5345458984375,
"KL/std": 236.4288787841797,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 85.58721160888672,
"fcm_dpo/q_t": 0.4944261908531189,
"grad_norm": 2.9436886310577393,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.15956711769104004,
"logits/rejected": -0.16407650709152222,
"logps/chosen": -345.4923095703125,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -449.1686096191406,
"loss": 1.3658,
"margin_dpo/margin_mean": 85.58720397949219,
"margin_dpo/margin_std": 308.574462890625,
"step": 184
},
{
"KL/chosen_KL_mean": -323.21502685546875,
"KL/mean": -346.2340393066406,
"KL/rejected_KL_mean": -369.2530212402344,
"KL/std": 302.51226806640625,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 46.0379753112793,
"fcm_dpo/q_t": 0.49701324105262756,
"grad_norm": 4.648723602294922,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.11234143376350403,
"logits/rejected": -0.1124282032251358,
"logps/chosen": -385.24090576171875,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -443.015625,
"loss": 1.3767,
"margin_dpo/margin_mean": 46.03797912597656,
"margin_dpo/margin_std": 361.93035888671875,
"step": 185
},
{
"KL/chosen_KL_mean": -321.8814697265625,
"KL/mean": -380.7614440917969,
"KL/rejected_KL_mean": -439.64141845703125,
"KL/std": 288.25714111328125,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 117.75992584228516,
"fcm_dpo/q_t": 0.49232739210128784,
"grad_norm": 3.8786041736602783,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.1721055507659912,
"logits/rejected": -0.1947544515132904,
"logps/chosen": -391.23492431640625,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -527.7138671875,
"loss": 1.3584,
"margin_dpo/margin_mean": 117.75991821289062,
"margin_dpo/margin_std": 379.52716064453125,
"step": 186
},
{
"KL/chosen_KL_mean": -243.9990234375,
"KL/mean": -316.26837158203125,
"KL/rejected_KL_mean": -388.53765869140625,
"KL/std": 256.2058410644531,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 144.5386199951172,
"fcm_dpo/q_t": 0.49058419466018677,
"grad_norm": 3.2976255416870117,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.14343130588531494,
"logits/rejected": -0.15117818117141724,
"logps/chosen": -296.7554931640625,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -470.50677490234375,
"loss": 1.3503,
"margin_dpo/margin_mean": 144.53860473632812,
"margin_dpo/margin_std": 277.4365234375,
"step": 187
},
{
"KL/chosen_KL_mean": -212.0707550048828,
"KL/mean": -298.758544921875,
"KL/rejected_KL_mean": -385.44635009765625,
"KL/std": 242.26661682128906,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 173.37557983398438,
"fcm_dpo/q_t": 0.48871082067489624,
"grad_norm": 4.286935806274414,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.17238366603851318,
"logits/rejected": -0.197291761636734,
"logps/chosen": -261.4862365722656,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -474.98675537109375,
"loss": 1.3429,
"margin_dpo/margin_mean": 173.37557983398438,
"margin_dpo/margin_std": 285.1446533203125,
"step": 188
},
{
"KL/chosen_KL_mean": -252.53201293945312,
"KL/mean": -311.17919921875,
"KL/rejected_KL_mean": -369.826416015625,
"KL/std": 267.0799560546875,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 117.29438781738281,
"fcm_dpo/q_t": 0.49236994981765747,
"grad_norm": 3.5573782920837402,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.1831766963005066,
"logits/rejected": -0.19586911797523499,
"logps/chosen": -304.93096923828125,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -441.9937744140625,
"loss": 1.358,
"margin_dpo/margin_mean": 117.29438781738281,
"margin_dpo/margin_std": 346.103759765625,
"step": 189
},
{
"KL/chosen_KL_mean": -264.6123046875,
"KL/mean": -355.71697998046875,
"KL/rejected_KL_mean": -446.8216857910156,
"KL/std": 305.12103271484375,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 182.20941162109375,
"fcm_dpo/q_t": 0.48815953731536865,
"grad_norm": 3.5123867988586426,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.2051401287317276,
"logits/rejected": -0.2246163785457611,
"logps/chosen": -329.29534912109375,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -549.3721923828125,
"loss": 1.3418,
"margin_dpo/margin_mean": 182.20941162109375,
"margin_dpo/margin_std": 370.34747314453125,
"step": 190
},
{
"KL/chosen_KL_mean": -261.7254333496094,
"KL/mean": -384.62042236328125,
"KL/rejected_KL_mean": -507.5153503417969,
"KL/std": 342.0569152832031,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 245.78993225097656,
"fcm_dpo/q_t": 0.48404163122177124,
"grad_norm": 5.01483678817749,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.22717076539993286,
"logits/rejected": -0.2197323739528656,
"logps/chosen": -330.3843078613281,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -617.655029296875,
"loss": 1.3261,
"margin_dpo/margin_mean": 245.78994750976562,
"margin_dpo/margin_std": 414.4168701171875,
"step": 191
},
{
"KL/chosen_KL_mean": -326.44854736328125,
"KL/mean": -397.67572021484375,
"KL/rejected_KL_mean": -468.90289306640625,
"KL/std": 319.5667724609375,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 142.454345703125,
"fcm_dpo/q_t": 0.4907473027706146,
"grad_norm": 3.0714664459228516,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.22967159748077393,
"logits/rejected": -0.23665109276771545,
"logps/chosen": -396.17547607421875,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -572.2242431640625,
"loss": 1.3524,
"margin_dpo/margin_mean": 142.454345703125,
"margin_dpo/margin_std": 413.8336181640625,
"step": 192
},
{
"KL/chosen_KL_mean": -321.8466796875,
"KL/mean": -365.2450256347656,
"KL/rejected_KL_mean": -408.64337158203125,
"KL/std": 342.6612854003906,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 86.79666900634766,
"fcm_dpo/q_t": 0.494370698928833,
"grad_norm": 3.745593786239624,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.19167697429656982,
"logits/rejected": -0.2146417796611786,
"logps/chosen": -382.0372009277344,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -485.0509033203125,
"loss": 1.367,
"margin_dpo/margin_mean": 86.79667663574219,
"margin_dpo/margin_std": 428.4646911621094,
"step": 193
},
{
"KL/chosen_KL_mean": -199.5254364013672,
"KL/mean": -264.96240234375,
"KL/rejected_KL_mean": -330.3993835449219,
"KL/std": 220.21302795410156,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.8739471435547,
"fcm_dpo/q_t": 0.49146947264671326,
"grad_norm": 3.0979061126708984,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.12437999248504639,
"logits/rejected": -0.13489431142807007,
"logps/chosen": -237.36581420898438,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -391.08416748046875,
"loss": 1.3535,
"margin_dpo/margin_mean": 130.8739471435547,
"margin_dpo/margin_std": 246.00189208984375,
"step": 194
},
{
"KL/chosen_KL_mean": -305.24945068359375,
"KL/mean": -408.80322265625,
"KL/rejected_KL_mean": -512.35693359375,
"KL/std": 328.6646728515625,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 207.1074676513672,
"fcm_dpo/q_t": 0.4865289628505707,
"grad_norm": 5.246110439300537,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.18198040127754211,
"logits/rejected": -0.18706868588924408,
"logps/chosen": -360.14105224609375,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -609.1279296875,
"loss": 1.335,
"margin_dpo/margin_mean": 207.1074676513672,
"margin_dpo/margin_std": 345.01220703125,
"step": 195
},
{
"KL/chosen_KL_mean": -284.6699523925781,
"KL/mean": -360.74468994140625,
"KL/rejected_KL_mean": -436.8194885253906,
"KL/std": 296.30322265625,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 152.1495361328125,
"fcm_dpo/q_t": 0.4900960326194763,
"grad_norm": 3.9334256649017334,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.18070095777511597,
"logits/rejected": -0.1948184370994568,
"logps/chosen": -337.9151916503906,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -512.8724365234375,
"loss": 1.3498,
"margin_dpo/margin_mean": 152.1495361328125,
"margin_dpo/margin_std": 388.2755126953125,
"step": 196
},
{
"KL/chosen_KL_mean": -307.69537353515625,
"KL/mean": -364.24560546875,
"KL/rejected_KL_mean": -420.79583740234375,
"KL/std": 271.6278076171875,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 113.10043334960938,
"fcm_dpo/q_t": 0.49263206124305725,
"grad_norm": 3.3762285709381104,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.1415313184261322,
"logits/rejected": -0.15090778470039368,
"logps/chosen": -368.11572265625,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -498.0047302246094,
"loss": 1.3587,
"margin_dpo/margin_mean": 113.10043334960938,
"margin_dpo/margin_std": 311.4438781738281,
"step": 197
},
{
"KL/chosen_KL_mean": -300.6440124511719,
"KL/mean": -404.669921875,
"KL/rejected_KL_mean": -508.69580078125,
"KL/std": 325.8094482421875,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 208.05178833007812,
"fcm_dpo/q_t": 0.48646122217178345,
"grad_norm": 3.8525943756103516,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.17303922772407532,
"logits/rejected": -0.1829444169998169,
"logps/chosen": -355.6802062988281,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -605.9390869140625,
"loss": 1.3347,
"margin_dpo/margin_mean": 208.05178833007812,
"margin_dpo/margin_std": 336.3865966796875,
"step": 198
},
{
"KL/chosen_KL_mean": -283.0042724609375,
"KL/mean": -371.278076171875,
"KL/rejected_KL_mean": -459.5518798828125,
"KL/std": 279.841064453125,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 176.547607421875,
"fcm_dpo/q_t": 0.48851385712623596,
"grad_norm": 4.46752405166626,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.19464930891990662,
"logits/rejected": -0.18601861596107483,
"logps/chosen": -339.8330993652344,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -544.2000732421875,
"loss": 1.3432,
"margin_dpo/margin_mean": 176.547607421875,
"margin_dpo/margin_std": 378.5025634765625,
"step": 199
},
{
"KL/chosen_KL_mean": -289.37652587890625,
"KL/mean": -378.7618408203125,
"KL/rejected_KL_mean": -468.1471862792969,
"KL/std": 326.5268249511719,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 178.77064514160156,
"fcm_dpo/q_t": 0.48838722705841064,
"grad_norm": 4.424515724182129,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.16734561324119568,
"logits/rejected": -0.1917646825313568,
"logps/chosen": -342.4435729980469,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -548.755615234375,
"loss": 1.3438,
"margin_dpo/margin_mean": 178.77066040039062,
"margin_dpo/margin_std": 455.8370056152344,
"step": 200
},
{
"KL/chosen_KL_mean": -342.85003662109375,
"KL/mean": -442.3621520996094,
"KL/rejected_KL_mean": -541.874267578125,
"KL/std": 393.974609375,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 199.02426147460938,
"fcm_dpo/q_t": 0.48709234595298767,
"grad_norm": 4.1248321533203125,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.28452351689338684,
"logits/rejected": -0.2933052182197571,
"logps/chosen": -418.25225830078125,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -656.6824951171875,
"loss": 1.3392,
"margin_dpo/margin_mean": 199.02427673339844,
"margin_dpo/margin_std": 483.22381591796875,
"step": 201
},
{
"KL/chosen_KL_mean": -308.9986877441406,
"KL/mean": -377.71002197265625,
"KL/rejected_KL_mean": -446.42132568359375,
"KL/std": 325.0315246582031,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 137.42266845703125,
"fcm_dpo/q_t": 0.49106624722480774,
"grad_norm": 3.5341696739196777,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.19032201170921326,
"logits/rejected": -0.20689848065376282,
"logps/chosen": -359.1000061035156,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -533.4063720703125,
"loss": 1.3534,
"margin_dpo/margin_mean": 137.42266845703125,
"margin_dpo/margin_std": 391.82958984375,
"step": 202
},
{
"KL/chosen_KL_mean": -348.4794006347656,
"KL/mean": -433.90374755859375,
"KL/rejected_KL_mean": -519.3280639648438,
"KL/std": 378.5124816894531,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 170.84869384765625,
"fcm_dpo/q_t": 0.4889407753944397,
"grad_norm": 3.832104206085205,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.17535123229026794,
"logits/rejected": -0.18367072939872742,
"logps/chosen": -409.089111328125,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -605.2239990234375,
"loss": 1.3457,
"margin_dpo/margin_mean": 170.84869384765625,
"margin_dpo/margin_std": 453.1564636230469,
"step": 203
},
{
"KL/chosen_KL_mean": -439.98712158203125,
"KL/mean": -480.7139892578125,
"KL/rejected_KL_mean": -521.4407958984375,
"KL/std": 433.0479736328125,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 81.45366668701172,
"fcm_dpo/q_t": 0.4946914613246918,
"grad_norm": 7.0813751220703125,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.2240895926952362,
"logits/rejected": -0.23155343532562256,
"logps/chosen": -520.152099609375,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -609.13671875,
"loss": 1.37,
"margin_dpo/margin_mean": 81.45367431640625,
"margin_dpo/margin_std": 518.8585205078125,
"step": 204
},
{
"KL/chosen_KL_mean": -384.1351013183594,
"KL/mean": -467.84967041015625,
"KL/rejected_KL_mean": -551.564208984375,
"KL/std": 466.6665344238281,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 167.42913818359375,
"fcm_dpo/q_t": 0.48906105756759644,
"grad_norm": 4.989256381988525,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.13244661688804626,
"logits/rejected": -0.1470334827899933,
"logps/chosen": -443.51983642578125,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -636.6893310546875,
"loss": 1.3482,
"margin_dpo/margin_mean": 167.42913818359375,
"margin_dpo/margin_std": 530.16162109375,
"step": 205
},
{
"KL/chosen_KL_mean": -307.87725830078125,
"KL/mean": -408.16168212890625,
"KL/rejected_KL_mean": -508.4461364746094,
"KL/std": 317.0555419921875,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 200.56893920898438,
"fcm_dpo/q_t": 0.48691922426223755,
"grad_norm": 4.713033676147461,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.13550423085689545,
"logits/rejected": -0.14084811508655548,
"logps/chosen": -354.84173583984375,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -607.399658203125,
"loss": 1.337,
"margin_dpo/margin_mean": 200.56893920898438,
"margin_dpo/margin_std": 370.0242919921875,
"step": 206
},
{
"KL/chosen_KL_mean": -296.52545166015625,
"KL/mean": -404.8011169433594,
"KL/rejected_KL_mean": -513.0767822265625,
"KL/std": 367.4708557128906,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 216.55137634277344,
"fcm_dpo/q_t": 0.485943078994751,
"grad_norm": 5.192399501800537,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.17912110686302185,
"logits/rejected": -0.18409845232963562,
"logps/chosen": -352.5816955566406,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -597.5245971679688,
"loss": 1.3342,
"margin_dpo/margin_mean": 216.55136108398438,
"margin_dpo/margin_std": 459.0970458984375,
"step": 207
},
{
"KL/chosen_KL_mean": -403.71490478515625,
"KL/mean": -488.7073974609375,
"KL/rejected_KL_mean": -573.699951171875,
"KL/std": 352.8353271484375,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 169.98507690429688,
"fcm_dpo/q_t": 0.48894062638282776,
"grad_norm": 3.8758647441864014,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.16348713636398315,
"logits/rejected": -0.17789456248283386,
"logps/chosen": -470.78253173828125,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -667.98681640625,
"loss": 1.3459,
"margin_dpo/margin_mean": 169.98507690429688,
"margin_dpo/margin_std": 450.72607421875,
"step": 208
},
{
"KL/chosen_KL_mean": -328.24884033203125,
"KL/mean": -394.6558532714844,
"KL/rejected_KL_mean": -461.0628662109375,
"KL/std": 306.83245849609375,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 132.81405639648438,
"fcm_dpo/q_t": 0.49136117100715637,
"grad_norm": 4.020294666290283,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.1070629358291626,
"logits/rejected": -0.12986576557159424,
"logps/chosen": -384.4305114746094,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -542.00439453125,
"loss": 1.3545,
"margin_dpo/margin_mean": 132.81405639648438,
"margin_dpo/margin_std": 382.304443359375,
"step": 209
},
{
"KL/chosen_KL_mean": -257.1470031738281,
"KL/mean": -342.8197021484375,
"KL/rejected_KL_mean": -428.4924011230469,
"KL/std": 300.5461730957031,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 171.34536743164062,
"fcm_dpo/q_t": 0.48883914947509766,
"grad_norm": 4.435647487640381,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.15906672179698944,
"logits/rejected": -0.18111161887645721,
"logps/chosen": -303.5188293457031,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -505.17401123046875,
"loss": 1.3446,
"margin_dpo/margin_mean": 171.34536743164062,
"margin_dpo/margin_std": 377.39703369140625,
"step": 210
},
{
"KL/chosen_KL_mean": -396.564208984375,
"KL/mean": -462.0057373046875,
"KL/rejected_KL_mean": -527.4472045898438,
"KL/std": 364.4835510253906,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.88299560546875,
"fcm_dpo/q_t": 0.4915020167827606,
"grad_norm": 4.021511554718018,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.13742633163928986,
"logits/rejected": -0.1542784869670868,
"logps/chosen": -475.49658203125,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -614.2681884765625,
"loss": 1.356,
"margin_dpo/margin_mean": 130.88299560546875,
"margin_dpo/margin_std": 444.1982116699219,
"step": 211
},
{
"KL/chosen_KL_mean": -325.6275634765625,
"KL/mean": -471.6920471191406,
"KL/rejected_KL_mean": -617.7564697265625,
"KL/std": 407.026123046875,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 292.12890625,
"fcm_dpo/q_t": 0.4810423254966736,
"grad_norm": 6.869906902313232,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.14784768223762512,
"logits/rejected": -0.159469336271286,
"logps/chosen": -383.8245849609375,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -720.8143310546875,
"loss": 1.3156,
"margin_dpo/margin_mean": 292.12890625,
"margin_dpo/margin_std": 470.7772216796875,
"step": 212
},
{
"KL/chosen_KL_mean": -360.6119384765625,
"KL/mean": -459.34661865234375,
"KL/rejected_KL_mean": -558.0812377929688,
"KL/std": 398.5190124511719,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 197.4693603515625,
"fcm_dpo/q_t": 0.48718905448913574,
"grad_norm": 4.646012783050537,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.11058177053928375,
"logits/rejected": -0.1331164687871933,
"logps/chosen": -428.1246337890625,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -651.9959716796875,
"loss": 1.3402,
"margin_dpo/margin_mean": 197.4693603515625,
"margin_dpo/margin_std": 528.8551635742188,
"step": 213
},
{
"KL/chosen_KL_mean": -279.425537109375,
"KL/mean": -385.142333984375,
"KL/rejected_KL_mean": -490.85919189453125,
"KL/std": 298.0137939453125,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 211.43362426757812,
"fcm_dpo/q_t": 0.4862461984157562,
"grad_norm": 4.712491512298584,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.09634272009134293,
"logits/rejected": -0.10510338842868805,
"logps/chosen": -321.0304260253906,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -568.3765869140625,
"loss": 1.3342,
"margin_dpo/margin_mean": 211.43362426757812,
"margin_dpo/margin_std": 372.36749267578125,
"step": 214
},
{
"KL/chosen_KL_mean": -357.0162658691406,
"KL/mean": -462.54559326171875,
"KL/rejected_KL_mean": -568.074951171875,
"KL/std": 371.6539306640625,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 211.05868530273438,
"fcm_dpo/q_t": 0.48629534244537354,
"grad_norm": 7.907057762145996,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.1348808854818344,
"logits/rejected": -0.14514675736427307,
"logps/chosen": -410.2955322265625,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -658.03955078125,
"loss": 1.3349,
"margin_dpo/margin_mean": 211.05868530273438,
"margin_dpo/margin_std": 410.6441650390625,
"step": 215
},
{
"KL/chosen_KL_mean": -329.777587890625,
"KL/mean": -422.9111328125,
"KL/rejected_KL_mean": -516.0447387695312,
"KL/std": 320.753662109375,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 186.26715087890625,
"fcm_dpo/q_t": 0.4878776967525482,
"grad_norm": 4.5853495597839355,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.1529572308063507,
"logits/rejected": -0.1603568196296692,
"logps/chosen": -378.6653747558594,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -593.24365234375,
"loss": 1.3412,
"margin_dpo/margin_mean": 186.26715087890625,
"margin_dpo/margin_std": 412.505126953125,
"step": 216
},
{
"KL/chosen_KL_mean": -363.7659912109375,
"KL/mean": -485.22747802734375,
"KL/rejected_KL_mean": -606.68896484375,
"KL/std": 382.6268005371094,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 242.92294311523438,
"fcm_dpo/q_t": 0.48422378301620483,
"grad_norm": 4.863481044769287,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.0929611325263977,
"logits/rejected": -0.10462786257266998,
"logps/chosen": -413.6112976074219,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -706.7672729492188,
"loss": 1.3279,
"margin_dpo/margin_mean": 242.92294311523438,
"margin_dpo/margin_std": 480.1194152832031,
"step": 217
},
{
"KL/chosen_KL_mean": -377.3120422363281,
"KL/mean": -467.94586181640625,
"KL/rejected_KL_mean": -558.5796508789062,
"KL/std": 384.8089904785156,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 181.26760864257812,
"fcm_dpo/q_t": 0.4882817268371582,
"grad_norm": 5.610445022583008,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.1369292289018631,
"logits/rejected": -0.1515660285949707,
"logps/chosen": -435.88873291015625,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -646.426025390625,
"loss": 1.344,
"margin_dpo/margin_mean": 181.26760864257812,
"margin_dpo/margin_std": 512.280517578125,
"step": 218
},
{
"KL/chosen_KL_mean": -393.2535400390625,
"KL/mean": -450.13006591796875,
"KL/rejected_KL_mean": -507.00653076171875,
"KL/std": 405.0634765625,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 113.75296020507812,
"fcm_dpo/q_t": 0.49264246225357056,
"grad_norm": 5.182664394378662,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.15762826800346375,
"logits/rejected": -0.18174490332603455,
"logps/chosen": -454.33740234375,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -592.8369750976562,
"loss": 1.3619,
"margin_dpo/margin_mean": 113.75296020507812,
"margin_dpo/margin_std": 537.340087890625,
"step": 219
},
{
"KL/chosen_KL_mean": -386.52679443359375,
"KL/mean": -445.1171875,
"KL/rejected_KL_mean": -503.7076110839844,
"KL/std": 314.7594909667969,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 117.18080139160156,
"fcm_dpo/q_t": 0.4923757314682007,
"grad_norm": 4.579725742340088,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.16421407461166382,
"logits/rejected": -0.16229598224163055,
"logps/chosen": -456.55810546875,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -591.3931274414062,
"loss": 1.3582,
"margin_dpo/margin_mean": 117.18080139160156,
"margin_dpo/margin_std": 367.78851318359375,
"step": 220
},
{
"KL/chosen_KL_mean": -283.6891784667969,
"KL/mean": -439.31463623046875,
"KL/rejected_KL_mean": -594.9400634765625,
"KL/std": 372.5516357421875,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 311.25091552734375,
"fcm_dpo/q_t": 0.47978127002716064,
"grad_norm": 6.784043312072754,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.17648911476135254,
"logits/rejected": -0.1816864311695099,
"logps/chosen": -335.8438720703125,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -701.40771484375,
"loss": 1.3099,
"margin_dpo/margin_mean": 311.25091552734375,
"margin_dpo/margin_std": 432.03814697265625,
"step": 221
},
{
"KL/chosen_KL_mean": -308.0732421875,
"KL/mean": -414.90313720703125,
"KL/rejected_KL_mean": -521.7329711914062,
"KL/std": 339.31243896484375,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 213.6597442626953,
"fcm_dpo/q_t": 0.48610588908195496,
"grad_norm": 4.452853202819824,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.1906522810459137,
"logits/rejected": -0.22085875272750854,
"logps/chosen": -369.04437255859375,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -621.734130859375,
"loss": 1.3344,
"margin_dpo/margin_mean": 213.6597442626953,
"margin_dpo/margin_std": 419.52435302734375,
"step": 222
},
{
"KL/chosen_KL_mean": -299.1932373046875,
"KL/mean": -382.9361572265625,
"KL/rejected_KL_mean": -466.6790771484375,
"KL/std": 313.127685546875,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 167.48585510253906,
"fcm_dpo/q_t": 0.4891514778137207,
"grad_norm": 4.128637313842773,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.1469314992427826,
"logits/rejected": -0.16430073976516724,
"logps/chosen": -351.83380126953125,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -549.504150390625,
"loss": 1.3463,
"margin_dpo/margin_mean": 167.48585510253906,
"margin_dpo/margin_std": 419.4111328125,
"step": 223
},
{
"KL/chosen_KL_mean": -290.7779541015625,
"KL/mean": -380.89788818359375,
"KL/rejected_KL_mean": -471.0178527832031,
"KL/std": 377.8497314453125,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 180.23985290527344,
"fcm_dpo/q_t": 0.48835575580596924,
"grad_norm": 4.23874568939209,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.17558273673057556,
"logits/rejected": -0.1842508316040039,
"logps/chosen": -339.3733825683594,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -548.1343383789062,
"loss": 1.3437,
"margin_dpo/margin_mean": 180.23985290527344,
"margin_dpo/margin_std": 466.7406005859375,
"step": 224
},
{
"KL/chosen_KL_mean": -341.2305603027344,
"KL/mean": -462.2286376953125,
"KL/rejected_KL_mean": -583.2266845703125,
"KL/std": 376.18865966796875,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 241.99610900878906,
"fcm_dpo/q_t": 0.4843394160270691,
"grad_norm": 5.250749111175537,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.13447608053684235,
"logits/rejected": -0.1467527151107788,
"logps/chosen": -399.23101806640625,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -683.129638671875,
"loss": 1.3286,
"margin_dpo/margin_mean": 241.99610900878906,
"margin_dpo/margin_std": 503.9619140625,
"step": 225
},
{
"KL/chosen_KL_mean": -325.04632568359375,
"KL/mean": -414.34527587890625,
"KL/rejected_KL_mean": -503.6441650390625,
"KL/std": 336.3865966796875,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 178.5978240966797,
"fcm_dpo/q_t": 0.48840048909187317,
"grad_norm": 3.9235055446624756,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.23346635699272156,
"logits/rejected": -0.2388550341129303,
"logps/chosen": -383.94512939453125,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -582.3319091796875,
"loss": 1.3435,
"margin_dpo/margin_mean": 178.5978240966797,
"margin_dpo/margin_std": 436.91937255859375,
"step": 226
},
{
"KL/chosen_KL_mean": -367.533447265625,
"KL/mean": -498.5723876953125,
"KL/rejected_KL_mean": -629.611328125,
"KL/std": 458.18707275390625,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 262.07794189453125,
"fcm_dpo/q_t": 0.48310425877571106,
"grad_norm": 4.806700229644775,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.21147508919239044,
"logits/rejected": -0.2103334367275238,
"logps/chosen": -426.6055908203125,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -729.023681640625,
"loss": 1.3237,
"margin_dpo/margin_mean": 262.07794189453125,
"margin_dpo/margin_std": 509.9268493652344,
"step": 227
},
{
"KL/chosen_KL_mean": -403.41619873046875,
"KL/mean": -481.063720703125,
"KL/rejected_KL_mean": -558.7113037109375,
"KL/std": 388.300537109375,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 155.29510498046875,
"fcm_dpo/q_t": 0.48991623520851135,
"grad_norm": 3.973229169845581,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.2081584632396698,
"logits/rejected": -0.20904359221458435,
"logps/chosen": -469.3074645996094,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -649.760009765625,
"loss": 1.3509,
"margin_dpo/margin_mean": 155.29510498046875,
"margin_dpo/margin_std": 524.5467529296875,
"step": 228
},
{
"KL/chosen_KL_mean": -462.6470642089844,
"KL/mean": -514.1942138671875,
"KL/rejected_KL_mean": -565.7413330078125,
"KL/std": 442.085205078125,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 103.0942611694336,
"fcm_dpo/q_t": 0.4933173358440399,
"grad_norm": 6.374516010284424,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.28295138478279114,
"logits/rejected": -0.2788426876068115,
"logps/chosen": -533.3534545898438,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -650.2687377929688,
"loss": 1.3638,
"margin_dpo/margin_mean": 103.09426879882812,
"margin_dpo/margin_std": 492.0677795410156,
"step": 229
},
{
"KL/chosen_KL_mean": -286.46820068359375,
"KL/mean": -433.4774169921875,
"KL/rejected_KL_mean": -580.486572265625,
"KL/std": 330.7721862792969,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 294.01837158203125,
"fcm_dpo/q_t": 0.48087483644485474,
"grad_norm": 5.674592018127441,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.21354076266288757,
"logits/rejected": -0.2245483547449112,
"logps/chosen": -325.75018310546875,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -666.1085205078125,
"loss": 1.3135,
"margin_dpo/margin_mean": 294.0184020996094,
"margin_dpo/margin_std": 378.5648498535156,
"step": 230
},
{
"KL/chosen_KL_mean": -383.29278564453125,
"KL/mean": -453.44512939453125,
"KL/rejected_KL_mean": -523.5975341796875,
"KL/std": 306.9707336425781,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 140.30477905273438,
"fcm_dpo/q_t": 0.4908882975578308,
"grad_norm": 5.705263137817383,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.2526136338710785,
"logits/rejected": -0.24818205833435059,
"logps/chosen": -446.5692138671875,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -597.721435546875,
"loss": 1.3527,
"margin_dpo/margin_mean": 140.30477905273438,
"margin_dpo/margin_std": 391.14666748046875,
"step": 231
},
{
"KL/chosen_KL_mean": -460.69354248046875,
"KL/mean": -498.51318359375,
"KL/rejected_KL_mean": -536.3327026367188,
"KL/std": 363.728515625,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 75.63915252685547,
"fcm_dpo/q_t": 0.49512791633605957,
"grad_norm": 7.27602481842041,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.24794329702854156,
"logits/rejected": -0.25373679399490356,
"logps/chosen": -531.4423217773438,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -620.309814453125,
"loss": 1.3698,
"margin_dpo/margin_mean": 75.63914489746094,
"margin_dpo/margin_std": 431.7769775390625,
"step": 232
},
{
"KL/chosen_KL_mean": -341.1106262207031,
"KL/mean": -483.42645263671875,
"KL/rejected_KL_mean": -625.7423095703125,
"KL/std": 399.1839599609375,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 284.6317138671875,
"fcm_dpo/q_t": 0.48159539699554443,
"grad_norm": 4.810127258300781,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.2372874766588211,
"logits/rejected": -0.254278302192688,
"logps/chosen": -395.9935607910156,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -733.222412109375,
"loss": 1.319,
"margin_dpo/margin_mean": 284.6316833496094,
"margin_dpo/margin_std": 552.460205078125,
"step": 233
},
{
"KL/chosen_KL_mean": -317.89398193359375,
"KL/mean": -468.377685546875,
"KL/rejected_KL_mean": -618.861328125,
"KL/std": 384.4058837890625,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 300.9674072265625,
"fcm_dpo/q_t": 0.4804891347885132,
"grad_norm": 4.5735554695129395,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.192276269197464,
"logits/rejected": -0.20555615425109863,
"logps/chosen": -361.98846435546875,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -718.8680419921875,
"loss": 1.3133,
"margin_dpo/margin_mean": 300.9674072265625,
"margin_dpo/margin_std": 458.0181884765625,
"step": 234
},
{
"KL/chosen_KL_mean": -396.1971740722656,
"KL/mean": -479.3715515136719,
"KL/rejected_KL_mean": -562.5459594726562,
"KL/std": 388.67431640625,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 166.3487548828125,
"fcm_dpo/q_t": 0.489266961812973,
"grad_norm": 4.237292289733887,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.1855693757534027,
"logits/rejected": -0.20113056898117065,
"logps/chosen": -458.4350891113281,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -652.9410400390625,
"loss": 1.3478,
"margin_dpo/margin_mean": 166.34877014160156,
"margin_dpo/margin_std": 514.4866333007812,
"step": 235
},
{
"KL/chosen_KL_mean": -272.7074279785156,
"KL/mean": -432.64501953125,
"KL/rejected_KL_mean": -592.5825805664062,
"KL/std": 387.78814697265625,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 319.8751220703125,
"fcm_dpo/q_t": 0.4792480170726776,
"grad_norm": 7.933678150177002,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.18187743425369263,
"logits/rejected": -0.18875324726104736,
"logps/chosen": -322.0487976074219,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -696.09423828125,
"loss": 1.3079,
"margin_dpo/margin_mean": 319.8751220703125,
"margin_dpo/margin_std": 438.8853759765625,
"step": 236
},
{
"KL/chosen_KL_mean": -399.15289306640625,
"KL/mean": -506.1591491699219,
"KL/rejected_KL_mean": -613.1654052734375,
"KL/std": 423.4207763671875,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 214.01246643066406,
"fcm_dpo/q_t": 0.4862016439437866,
"grad_norm": 5.225279808044434,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.1944710910320282,
"logits/rejected": -0.20811229944229126,
"logps/chosen": -453.321044921875,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -707.94580078125,
"loss": 1.3365,
"margin_dpo/margin_mean": 214.01246643066406,
"margin_dpo/margin_std": 556.74853515625,
"step": 237
},
{
"KL/chosen_KL_mean": -385.951171875,
"KL/mean": -494.75103759765625,
"KL/rejected_KL_mean": -603.5508422851562,
"KL/std": 423.44287109375,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 217.59967041015625,
"fcm_dpo/q_t": 0.48592108488082886,
"grad_norm": 4.655148029327393,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.18144190311431885,
"logits/rejected": -0.1798359453678131,
"logps/chosen": -439.92431640625,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -692.9688110351562,
"loss": 1.3345,
"margin_dpo/margin_mean": 217.59967041015625,
"margin_dpo/margin_std": 496.64794921875,
"step": 238
},
{
"KL/chosen_KL_mean": -443.1206359863281,
"KL/mean": -553.6134643554688,
"KL/rejected_KL_mean": -664.106201171875,
"KL/std": 398.3800048828125,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 220.98562622070312,
"fcm_dpo/q_t": 0.4856555461883545,
"grad_norm": 4.865357875823975,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.17183178663253784,
"logits/rejected": -0.1842924803495407,
"logps/chosen": -501.21844482421875,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -757.69921875,
"loss": 1.3339,
"margin_dpo/margin_mean": 220.98565673828125,
"margin_dpo/margin_std": 507.60162353515625,
"step": 239
},
{
"KL/chosen_KL_mean": -512.9976806640625,
"KL/mean": -566.2974243164062,
"KL/rejected_KL_mean": -619.5970458984375,
"KL/std": 433.83050537109375,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 106.59938049316406,
"fcm_dpo/q_t": 0.4930400848388672,
"grad_norm": 7.718948841094971,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.23433905839920044,
"logits/rejected": -0.23268157243728638,
"logps/chosen": -573.6121826171875,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -693.715576171875,
"loss": 1.3637,
"margin_dpo/margin_mean": 106.5993881225586,
"margin_dpo/margin_std": 538.6660766601562,
"step": 240
},
{
"KL/chosen_KL_mean": -482.7232360839844,
"KL/mean": -591.3538208007812,
"KL/rejected_KL_mean": -699.9844360351562,
"KL/std": 479.83416748046875,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 217.26119995117188,
"fcm_dpo/q_t": 0.4859463572502136,
"grad_norm": 7.044655799865723,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.22257652878761292,
"logits/rejected": -0.23118522763252258,
"logps/chosen": -548.8143310546875,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -788.0452880859375,
"loss": 1.3367,
"margin_dpo/margin_mean": 217.2611846923828,
"margin_dpo/margin_std": 595.4739990234375,
"step": 241
},
{
"KL/chosen_KL_mean": -509.2297058105469,
"KL/mean": -566.0556640625,
"KL/rejected_KL_mean": -622.8816528320312,
"KL/std": 476.64068603515625,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 113.65194702148438,
"fcm_dpo/q_t": 0.49247777462005615,
"grad_norm": 7.708585262298584,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.1928231418132782,
"logits/rejected": -0.19980955123901367,
"logps/chosen": -577.0936279296875,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -706.241943359375,
"loss": 1.3645,
"margin_dpo/margin_mean": 113.65194702148438,
"margin_dpo/margin_std": 659.6571655273438,
"step": 242
},
{
"KL/chosen_KL_mean": -418.51556396484375,
"KL/mean": -507.8591613769531,
"KL/rejected_KL_mean": -597.2027587890625,
"KL/std": 383.8472900390625,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 178.68716430664062,
"fcm_dpo/q_t": 0.4883681833744049,
"grad_norm": 6.65237283706665,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.1828111708164215,
"logits/rejected": -0.2104099690914154,
"logps/chosen": -481.5998229980469,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -673.538330078125,
"loss": 1.3436,
"margin_dpo/margin_mean": 178.68716430664062,
"margin_dpo/margin_std": 438.80706787109375,
"step": 243
},
{
"KL/chosen_KL_mean": -362.7891845703125,
"KL/mean": -491.2398681640625,
"KL/rejected_KL_mean": -619.6905517578125,
"KL/std": 432.4399719238281,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 256.90142822265625,
"fcm_dpo/q_t": 0.48334479331970215,
"grad_norm": 4.84958028793335,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.19160521030426025,
"logits/rejected": -0.21015608310699463,
"logps/chosen": -423.92987060546875,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -714.58251953125,
"loss": 1.3242,
"margin_dpo/margin_mean": 256.90142822265625,
"margin_dpo/margin_std": 459.79437255859375,
"step": 244
},
{
"KL/chosen_KL_mean": -390.1641540527344,
"KL/mean": -448.14697265625,
"KL/rejected_KL_mean": -506.1298522949219,
"KL/std": 339.2786865234375,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 115.9656753540039,
"fcm_dpo/q_t": 0.4924522638320923,
"grad_norm": 4.89885139465332,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.19130942225456238,
"logits/rejected": -0.19781740009784698,
"logps/chosen": -457.42645263671875,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -593.7699584960938,
"loss": 1.3592,
"margin_dpo/margin_mean": 115.9656753540039,
"margin_dpo/margin_std": 399.8912353515625,
"step": 245
},
{
"KL/chosen_KL_mean": -332.69659423828125,
"KL/mean": -417.5989685058594,
"KL/rejected_KL_mean": -502.5013427734375,
"KL/std": 386.60040283203125,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 169.80477905273438,
"fcm_dpo/q_t": 0.48896580934524536,
"grad_norm": 4.2765350341796875,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.20524047315120697,
"logits/rejected": -0.21078921854496002,
"logps/chosen": -399.3935546875,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -586.8477172851562,
"loss": 1.3457,
"margin_dpo/margin_mean": 169.80477905273438,
"margin_dpo/margin_std": 431.6588134765625,
"step": 246
},
{
"KL/chosen_KL_mean": -336.91021728515625,
"KL/mean": -498.5659484863281,
"KL/rejected_KL_mean": -660.2217407226562,
"KL/std": 401.452880859375,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 323.31146240234375,
"fcm_dpo/q_t": 0.4790780544281006,
"grad_norm": 6.638669490814209,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.16730782389640808,
"logits/rejected": -0.17316174507141113,
"logps/chosen": -393.51556396484375,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -766.5150146484375,
"loss": 1.3086,
"margin_dpo/margin_mean": 323.31146240234375,
"margin_dpo/margin_std": 497.8956298828125,
"step": 247
},
{
"KL/chosen_KL_mean": -257.7315368652344,
"KL/mean": -389.46868896484375,
"KL/rejected_KL_mean": -521.205810546875,
"KL/std": 344.24053955078125,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 263.47430419921875,
"fcm_dpo/q_t": 0.48291152715682983,
"grad_norm": 7.276768684387207,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.16964401304721832,
"logits/rejected": -0.1837081015110016,
"logps/chosen": -301.7747497558594,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -613.0626831054688,
"loss": 1.3216,
"margin_dpo/margin_mean": 263.47430419921875,
"margin_dpo/margin_std": 388.28094482421875,
"step": 248
},
{
"KL/chosen_KL_mean": -385.25128173828125,
"KL/mean": -434.3623046875,
"KL/rejected_KL_mean": -483.4732666015625,
"KL/std": 371.12042236328125,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 98.22201538085938,
"fcm_dpo/q_t": 0.49368637800216675,
"grad_norm": 3.791370391845703,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.1448793113231659,
"logits/rejected": -0.15905806422233582,
"logps/chosen": -447.693603515625,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -563.9413452148438,
"loss": 1.3647,
"margin_dpo/margin_mean": 98.2220230102539,
"margin_dpo/margin_std": 475.65869140625,
"step": 249
},
{
"KL/chosen_KL_mean": -364.223388671875,
"KL/mean": -443.6612548828125,
"KL/rejected_KL_mean": -523.0990600585938,
"KL/std": 361.03973388671875,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 158.87567138671875,
"fcm_dpo/q_t": 0.4896411597728729,
"grad_norm": 4.5081467628479,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.1620238721370697,
"logits/rejected": -0.17770114541053772,
"logps/chosen": -429.860107421875,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -596.970947265625,
"loss": 1.3485,
"margin_dpo/margin_mean": 158.87567138671875,
"margin_dpo/margin_std": 429.6935729980469,
"step": 250
},
{
"KL/chosen_KL_mean": -367.9019470214844,
"KL/mean": -433.2972412109375,
"KL/rejected_KL_mean": -498.6925048828125,
"KL/std": 377.6723327636719,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 130.79058837890625,
"fcm_dpo/q_t": 0.4915042221546173,
"grad_norm": 5.190096855163574,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.17455437779426575,
"logits/rejected": -0.18709741532802582,
"logps/chosen": -425.08465576171875,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -576.35595703125,
"loss": 1.356,
"margin_dpo/margin_mean": 130.79058837890625,
"margin_dpo/margin_std": 441.9692077636719,
"step": 251
},
{
"KL/chosen_KL_mean": -409.4916687011719,
"KL/mean": -503.2325439453125,
"KL/rejected_KL_mean": -596.973388671875,
"KL/std": 341.1922302246094,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 187.481689453125,
"fcm_dpo/q_t": 0.48779380321502686,
"grad_norm": 5.0120744705200195,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.15551039576530457,
"logits/rejected": -0.14695081114768982,
"logps/chosen": -481.17730712890625,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -681.7313232421875,
"loss": 1.3404,
"margin_dpo/margin_mean": 187.481689453125,
"margin_dpo/margin_std": 378.55426025390625,
"step": 252
},
{
"KL/chosen_KL_mean": -384.7890625,
"KL/mean": -510.7060546875,
"KL/rejected_KL_mean": -636.6231079101562,
"KL/std": 366.9384765625,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 251.8341064453125,
"fcm_dpo/q_t": 0.4836481809616089,
"grad_norm": 5.239863872528076,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.24072858691215515,
"logits/rejected": -0.24128055572509766,
"logps/chosen": -453.9229736328125,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -735.32568359375,
"loss": 1.3252,
"margin_dpo/margin_mean": 251.8341064453125,
"margin_dpo/margin_std": 442.5596923828125,
"step": 253
},
{
"KL/chosen_KL_mean": -414.69989013671875,
"KL/mean": -491.3453369140625,
"KL/rejected_KL_mean": -567.9908447265625,
"KL/std": 399.92486572265625,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 153.29092407226562,
"fcm_dpo/q_t": 0.49005135893821716,
"grad_norm": 6.106077194213867,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.17732489109039307,
"logits/rejected": -0.17891092598438263,
"logps/chosen": -468.8548889160156,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -658.2984619140625,
"loss": 1.3504,
"margin_dpo/margin_mean": 153.29092407226562,
"margin_dpo/margin_std": 465.193115234375,
"step": 254
},
{
"KL/chosen_KL_mean": -445.1566162109375,
"KL/mean": -541.7379150390625,
"KL/rejected_KL_mean": -638.3192138671875,
"KL/std": 423.0499572753906,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 193.16268920898438,
"fcm_dpo/q_t": 0.4874666929244995,
"grad_norm": 7.234618663787842,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.20868191123008728,
"logits/rejected": -0.20996426045894623,
"logps/chosen": -502.29827880859375,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -728.52783203125,
"loss": 1.3419,
"margin_dpo/margin_mean": 193.16268920898438,
"margin_dpo/margin_std": 552.7227783203125,
"step": 255
},
{
"KL/chosen_KL_mean": -393.31829833984375,
"KL/mean": -517.4535522460938,
"KL/rejected_KL_mean": -641.5887451171875,
"KL/std": 461.53656005859375,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 248.27044677734375,
"fcm_dpo/q_t": 0.48388510942459106,
"grad_norm": 5.720200061798096,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.2003256380558014,
"logits/rejected": -0.22089025378227234,
"logps/chosen": -448.4818115234375,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -734.1517333984375,
"loss": 1.327,
"margin_dpo/margin_mean": 248.2704315185547,
"margin_dpo/margin_std": 509.9310607910156,
"step": 256
},
{
"KL/chosen_KL_mean": -340.8580322265625,
"KL/mean": -473.32391357421875,
"KL/rejected_KL_mean": -605.7897338867188,
"KL/std": 385.75244140625,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 264.93170166015625,
"fcm_dpo/q_t": 0.48283612728118896,
"grad_norm": 6.3806633949279785,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.18380647897720337,
"logits/rejected": -0.1893559843301773,
"logps/chosen": -390.28173828125,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -685.32763671875,
"loss": 1.3228,
"margin_dpo/margin_mean": 264.9317321777344,
"margin_dpo/margin_std": 493.8131103515625,
"step": 257
},
{
"KL/chosen_KL_mean": -456.58740234375,
"KL/mean": -573.7291259765625,
"KL/rejected_KL_mean": -690.870849609375,
"KL/std": 422.5630187988281,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 234.2834014892578,
"fcm_dpo/q_t": 0.4847896099090576,
"grad_norm": 6.556681156158447,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.16496992111206055,
"logits/rejected": -0.16959482431411743,
"logps/chosen": -515.9715576171875,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -786.8609619140625,
"loss": 1.331,
"margin_dpo/margin_mean": 234.2834014892578,
"margin_dpo/margin_std": 536.2554321289062,
"step": 258
},
{
"KL/chosen_KL_mean": -437.74139404296875,
"KL/mean": -527.7792358398438,
"KL/rejected_KL_mean": -617.8170776367188,
"KL/std": 397.44317626953125,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 180.0756378173828,
"fcm_dpo/q_t": 0.4883081018924713,
"grad_norm": 5.156373500823975,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.2657771706581116,
"logits/rejected": -0.2764046788215637,
"logps/chosen": -490.56976318359375,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -707.0087280273438,
"loss": 1.3441,
"margin_dpo/margin_mean": 180.0756378173828,
"margin_dpo/margin_std": 496.28350830078125,
"step": 259
},
{
"KL/chosen_KL_mean": -408.2018737792969,
"KL/mean": -567.6690673828125,
"KL/rejected_KL_mean": -727.13623046875,
"KL/std": 518.0980224609375,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 318.9344177246094,
"fcm_dpo/q_t": 0.47985440492630005,
"grad_norm": 6.141451835632324,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.21567201614379883,
"logits/rejected": -0.22696366906166077,
"logps/chosen": -455.61956787109375,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -822.22607421875,
"loss": 1.3125,
"margin_dpo/margin_mean": 318.9344177246094,
"margin_dpo/margin_std": 634.315673828125,
"step": 260
},
{
"KL/chosen_KL_mean": -453.39117431640625,
"KL/mean": -568.876953125,
"KL/rejected_KL_mean": -684.36279296875,
"KL/std": 498.5777587890625,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 230.97158813476562,
"fcm_dpo/q_t": 0.48512423038482666,
"grad_norm": 6.820346832275391,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.19904325902462006,
"logits/rejected": -0.212869793176651,
"logps/chosen": -506.4225769042969,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -772.877685546875,
"loss": 1.3336,
"margin_dpo/margin_mean": 230.9715576171875,
"margin_dpo/margin_std": 618.2874755859375,
"step": 261
},
{
"KL/chosen_KL_mean": -557.084228515625,
"KL/mean": -635.599853515625,
"KL/rejected_KL_mean": -714.1155395507812,
"KL/std": 484.9010314941406,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 157.03131103515625,
"fcm_dpo/q_t": 0.48986271023750305,
"grad_norm": 11.662672996520996,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.2308690845966339,
"logits/rejected": -0.2301132082939148,
"logps/chosen": -616.704345703125,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -800.5340576171875,
"loss": 1.3504,
"margin_dpo/margin_mean": 157.03131103515625,
"margin_dpo/margin_std": 523.501953125,
"step": 262
},
{
"KL/chosen_KL_mean": -488.0692138671875,
"KL/mean": -623.00439453125,
"KL/rejected_KL_mean": -757.9396362304688,
"KL/std": 583.6214599609375,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 269.87042236328125,
"fcm_dpo/q_t": 0.4825875461101532,
"grad_norm": 8.312024116516113,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.22481516003608704,
"logits/rejected": -0.23364132642745972,
"logps/chosen": -547.4901733398438,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -854.796875,
"loss": 1.3262,
"margin_dpo/margin_mean": 269.87042236328125,
"margin_dpo/margin_std": 712.5626831054688,
"step": 263
},
{
"KL/chosen_KL_mean": -497.3571472167969,
"KL/mean": -621.4713134765625,
"KL/rejected_KL_mean": -745.5855102539062,
"KL/std": 472.39825439453125,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 248.22833251953125,
"fcm_dpo/q_t": 0.48396652936935425,
"grad_norm": 6.751438617706299,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.23678645491600037,
"logits/rejected": -0.2437746822834015,
"logps/chosen": -560.0792236328125,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -839.4417114257812,
"loss": 1.3291,
"margin_dpo/margin_mean": 248.22830200195312,
"margin_dpo/margin_std": 611.2507934570312,
"step": 264
},
{
"KL/chosen_KL_mean": -507.474365234375,
"KL/mean": -589.9117431640625,
"KL/rejected_KL_mean": -672.34912109375,
"KL/std": 430.08966064453125,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 164.87478637695312,
"fcm_dpo/q_t": 0.48930346965789795,
"grad_norm": 7.545551300048828,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.15824304521083832,
"logits/rejected": -0.16281482577323914,
"logps/chosen": -569.44580078125,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -760.3697509765625,
"loss": 1.3489,
"margin_dpo/margin_mean": 164.8748016357422,
"margin_dpo/margin_std": 550.4019165039062,
"step": 265
},
{
"KL/chosen_KL_mean": -501.43597412109375,
"KL/mean": -529.8870849609375,
"KL/rejected_KL_mean": -558.3381958007812,
"KL/std": 390.973388671875,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 56.9022102355957,
"fcm_dpo/q_t": 0.4962887465953827,
"grad_norm": 11.123693466186523,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.17855733633041382,
"logits/rejected": -0.19599322974681854,
"logps/chosen": -568.53564453125,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -626.3094482421875,
"loss": 1.3757,
"margin_dpo/margin_mean": 56.90220260620117,
"margin_dpo/margin_std": 493.3840026855469,
"step": 266
},
{
"KL/chosen_KL_mean": -404.4932556152344,
"KL/mean": -489.49261474609375,
"KL/rejected_KL_mean": -574.491943359375,
"KL/std": 411.52874755859375,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 169.99867248535156,
"fcm_dpo/q_t": 0.4889971613883972,
"grad_norm": 6.4354047775268555,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.15721634030342102,
"logits/rejected": -0.1611420214176178,
"logps/chosen": -473.4639892578125,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -664.660400390625,
"loss": 1.3474,
"margin_dpo/margin_mean": 169.99867248535156,
"margin_dpo/margin_std": 537.1679077148438,
"step": 267
},
{
"KL/chosen_KL_mean": -367.1216125488281,
"KL/mean": -490.97198486328125,
"KL/rejected_KL_mean": -614.822265625,
"KL/std": 461.61407470703125,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 247.70066833496094,
"fcm_dpo/q_t": 0.483919620513916,
"grad_norm": 8.824798583984375,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.13857878744602203,
"logits/rejected": -0.136179119348526,
"logps/chosen": -423.02191162109375,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -716.469970703125,
"loss": 1.3277,
"margin_dpo/margin_mean": 247.70066833496094,
"margin_dpo/margin_std": 543.2017211914062,
"step": 268
},
{
"KL/chosen_KL_mean": -424.86187744140625,
"KL/mean": -545.9503173828125,
"KL/rejected_KL_mean": -667.038818359375,
"KL/std": 403.55706787109375,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 242.17694091796875,
"fcm_dpo/q_t": 0.4843018651008606,
"grad_norm": 8.456543922424316,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.16604793071746826,
"logits/rejected": -0.18448489904403687,
"logps/chosen": -494.90142822265625,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -774.38818359375,
"loss": 1.3295,
"margin_dpo/margin_mean": 242.1769256591797,
"margin_dpo/margin_std": 559.4677124023438,
"step": 269
},
{
"KL/chosen_KL_mean": -406.35626220703125,
"KL/mean": -518.3023071289062,
"KL/rejected_KL_mean": -630.248291015625,
"KL/std": 403.0386962890625,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 223.89202880859375,
"fcm_dpo/q_t": 0.4855157136917114,
"grad_norm": 6.902976036071777,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.18907925486564636,
"logits/rejected": -0.20055195689201355,
"logps/chosen": -475.8897705078125,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -740.1769409179688,
"loss": 1.3342,
"margin_dpo/margin_mean": 223.89202880859375,
"margin_dpo/margin_std": 561.119384765625,
"step": 270
},
{
"KL/chosen_KL_mean": -332.3415832519531,
"KL/mean": -452.5819091796875,
"KL/rejected_KL_mean": -572.822265625,
"KL/std": 376.9450378417969,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 240.48062133789062,
"fcm_dpo/q_t": 0.484383761882782,
"grad_norm": 7.943375110626221,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.2072502076625824,
"logits/rejected": -0.2298785150051117,
"logps/chosen": -389.10614013671875,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -665.3360595703125,
"loss": 1.3284,
"margin_dpo/margin_mean": 240.48062133789062,
"margin_dpo/margin_std": 473.6417236328125,
"step": 271
},
{
"KL/chosen_KL_mean": -324.92218017578125,
"KL/mean": -495.7729187011719,
"KL/rejected_KL_mean": -666.6236572265625,
"KL/std": 421.35009765625,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 341.70147705078125,
"fcm_dpo/q_t": 0.4778040647506714,
"grad_norm": 15.071242332458496,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.19679825007915497,
"logits/rejected": -0.20816272497177124,
"logps/chosen": -374.41937255859375,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -772.16650390625,
"loss": 1.3028,
"margin_dpo/margin_mean": 341.70147705078125,
"margin_dpo/margin_std": 460.557373046875,
"step": 272
},
{
"KL/chosen_KL_mean": -434.1469421386719,
"KL/mean": -575.2939453125,
"KL/rejected_KL_mean": -716.4408569335938,
"KL/std": 456.7916259765625,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 282.29388427734375,
"fcm_dpo/q_t": 0.4816928505897522,
"grad_norm": 7.3902764320373535,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.26582562923431396,
"logits/rejected": -0.2689433991909027,
"logps/chosen": -497.1223449707031,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -808.939453125,
"loss": 1.3187,
"margin_dpo/margin_mean": 282.2939147949219,
"margin_dpo/margin_std": 524.8466796875,
"step": 273
},
{
"KL/chosen_KL_mean": -461.13970947265625,
"KL/mean": -559.404052734375,
"KL/rejected_KL_mean": -657.66845703125,
"KL/std": 382.537841796875,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 196.5286865234375,
"fcm_dpo/q_t": 0.48726481199264526,
"grad_norm": 8.061701774597168,
"learning_rate": 3.75e-07,
"logits/chosen": -0.2275162637233734,
"logits/rejected": -0.22361652553081512,
"logps/chosen": -516.8074340820312,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -735.00146484375,
"loss": 1.3397,
"margin_dpo/margin_mean": 196.52871704101562,
"margin_dpo/margin_std": 475.701171875,
"step": 274
},
{
"KL/chosen_KL_mean": -403.9293518066406,
"KL/mean": -539.42724609375,
"KL/rejected_KL_mean": -674.9251708984375,
"KL/std": 428.0262145996094,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 270.99578857421875,
"fcm_dpo/q_t": 0.4824160635471344,
"grad_norm": 9.460725784301758,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.27084943652153015,
"logits/rejected": -0.2707024812698364,
"logps/chosen": -452.5240478515625,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -768.2288818359375,
"loss": 1.3215,
"margin_dpo/margin_mean": 270.99578857421875,
"margin_dpo/margin_std": 521.1408081054688,
"step": 275
},
{
"KL/chosen_KL_mean": -461.385986328125,
"KL/mean": -551.3458862304688,
"KL/rejected_KL_mean": -641.3057861328125,
"KL/std": 443.9175109863281,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 179.91978454589844,
"fcm_dpo/q_t": 0.48834025859832764,
"grad_norm": 5.902976036071777,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.275311142206192,
"logits/rejected": -0.2855873107910156,
"logps/chosen": -517.96337890625,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -711.6714477539062,
"loss": 1.3466,
"margin_dpo/margin_mean": 179.9197998046875,
"margin_dpo/margin_std": 613.080810546875,
"step": 276
},
{
"KL/chosen_KL_mean": -490.01446533203125,
"KL/mean": -628.3453979492188,
"KL/rejected_KL_mean": -766.67626953125,
"KL/std": 460.1982421875,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 276.66180419921875,
"fcm_dpo/q_t": 0.4820600152015686,
"grad_norm": 5.049780368804932,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.2941455841064453,
"logits/rejected": -0.2855493426322937,
"logps/chosen": -546.2860107421875,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -859.5575561523438,
"loss": 1.3207,
"margin_dpo/margin_mean": 276.66180419921875,
"margin_dpo/margin_std": 558.4758911132812,
"step": 277
},
{
"KL/chosen_KL_mean": -490.9912109375,
"KL/mean": -647.95458984375,
"KL/rejected_KL_mean": -804.91796875,
"KL/std": 546.8567504882812,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 313.92669677734375,
"fcm_dpo/q_t": 0.4797661304473877,
"grad_norm": 7.039979934692383,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.26116132736206055,
"logits/rejected": -0.2724123001098633,
"logps/chosen": -543.93310546875,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -896.1715087890625,
"loss": 1.3148,
"margin_dpo/margin_mean": 313.92669677734375,
"margin_dpo/margin_std": 714.0025634765625,
"step": 278
},
{
"KL/chosen_KL_mean": -528.8704833984375,
"KL/mean": -699.5042724609375,
"KL/rejected_KL_mean": -870.1380615234375,
"KL/std": 579.2550659179688,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 341.2675476074219,
"fcm_dpo/q_t": 0.47832390666007996,
"grad_norm": 8.836512565612793,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.2435876727104187,
"logits/rejected": -0.2540760934352875,
"logps/chosen": -577.5117797851562,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -957.989501953125,
"loss": 1.3106,
"margin_dpo/margin_mean": 341.26751708984375,
"margin_dpo/margin_std": 805.846435546875,
"step": 279
},
{
"KL/chosen_KL_mean": -519.3259887695312,
"KL/mean": -680.1437377929688,
"KL/rejected_KL_mean": -840.96142578125,
"KL/std": 494.4556884765625,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 321.635498046875,
"fcm_dpo/q_t": 0.4792141318321228,
"grad_norm": 7.04791784286499,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.1967657059431076,
"logits/rejected": -0.21048101782798767,
"logps/chosen": -578.1231079101562,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -939.580322265625,
"loss": 1.3113,
"margin_dpo/margin_mean": 321.635498046875,
"margin_dpo/margin_std": 650.3975830078125,
"step": 280
},
{
"KL/chosen_KL_mean": -519.2962646484375,
"KL/mean": -635.3290405273438,
"KL/rejected_KL_mean": -751.3618774414062,
"KL/std": 475.23504638671875,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 232.0655975341797,
"fcm_dpo/q_t": 0.48489832878112793,
"grad_norm": 7.778337478637695,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.19264814257621765,
"logits/rejected": -0.20138809084892273,
"logps/chosen": -574.7847900390625,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -832.2444458007812,
"loss": 1.3332,
"margin_dpo/margin_mean": 232.0655975341797,
"margin_dpo/margin_std": 605.12109375,
"step": 281
},
{
"KL/chosen_KL_mean": -589.7445068359375,
"KL/mean": -695.1300048828125,
"KL/rejected_KL_mean": -800.515380859375,
"KL/std": 656.7650146484375,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 210.77078247070312,
"fcm_dpo/q_t": 0.4869770109653473,
"grad_norm": 6.628312110900879,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.22694750130176544,
"logits/rejected": -0.23224890232086182,
"logps/chosen": -662.814697265625,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -895.8663330078125,
"loss": 1.3427,
"margin_dpo/margin_mean": 210.7707977294922,
"margin_dpo/margin_std": 778.447998046875,
"step": 282
},
{
"KL/chosen_KL_mean": -562.0560913085938,
"KL/mean": -729.2489013671875,
"KL/rejected_KL_mean": -896.4417724609375,
"KL/std": 537.7421264648438,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 334.38568115234375,
"fcm_dpo/q_t": 0.47830215096473694,
"grad_norm": 8.030158996582031,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.15689605474472046,
"logits/rejected": -0.16659200191497803,
"logps/chosen": -623.9545288085938,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -993.4283447265625,
"loss": 1.3066,
"margin_dpo/margin_mean": 334.38568115234375,
"margin_dpo/margin_std": 558.3077392578125,
"step": 283
},
{
"KL/chosen_KL_mean": -445.6246337890625,
"KL/mean": -596.072509765625,
"KL/rejected_KL_mean": -746.5203857421875,
"KL/std": 443.46435546875,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 300.89569091796875,
"fcm_dpo/q_t": 0.48052817583084106,
"grad_norm": 7.716520309448242,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.22630912065505981,
"logits/rejected": -0.2253245860338211,
"logps/chosen": -504.0601806640625,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -839.9896240234375,
"loss": 1.3141,
"margin_dpo/margin_mean": 300.89569091796875,
"margin_dpo/margin_std": 518.4878540039062,
"step": 284
},
{
"KL/chosen_KL_mean": -528.9168090820312,
"KL/mean": -623.526611328125,
"KL/rejected_KL_mean": -718.1364135742188,
"KL/std": 397.197509765625,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 189.2196502685547,
"fcm_dpo/q_t": 0.4876604974269867,
"grad_norm": 5.944766044616699,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.1648595631122589,
"logits/rejected": -0.16167069971561432,
"logps/chosen": -595.1489868164062,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -817.2632446289062,
"loss": 1.3422,
"margin_dpo/margin_mean": 189.2196502685547,
"margin_dpo/margin_std": 515.746337890625,
"step": 285
},
{
"KL/chosen_KL_mean": -525.88720703125,
"KL/mean": -641.6607666015625,
"KL/rejected_KL_mean": -757.434326171875,
"KL/std": 519.7754516601562,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 231.547119140625,
"fcm_dpo/q_t": 0.4849644899368286,
"grad_norm": 6.175311088562012,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.17296646535396576,
"logits/rejected": -0.17703449726104736,
"logps/chosen": -598.8381958007812,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -846.022705078125,
"loss": 1.3339,
"margin_dpo/margin_mean": 231.54708862304688,
"margin_dpo/margin_std": 646.5220336914062,
"step": 286
},
{
"KL/chosen_KL_mean": -439.7398986816406,
"KL/mean": -516.02294921875,
"KL/rejected_KL_mean": -592.3060302734375,
"KL/std": 409.7009582519531,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 152.56610107421875,
"fcm_dpo/q_t": 0.49012070894241333,
"grad_norm": 5.386925220489502,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.18158751726150513,
"logits/rejected": -0.1914394050836563,
"logps/chosen": -501.28106689453125,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -670.0020751953125,
"loss": 1.3512,
"margin_dpo/margin_mean": 152.56613159179688,
"margin_dpo/margin_std": 495.0728759765625,
"step": 287
},
{
"KL/chosen_KL_mean": -347.9739074707031,
"KL/mean": -471.96112060546875,
"KL/rejected_KL_mean": -595.9483642578125,
"KL/std": 354.6069030761719,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 247.97445678710938,
"fcm_dpo/q_t": 0.48386895656585693,
"grad_norm": 5.900707244873047,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.20927947759628296,
"logits/rejected": -0.21391412615776062,
"logps/chosen": -404.6351318359375,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -683.2840576171875,
"loss": 1.3261,
"margin_dpo/margin_mean": 247.97445678710938,
"margin_dpo/margin_std": 437.38543701171875,
"step": 288
},
{
"KL/chosen_KL_mean": -309.76373291015625,
"KL/mean": -466.19610595703125,
"KL/rejected_KL_mean": -622.6285400390625,
"KL/std": 418.3388977050781,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 312.86480712890625,
"fcm_dpo/q_t": 0.4797741770744324,
"grad_norm": 7.995468616485596,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.15004077553749084,
"logits/rejected": -0.16116581857204437,
"logps/chosen": -354.9941101074219,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -710.2711791992188,
"loss": 1.3106,
"margin_dpo/margin_mean": 312.86480712890625,
"margin_dpo/margin_std": 488.91461181640625,
"step": 289
},
{
"KL/chosen_KL_mean": -354.0469055175781,
"KL/mean": -538.64892578125,
"KL/rejected_KL_mean": -723.2508544921875,
"KL/std": 475.892822265625,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 369.20404052734375,
"fcm_dpo/q_t": 0.4761792719364166,
"grad_norm": 10.0774564743042,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.1508667916059494,
"logits/rejected": -0.17154675722122192,
"logps/chosen": -409.5184020996094,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -839.95947265625,
"loss": 1.2979,
"margin_dpo/margin_mean": 369.2040100097656,
"margin_dpo/margin_std": 568.3728637695312,
"step": 290
},
{
"KL/chosen_KL_mean": -348.29998779296875,
"KL/mean": -480.74700927734375,
"KL/rejected_KL_mean": -613.194091796875,
"KL/std": 381.81988525390625,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 264.89410400390625,
"fcm_dpo/q_t": 0.48280757665634155,
"grad_norm": 7.444272994995117,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.1743829846382141,
"logits/rejected": -0.18439102172851562,
"logps/chosen": -411.56036376953125,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -702.4911499023438,
"loss": 1.3228,
"margin_dpo/margin_mean": 264.89410400390625,
"margin_dpo/margin_std": 509.3777160644531,
"step": 291
},
{
"KL/chosen_KL_mean": -375.5555419921875,
"KL/mean": -526.6381225585938,
"KL/rejected_KL_mean": -677.7206420898438,
"KL/std": 419.32891845703125,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 302.1651306152344,
"fcm_dpo/q_t": 0.480354905128479,
"grad_norm": 6.041011333465576,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.2032952606678009,
"logits/rejected": -0.21287089586257935,
"logps/chosen": -429.47406005859375,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -767.6820068359375,
"loss": 1.3134,
"margin_dpo/margin_mean": 302.1651306152344,
"margin_dpo/margin_std": 505.92315673828125,
"step": 292
},
{
"KL/chosen_KL_mean": -508.30621337890625,
"KL/mean": -616.5218505859375,
"KL/rejected_KL_mean": -724.7373657226562,
"KL/std": 533.0728759765625,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 216.43118286132812,
"fcm_dpo/q_t": 0.48600873351097107,
"grad_norm": 6.5855913162231445,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.21148748695850372,
"logits/rejected": -0.22140976786613464,
"logps/chosen": -568.6822509765625,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -802.58984375,
"loss": 1.3359,
"margin_dpo/margin_mean": 216.43116760253906,
"margin_dpo/margin_std": 558.06982421875,
"step": 293
},
{
"KL/chosen_KL_mean": -426.6471862792969,
"KL/mean": -560.2008666992188,
"KL/rejected_KL_mean": -693.7545776367188,
"KL/std": 453.28826904296875,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 267.1073303222656,
"fcm_dpo/q_t": 0.48279574513435364,
"grad_norm": 7.555781841278076,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.24115553498268127,
"logits/rejected": -0.25332289934158325,
"logps/chosen": -474.7347106933594,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -775.6515502929688,
"loss": 1.3246,
"margin_dpo/margin_mean": 267.1073303222656,
"margin_dpo/margin_std": 627.17822265625,
"step": 294
},
{
"KL/chosen_KL_mean": -553.238037109375,
"KL/mean": -707.3406982421875,
"KL/rejected_KL_mean": -861.443359375,
"KL/std": 558.95654296875,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 308.205322265625,
"fcm_dpo/q_t": 0.4802568554878235,
"grad_norm": 6.732223987579346,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.2826892137527466,
"logits/rejected": -0.28627097606658936,
"logps/chosen": -603.1626586914062,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -948.899658203125,
"loss": 1.3164,
"margin_dpo/margin_mean": 308.205322265625,
"margin_dpo/margin_std": 724.8931884765625,
"step": 295
},
{
"KL/chosen_KL_mean": -672.35009765625,
"KL/mean": -775.7928466796875,
"KL/rejected_KL_mean": -879.235595703125,
"KL/std": 503.78070068359375,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 206.88555908203125,
"fcm_dpo/q_t": 0.486669659614563,
"grad_norm": 8.33739185333252,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.2496371865272522,
"logits/rejected": -0.24974963068962097,
"logps/chosen": -737.84130859375,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -972.32470703125,
"loss": 1.3406,
"margin_dpo/margin_mean": 206.88555908203125,
"margin_dpo/margin_std": 652.9727783203125,
"step": 296
},
{
"KL/chosen_KL_mean": -614.4139404296875,
"KL/mean": -749.4168090820312,
"KL/rejected_KL_mean": -884.419677734375,
"KL/std": 595.4171752929688,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 270.00567626953125,
"fcm_dpo/q_t": 0.4827141761779785,
"grad_norm": 8.87056827545166,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.2674667239189148,
"logits/rejected": -0.27443575859069824,
"logps/chosen": -670.890869140625,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -979.5582275390625,
"loss": 1.3246,
"margin_dpo/margin_mean": 270.00567626953125,
"margin_dpo/margin_std": 666.51513671875,
"step": 297
},
{
"KL/chosen_KL_mean": -678.1602783203125,
"KL/mean": -886.756591796875,
"KL/rejected_KL_mean": -1095.3529052734375,
"KL/std": 748.703857421875,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 417.192626953125,
"fcm_dpo/q_t": 0.4735215902328491,
"grad_norm": 8.397970199584961,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.2270020991563797,
"logits/rejected": -0.23736727237701416,
"logps/chosen": -745.4854736328125,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -1212.01513671875,
"loss": 1.2938,
"margin_dpo/margin_mean": 417.192626953125,
"margin_dpo/margin_std": 878.2392578125,
"step": 298
},
{
"KL/chosen_KL_mean": -549.8248291015625,
"KL/mean": -667.109130859375,
"KL/rejected_KL_mean": -784.3934936523438,
"KL/std": 495.10968017578125,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 234.56871032714844,
"fcm_dpo/q_t": 0.4848034977912903,
"grad_norm": 8.802879333496094,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.25002071261405945,
"logits/rejected": -0.2646872401237488,
"logps/chosen": -598.7869262695312,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -868.7217407226562,
"loss": 1.3337,
"margin_dpo/margin_mean": 234.5686798095703,
"margin_dpo/margin_std": 640.656982421875,
"step": 299
},
{
"KL/chosen_KL_mean": -645.182861328125,
"KL/mean": -834.207763671875,
"KL/rejected_KL_mean": -1023.232666015625,
"KL/std": 587.900390625,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 378.0498352050781,
"fcm_dpo/q_t": 0.4755672812461853,
"grad_norm": 7.947546005249023,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.1696600764989853,
"logits/rejected": -0.18944713473320007,
"logps/chosen": -704.256591796875,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -1119.1990966796875,
"loss": 1.2997,
"margin_dpo/margin_mean": 378.0498046875,
"margin_dpo/margin_std": 751.8477783203125,
"step": 300
},
{
"KL/chosen_KL_mean": -525.6502685546875,
"KL/mean": -696.746337890625,
"KL/rejected_KL_mean": -867.8424072265625,
"KL/std": 537.3768310546875,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 342.19207763671875,
"fcm_dpo/q_t": 0.47806602716445923,
"grad_norm": 7.660505771636963,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.19091829657554626,
"logits/rejected": -0.20372003316879272,
"logps/chosen": -582.899658203125,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -960.1959228515625,
"loss": 1.3081,
"margin_dpo/margin_mean": 342.1921081542969,
"margin_dpo/margin_std": 729.956298828125,
"step": 301
},
{
"KL/chosen_KL_mean": -453.0435791015625,
"KL/mean": -614.121337890625,
"KL/rejected_KL_mean": -775.19921875,
"KL/std": 527.9852294921875,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 322.1556396484375,
"fcm_dpo/q_t": 0.479292631149292,
"grad_norm": 8.750789642333984,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.15997496247291565,
"logits/rejected": -0.15943099558353424,
"logps/chosen": -504.2415771484375,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -872.425537109375,
"loss": 1.3124,
"margin_dpo/margin_mean": 322.1556396484375,
"margin_dpo/margin_std": 693.7476806640625,
"step": 302
},
{
"KL/chosen_KL_mean": -511.125244140625,
"KL/mean": -612.9938354492188,
"KL/rejected_KL_mean": -714.8624877929688,
"KL/std": 489.7009582519531,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 203.7372589111328,
"fcm_dpo/q_t": 0.48675549030303955,
"grad_norm": 6.2934136390686035,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.16826504468917847,
"logits/rejected": -0.1783979833126068,
"logps/chosen": -577.8391723632812,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -801.8079223632812,
"loss": 1.3402,
"margin_dpo/margin_mean": 203.73727416992188,
"margin_dpo/margin_std": 608.4408569335938,
"step": 303
},
{
"KL/chosen_KL_mean": -474.92987060546875,
"KL/mean": -585.3536376953125,
"KL/rejected_KL_mean": -695.7774658203125,
"KL/std": 415.7464599609375,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 220.84759521484375,
"fcm_dpo/q_t": 0.4856563210487366,
"grad_norm": 7.61824893951416,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.1859014928340912,
"logits/rejected": -0.1942559778690338,
"logps/chosen": -546.8805541992188,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -786.24951171875,
"loss": 1.3334,
"margin_dpo/margin_mean": 220.84759521484375,
"margin_dpo/margin_std": 473.25732421875,
"step": 304
},
{
"KL/chosen_KL_mean": -428.038330078125,
"KL/mean": -549.772705078125,
"KL/rejected_KL_mean": -671.5070190429688,
"KL/std": 450.9216003417969,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 243.46868896484375,
"fcm_dpo/q_t": 0.4842411279678345,
"grad_norm": 5.930902004241943,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.22110234200954437,
"logits/rejected": -0.23830103874206543,
"logps/chosen": -494.8335876464844,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -764.2615966796875,
"loss": 1.3293,
"margin_dpo/margin_mean": 243.46868896484375,
"margin_dpo/margin_std": 573.030029296875,
"step": 305
},
{
"KL/chosen_KL_mean": -461.40081787109375,
"KL/mean": -561.10595703125,
"KL/rejected_KL_mean": -660.81103515625,
"KL/std": 412.7308654785156,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 199.41024780273438,
"fcm_dpo/q_t": 0.4870589077472687,
"grad_norm": 6.410506725311279,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.1642266809940338,
"logits/rejected": -0.17837274074554443,
"logps/chosen": -531.084716796875,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -745.97021484375,
"loss": 1.3388,
"margin_dpo/margin_mean": 199.41024780273438,
"margin_dpo/margin_std": 468.4999084472656,
"step": 306
},
{
"KL/chosen_KL_mean": -415.28350830078125,
"KL/mean": -531.0717163085938,
"KL/rejected_KL_mean": -646.8599243164062,
"KL/std": 411.35687255859375,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 231.57640075683594,
"fcm_dpo/q_t": 0.4849886894226074,
"grad_norm": 8.488301277160645,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.18022067844867706,
"logits/rejected": -0.1865011602640152,
"logps/chosen": -485.44891357421875,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -733.832275390625,
"loss": 1.3317,
"margin_dpo/margin_mean": 231.576416015625,
"margin_dpo/margin_std": 537.4920654296875,
"step": 307
},
{
"KL/chosen_KL_mean": -447.8622741699219,
"KL/mean": -599.67041015625,
"KL/rejected_KL_mean": -751.478515625,
"KL/std": 453.7755432128906,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 303.61627197265625,
"fcm_dpo/q_t": 0.48028409481048584,
"grad_norm": 12.173463821411133,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.20448637008666992,
"logits/rejected": -0.21775467693805695,
"logps/chosen": -503.1072692871094,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -830.8507690429688,
"loss": 1.3125,
"margin_dpo/margin_mean": 303.6162414550781,
"margin_dpo/margin_std": 472.8870849609375,
"step": 308
},
{
"KL/chosen_KL_mean": -451.45013427734375,
"KL/mean": -587.4209594726562,
"KL/rejected_KL_mean": -723.3917846679688,
"KL/std": 521.2839965820312,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 271.9416809082031,
"fcm_dpo/q_t": 0.4822486639022827,
"grad_norm": 8.317163467407227,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.1586446464061737,
"logits/rejected": -0.17579534649848938,
"logps/chosen": -500.4092102050781,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -805.7325439453125,
"loss": 1.3225,
"margin_dpo/margin_mean": 271.941650390625,
"margin_dpo/margin_std": 569.9622802734375,
"step": 309
},
{
"KL/chosen_KL_mean": -649.622802734375,
"KL/mean": -782.8016357421875,
"KL/rejected_KL_mean": -915.9805297851562,
"KL/std": 569.54541015625,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 266.3577575683594,
"fcm_dpo/q_t": 0.48275303840637207,
"grad_norm": 11.456547737121582,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.16804948449134827,
"logits/rejected": -0.19079285860061646,
"logps/chosen": -712.364501953125,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -995.9108276367188,
"loss": 1.3254,
"margin_dpo/margin_mean": 266.3577575683594,
"margin_dpo/margin_std": 641.886962890625,
"step": 310
},
{
"KL/chosen_KL_mean": -619.607421875,
"KL/mean": -746.11328125,
"KL/rejected_KL_mean": -872.619140625,
"KL/std": 541.6986694335938,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 253.01168823242188,
"fcm_dpo/q_t": 0.4836409389972687,
"grad_norm": 12.55339241027832,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.2095467746257782,
"logits/rejected": -0.21634000539779663,
"logps/chosen": -672.6354370117188,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -950.057373046875,
"loss": 1.3278,
"margin_dpo/margin_mean": 253.0116729736328,
"margin_dpo/margin_std": 614.6041870117188,
"step": 311
},
{
"KL/chosen_KL_mean": -701.8001708984375,
"KL/mean": -825.3548583984375,
"KL/rejected_KL_mean": -948.909423828125,
"KL/std": 627.0281982421875,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 247.10928344726562,
"fcm_dpo/q_t": 0.4842231571674347,
"grad_norm": 8.390949249267578,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -0.19486571848392487,
"logits/rejected": -0.20282875001430511,
"logps/chosen": -751.1923828125,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -1024.7022705078125,
"loss": 1.3342,
"margin_dpo/margin_mean": 247.10926818847656,
"margin_dpo/margin_std": 803.0827026367188,
"step": 312
},
{
"KL/chosen_KL_mean": -695.1138916015625,
"KL/mean": -879.8554077148438,
"KL/rejected_KL_mean": -1064.596923828125,
"KL/std": 770.5494384765625,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 369.48297119140625,
"fcm_dpo/q_t": 0.47646960616111755,
"grad_norm": 7.17667818069458,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -0.25826430320739746,
"logits/rejected": -0.27332425117492676,
"logps/chosen": -745.2666625976562,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -1151.003173828125,
"loss": 1.3055,
"margin_dpo/margin_mean": 369.4830017089844,
"margin_dpo/margin_std": 858.5765380859375,
"step": 313
},
{
"KL/chosen_KL_mean": -779.741455078125,
"KL/mean": -921.0244750976562,
"KL/rejected_KL_mean": -1062.307373046875,
"KL/std": 699.74462890625,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 282.5658874511719,
"fcm_dpo/q_t": 0.4820650815963745,
"grad_norm": 11.305423736572266,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.2745167315006256,
"logits/rejected": -0.2868790030479431,
"logps/chosen": -836.9790649414062,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -1159.904052734375,
"loss": 1.329,
"margin_dpo/margin_mean": 282.56591796875,
"margin_dpo/margin_std": 936.9970703125,
"step": 314
},
{
"KL/chosen_KL_mean": -648.9210815429688,
"KL/mean": -783.290283203125,
"KL/rejected_KL_mean": -917.6595458984375,
"KL/std": 683.492431640625,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 268.738525390625,
"fcm_dpo/q_t": 0.48275700211524963,
"grad_norm": 11.812366485595703,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.23237919807434082,
"logits/rejected": -0.23814760148525238,
"logps/chosen": -697.9906616210938,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -1003.3404541015625,
"loss": 1.3289,
"margin_dpo/margin_mean": 268.738525390625,
"margin_dpo/margin_std": 817.6375732421875,
"step": 315
},
{
"KL/chosen_KL_mean": -583.3734130859375,
"KL/mean": -823.87109375,
"KL/rejected_KL_mean": -1064.3687744140625,
"KL/std": 654.394775390625,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 480.99542236328125,
"fcm_dpo/q_t": 0.4692395031452179,
"grad_norm": 9.822944641113281,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.27629613876342773,
"logits/rejected": -0.2879878878593445,
"logps/chosen": -637.6341552734375,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -1165.650146484375,
"loss": 1.2759,
"margin_dpo/margin_mean": 480.995361328125,
"margin_dpo/margin_std": 791.6807861328125,
"step": 316
},
{
"KL/chosen_KL_mean": -654.828369140625,
"KL/mean": -917.2320556640625,
"KL/rejected_KL_mean": -1179.6356201171875,
"KL/std": 796.4964599609375,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 524.8072509765625,
"fcm_dpo/q_t": 0.46749019622802734,
"grad_norm": 13.96440601348877,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.2654808461666107,
"logits/rejected": -0.27527332305908203,
"logps/chosen": -710.922607421875,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -1280.334716796875,
"loss": 1.2729,
"margin_dpo/margin_mean": 524.8072509765625,
"margin_dpo/margin_std": 1009.199462890625,
"step": 317
},
{
"KL/chosen_KL_mean": -712.6109619140625,
"KL/mean": -797.7545166015625,
"KL/rejected_KL_mean": -882.8980712890625,
"KL/std": 595.2559814453125,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 170.2871551513672,
"fcm_dpo/q_t": 0.48896729946136475,
"grad_norm": 11.569266319274902,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.3017517626285553,
"logits/rejected": -0.29757431149482727,
"logps/chosen": -777.256591796875,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -965.662353515625,
"loss": 1.3506,
"margin_dpo/margin_mean": 170.28717041015625,
"margin_dpo/margin_std": 690.7733154296875,
"step": 318
},
{
"KL/chosen_KL_mean": -496.1173095703125,
"KL/mean": -747.5867309570312,
"KL/rejected_KL_mean": -999.0562744140625,
"KL/std": 743.3543701171875,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 502.9389343261719,
"fcm_dpo/q_t": 0.46803224086761475,
"grad_norm": 14.8353271484375,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.2799922227859497,
"logits/rejected": -0.2898196578025818,
"logps/chosen": -545.5010375976562,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -1112.9627685546875,
"loss": 1.2732,
"margin_dpo/margin_mean": 502.9389343261719,
"margin_dpo/margin_std": 895.0174560546875,
"step": 319
},
{
"KL/chosen_KL_mean": -562.5033569335938,
"KL/mean": -736.6514282226562,
"KL/rejected_KL_mean": -910.7994384765625,
"KL/std": 562.046142578125,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 348.2961120605469,
"fcm_dpo/q_t": 0.47749677300453186,
"grad_norm": 7.6283860206604,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.2599332332611084,
"logits/rejected": -0.274741530418396,
"logps/chosen": -622.00830078125,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -1008.4666748046875,
"loss": 1.3054,
"margin_dpo/margin_mean": 348.2961120605469,
"margin_dpo/margin_std": 674.3231201171875,
"step": 320
},
{
"KL/chosen_KL_mean": -725.1861572265625,
"KL/mean": -895.68994140625,
"KL/rejected_KL_mean": -1066.193603515625,
"KL/std": 716.3073120117188,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 341.00750732421875,
"fcm_dpo/q_t": 0.4783346951007843,
"grad_norm": 9.341181755065918,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -0.23902887105941772,
"logits/rejected": -0.2461167275905609,
"logps/chosen": -786.73486328125,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -1157.834716796875,
"loss": 1.3123,
"margin_dpo/margin_mean": 341.00750732421875,
"margin_dpo/margin_std": 881.1239013671875,
"step": 321
},
{
"KL/chosen_KL_mean": -587.3211669921875,
"KL/mean": -776.135986328125,
"KL/rejected_KL_mean": -964.9507446289062,
"KL/std": 558.770751953125,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 377.6296081542969,
"fcm_dpo/q_t": 0.4755861759185791,
"grad_norm": 13.473780632019043,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -0.26182982325553894,
"logits/rejected": -0.2680904269218445,
"logps/chosen": -644.6112060546875,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -1060.70068359375,
"loss": 1.2985,
"margin_dpo/margin_mean": 377.629638671875,
"margin_dpo/margin_std": 694.3011474609375,
"step": 322
},
{
"KL/chosen_KL_mean": -609.4126586914062,
"KL/mean": -739.869140625,
"KL/rejected_KL_mean": -870.32568359375,
"KL/std": 562.13623046875,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 260.9129943847656,
"fcm_dpo/q_t": 0.4830913543701172,
"grad_norm": 9.924291610717773,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -0.26495540142059326,
"logits/rejected": -0.27675527334213257,
"logps/chosen": -660.6466064453125,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -945.3875732421875,
"loss": 1.3277,
"margin_dpo/margin_mean": 260.9129638671875,
"margin_dpo/margin_std": 673.8228149414062,
"step": 323
},
{
"KL/chosen_KL_mean": -724.944091796875,
"KL/mean": -845.3299560546875,
"KL/rejected_KL_mean": -965.7159423828125,
"KL/std": 667.5174560546875,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 240.7718505859375,
"fcm_dpo/q_t": 0.4845079779624939,
"grad_norm": 11.485196113586426,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -0.26390862464904785,
"logits/rejected": -0.26681196689605713,
"logps/chosen": -790.0792236328125,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -1052.193359375,
"loss": 1.3385,
"margin_dpo/margin_mean": 240.77186584472656,
"margin_dpo/margin_std": 892.4651489257812,
"step": 324
},
{
"KL/chosen_KL_mean": -495.9256896972656,
"KL/mean": -597.509033203125,
"KL/rejected_KL_mean": -699.09228515625,
"KL/std": 471.72412109375,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 203.16659545898438,
"fcm_dpo/q_t": 0.4868882894515991,
"grad_norm": 10.78111743927002,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.29811710119247437,
"logits/rejected": -0.3030798137187958,
"logps/chosen": -552.1412963867188,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -769.17822265625,
"loss": 1.3397,
"margin_dpo/margin_mean": 203.16659545898438,
"margin_dpo/margin_std": 564.2322998046875,
"step": 325
},
{
"KL/chosen_KL_mean": -589.589599609375,
"KL/mean": -696.318359375,
"KL/rejected_KL_mean": -803.0471801757812,
"KL/std": 521.7206420898438,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 213.4575653076172,
"fcm_dpo/q_t": 0.48614704608917236,
"grad_norm": 9.612821578979492,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.305173397064209,
"logits/rejected": -0.3111916780471802,
"logps/chosen": -662.3145751953125,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -882.8939208984375,
"loss": 1.3363,
"margin_dpo/margin_mean": 213.45758056640625,
"margin_dpo/margin_std": 538.012939453125,
"step": 326
},
{
"KL/chosen_KL_mean": -558.3499755859375,
"KL/mean": -724.33056640625,
"KL/rejected_KL_mean": -890.311279296875,
"KL/std": 517.6776123046875,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 331.9613037109375,
"fcm_dpo/q_t": 0.47847986221313477,
"grad_norm": 15.699017524719238,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.2681811451911926,
"logits/rejected": -0.27183783054351807,
"logps/chosen": -627.484375,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -1002.2450561523438,
"loss": 1.3086,
"margin_dpo/margin_mean": 331.9612731933594,
"margin_dpo/margin_std": 640.1740112304688,
"step": 327
},
{
"KL/chosen_KL_mean": -558.3543701171875,
"KL/mean": -749.8038330078125,
"KL/rejected_KL_mean": -941.2532348632812,
"KL/std": 636.8343505859375,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 382.89892578125,
"fcm_dpo/q_t": 0.4753231406211853,
"grad_norm": 14.158612251281738,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.2592814862728119,
"logits/rejected": -0.2679283022880554,
"logps/chosen": -618.0415649414062,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -1032.1082763671875,
"loss": 1.2986,
"margin_dpo/margin_mean": 382.8988952636719,
"margin_dpo/margin_std": 756.589599609375,
"step": 328
},
{
"KL/chosen_KL_mean": -674.1375732421875,
"KL/mean": -896.6131591796875,
"KL/rejected_KL_mean": -1119.0888671875,
"KL/std": 738.45849609375,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 444.9512023925781,
"fcm_dpo/q_t": 0.47157514095306396,
"grad_norm": 9.89807415008545,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -0.25955867767333984,
"logits/rejected": -0.2711198329925537,
"logps/chosen": -739.3837890625,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -1219.7864990234375,
"loss": 1.287,
"margin_dpo/margin_mean": 444.951171875,
"margin_dpo/margin_std": 891.010986328125,
"step": 329
},
{
"KL/chosen_KL_mean": -574.3134765625,
"KL/mean": -772.6851196289062,
"KL/rejected_KL_mean": -971.0567626953125,
"KL/std": 623.42626953125,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 396.7432861328125,
"fcm_dpo/q_t": 0.4744586944580078,
"grad_norm": 11.012582778930664,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -0.2380240261554718,
"logits/rejected": -0.23784172534942627,
"logps/chosen": -621.3118286132812,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -1057.93359375,
"loss": 1.2955,
"margin_dpo/margin_mean": 396.7432861328125,
"margin_dpo/margin_std": 763.457763671875,
"step": 330
},
{
"KL/chosen_KL_mean": -734.039306640625,
"KL/mean": -967.9661865234375,
"KL/rejected_KL_mean": -1201.8931884765625,
"KL/std": 592.183837890625,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 467.8538818359375,
"fcm_dpo/q_t": 0.46986526250839233,
"grad_norm": 14.263129234313965,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -0.23857057094573975,
"logits/rejected": -0.23989224433898926,
"logps/chosen": -784.5635375976562,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -1290.90869140625,
"loss": 1.2768,
"margin_dpo/margin_mean": 467.85394287109375,
"margin_dpo/margin_std": 727.0281982421875,
"step": 331
},
{
"KL/chosen_KL_mean": -898.7767333984375,
"KL/mean": -1009.329345703125,
"KL/rejected_KL_mean": -1119.8819580078125,
"KL/std": 775.726318359375,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 221.10525512695312,
"fcm_dpo/q_t": 0.4862329959869385,
"grad_norm": 11.460166931152344,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -0.2585371732711792,
"logits/rejected": -0.2638899087905884,
"logps/chosen": -947.95703125,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -1196.3671875,
"loss": 1.3453,
"margin_dpo/margin_mean": 221.10525512695312,
"margin_dpo/margin_std": 970.4208374023438,
"step": 332
},
{
"KL/chosen_KL_mean": -1128.766845703125,
"KL/mean": -1313.67626953125,
"KL/rejected_KL_mean": -1498.585693359375,
"KL/std": 929.1944580078125,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 369.8189697265625,
"fcm_dpo/q_t": 0.4764955937862396,
"grad_norm": 28.53215980529785,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -0.29855263233184814,
"logits/rejected": -0.3038429021835327,
"logps/chosen": -1192.5224609375,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -1593.6298828125,
"loss": 1.3193,
"margin_dpo/margin_mean": 369.8189697265625,
"margin_dpo/margin_std": 1217.3851318359375,
"step": 333
},
{
"KL/chosen_KL_mean": -1276.139892578125,
"KL/mean": -1387.209716796875,
"KL/rejected_KL_mean": -1498.279541015625,
"KL/std": 1128.24169921875,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 222.13943481445312,
"fcm_dpo/q_t": 0.4864605665206909,
"grad_norm": 47.288944244384766,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.2978268265724182,
"logits/rejected": -0.3007590174674988,
"logps/chosen": -1343.1197509765625,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -1593.596435546875,
"loss": 1.3669,
"margin_dpo/margin_mean": 222.139404296875,
"margin_dpo/margin_std": 1478.4951171875,
"step": 334
},
{
"KL/chosen_KL_mean": -1121.868408203125,
"KL/mean": -1232.154052734375,
"KL/rejected_KL_mean": -1342.439453125,
"KL/std": 981.7579345703125,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 220.57101440429688,
"fcm_dpo/q_t": 0.48608070611953735,
"grad_norm": 25.209972381591797,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -0.32386332750320435,
"logits/rejected": -0.3207986354827881,
"logps/chosen": -1184.410888671875,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -1430.05712890625,
"loss": 1.3467,
"margin_dpo/margin_mean": 220.571044921875,
"margin_dpo/margin_std": 993.0822143554688,
"step": 335
},
{
"KL/chosen_KL_mean": -1050.422607421875,
"KL/mean": -1337.56591796875,
"KL/rejected_KL_mean": -1624.709228515625,
"KL/std": 1164.25244140625,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 574.28662109375,
"fcm_dpo/q_t": 0.46436643600463867,
"grad_norm": 14.840729713439941,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -0.30795538425445557,
"logits/rejected": -0.31845933198928833,
"logps/chosen": -1104.9537353515625,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -1729.113525390625,
"loss": 1.2781,
"margin_dpo/margin_mean": 574.28662109375,
"margin_dpo/margin_std": 1452.664306640625,
"step": 336
},
{
"KL/chosen_KL_mean": -855.2294921875,
"KL/mean": -1081.739501953125,
"KL/rejected_KL_mean": -1308.249755859375,
"KL/std": 739.3917236328125,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 453.0201721191406,
"fcm_dpo/q_t": 0.4711313247680664,
"grad_norm": 11.842601776123047,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -0.3099353313446045,
"logits/rejected": -0.3142092227935791,
"logps/chosen": -920.3582153320312,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -1409.9766845703125,
"loss": 1.2874,
"margin_dpo/margin_mean": 453.02020263671875,
"margin_dpo/margin_std": 964.009765625,
"step": 337
},
{
"KL/chosen_KL_mean": -630.074951171875,
"KL/mean": -862.7835083007812,
"KL/rejected_KL_mean": -1095.491943359375,
"KL/std": 717.5847778320312,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 465.41705322265625,
"fcm_dpo/q_t": 0.4699110984802246,
"grad_norm": 10.382853507995605,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -0.272180438041687,
"logits/rejected": -0.28474897146224976,
"logps/chosen": -688.4976806640625,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -1184.560546875,
"loss": 1.2764,
"margin_dpo/margin_mean": 465.4170227050781,
"margin_dpo/margin_std": 681.0792846679688,
"step": 338
},
{
"KL/chosen_KL_mean": -607.092041015625,
"KL/mean": -813.3968505859375,
"KL/rejected_KL_mean": -1019.7017211914062,
"KL/std": 651.6005859375,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 412.6096496582031,
"fcm_dpo/q_t": 0.47347021102905273,
"grad_norm": 14.31478500366211,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -0.25229567289352417,
"logits/rejected": -0.26170292496681213,
"logps/chosen": -667.0873413085938,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -1123.612548828125,
"loss": 1.2931,
"margin_dpo/margin_mean": 412.6096496582031,
"margin_dpo/margin_std": 765.8604736328125,
"step": 339
},
{
"KL/chosen_KL_mean": -682.767822265625,
"KL/mean": -883.4863891601562,
"KL/rejected_KL_mean": -1084.2049560546875,
"KL/std": 732.8419189453125,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 401.4371337890625,
"fcm_dpo/q_t": 0.4744035601615906,
"grad_norm": 15.460363388061523,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -0.2546389698982239,
"logits/rejected": -0.25613105297088623,
"logps/chosen": -735.5980224609375,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -1157.312255859375,
"loss": 1.299,
"margin_dpo/margin_mean": 401.4371337890625,
"margin_dpo/margin_std": 928.7999267578125,
"step": 340
},
{
"KL/chosen_KL_mean": -624.3984375,
"KL/mean": -853.274169921875,
"KL/rejected_KL_mean": -1082.1497802734375,
"KL/std": 722.1390380859375,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 457.75128173828125,
"fcm_dpo/q_t": 0.4708910584449768,
"grad_norm": 21.002342224121094,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.23577266931533813,
"logits/rejected": -0.24404844641685486,
"logps/chosen": -672.2982788085938,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -1183.959716796875,
"loss": 1.2852,
"margin_dpo/margin_mean": 457.7512512207031,
"margin_dpo/margin_std": 918.9498291015625,
"step": 341
},
{
"KL/chosen_KL_mean": -543.6002197265625,
"KL/mean": -688.5452880859375,
"KL/rejected_KL_mean": -833.4903564453125,
"KL/std": 547.8942260742188,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 289.8901672363281,
"fcm_dpo/q_t": 0.48126789927482605,
"grad_norm": 9.170316696166992,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.30688920617103577,
"logits/rejected": -0.30778950452804565,
"logps/chosen": -615.5968017578125,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -926.0799560546875,
"loss": 1.3206,
"margin_dpo/margin_mean": 289.8901672363281,
"margin_dpo/margin_std": 702.553955078125,
"step": 342
},
{
"KL/chosen_KL_mean": -471.5529479980469,
"KL/mean": -678.2440185546875,
"KL/rejected_KL_mean": -884.93505859375,
"KL/std": 528.4884033203125,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 413.38214111328125,
"fcm_dpo/q_t": 0.47324374318122864,
"grad_norm": 11.689512252807617,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.24627618491649628,
"logits/rejected": -0.25361427664756775,
"logps/chosen": -525.9585571289062,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -995.9764404296875,
"loss": 1.2863,
"margin_dpo/margin_mean": 413.38214111328125,
"margin_dpo/margin_std": 545.55517578125,
"step": 343
},
{
"KL/chosen_KL_mean": -607.8341064453125,
"KL/mean": -803.11181640625,
"KL/rejected_KL_mean": -998.3895263671875,
"KL/std": 760.2177734375,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 390.555419921875,
"fcm_dpo/q_t": 0.4753226637840271,
"grad_norm": 9.874456405639648,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.2654206454753876,
"logits/rejected": -0.2721824645996094,
"logps/chosen": -661.798828125,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -1089.012939453125,
"loss": 1.3019,
"margin_dpo/margin_mean": 390.555419921875,
"margin_dpo/margin_std": 907.13916015625,
"step": 344
},
{
"KL/chosen_KL_mean": -735.56591796875,
"KL/mean": -957.1832275390625,
"KL/rejected_KL_mean": -1178.8006591796875,
"KL/std": 641.4892578125,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 443.2347412109375,
"fcm_dpo/q_t": 0.4715315103530884,
"grad_norm": 11.198358535766602,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.27164191007614136,
"logits/rejected": -0.27693307399749756,
"logps/chosen": -797.2515869140625,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -1278.291015625,
"loss": 1.2858,
"margin_dpo/margin_mean": 443.2347412109375,
"margin_dpo/margin_std": 839.5887451171875,
"step": 345
},
{
"KL/chosen_KL_mean": -720.7941284179688,
"KL/mean": -924.3731079101562,
"KL/rejected_KL_mean": -1127.9521484375,
"KL/std": 670.3480224609375,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 407.1580505371094,
"fcm_dpo/q_t": 0.4737437069416046,
"grad_norm": 13.88206672668457,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -0.2596754729747772,
"logits/rejected": -0.26096630096435547,
"logps/chosen": -776.0504150390625,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -1205.367431640625,
"loss": 1.2905,
"margin_dpo/margin_mean": 407.15802001953125,
"margin_dpo/margin_std": 673.2942504882812,
"step": 346
},
{
"KL/chosen_KL_mean": -865.6432495117188,
"KL/mean": -1022.4780883789062,
"KL/rejected_KL_mean": -1179.31298828125,
"KL/std": 805.083251953125,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 313.669677734375,
"fcm_dpo/q_t": 0.47981417179107666,
"grad_norm": 10.583271026611328,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -0.27997511625289917,
"logits/rejected": -0.28578126430511475,
"logps/chosen": -923.20947265625,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -1271.6680908203125,
"loss": 1.3201,
"margin_dpo/margin_mean": 313.669677734375,
"margin_dpo/margin_std": 908.4871826171875,
"step": 347
},
{
"KL/chosen_KL_mean": -720.2841796875,
"KL/mean": -897.8193359375,
"KL/rejected_KL_mean": -1075.3543701171875,
"KL/std": 628.7544555664062,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 355.0701904296875,
"fcm_dpo/q_t": 0.4771634340286255,
"grad_norm": 10.66627311706543,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -0.2865196466445923,
"logits/rejected": -0.2877465486526489,
"logps/chosen": -776.6019287109375,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -1164.4927978515625,
"loss": 1.307,
"margin_dpo/margin_mean": 355.0701904296875,
"margin_dpo/margin_std": 798.5880126953125,
"step": 348
},
{
"KL/chosen_KL_mean": -805.5579223632812,
"KL/mean": -1064.7337646484375,
"KL/rejected_KL_mean": -1323.90966796875,
"KL/std": 1003.7889404296875,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 518.3516845703125,
"fcm_dpo/q_t": 0.46881359815597534,
"grad_norm": 15.495862007141113,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -0.27927365899086,
"logits/rejected": -0.28703418374061584,
"logps/chosen": -863.5834350585938,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -1421.414794921875,
"loss": 1.2844,
"margin_dpo/margin_mean": 518.3516235351562,
"margin_dpo/margin_std": 1315.3780517578125,
"step": 349
},
{
"KL/chosen_KL_mean": -923.5478515625,
"KL/mean": -1110.17333984375,
"KL/rejected_KL_mean": -1296.798828125,
"KL/std": 911.0496826171875,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 373.2510070800781,
"fcm_dpo/q_t": 0.475941002368927,
"grad_norm": 21.998889923095703,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -0.3065475821495056,
"logits/rejected": -0.31878340244293213,
"logps/chosen": -987.8783569335938,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -1386.67041015625,
"loss": 1.3163,
"margin_dpo/margin_mean": 373.2510070800781,
"margin_dpo/margin_std": 1196.018798828125,
"step": 350
},
{
"KL/chosen_KL_mean": -735.504638671875,
"KL/mean": -1000.615234375,
"KL/rejected_KL_mean": -1265.725830078125,
"KL/std": 860.2415771484375,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 530.22119140625,
"fcm_dpo/q_t": 0.46662163734436035,
"grad_norm": 10.427867889404297,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -0.3097544312477112,
"logits/rejected": -0.32192495465278625,
"logps/chosen": -796.1767578125,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -1367.291259765625,
"loss": 1.2742,
"margin_dpo/margin_mean": 530.22119140625,
"margin_dpo/margin_std": 1098.137939453125,
"step": 351
},
{
"KL/chosen_KL_mean": -911.3470458984375,
"KL/mean": -1003.693359375,
"KL/rejected_KL_mean": -1096.0396728515625,
"KL/std": 876.0338134765625,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 184.69261169433594,
"fcm_dpo/q_t": 0.4880898594856262,
"grad_norm": 19.379594802856445,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -0.30126988887786865,
"logits/rejected": -0.2974497675895691,
"logps/chosen": -982.29052734375,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -1172.681640625,
"loss": 1.3548,
"margin_dpo/margin_mean": 184.69261169433594,
"margin_dpo/margin_std": 969.9511108398438,
"step": 352
},
{
"KL/chosen_KL_mean": -786.5258178710938,
"KL/mean": -942.9754028320312,
"KL/rejected_KL_mean": -1099.425048828125,
"KL/std": 793.8702392578125,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 312.899169921875,
"fcm_dpo/q_t": 0.4803735613822937,
"grad_norm": 14.741339683532715,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -0.2722886800765991,
"logits/rejected": -0.2665886878967285,
"logps/chosen": -844.921142578125,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -1179.760498046875,
"loss": 1.3249,
"margin_dpo/margin_mean": 312.899169921875,
"margin_dpo/margin_std": 1029.1500244140625,
"step": 353
},
{
"KL/chosen_KL_mean": -602.9953002929688,
"KL/mean": -845.4747924804688,
"KL/rejected_KL_mean": -1087.9542236328125,
"KL/std": 870.7578125,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 484.9588928222656,
"fcm_dpo/q_t": 0.4696187674999237,
"grad_norm": 23.143972396850586,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -0.2781677842140198,
"logits/rejected": -0.2879522442817688,
"logps/chosen": -662.79833984375,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -1176.711669921875,
"loss": 1.2783,
"margin_dpo/margin_mean": 484.9588623046875,
"margin_dpo/margin_std": 908.888916015625,
"step": 354
},
{
"KL/chosen_KL_mean": -570.4051513671875,
"KL/mean": -812.153564453125,
"KL/rejected_KL_mean": -1053.9019775390625,
"KL/std": 624.933349609375,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 483.4968566894531,
"fcm_dpo/q_t": 0.468807578086853,
"grad_norm": 22.532615661621094,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -0.26445209980010986,
"logits/rejected": -0.27031999826431274,
"logps/chosen": -624.5336303710938,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -1136.30810546875,
"loss": 1.2727,
"margin_dpo/margin_mean": 483.496826171875,
"margin_dpo/margin_std": 713.3695068359375,
"step": 355
},
{
"KL/chosen_KL_mean": -677.2150268554688,
"KL/mean": -766.095703125,
"KL/rejected_KL_mean": -854.9764404296875,
"KL/std": 658.3951416015625,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 177.76144409179688,
"fcm_dpo/q_t": 0.4887309670448303,
"grad_norm": 10.51893138885498,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -0.2572939395904541,
"logits/rejected": -0.2499064952135086,
"logps/chosen": -741.8887939453125,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -930.875732421875,
"loss": 1.3528,
"margin_dpo/margin_mean": 177.76144409179688,
"margin_dpo/margin_std": 852.14697265625,
"step": 356
},
{
"KL/chosen_KL_mean": -588.591552734375,
"KL/mean": -709.5107421875,
"KL/rejected_KL_mean": -830.4298095703125,
"KL/std": 630.9422607421875,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 241.83822631835938,
"fcm_dpo/q_t": 0.4842875003814697,
"grad_norm": 20.39065170288086,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -0.3094186782836914,
"logits/rejected": -0.310663104057312,
"logps/chosen": -641.3173828125,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -917.27099609375,
"loss": 1.3353,
"margin_dpo/margin_mean": 241.83822631835938,
"margin_dpo/margin_std": 793.8090209960938,
"step": 357
},
{
"KL/chosen_KL_mean": -518.7498779296875,
"KL/mean": -689.32568359375,
"KL/rejected_KL_mean": -859.9015502929688,
"KL/std": 604.7923583984375,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 341.15167236328125,
"fcm_dpo/q_t": 0.4782078266143799,
"grad_norm": 12.143132209777832,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.2932926118373871,
"logits/rejected": -0.29479849338531494,
"logps/chosen": -581.955322265625,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -948.2748413085938,
"loss": 1.3097,
"margin_dpo/margin_mean": 341.15167236328125,
"margin_dpo/margin_std": 780.6080322265625,
"step": 358
},
{
"KL/chosen_KL_mean": -671.3048706054688,
"KL/mean": -872.405029296875,
"KL/rejected_KL_mean": -1073.505126953125,
"KL/std": 705.5187377929688,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 402.20025634765625,
"fcm_dpo/q_t": 0.4743681848049164,
"grad_norm": 13.933921813964844,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -0.25826162099838257,
"logits/rejected": -0.2619970738887787,
"logps/chosen": -727.6751098632812,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -1155.678955078125,
"loss": 1.3002,
"margin_dpo/margin_mean": 402.2002258300781,
"margin_dpo/margin_std": 950.1470947265625,
"step": 359
},
{
"KL/chosen_KL_mean": -685.37353515625,
"KL/mean": -844.9564208984375,
"KL/rejected_KL_mean": -1004.539306640625,
"KL/std": 776.1425170898438,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 319.16571044921875,
"fcm_dpo/q_t": 0.47939565777778625,
"grad_norm": 11.378792762756348,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -0.2582198679447174,
"logits/rejected": -0.257494181394577,
"logps/chosen": -736.8339233398438,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -1074.378173828125,
"loss": 1.3129,
"margin_dpo/margin_mean": 319.1656494140625,
"margin_dpo/margin_std": 692.6895751953125,
"step": 360
},
{
"KL/chosen_KL_mean": -665.7442626953125,
"KL/mean": -860.533935546875,
"KL/rejected_KL_mean": -1055.323486328125,
"KL/std": 791.1251220703125,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 389.5793151855469,
"fcm_dpo/q_t": 0.47519195079803467,
"grad_norm": 16.64234733581543,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -0.26100149750709534,
"logits/rejected": -0.2740827798843384,
"logps/chosen": -719.61376953125,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -1146.0927734375,
"loss": 1.302,
"margin_dpo/margin_mean": 389.5793151855469,
"margin_dpo/margin_std": 934.74609375,
"step": 361
},
{
"KL/chosen_KL_mean": -607.8038330078125,
"KL/mean": -897.657958984375,
"KL/rejected_KL_mean": -1187.5120849609375,
"KL/std": 833.4835815429688,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 579.708251953125,
"fcm_dpo/q_t": 0.46335670351982117,
"grad_norm": 23.430381774902344,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -0.28775614500045776,
"logits/rejected": -0.2993890643119812,
"logps/chosen": -666.44287109375,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -1293.093994140625,
"loss": 1.2568,
"margin_dpo/margin_mean": 579.708251953125,
"margin_dpo/margin_std": 957.5960693359375,
"step": 362
},
{
"KL/chosen_KL_mean": -668.0380859375,
"KL/mean": -925.823974609375,
"KL/rejected_KL_mean": -1183.6099853515625,
"KL/std": 786.2000732421875,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 515.5718994140625,
"fcm_dpo/q_t": 0.4670901298522949,
"grad_norm": 13.35830020904541,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -0.2682288885116577,
"logits/rejected": -0.2802722454071045,
"logps/chosen": -712.596435546875,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -1258.304931640625,
"loss": 1.2705,
"margin_dpo/margin_mean": 515.5718994140625,
"margin_dpo/margin_std": 912.45263671875,
"step": 363
},
{
"KL/chosen_KL_mean": -679.378662109375,
"KL/mean": -908.24169921875,
"KL/rejected_KL_mean": -1137.104736328125,
"KL/std": 799.5364379882812,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 457.7259826660156,
"fcm_dpo/q_t": 0.4708824157714844,
"grad_norm": 17.259096145629883,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.30234140157699585,
"logits/rejected": -0.31423497200012207,
"logps/chosen": -728.2733154296875,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -1228.50048828125,
"loss": 1.2858,
"margin_dpo/margin_mean": 457.7259826660156,
"margin_dpo/margin_std": 911.04443359375,
"step": 364
},
{
"KL/chosen_KL_mean": -783.33154296875,
"KL/mean": -1068.940185546875,
"KL/rejected_KL_mean": -1354.548828125,
"KL/std": 1015.5641479492188,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.00026103141135536134,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 571.2171630859375,
"fcm_dpo/q_t": 0.46466195583343506,
"grad_norm": 16.475479125976562,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -0.2803174555301666,
"logits/rejected": -0.30174314975738525,
"logps/chosen": -834.8242797851562,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -1447.25048828125,
"loss": 1.2675,
"margin_dpo/margin_mean": 571.2171630859375,
"margin_dpo/margin_std": 1223.385498046875,
"step": 365
},
{
"KL/chosen_KL_mean": -831.8895263671875,
"KL/mean": -1130.086669921875,
"KL/rejected_KL_mean": -1428.2838134765625,
"KL/std": 1027.395263671875,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.0014974254881963134,
"fcm_dpo/delta": 0.12657544016838074,
"fcm_dpo/margin": 596.3943481445312,
"fcm_dpo/q_t": 0.446040540933609,
"grad_norm": 130.77503967285156,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -0.324015349149704,
"logits/rejected": -0.34143155813217163,
"logps/chosen": -876.610107421875,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -1511.59423828125,
"loss": 1.4374,
"margin_dpo/margin_mean": 596.3943481445312,
"margin_dpo/margin_std": 1373.01025390625,
"step": 366
},
{
"KL/chosen_KL_mean": -953.1829223632812,
"KL/mean": -1134.24951171875,
"KL/rejected_KL_mean": -1315.31591796875,
"KL/std": 867.8139038085938,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.0010564139811322093,
"fcm_dpo/delta": -0.019750438630580902,
"fcm_dpo/margin": 362.13287353515625,
"fcm_dpo/q_t": 0.42206183075904846,
"grad_norm": 113.0547103881836,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -0.31291621923446655,
"logits/rejected": -0.31141310930252075,
"logps/chosen": -1011.58837890625,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -1392.067138671875,
"loss": 1.322,
"margin_dpo/margin_mean": 362.1329345703125,
"margin_dpo/margin_std": 1070.106201171875,
"step": 367
},
{
"KL/chosen_KL_mean": -768.363037109375,
"KL/mean": -1200.409912109375,
"KL/rejected_KL_mean": -1632.4569091796875,
"KL/std": 953.82568359375,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": -0.44886526465415955,
"fcm_dpo/margin": 864.0938720703125,
"fcm_dpo/q_t": 0.39556068181991577,
"grad_norm": 0.13906408846378326,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -0.3350134491920471,
"logits/rejected": -0.3492249548435211,
"logps/chosen": -812.8155517578125,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -1731.01220703125,
"loss": 1.3853,
"margin_dpo/margin_mean": 864.0938720703125,
"margin_dpo/margin_std": 1078.280029296875,
"step": 368
},
{
"KL/chosen_KL_mean": -998.696533203125,
"KL/mean": -1232.2938232421875,
"KL/rejected_KL_mean": -1465.89111328125,
"KL/std": 959.7288208007812,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 467.19464111328125,
"fcm_dpo/q_t": 0.4998719394207001,
"grad_norm": 0.0773184671998024,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -0.3440535366535187,
"logits/rejected": -0.3447309732437134,
"logps/chosen": -1070.0780029296875,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -1557.18701171875,
"loss": 1.3858,
"margin_dpo/margin_mean": 467.19464111328125,
"margin_dpo/margin_std": 1237.226806640625,
"step": 369
},
{
"KL/chosen_KL_mean": -1075.063720703125,
"KL/mean": -1165.5103759765625,
"KL/rejected_KL_mean": -1255.95703125,
"KL/std": 1011.834228515625,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 180.89346313476562,
"fcm_dpo/q_t": 0.4999504089355469,
"grad_norm": 0.07674546539783478,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -0.33612939715385437,
"logits/rejected": -0.3382912278175354,
"logps/chosen": -1146.671142578125,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -1353.216796875,
"loss": 1.3861,
"margin_dpo/margin_mean": 180.89344787597656,
"margin_dpo/margin_std": 1252.171630859375,
"step": 370
},
{
"KL/chosen_KL_mean": -1031.171142578125,
"KL/mean": -1296.2216796875,
"KL/rejected_KL_mean": -1561.272216796875,
"KL/std": 1119.242919921875,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 530.1011352539062,
"fcm_dpo/q_t": 0.4998546838760376,
"grad_norm": 0.06114376336336136,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -0.342197060585022,
"logits/rejected": -0.3496757447719574,
"logps/chosen": -1100.5855712890625,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -1660.4443359375,
"loss": 1.3857,
"margin_dpo/margin_mean": 530.1011962890625,
"margin_dpo/margin_std": 1297.156494140625,
"step": 371
},
{
"KL/chosen_KL_mean": -913.1492919921875,
"KL/mean": -1198.9420166015625,
"KL/rejected_KL_mean": -1484.73486328125,
"KL/std": 1118.00634765625,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 571.58544921875,
"fcm_dpo/q_t": 0.4998433291912079,
"grad_norm": 0.058624256402254105,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -0.2874826192855835,
"logits/rejected": -0.3007184565067291,
"logps/chosen": -974.9673461914062,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -1563.2742919921875,
"loss": 1.3857,
"margin_dpo/margin_mean": 571.58544921875,
"margin_dpo/margin_std": 1257.53955078125,
"step": 372
},
{
"KL/chosen_KL_mean": -990.4146118164062,
"KL/mean": -1182.953857421875,
"KL/rejected_KL_mean": -1375.4930419921875,
"KL/std": 983.2168579101562,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 385.07843017578125,
"fcm_dpo/q_t": 0.49989446997642517,
"grad_norm": 0.07196224480867386,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -0.27686309814453125,
"logits/rejected": -0.27830153703689575,
"logps/chosen": -1054.6317138671875,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -1461.45263671875,
"loss": 1.3859,
"margin_dpo/margin_mean": 385.07843017578125,
"margin_dpo/margin_std": 1091.12890625,
"step": 373
},
{
"KL/chosen_KL_mean": -760.64453125,
"KL/mean": -1015.76416015625,
"KL/rejected_KL_mean": -1270.8837890625,
"KL/std": 982.1947021484375,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 510.2392883300781,
"fcm_dpo/q_t": 0.49986016750335693,
"grad_norm": 0.0749947726726532,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -0.3357512354850769,
"logits/rejected": -0.33956992626190186,
"logps/chosen": -831.2947387695312,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -1364.52392578125,
"loss": 1.3857,
"margin_dpo/margin_mean": 510.23931884765625,
"margin_dpo/margin_std": 1257.8927001953125,
"step": 374
},
{
"KL/chosen_KL_mean": -773.878662109375,
"KL/mean": -1017.3250732421875,
"KL/rejected_KL_mean": -1260.771484375,
"KL/std": 752.9087524414062,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 486.89276123046875,
"fcm_dpo/q_t": 0.4998665452003479,
"grad_norm": 0.10702349245548248,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -0.2944202721118927,
"logits/rejected": -0.3069683909416199,
"logps/chosen": -833.9588623046875,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -1349.709716796875,
"loss": 1.3858,
"margin_dpo/margin_mean": 486.8927917480469,
"margin_dpo/margin_std": 893.77978515625,
"step": 375
},
{
"KL/chosen_KL_mean": -906.502685546875,
"KL/mean": -1175.470703125,
"KL/rejected_KL_mean": -1444.438720703125,
"KL/std": 964.0004272460938,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 537.9361572265625,
"fcm_dpo/q_t": 0.49985256791114807,
"grad_norm": 0.08315837383270264,
"learning_rate": 2.5e-07,
"logits/chosen": -0.30217331647872925,
"logits/rejected": -0.30075645446777344,
"logps/chosen": -969.1629638671875,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -1549.96533203125,
"loss": 1.3857,
"margin_dpo/margin_mean": 537.9361572265625,
"margin_dpo/margin_std": 1077.5111083984375,
"step": 376
},
{
"KL/chosen_KL_mean": -898.3602905273438,
"KL/mean": -1259.601318359375,
"KL/rejected_KL_mean": -1620.84228515625,
"KL/std": 1318.417724609375,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 722.4820556640625,
"fcm_dpo/q_t": 0.49980199337005615,
"grad_norm": 0.2360827922821045,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -0.2579605281352997,
"logits/rejected": -0.27427393198013306,
"logps/chosen": -952.8390502929688,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -1719.545654296875,
"loss": 1.3855,
"margin_dpo/margin_mean": 722.4820556640625,
"margin_dpo/margin_std": 1614.525634765625,
"step": 377
},
{
"KL/chosen_KL_mean": -706.89794921875,
"KL/mean": -1019.0738525390625,
"KL/rejected_KL_mean": -1331.2496337890625,
"KL/std": 896.5899047851562,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 624.3516845703125,
"fcm_dpo/q_t": 0.49982887506484985,
"grad_norm": 0.07591956853866577,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -0.2470046877861023,
"logits/rejected": -0.26460781693458557,
"logps/chosen": -751.9185180664062,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -1419.296630859375,
"loss": 1.3856,
"margin_dpo/margin_mean": 624.3516235351562,
"margin_dpo/margin_std": 1124.9613037109375,
"step": 378
},
{
"KL/chosen_KL_mean": -1031.705078125,
"KL/mean": -1375.2432861328125,
"KL/rejected_KL_mean": -1718.7816162109375,
"KL/std": 1386.5126953125,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 687.07666015625,
"fcm_dpo/q_t": 0.49981167912483215,
"grad_norm": 0.19187086820602417,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -0.2563135623931885,
"logits/rejected": -0.2795560956001282,
"logps/chosen": -1084.88720703125,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -1833.081787109375,
"loss": 1.3855,
"margin_dpo/margin_mean": 687.0765991210938,
"margin_dpo/margin_std": 1713.9254150390625,
"step": 379
},
{
"KL/chosen_KL_mean": -902.8639526367188,
"KL/mean": -1237.943359375,
"KL/rejected_KL_mean": -1573.0228271484375,
"KL/std": 1124.9130859375,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 670.158935546875,
"fcm_dpo/q_t": 0.49981629848480225,
"grad_norm": 0.0999961718916893,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -0.2708591818809509,
"logits/rejected": -0.29440176486968994,
"logps/chosen": -954.2169799804688,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -1677.214599609375,
"loss": 1.3856,
"margin_dpo/margin_mean": 670.158935546875,
"margin_dpo/margin_std": 1411.98486328125,
"step": 380
},
{
"KL/chosen_KL_mean": -949.5911865234375,
"KL/mean": -1092.4285888671875,
"KL/rejected_KL_mean": -1235.26611328125,
"KL/std": 1007.7601318359375,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 285.6749267578125,
"fcm_dpo/q_t": 0.49992167949676514,
"grad_norm": 0.06599840521812439,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -0.28272417187690735,
"logits/rejected": -0.2873557209968567,
"logps/chosen": -1007.394287109375,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -1314.4854736328125,
"loss": 1.386,
"margin_dpo/margin_mean": 285.6749267578125,
"margin_dpo/margin_std": 1382.336181640625,
"step": 381
},
{
"KL/chosen_KL_mean": -899.02978515625,
"KL/mean": -1149.078369140625,
"KL/rejected_KL_mean": -1399.1270751953125,
"KL/std": 840.8511962890625,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 500.0972900390625,
"fcm_dpo/q_t": 0.4998629093170166,
"grad_norm": 0.14141029119491577,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -0.3087759017944336,
"logits/rejected": -0.3224329948425293,
"logps/chosen": -965.050048828125,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -1509.837158203125,
"loss": 1.3857,
"margin_dpo/margin_mean": 500.0972595214844,
"margin_dpo/margin_std": 881.119384765625,
"step": 382
},
{
"KL/chosen_KL_mean": -956.48486328125,
"KL/mean": -1221.72021484375,
"KL/rejected_KL_mean": -1486.9556884765625,
"KL/std": 1089.745849609375,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 530.4708862304688,
"fcm_dpo/q_t": 0.49985459446907043,
"grad_norm": 0.18441635370254517,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -0.2525390684604645,
"logits/rejected": -0.2726495563983917,
"logps/chosen": -1006.8762817382812,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -1580.671630859375,
"loss": 1.3857,
"margin_dpo/margin_mean": 530.4708862304688,
"margin_dpo/margin_std": 1537.28515625,
"step": 383
},
{
"KL/chosen_KL_mean": -968.1121826171875,
"KL/mean": -1152.972900390625,
"KL/rejected_KL_mean": -1337.8336181640625,
"KL/std": 965.8948364257812,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 369.7212829589844,
"fcm_dpo/q_t": 0.49989867210388184,
"grad_norm": 0.07994517683982849,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -0.31392794847488403,
"logits/rejected": -0.31402724981307983,
"logps/chosen": -1020.1583251953125,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -1423.594482421875,
"loss": 1.3859,
"margin_dpo/margin_mean": 369.7212829589844,
"margin_dpo/margin_std": 968.5846557617188,
"step": 384
},
{
"KL/chosen_KL_mean": -954.828369140625,
"KL/mean": -1170.8798828125,
"KL/rejected_KL_mean": -1386.931396484375,
"KL/std": 830.7667236328125,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 432.1029357910156,
"fcm_dpo/q_t": 0.4998815655708313,
"grad_norm": 0.0972379669547081,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -0.3255078196525574,
"logits/rejected": -0.3330804109573364,
"logps/chosen": -1020.3804931640625,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -1464.75927734375,
"loss": 1.3858,
"margin_dpo/margin_mean": 432.1029357910156,
"margin_dpo/margin_std": 1054.419921875,
"step": 385
},
{
"KL/chosen_KL_mean": -900.58447265625,
"KL/mean": -1217.9410400390625,
"KL/rejected_KL_mean": -1535.2978515625,
"KL/std": 988.79345703125,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 634.7132568359375,
"fcm_dpo/q_t": 0.499826043844223,
"grad_norm": 0.1786276251077652,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -0.30368974804878235,
"logits/rejected": -0.3101522922515869,
"logps/chosen": -958.8062744140625,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -1627.625244140625,
"loss": 1.3856,
"margin_dpo/margin_mean": 634.7132568359375,
"margin_dpo/margin_std": 1239.7891845703125,
"step": 386
},
{
"KL/chosen_KL_mean": -1167.38916015625,
"KL/mean": -1465.2535400390625,
"KL/rejected_KL_mean": -1763.11767578125,
"KL/std": 1312.04248046875,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 595.728515625,
"fcm_dpo/q_t": 0.4998367428779602,
"grad_norm": 0.07909517735242844,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -0.3058868646621704,
"logits/rejected": -0.30179405212402344,
"logps/chosen": -1233.8087158203125,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -1855.286865234375,
"loss": 1.3856,
"margin_dpo/margin_mean": 595.728515625,
"margin_dpo/margin_std": 1582.2099609375,
"step": 387
},
{
"KL/chosen_KL_mean": -1178.15576171875,
"KL/mean": -1469.792236328125,
"KL/rejected_KL_mean": -1761.428466796875,
"KL/std": 1194.9736328125,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 583.2728271484375,
"fcm_dpo/q_t": 0.49984011054039,
"grad_norm": 0.13137727975845337,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -0.277870774269104,
"logits/rejected": -0.29402369260787964,
"logps/chosen": -1228.28515625,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -1865.861572265625,
"loss": 1.3857,
"margin_dpo/margin_mean": 583.2728271484375,
"margin_dpo/margin_std": 1514.8427734375,
"step": 388
},
{
"KL/chosen_KL_mean": -1263.1337890625,
"KL/mean": -1510.9075927734375,
"KL/rejected_KL_mean": -1758.681396484375,
"KL/std": 1231.6842041015625,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 495.54779052734375,
"fcm_dpo/q_t": 0.4998641610145569,
"grad_norm": 0.13468711078166962,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -0.316806823015213,
"logits/rejected": -0.3238714635372162,
"logps/chosen": -1321.040283203125,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -1836.595947265625,
"loss": 1.3858,
"margin_dpo/margin_mean": 495.5478210449219,
"margin_dpo/margin_std": 1563.379638671875,
"step": 389
},
{
"KL/chosen_KL_mean": -1154.65478515625,
"KL/mean": -1450.0166015625,
"KL/rejected_KL_mean": -1745.378173828125,
"KL/std": 1158.6177978515625,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 590.7235107421875,
"fcm_dpo/q_t": 0.49983808398246765,
"grad_norm": 0.0688537061214447,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -0.29213494062423706,
"logits/rejected": -0.30260345339775085,
"logps/chosen": -1203.880615234375,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -1830.906494140625,
"loss": 1.3856,
"margin_dpo/margin_mean": 590.7235107421875,
"margin_dpo/margin_std": 1621.9512939453125,
"step": 390
},
{
"KL/chosen_KL_mean": -1452.38134765625,
"KL/mean": -1540.049072265625,
"KL/rejected_KL_mean": -1627.7166748046875,
"KL/std": 1353.0072021484375,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 175.33538818359375,
"fcm_dpo/q_t": 0.49995192885398865,
"grad_norm": 0.1264980137348175,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -0.311432421207428,
"logits/rejected": -0.30648520588874817,
"logps/chosen": -1516.7109375,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -1714.454833984375,
"loss": 1.3861,
"margin_dpo/margin_mean": 175.3353729248047,
"margin_dpo/margin_std": 1589.905517578125,
"step": 391
},
{
"KL/chosen_KL_mean": -981.5897216796875,
"KL/mean": -1465.202880859375,
"KL/rejected_KL_mean": -1948.816162109375,
"KL/std": 1405.0225830078125,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 967.2265625,
"fcm_dpo/q_t": 0.49973487854003906,
"grad_norm": 0.13413800299167633,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -0.3217155933380127,
"logits/rejected": -0.33385881781578064,
"logps/chosen": -1035.09375,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -2051.162109375,
"loss": 1.3852,
"margin_dpo/margin_mean": 967.2265014648438,
"margin_dpo/margin_std": 1548.9447021484375,
"step": 392
},
{
"KL/chosen_KL_mean": -1129.197998046875,
"KL/mean": -1415.0509033203125,
"KL/rejected_KL_mean": -1700.9039306640625,
"KL/std": 1278.572998046875,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 571.7060546875,
"fcm_dpo/q_t": 0.4998432993888855,
"grad_norm": 0.09915830194950104,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -0.30509477853775024,
"logits/rejected": -0.3083847165107727,
"logps/chosen": -1175.671875,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -1772.872802734375,
"loss": 1.3857,
"margin_dpo/margin_mean": 571.7060546875,
"margin_dpo/margin_std": 1579.58154296875,
"step": 393
},
{
"KL/chosen_KL_mean": -1186.830322265625,
"KL/mean": -1445.2498779296875,
"KL/rejected_KL_mean": -1703.66943359375,
"KL/std": 1274.763671875,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 516.8390502929688,
"fcm_dpo/q_t": 0.4998583197593689,
"grad_norm": 0.06946764141321182,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -0.3197329044342041,
"logits/rejected": -0.3295214772224426,
"logps/chosen": -1239.741943359375,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -1794.4920654296875,
"loss": 1.3857,
"margin_dpo/margin_mean": 516.839111328125,
"margin_dpo/margin_std": 1375.14990234375,
"step": 394
},
{
"KL/chosen_KL_mean": -1227.3748779296875,
"KL/mean": -1495.52783203125,
"KL/rejected_KL_mean": -1763.6806640625,
"KL/std": 1176.624267578125,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 536.3056640625,
"fcm_dpo/q_t": 0.4998530149459839,
"grad_norm": 0.11004586517810822,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -0.29644620418548584,
"logits/rejected": -0.2991763949394226,
"logps/chosen": -1289.9210205078125,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -1847.4632568359375,
"loss": 1.3857,
"margin_dpo/margin_mean": 536.3056640625,
"margin_dpo/margin_std": 1487.039794921875,
"step": 395
},
{
"KL/chosen_KL_mean": -1308.0919189453125,
"KL/mean": -1489.021240234375,
"KL/rejected_KL_mean": -1669.950439453125,
"KL/std": 1317.3895263671875,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 361.8585205078125,
"fcm_dpo/q_t": 0.49990081787109375,
"grad_norm": 0.08168221265077591,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.3633381128311157,
"logits/rejected": -0.36133861541748047,
"logps/chosen": -1377.087890625,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -1758.59716796875,
"loss": 1.3859,
"margin_dpo/margin_mean": 361.8585205078125,
"margin_dpo/margin_std": 1191.16259765625,
"step": 396
},
{
"KL/chosen_KL_mean": -1209.757568359375,
"KL/mean": -1688.31787109375,
"KL/rejected_KL_mean": -2166.8779296875,
"KL/std": 1485.7529296875,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 957.1202392578125,
"fcm_dpo/q_t": 0.4997376799583435,
"grad_norm": 0.16516095399856567,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -0.28989318013191223,
"logits/rejected": -0.30872002243995667,
"logps/chosen": -1271.0347900390625,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -2269.994140625,
"loss": 1.3852,
"margin_dpo/margin_mean": 957.1203002929688,
"margin_dpo/margin_std": 1786.31591796875,
"step": 397
},
{
"KL/chosen_KL_mean": -1300.635986328125,
"KL/mean": -1687.136962890625,
"KL/rejected_KL_mean": -2073.637451171875,
"KL/std": 1495.1741943359375,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 773.0015258789062,
"fcm_dpo/q_t": 0.4997881352901459,
"grad_norm": 0.11998479068279266,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.3601279854774475,
"logits/rejected": -0.37861526012420654,
"logps/chosen": -1368.78759765625,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -2182.1611328125,
"loss": 1.3854,
"margin_dpo/margin_mean": 773.0015258789062,
"margin_dpo/margin_std": 1943.517822265625,
"step": 398
},
{
"KL/chosen_KL_mean": -1110.9375,
"KL/mean": -1328.4151611328125,
"KL/rejected_KL_mean": -1545.89306640625,
"KL/std": 1136.963134765625,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 434.9555358886719,
"fcm_dpo/q_t": 0.4998807907104492,
"grad_norm": 0.17861014604568481,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -0.3131040334701538,
"logits/rejected": -0.3180689215660095,
"logps/chosen": -1171.8271484375,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -1623.858642578125,
"loss": 1.3858,
"margin_dpo/margin_mean": 434.95556640625,
"margin_dpo/margin_std": 1538.548583984375,
"step": 399
},
{
"KL/chosen_KL_mean": -1076.3807373046875,
"KL/mean": -1457.690673828125,
"KL/rejected_KL_mean": -1839.00048828125,
"KL/std": 1168.469482421875,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 762.619873046875,
"fcm_dpo/q_t": 0.4997909665107727,
"grad_norm": 0.11573357880115509,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.2832724153995514,
"logits/rejected": -0.2879381775856018,
"logps/chosen": -1140.0242919921875,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -1944.253173828125,
"loss": 1.3855,
"margin_dpo/margin_mean": 762.619873046875,
"margin_dpo/margin_std": 1511.848388671875,
"step": 400
},
{
"KL/chosen_KL_mean": -1283.578369140625,
"KL/mean": -1460.8790283203125,
"KL/rejected_KL_mean": -1638.1796875,
"KL/std": 1255.499267578125,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 354.60137939453125,
"fcm_dpo/q_t": 0.4999028146266937,
"grad_norm": 0.0686601996421814,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -0.29005157947540283,
"logits/rejected": -0.28908443450927734,
"logps/chosen": -1340.7413330078125,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -1721.97216796875,
"loss": 1.3859,
"margin_dpo/margin_mean": 354.60137939453125,
"margin_dpo/margin_std": 1711.057861328125,
"step": 401
},
{
"KL/chosen_KL_mean": -1143.9306640625,
"KL/mean": -1476.869384765625,
"KL/rejected_KL_mean": -1809.80810546875,
"KL/std": 1272.91845703125,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 665.8773193359375,
"fcm_dpo/q_t": 0.49981752038002014,
"grad_norm": 0.14180243015289307,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -0.2916412651538849,
"logits/rejected": -0.29614734649658203,
"logps/chosen": -1194.671142578125,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -1890.8541259765625,
"loss": 1.3856,
"margin_dpo/margin_mean": 665.8773193359375,
"margin_dpo/margin_std": 1277.2041015625,
"step": 402
},
{
"KL/chosen_KL_mean": -1045.534912109375,
"KL/mean": -1374.931396484375,
"KL/rejected_KL_mean": -1704.3280029296875,
"KL/std": 1424.170654296875,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 658.793212890625,
"fcm_dpo/q_t": 0.49981942772865295,
"grad_norm": 0.09196604043245316,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -0.2864852547645569,
"logits/rejected": -0.3000401258468628,
"logps/chosen": -1092.68212890625,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -1781.5946044921875,
"loss": 1.3856,
"margin_dpo/margin_mean": 658.7931518554688,
"margin_dpo/margin_std": 1556.16015625,
"step": 403
},
{
"KL/chosen_KL_mean": -1062.3935546875,
"KL/mean": -1348.3885498046875,
"KL/rejected_KL_mean": -1634.3836669921875,
"KL/std": 1081.5550537109375,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 571.990234375,
"fcm_dpo/q_t": 0.49984320998191833,
"grad_norm": 0.08485666662454605,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -0.27411210536956787,
"logits/rejected": -0.2838093042373657,
"logps/chosen": -1110.268798828125,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -1711.53857421875,
"loss": 1.3857,
"margin_dpo/margin_mean": 571.990234375,
"margin_dpo/margin_std": 1484.571533203125,
"step": 404
},
{
"KL/chosen_KL_mean": -1302.394775390625,
"KL/mean": -1553.5927734375,
"KL/rejected_KL_mean": -1804.7906494140625,
"KL/std": 1502.25146484375,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 502.3959655761719,
"fcm_dpo/q_t": 0.49986231327056885,
"grad_norm": 0.18242168426513672,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -0.30928295850753784,
"logits/rejected": -0.319208562374115,
"logps/chosen": -1367.5576171875,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -1891.9774169921875,
"loss": 1.3857,
"margin_dpo/margin_mean": 502.3959655761719,
"margin_dpo/margin_std": 1869.696044921875,
"step": 405
},
{
"KL/chosen_KL_mean": -1168.0423583984375,
"KL/mean": -1538.92822265625,
"KL/rejected_KL_mean": -1909.814208984375,
"KL/std": 1370.6793212890625,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 741.771728515625,
"fcm_dpo/q_t": 0.49979668855667114,
"grad_norm": 0.15367038547992706,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -0.2888413965702057,
"logits/rejected": -0.3083066940307617,
"logps/chosen": -1217.783203125,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -2001.892822265625,
"loss": 1.3855,
"margin_dpo/margin_mean": 741.771728515625,
"margin_dpo/margin_std": 1697.0975341796875,
"step": 406
},
{
"KL/chosen_KL_mean": -1067.0177001953125,
"KL/mean": -1228.9609375,
"KL/rejected_KL_mean": -1390.904296875,
"KL/std": 1061.7740478515625,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 323.8866271972656,
"fcm_dpo/q_t": 0.49991121888160706,
"grad_norm": 0.0722283273935318,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -0.26621389389038086,
"logits/rejected": -0.267297625541687,
"logps/chosen": -1123.348388671875,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -1468.4163818359375,
"loss": 1.3859,
"margin_dpo/margin_mean": 323.88665771484375,
"margin_dpo/margin_std": 1483.5982666015625,
"step": 407
},
{
"KL/chosen_KL_mean": -1197.566162109375,
"KL/mean": -1406.6339111328125,
"KL/rejected_KL_mean": -1615.701416015625,
"KL/std": 1249.1373291015625,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 418.13525390625,
"fcm_dpo/q_t": 0.4998853802680969,
"grad_norm": 0.08033311367034912,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -0.2984810471534729,
"logits/rejected": -0.3063961863517761,
"logps/chosen": -1267.35546875,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -1705.7984619140625,
"loss": 1.3858,
"margin_dpo/margin_mean": 418.13525390625,
"margin_dpo/margin_std": 1451.70849609375,
"step": 408
},
{
"KL/chosen_KL_mean": -1052.406982421875,
"KL/mean": -1284.6824951171875,
"KL/rejected_KL_mean": -1516.9578857421875,
"KL/std": 1181.7725830078125,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 464.55096435546875,
"fcm_dpo/q_t": 0.49987268447875977,
"grad_norm": 0.09263404458761215,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -0.3008713126182556,
"logits/rejected": -0.3142474293708801,
"logps/chosen": -1119.724365234375,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -1601.8621826171875,
"loss": 1.3858,
"margin_dpo/margin_mean": 464.5509033203125,
"margin_dpo/margin_std": 1583.107177734375,
"step": 409
},
{
"KL/chosen_KL_mean": -1056.844482421875,
"KL/mean": -1352.3720703125,
"KL/rejected_KL_mean": -1647.899658203125,
"KL/std": 1171.762451171875,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 591.05517578125,
"fcm_dpo/q_t": 0.4998379945755005,
"grad_norm": 0.08020097762346268,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -0.27830201387405396,
"logits/rejected": -0.28551220893859863,
"logps/chosen": -1108.309814453125,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -1731.0986328125,
"loss": 1.3856,
"margin_dpo/margin_mean": 591.05517578125,
"margin_dpo/margin_std": 1348.9642333984375,
"step": 410
},
{
"KL/chosen_KL_mean": -1087.2542724609375,
"KL/mean": -1336.627685546875,
"KL/rejected_KL_mean": -1586.0010986328125,
"KL/std": 1284.406494140625,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 498.746826171875,
"fcm_dpo/q_t": 0.49986329674720764,
"grad_norm": 0.06663346290588379,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -0.24104216694831848,
"logits/rejected": -0.24600106477737427,
"logps/chosen": -1139.5615234375,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -1666.696044921875,
"loss": 1.3857,
"margin_dpo/margin_mean": 498.7467956542969,
"margin_dpo/margin_std": 1673.5802001953125,
"step": 411
},
{
"KL/chosen_KL_mean": -1194.093994140625,
"KL/mean": -1541.434326171875,
"KL/rejected_KL_mean": -1888.7745361328125,
"KL/std": 1356.6710205078125,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 694.6804809570312,
"fcm_dpo/q_t": 0.499809592962265,
"grad_norm": 0.16892312467098236,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -0.2868436574935913,
"logits/rejected": -0.3128548562526703,
"logps/chosen": -1247.2381591796875,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -1988.8353271484375,
"loss": 1.3855,
"margin_dpo/margin_mean": 694.6804809570312,
"margin_dpo/margin_std": 1838.8818359375,
"step": 412
},
{
"KL/chosen_KL_mean": -1412.858642578125,
"KL/mean": -1703.552734375,
"KL/rejected_KL_mean": -1994.246826171875,
"KL/std": 1468.816162109375,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 581.3883056640625,
"fcm_dpo/q_t": 0.499840646982193,
"grad_norm": 0.09038924425840378,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -0.27624720335006714,
"logits/rejected": -0.2850080728530884,
"logps/chosen": -1474.440673828125,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -2093.72021484375,
"loss": 1.3857,
"margin_dpo/margin_mean": 581.3883056640625,
"margin_dpo/margin_std": 1843.23681640625,
"step": 413
},
{
"KL/chosen_KL_mean": -1188.63134765625,
"KL/mean": -1421.198974609375,
"KL/rejected_KL_mean": -1653.7666015625,
"KL/std": 1262.495849609375,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 465.1351318359375,
"fcm_dpo/q_t": 0.49987250566482544,
"grad_norm": 0.09474116563796997,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -0.255864679813385,
"logits/rejected": -0.2618110179901123,
"logps/chosen": -1235.262939453125,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -1741.4130859375,
"loss": 1.3858,
"margin_dpo/margin_mean": 465.13507080078125,
"margin_dpo/margin_std": 1664.598876953125,
"step": 414
},
{
"KL/chosen_KL_mean": -1491.892578125,
"KL/mean": -1723.411865234375,
"KL/rejected_KL_mean": -1954.93115234375,
"KL/std": 1511.8848876953125,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 463.03875732421875,
"fcm_dpo/q_t": 0.4998731017112732,
"grad_norm": 0.09545289725065231,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -0.3036535382270813,
"logits/rejected": -0.3018915057182312,
"logps/chosen": -1570.5107421875,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -2055.40869140625,
"loss": 1.3858,
"margin_dpo/margin_mean": 463.03875732421875,
"margin_dpo/margin_std": 1996.5362548828125,
"step": 415
},
{
"KL/chosen_KL_mean": -1218.62744140625,
"KL/mean": -1573.49609375,
"KL/rejected_KL_mean": -1928.3646240234375,
"KL/std": 1414.0084228515625,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 709.7371215820312,
"fcm_dpo/q_t": 0.4998054504394531,
"grad_norm": 0.09039551764726639,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -0.2990315556526184,
"logits/rejected": -0.3177641034126282,
"logps/chosen": -1276.9066162109375,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -2018.933349609375,
"loss": 1.3855,
"margin_dpo/margin_mean": 709.7371215820312,
"margin_dpo/margin_std": 1649.296630859375,
"step": 416
},
{
"KL/chosen_KL_mean": -1047.98291015625,
"KL/mean": -1337.9010009765625,
"KL/rejected_KL_mean": -1627.819091796875,
"KL/std": 1125.5438232421875,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 579.8360595703125,
"fcm_dpo/q_t": 0.4998410642147064,
"grad_norm": 0.07524900883436203,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -0.26090145111083984,
"logits/rejected": -0.26464271545410156,
"logps/chosen": -1098.181640625,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -1695.970947265625,
"loss": 1.3857,
"margin_dpo/margin_mean": 579.8360595703125,
"margin_dpo/margin_std": 1336.595458984375,
"step": 417
},
{
"KL/chosen_KL_mean": -1321.910400390625,
"KL/mean": -1603.6380615234375,
"KL/rejected_KL_mean": -1885.36572265625,
"KL/std": 1282.54248046875,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 563.455322265625,
"fcm_dpo/q_t": 0.49984556436538696,
"grad_norm": 0.10205204784870148,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -0.3112577199935913,
"logits/rejected": -0.31804031133651733,
"logps/chosen": -1403.888916015625,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -1990.05712890625,
"loss": 1.3857,
"margin_dpo/margin_mean": 563.4553833007812,
"margin_dpo/margin_std": 1835.57958984375,
"step": 418
},
{
"KL/chosen_KL_mean": -1055.2227783203125,
"KL/mean": -1440.5726318359375,
"KL/rejected_KL_mean": -1825.9224853515625,
"KL/std": 1226.0582275390625,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 770.6995849609375,
"fcm_dpo/q_t": 0.49978873133659363,
"grad_norm": 0.13206477463245392,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -0.32074233889579773,
"logits/rejected": -0.3380177319049835,
"logps/chosen": -1108.17138671875,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -1917.505615234375,
"loss": 1.3855,
"margin_dpo/margin_mean": 770.6995849609375,
"margin_dpo/margin_std": 1524.392333984375,
"step": 419
},
{
"KL/chosen_KL_mean": -1762.050537109375,
"KL/mean": -1727.145263671875,
"KL/rejected_KL_mean": -1692.239990234375,
"KL/std": 1848.707763671875,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -69.8105697631836,
"fcm_dpo/q_t": 0.5000191330909729,
"grad_norm": 0.40303945541381836,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -0.30101293325424194,
"logits/rejected": -0.30338841676712036,
"logps/chosen": -1839.8204345703125,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -1761.559814453125,
"loss": 1.3864,
"margin_dpo/margin_mean": -69.81057739257812,
"margin_dpo/margin_std": 2532.6513671875,
"step": 420
},
{
"KL/chosen_KL_mean": -1044.2967529296875,
"KL/mean": -1356.22119140625,
"KL/rejected_KL_mean": -1668.145751953125,
"KL/std": 1289.6182861328125,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 623.848876953125,
"fcm_dpo/q_t": 0.4998289942741394,
"grad_norm": 0.21130794286727905,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -0.30749380588531494,
"logits/rejected": -0.3160657286643982,
"logps/chosen": -1098.062744140625,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -1757.4271240234375,
"loss": 1.3856,
"margin_dpo/margin_mean": 623.848876953125,
"margin_dpo/margin_std": 1501.941162109375,
"step": 421
},
{
"KL/chosen_KL_mean": -1388.9171142578125,
"KL/mean": -1671.041015625,
"KL/rejected_KL_mean": -1953.1650390625,
"KL/std": 1397.2171630859375,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 564.2479248046875,
"fcm_dpo/q_t": 0.49984532594680786,
"grad_norm": 0.08393198996782303,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -0.3038041591644287,
"logits/rejected": -0.3160828649997711,
"logps/chosen": -1457.5509033203125,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -2041.0284423828125,
"loss": 1.3857,
"margin_dpo/margin_mean": 564.2479248046875,
"margin_dpo/margin_std": 1596.736572265625,
"step": 422
},
{
"KL/chosen_KL_mean": -1274.769287109375,
"KL/mean": -1504.78369140625,
"KL/rejected_KL_mean": -1734.798095703125,
"KL/std": 1181.0693359375,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 460.02880859375,
"fcm_dpo/q_t": 0.49987390637397766,
"grad_norm": 0.11960334330797195,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -0.2938622534275055,
"logits/rejected": -0.3022230267524719,
"logps/chosen": -1329.7596435546875,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -1821.104736328125,
"loss": 1.3858,
"margin_dpo/margin_mean": 460.0288391113281,
"margin_dpo/margin_std": 1325.856201171875,
"step": 423
},
{
"KL/chosen_KL_mean": -1139.594970703125,
"KL/mean": -1338.92041015625,
"KL/rejected_KL_mean": -1538.2457275390625,
"KL/std": 1100.13720703125,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 398.6508483886719,
"fcm_dpo/q_t": 0.4998907446861267,
"grad_norm": 0.21664263308048248,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -0.24696838855743408,
"logits/rejected": -0.24573785066604614,
"logps/chosen": -1195.6068115234375,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -1604.7247314453125,
"loss": 1.3859,
"margin_dpo/margin_mean": 398.6508483886719,
"margin_dpo/margin_std": 1355.67431640625,
"step": 424
},
{
"KL/chosen_KL_mean": -1245.33984375,
"KL/mean": -1545.072998046875,
"KL/rejected_KL_mean": -1844.806396484375,
"KL/std": 1275.583251953125,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 599.4664306640625,
"fcm_dpo/q_t": 0.49983566999435425,
"grad_norm": 0.0939221978187561,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -0.2669602334499359,
"logits/rejected": -0.2846640348434448,
"logps/chosen": -1292.2088623046875,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -1940.7318115234375,
"loss": 1.3856,
"margin_dpo/margin_mean": 599.4664306640625,
"margin_dpo/margin_std": 1722.2542724609375,
"step": 425
},
{
"KL/chosen_KL_mean": -1096.74462890625,
"KL/mean": -1299.535888671875,
"KL/rejected_KL_mean": -1502.327392578125,
"KL/std": 1008.5482177734375,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 405.5826416015625,
"fcm_dpo/q_t": 0.4998888373374939,
"grad_norm": 0.09973110258579254,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -0.27449798583984375,
"logits/rejected": -0.27245032787323,
"logps/chosen": -1173.328125,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -1583.5938720703125,
"loss": 1.3859,
"margin_dpo/margin_mean": 405.5826721191406,
"margin_dpo/margin_std": 1211.4677734375,
"step": 426
},
{
"KL/chosen_KL_mean": -1049.704345703125,
"KL/mean": -1220.3751220703125,
"KL/rejected_KL_mean": -1391.0460205078125,
"KL/std": 1035.2784423828125,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 341.3416748046875,
"fcm_dpo/q_t": 0.49990642070770264,
"grad_norm": 0.07983623445034027,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -0.2558949291706085,
"logits/rejected": -0.25835371017456055,
"logps/chosen": -1114.55810546875,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -1469.612060546875,
"loss": 1.3859,
"margin_dpo/margin_mean": 341.34161376953125,
"margin_dpo/margin_std": 1104.6048583984375,
"step": 427
},
{
"KL/chosen_KL_mean": -1136.0379638671875,
"KL/mean": -1533.8326416015625,
"KL/rejected_KL_mean": -1931.62744140625,
"KL/std": 1306.2242431640625,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 795.5894775390625,
"fcm_dpo/q_t": 0.49978193640708923,
"grad_norm": 0.18045121431350708,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -0.2967488467693329,
"logits/rejected": -0.3061639666557312,
"logps/chosen": -1198.674560546875,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -2034.9091796875,
"loss": 1.3854,
"margin_dpo/margin_mean": 795.5894775390625,
"margin_dpo/margin_std": 1710.707275390625,
"step": 428
},
{
"KL/chosen_KL_mean": -1484.685302734375,
"KL/mean": -1713.23291015625,
"KL/rejected_KL_mean": -1941.780517578125,
"KL/std": 1404.762939453125,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 457.09521484375,
"fcm_dpo/q_t": 0.49987471103668213,
"grad_norm": 0.10288428515195847,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -0.2776823043823242,
"logits/rejected": -0.27903062105178833,
"logps/chosen": -1565.919189453125,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -2033.575439453125,
"loss": 1.3858,
"margin_dpo/margin_mean": 457.09527587890625,
"margin_dpo/margin_std": 1604.7532958984375,
"step": 429
},
{
"KL/chosen_KL_mean": -1077.0499267578125,
"KL/mean": -1380.4365234375,
"KL/rejected_KL_mean": -1683.822998046875,
"KL/std": 1355.672119140625,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 606.7730712890625,
"fcm_dpo/q_t": 0.4998336732387543,
"grad_norm": 0.13458018004894257,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.3125556707382202,
"logits/rejected": -0.3251245617866516,
"logps/chosen": -1137.97021484375,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -1788.2457275390625,
"loss": 1.3856,
"margin_dpo/margin_mean": 606.7731323242188,
"margin_dpo/margin_std": 1768.553955078125,
"step": 430
},
{
"KL/chosen_KL_mean": -989.0621337890625,
"KL/mean": -1313.78857421875,
"KL/rejected_KL_mean": -1638.514892578125,
"KL/std": 1129.193115234375,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 649.452880859375,
"fcm_dpo/q_t": 0.4998219907283783,
"grad_norm": 0.10206551104784012,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -0.2263118028640747,
"logits/rejected": -0.23984399437904358,
"logps/chosen": -1046.410888671875,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -1731.355224609375,
"loss": 1.3856,
"margin_dpo/margin_mean": 649.452880859375,
"margin_dpo/margin_std": 1453.598388671875,
"step": 431
},
{
"KL/chosen_KL_mean": -953.16552734375,
"KL/mean": -1365.68798828125,
"KL/rejected_KL_mean": -1778.210693359375,
"KL/std": 1248.4228515625,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 825.0450439453125,
"fcm_dpo/q_t": 0.49977385997772217,
"grad_norm": 0.11800537258386612,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -0.2888324558734894,
"logits/rejected": -0.300900399684906,
"logps/chosen": -1024.240234375,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -1876.7901611328125,
"loss": 1.3854,
"margin_dpo/margin_mean": 825.0451049804688,
"margin_dpo/margin_std": 1362.866943359375,
"step": 432
},
{
"KL/chosen_KL_mean": -1171.21923828125,
"KL/mean": -1499.420166015625,
"KL/rejected_KL_mean": -1827.62109375,
"KL/std": 1200.9256591796875,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 656.4017944335938,
"fcm_dpo/q_t": 0.49982011318206787,
"grad_norm": 0.0767941102385521,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -0.28949469327926636,
"logits/rejected": -0.30554211139678955,
"logps/chosen": -1229.492431640625,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -1923.572021484375,
"loss": 1.3856,
"margin_dpo/margin_mean": 656.4017944335938,
"margin_dpo/margin_std": 1420.9886474609375,
"step": 433
},
{
"KL/chosen_KL_mean": -1108.557861328125,
"KL/mean": -1367.704345703125,
"KL/rejected_KL_mean": -1626.8509521484375,
"KL/std": 1420.2242431640625,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 518.2930908203125,
"fcm_dpo/q_t": 0.49985793232917786,
"grad_norm": 0.12584568560123444,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -0.2219351828098297,
"logits/rejected": -0.23550885915756226,
"logps/chosen": -1170.531494140625,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -1705.349609375,
"loss": 1.3857,
"margin_dpo/margin_mean": 518.2930908203125,
"margin_dpo/margin_std": 2014.797607421875,
"step": 434
},
{
"KL/chosen_KL_mean": -905.6617431640625,
"KL/mean": -1253.97802734375,
"KL/rejected_KL_mean": -1602.2943115234375,
"KL/std": 1168.88720703125,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 696.6324462890625,
"fcm_dpo/q_t": 0.49980905652046204,
"grad_norm": 0.11213953047990799,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -0.22496365010738373,
"logits/rejected": -0.22140967845916748,
"logps/chosen": -957.163818359375,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -1689.861083984375,
"loss": 1.3855,
"margin_dpo/margin_mean": 696.6324462890625,
"margin_dpo/margin_std": 1404.775390625,
"step": 435
},
{
"KL/chosen_KL_mean": -951.4571533203125,
"KL/mean": -1099.720947265625,
"KL/rejected_KL_mean": -1247.9847412109375,
"KL/std": 928.330810546875,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 296.52752685546875,
"fcm_dpo/q_t": 0.49991875886917114,
"grad_norm": 0.06883503496646881,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -0.27546530961990356,
"logits/rejected": -0.2770288288593292,
"logps/chosen": -1022.86083984375,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -1330.71240234375,
"loss": 1.386,
"margin_dpo/margin_mean": 296.5275573730469,
"margin_dpo/margin_std": 1227.0777587890625,
"step": 436
},
{
"KL/chosen_KL_mean": -1031.511962890625,
"KL/mean": -1204.827392578125,
"KL/rejected_KL_mean": -1378.1427001953125,
"KL/std": 1022.2294311523438,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 346.63067626953125,
"fcm_dpo/q_t": 0.499904990196228,
"grad_norm": 0.06253945827484131,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -0.2463058978319168,
"logits/rejected": -0.2471463531255722,
"logps/chosen": -1096.2562255859375,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -1460.186279296875,
"loss": 1.3859,
"margin_dpo/margin_mean": 346.63067626953125,
"margin_dpo/margin_std": 1298.53076171875,
"step": 437
},
{
"KL/chosen_KL_mean": -917.509765625,
"KL/mean": -1201.0556640625,
"KL/rejected_KL_mean": -1484.6015625,
"KL/std": 1046.44921875,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 567.091796875,
"fcm_dpo/q_t": 0.4998445510864258,
"grad_norm": 0.06463596969842911,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -0.23667128384113312,
"logits/rejected": -0.24160590767860413,
"logps/chosen": -976.5284423828125,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -1567.678466796875,
"loss": 1.3857,
"margin_dpo/margin_mean": 567.0918579101562,
"margin_dpo/margin_std": 1066.5599365234375,
"step": 438
},
{
"KL/chosen_KL_mean": -925.7264404296875,
"KL/mean": -1227.3575439453125,
"KL/rejected_KL_mean": -1528.988525390625,
"KL/std": 1210.546630859375,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 603.2620849609375,
"fcm_dpo/q_t": 0.49983465671539307,
"grad_norm": 0.1023441031575203,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -0.24012020230293274,
"logits/rejected": -0.25954288244247437,
"logps/chosen": -979.510498046875,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -1612.9739990234375,
"loss": 1.3856,
"margin_dpo/margin_mean": 603.2620849609375,
"margin_dpo/margin_std": 1559.177490234375,
"step": 439
},
{
"KL/chosen_KL_mean": -1179.697998046875,
"KL/mean": -1430.30322265625,
"KL/rejected_KL_mean": -1680.908447265625,
"KL/std": 1239.806884765625,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 501.2104797363281,
"fcm_dpo/q_t": 0.4998626410961151,
"grad_norm": 0.1351250261068344,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -0.2789965271949768,
"logits/rejected": -0.28014230728149414,
"logps/chosen": -1258.2646484375,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -1777.40625,
"loss": 1.3857,
"margin_dpo/margin_mean": 501.21051025390625,
"margin_dpo/margin_std": 1619.5087890625,
"step": 440
},
{
"KL/chosen_KL_mean": -1438.1356201171875,
"KL/mean": -1626.278076171875,
"KL/rejected_KL_mean": -1814.420654296875,
"KL/std": 1308.76953125,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 376.28509521484375,
"fcm_dpo/q_t": 0.4998968541622162,
"grad_norm": 0.09744305908679962,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -0.303938090801239,
"logits/rejected": -0.30175545811653137,
"logps/chosen": -1498.9600830078125,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -1910.8914794921875,
"loss": 1.3859,
"margin_dpo/margin_mean": 376.28509521484375,
"margin_dpo/margin_std": 1590.92333984375,
"step": 441
},
{
"KL/chosen_KL_mean": -1014.8897094726562,
"KL/mean": -1272.22607421875,
"KL/rejected_KL_mean": -1529.562255859375,
"KL/std": 1172.866943359375,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 514.672607421875,
"fcm_dpo/q_t": 0.49985894560813904,
"grad_norm": 0.0797480046749115,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -0.29978764057159424,
"logits/rejected": -0.3030720353126526,
"logps/chosen": -1061.90087890625,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -1606.1015625,
"loss": 1.3857,
"margin_dpo/margin_mean": 514.672607421875,
"margin_dpo/margin_std": 1585.113525390625,
"step": 442
},
{
"KL/chosen_KL_mean": -1156.9581298828125,
"KL/mean": -1253.765869140625,
"KL/rejected_KL_mean": -1350.57373046875,
"KL/std": 1065.82373046875,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 193.615478515625,
"fcm_dpo/q_t": 0.4999469518661499,
"grad_norm": 0.1120293065905571,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -0.246562659740448,
"logits/rejected": -0.25994592905044556,
"logps/chosen": -1228.231201171875,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -1437.253662109375,
"loss": 1.3861,
"margin_dpo/margin_mean": 193.615478515625,
"margin_dpo/margin_std": 1421.083251953125,
"step": 443
},
{
"KL/chosen_KL_mean": -1464.67724609375,
"KL/mean": -1763.21875,
"KL/rejected_KL_mean": -2061.76025390625,
"KL/std": 1581.70458984375,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 597.0831909179688,
"fcm_dpo/q_t": 0.4998363256454468,
"grad_norm": 0.13168448209762573,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -0.26163217425346375,
"logits/rejected": -0.26668477058410645,
"logps/chosen": -1521.890869140625,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -2159.015380859375,
"loss": 1.3856,
"margin_dpo/margin_mean": 597.0831909179688,
"margin_dpo/margin_std": 1862.0196533203125,
"step": 444
},
{
"KL/chosen_KL_mean": -1140.0430908203125,
"KL/mean": -1497.06689453125,
"KL/rejected_KL_mean": -1854.090576171875,
"KL/std": 1608.1549072265625,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 714.047607421875,
"fcm_dpo/q_t": 0.4998042583465576,
"grad_norm": 0.12488368898630142,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -0.28203797340393066,
"logits/rejected": -0.284241646528244,
"logps/chosen": -1207.3427734375,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -1946.7733154296875,
"loss": 1.3855,
"margin_dpo/margin_mean": 714.0476684570312,
"margin_dpo/margin_std": 2080.40625,
"step": 445
},
{
"KL/chosen_KL_mean": -1288.3699951171875,
"KL/mean": -1554.406494140625,
"KL/rejected_KL_mean": -1820.443115234375,
"KL/std": 1258.46044921875,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 532.0732421875,
"fcm_dpo/q_t": 0.4998541474342346,
"grad_norm": 0.0856269970536232,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -0.2942993938922882,
"logits/rejected": -0.30123913288116455,
"logps/chosen": -1347.468505859375,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -1921.707275390625,
"loss": 1.3857,
"margin_dpo/margin_mean": 532.0732421875,
"margin_dpo/margin_std": 1500.825927734375,
"step": 446
},
{
"KL/chosen_KL_mean": -1074.880126953125,
"KL/mean": -1467.91357421875,
"KL/rejected_KL_mean": -1860.9471435546875,
"KL/std": 1457.51513671875,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 786.0670776367188,
"fcm_dpo/q_t": 0.49978452920913696,
"grad_norm": 0.19844543933868408,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -0.2915193438529968,
"logits/rejected": -0.2992427349090576,
"logps/chosen": -1130.955322265625,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -1959.641845703125,
"loss": 1.3854,
"margin_dpo/margin_mean": 786.0670776367188,
"margin_dpo/margin_std": 1644.028564453125,
"step": 447
},
{
"KL/chosen_KL_mean": -1095.27685546875,
"KL/mean": -1506.1376953125,
"KL/rejected_KL_mean": -1916.998291015625,
"KL/std": 1311.316650390625,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 821.721435546875,
"fcm_dpo/q_t": 0.4997747540473938,
"grad_norm": 0.13404960930347443,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -0.25953444838523865,
"logits/rejected": -0.26477596163749695,
"logps/chosen": -1155.28076171875,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -2019.262939453125,
"loss": 1.3854,
"margin_dpo/margin_mean": 821.721435546875,
"margin_dpo/margin_std": 1332.540771484375,
"step": 448
},
{
"KL/chosen_KL_mean": -1298.99755859375,
"KL/mean": -1579.174560546875,
"KL/rejected_KL_mean": -1859.3515625,
"KL/std": 1209.3814697265625,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 560.3539428710938,
"fcm_dpo/q_t": 0.4998464286327362,
"grad_norm": 0.09452203661203384,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -0.3198021352291107,
"logits/rejected": -0.3200905919075012,
"logps/chosen": -1366.524169921875,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -1947.948486328125,
"loss": 1.3857,
"margin_dpo/margin_mean": 560.3539428710938,
"margin_dpo/margin_std": 1519.3251953125,
"step": 449
},
{
"KL/chosen_KL_mean": -882.3565673828125,
"KL/mean": -1320.7303466796875,
"KL/rejected_KL_mean": -1759.1041259765625,
"KL/std": 1190.139892578125,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 876.7476806640625,
"fcm_dpo/q_t": 0.4997596740722656,
"grad_norm": 0.14239948987960815,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -0.2664763331413269,
"logits/rejected": -0.2864212989807129,
"logps/chosen": -939.4646606445312,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -1861.859130859375,
"loss": 1.3853,
"margin_dpo/margin_mean": 876.7476806640625,
"margin_dpo/margin_std": 1359.66943359375,
"step": 450
},
{
"KL/chosen_KL_mean": -1219.8216552734375,
"KL/mean": -1379.736328125,
"KL/rejected_KL_mean": -1539.651123046875,
"KL/std": 1262.7548828125,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 319.829345703125,
"fcm_dpo/q_t": 0.4999123513698578,
"grad_norm": 0.06787317246198654,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -0.235478937625885,
"logits/rejected": -0.23634299635887146,
"logps/chosen": -1278.29052734375,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -1612.58056640625,
"loss": 1.3859,
"margin_dpo/margin_mean": 319.829345703125,
"margin_dpo/margin_std": 1746.6220703125,
"step": 451
},
{
"KL/chosen_KL_mean": -1223.572265625,
"KL/mean": -1464.432861328125,
"KL/rejected_KL_mean": -1705.29345703125,
"KL/std": 1323.29638671875,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 481.72125244140625,
"fcm_dpo/q_t": 0.4998679757118225,
"grad_norm": 0.1068187803030014,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -0.24899950623512268,
"logits/rejected": -0.26239603757858276,
"logps/chosen": -1273.9627685546875,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -1803.06494140625,
"loss": 1.3858,
"margin_dpo/margin_mean": 481.7213134765625,
"margin_dpo/margin_std": 1598.2059326171875,
"step": 452
},
{
"KL/chosen_KL_mean": -1116.9976806640625,
"KL/mean": -1405.4814453125,
"KL/rejected_KL_mean": -1693.96533203125,
"KL/std": 1186.461669921875,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 576.9676513671875,
"fcm_dpo/q_t": 0.4998418688774109,
"grad_norm": 0.11661150306463242,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -0.29134508967399597,
"logits/rejected": -0.28827986121177673,
"logps/chosen": -1174.7125244140625,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -1776.1727294921875,
"loss": 1.3857,
"margin_dpo/margin_mean": 576.9676513671875,
"margin_dpo/margin_std": 1536.704833984375,
"step": 453
},
{
"KL/chosen_KL_mean": -1351.572021484375,
"KL/mean": -1602.408447265625,
"KL/rejected_KL_mean": -1853.244873046875,
"KL/std": 1257.90771484375,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 501.67279052734375,
"fcm_dpo/q_t": 0.4998624920845032,
"grad_norm": 0.10205873847007751,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -0.2497563660144806,
"logits/rejected": -0.25889086723327637,
"logps/chosen": -1412.517822265625,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -1938.19580078125,
"loss": 1.3857,
"margin_dpo/margin_mean": 501.67279052734375,
"margin_dpo/margin_std": 1519.4886474609375,
"step": 454
},
{
"KL/chosen_KL_mean": -1146.478271484375,
"KL/mean": -1673.6832275390625,
"KL/rejected_KL_mean": -2200.88818359375,
"KL/std": 1628.58154296875,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1054.409912109375,
"fcm_dpo/q_t": 0.49971097707748413,
"grad_norm": 0.21247386932373047,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -0.31010985374450684,
"logits/rejected": -0.32810282707214355,
"logps/chosen": -1191.364990234375,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -2316.189697265625,
"loss": 1.3851,
"margin_dpo/margin_mean": 1054.4097900390625,
"margin_dpo/margin_std": 2063.35009765625,
"step": 455
},
{
"KL/chosen_KL_mean": -1452.318603515625,
"KL/mean": -1761.364501953125,
"KL/rejected_KL_mean": -2070.41064453125,
"KL/std": 1583.523193359375,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 618.0919799804688,
"fcm_dpo/q_t": 0.49983057379722595,
"grad_norm": 0.1313888132572174,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -0.28538087010383606,
"logits/rejected": -0.29785096645355225,
"logps/chosen": -1509.35546875,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -2175.62841796875,
"loss": 1.3856,
"margin_dpo/margin_mean": 618.0919189453125,
"margin_dpo/margin_std": 1710.4136962890625,
"step": 456
},
{
"KL/chosen_KL_mean": -1115.773193359375,
"KL/mean": -1518.5028076171875,
"KL/rejected_KL_mean": -1921.232421875,
"KL/std": 1319.357177734375,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 805.4591674804688,
"fcm_dpo/q_t": 0.49977922439575195,
"grad_norm": 0.09444835782051086,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -0.3091793656349182,
"logits/rejected": -0.3152327537536621,
"logps/chosen": -1170.0157470703125,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -2006.342041015625,
"loss": 1.3854,
"margin_dpo/margin_mean": 805.4591064453125,
"margin_dpo/margin_std": 1653.950439453125,
"step": 457
},
{
"KL/chosen_KL_mean": -1173.20263671875,
"KL/mean": -1522.257568359375,
"KL/rejected_KL_mean": -1871.312255859375,
"KL/std": 1152.4215087890625,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 698.1096801757812,
"fcm_dpo/q_t": 0.499808669090271,
"grad_norm": 0.09302345663309097,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -0.27563565969467163,
"logits/rejected": -0.2783600091934204,
"logps/chosen": -1228.611572265625,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -1968.99560546875,
"loss": 1.3855,
"margin_dpo/margin_mean": 698.1096801757812,
"margin_dpo/margin_std": 1354.467041015625,
"step": 458
},
{
"KL/chosen_KL_mean": -1204.60009765625,
"KL/mean": -1605.72021484375,
"KL/rejected_KL_mean": -2006.840576171875,
"KL/std": 1491.602783203125,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 802.2403564453125,
"fcm_dpo/q_t": 0.4997801184654236,
"grad_norm": 0.07887598872184753,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -0.29390937089920044,
"logits/rejected": -0.3003736734390259,
"logps/chosen": -1251.1575927734375,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -2093.009033203125,
"loss": 1.3854,
"margin_dpo/margin_mean": 802.2403564453125,
"margin_dpo/margin_std": 1910.6488037109375,
"step": 459
},
{
"KL/chosen_KL_mean": -1455.231689453125,
"KL/mean": -1830.2767333984375,
"KL/rejected_KL_mean": -2205.32177734375,
"KL/std": 1589.1705322265625,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 750.090087890625,
"fcm_dpo/q_t": 0.4997944235801697,
"grad_norm": 0.14750739932060242,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -0.2926778197288513,
"logits/rejected": -0.31306982040405273,
"logps/chosen": -1506.8665771484375,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -2309.441162109375,
"loss": 1.3855,
"margin_dpo/margin_mean": 750.090087890625,
"margin_dpo/margin_std": 1895.516845703125,
"step": 460
},
{
"KL/chosen_KL_mean": -1283.6640625,
"KL/mean": -1520.5830078125,
"KL/rejected_KL_mean": -1757.5018310546875,
"KL/std": 1185.034423828125,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 473.8377685546875,
"fcm_dpo/q_t": 0.4998701214790344,
"grad_norm": 0.07519460469484329,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -0.3480767011642456,
"logits/rejected": -0.35095617175102234,
"logps/chosen": -1338.8460693359375,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -1843.978759765625,
"loss": 1.3858,
"margin_dpo/margin_mean": 473.83770751953125,
"margin_dpo/margin_std": 1498.89306640625,
"step": 461
},
{
"KL/chosen_KL_mean": -1435.103759765625,
"KL/mean": -1612.481689453125,
"KL/rejected_KL_mean": -1789.859375,
"KL/std": 1408.7646484375,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 354.7555847167969,
"fcm_dpo/q_t": 0.49990275502204895,
"grad_norm": 0.25926288962364197,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -0.29589688777923584,
"logits/rejected": -0.2953804135322571,
"logps/chosen": -1505.0318603515625,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -1868.700439453125,
"loss": 1.3859,
"margin_dpo/margin_mean": 354.75555419921875,
"margin_dpo/margin_std": 1666.0538330078125,
"step": 462
},
{
"KL/chosen_KL_mean": -1326.2193603515625,
"KL/mean": -1661.249267578125,
"KL/rejected_KL_mean": -1996.279052734375,
"KL/std": 1530.80078125,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 670.059814453125,
"fcm_dpo/q_t": 0.499816358089447,
"grad_norm": 0.15044397115707397,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -0.3050798773765564,
"logits/rejected": -0.31533971428871155,
"logps/chosen": -1381.49365234375,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -2085.303955078125,
"loss": 1.3856,
"margin_dpo/margin_mean": 670.0597534179688,
"margin_dpo/margin_std": 2028.2119140625,
"step": 463
},
{
"KL/chosen_KL_mean": -1342.7457275390625,
"KL/mean": -1907.53564453125,
"KL/rejected_KL_mean": -2472.32568359375,
"KL/std": 1601.189697265625,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1129.5799560546875,
"fcm_dpo/q_t": 0.4996904134750366,
"grad_norm": 0.16032731533050537,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -0.263306200504303,
"logits/rejected": -0.27996712923049927,
"logps/chosen": -1393.657958984375,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -2574.81494140625,
"loss": 1.3851,
"margin_dpo/margin_mean": 1129.5799560546875,
"margin_dpo/margin_std": 1819.3665771484375,
"step": 464
},
{
"KL/chosen_KL_mean": -1383.0517578125,
"KL/mean": -1747.37158203125,
"KL/rejected_KL_mean": -2111.691650390625,
"KL/std": 1785.874755859375,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 728.6397705078125,
"fcm_dpo/q_t": 0.49980029463768005,
"grad_norm": 0.10014206171035767,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -0.2919054627418518,
"logits/rejected": -0.30143067240715027,
"logps/chosen": -1443.1685791015625,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -2225.6376953125,
"loss": 1.3855,
"margin_dpo/margin_mean": 728.6398315429688,
"margin_dpo/margin_std": 2277.84765625,
"step": 465
},
{
"KL/chosen_KL_mean": -1208.7822265625,
"KL/mean": -1561.8572998046875,
"KL/rejected_KL_mean": -1914.9324951171875,
"KL/std": 1355.089599609375,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 706.1502685546875,
"fcm_dpo/q_t": 0.4998064637184143,
"grad_norm": 0.12803106009960175,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -0.272348016500473,
"logits/rejected": -0.28248733282089233,
"logps/chosen": -1261.703125,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -2005.2479248046875,
"loss": 1.3855,
"margin_dpo/margin_mean": 706.1502685546875,
"margin_dpo/margin_std": 1762.6644287109375,
"step": 466
},
{
"KL/chosen_KL_mean": -1984.1240234375,
"KL/mean": -2191.9873046875,
"KL/rejected_KL_mean": -2399.850341796875,
"KL/std": 1883.2333984375,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 415.726318359375,
"fcm_dpo/q_t": 0.49988603591918945,
"grad_norm": 0.10527905076742172,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -0.28988873958587646,
"logits/rejected": -0.29053300619125366,
"logps/chosen": -2062.83984375,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -2502.71044921875,
"loss": 1.3858,
"margin_dpo/margin_mean": 415.7263488769531,
"margin_dpo/margin_std": 2690.570556640625,
"step": 467
},
{
"KL/chosen_KL_mean": -1457.8660888671875,
"KL/mean": -1854.851318359375,
"KL/rejected_KL_mean": -2251.83642578125,
"KL/std": 1709.1533203125,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 793.97021484375,
"fcm_dpo/q_t": 0.49978235363960266,
"grad_norm": 0.15743249654769897,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -0.32493388652801514,
"logits/rejected": -0.32417386770248413,
"logps/chosen": -1527.801513671875,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -2352.865234375,
"loss": 1.3854,
"margin_dpo/margin_mean": 793.97021484375,
"margin_dpo/margin_std": 2327.3564453125,
"step": 468
},
{
"KL/chosen_KL_mean": -1288.8515625,
"KL/mean": -1759.3974609375,
"KL/rejected_KL_mean": -2229.943359375,
"KL/std": 1474.0218505859375,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 941.0918579101562,
"fcm_dpo/q_t": 0.4997420310974121,
"grad_norm": 0.0797450840473175,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -0.3195040225982666,
"logits/rejected": -0.3280268907546997,
"logps/chosen": -1356.976318359375,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -2334.729736328125,
"loss": 1.3853,
"margin_dpo/margin_mean": 941.091796875,
"margin_dpo/margin_std": 1737.1143798828125,
"step": 469
},
{
"KL/chosen_KL_mean": -1046.739501953125,
"KL/mean": -1347.437744140625,
"KL/rejected_KL_mean": -1648.1357421875,
"KL/std": 1214.6531982421875,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 601.3963623046875,
"fcm_dpo/q_t": 0.49983516335487366,
"grad_norm": 0.08385954052209854,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -0.262192964553833,
"logits/rejected": -0.2826134264469147,
"logps/chosen": -1090.531494140625,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -1730.838623046875,
"loss": 1.3856,
"margin_dpo/margin_mean": 601.3963012695312,
"margin_dpo/margin_std": 1419.621826171875,
"step": 470
},
{
"KL/chosen_KL_mean": -1479.0435791015625,
"KL/mean": -1652.86083984375,
"KL/rejected_KL_mean": -1826.677978515625,
"KL/std": 1403.427734375,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 347.63446044921875,
"fcm_dpo/q_t": 0.49990469217300415,
"grad_norm": 0.2278512865304947,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -0.3011685609817505,
"logits/rejected": -0.30319029092788696,
"logps/chosen": -1542.383056640625,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -1910.2884521484375,
"loss": 1.3859,
"margin_dpo/margin_mean": 347.6344909667969,
"margin_dpo/margin_std": 1685.4962158203125,
"step": 471
},
{
"KL/chosen_KL_mean": -1486.688232421875,
"KL/mean": -1817.849853515625,
"KL/rejected_KL_mean": -2149.01123046875,
"KL/std": 1643.24609375,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 662.3231811523438,
"fcm_dpo/q_t": 0.49981844425201416,
"grad_norm": 0.12714260816574097,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -0.36696937680244446,
"logits/rejected": -0.37076157331466675,
"logps/chosen": -1570.3543701171875,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -2266.220703125,
"loss": 1.3856,
"margin_dpo/margin_mean": 662.3231201171875,
"margin_dpo/margin_std": 2021.212646484375,
"step": 472
},
{
"KL/chosen_KL_mean": -1896.511474609375,
"KL/mean": -1903.3677978515625,
"KL/rejected_KL_mean": -1910.22412109375,
"KL/std": 1944.942626953125,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 13.71258544921875,
"fcm_dpo/q_t": 0.49999624490737915,
"grad_norm": 0.13963881134986877,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -0.24187920987606049,
"logits/rejected": -0.24330055713653564,
"logps/chosen": -1960.008544921875,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -1991.3707275390625,
"loss": 1.3863,
"margin_dpo/margin_mean": 13.712547302246094,
"margin_dpo/margin_std": 2561.2255859375,
"step": 473
},
{
"KL/chosen_KL_mean": -1129.59130859375,
"KL/mean": -1574.693359375,
"KL/rejected_KL_mean": -2019.795654296875,
"KL/std": 1632.31298828125,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 890.2041625976562,
"fcm_dpo/q_t": 0.49975600838661194,
"grad_norm": 0.13947923481464386,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -0.2791895866394043,
"logits/rejected": -0.2850819528102875,
"logps/chosen": -1182.2032470703125,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -2109.8759765625,
"loss": 1.3853,
"margin_dpo/margin_mean": 890.2042236328125,
"margin_dpo/margin_std": 1976.387939453125,
"step": 474
},
{
"KL/chosen_KL_mean": -839.9306030273438,
"KL/mean": -1260.02001953125,
"KL/rejected_KL_mean": -1680.109375,
"KL/std": 1373.2816162109375,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 840.1787109375,
"fcm_dpo/q_t": 0.4997697174549103,
"grad_norm": 0.15250550210475922,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -0.23655390739440918,
"logits/rejected": -0.24804091453552246,
"logps/chosen": -882.42578125,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -1770.17236328125,
"loss": 1.3854,
"margin_dpo/margin_mean": 840.1787109375,
"margin_dpo/margin_std": 1737.7989501953125,
"step": 475
},
{
"KL/chosen_KL_mean": -1057.037353515625,
"KL/mean": -1375.7066650390625,
"KL/rejected_KL_mean": -1694.3759765625,
"KL/std": 1179.803466796875,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 637.3386840820312,
"fcm_dpo/q_t": 0.49982529878616333,
"grad_norm": 0.06534449756145477,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -0.25173643231391907,
"logits/rejected": -0.26171934604644775,
"logps/chosen": -1099.9866943359375,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -1768.086181640625,
"loss": 1.3856,
"margin_dpo/margin_mean": 637.3387451171875,
"margin_dpo/margin_std": 1320.2890625,
"step": 476
},
{
"KL/chosen_KL_mean": -1168.0440673828125,
"KL/mean": -1340.4217529296875,
"KL/rejected_KL_mean": -1512.799560546875,
"KL/std": 1134.163818359375,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 344.75555419921875,
"fcm_dpo/q_t": 0.4999054968357086,
"grad_norm": 0.10434012115001678,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -0.27185824513435364,
"logits/rejected": -0.2656621038913727,
"logps/chosen": -1238.816650390625,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -1588.93701171875,
"loss": 1.3859,
"margin_dpo/margin_mean": 344.75555419921875,
"margin_dpo/margin_std": 1227.234130859375,
"step": 477
},
{
"KL/chosen_KL_mean": -786.3189697265625,
"KL/mean": -1181.30322265625,
"KL/rejected_KL_mean": -1576.28759765625,
"KL/std": 1160.535400390625,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 789.96875,
"fcm_dpo/q_t": 0.4997834861278534,
"grad_norm": 0.17800100147724152,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.24197494983673096,
"logits/rejected": -0.24932987987995148,
"logps/chosen": -827.7594604492188,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -1661.649658203125,
"loss": 1.3854,
"margin_dpo/margin_mean": 789.96875,
"margin_dpo/margin_std": 1413.09228515625,
"step": 478
},
{
"KL/chosen_KL_mean": -1186.480712890625,
"KL/mean": -1565.7657470703125,
"KL/rejected_KL_mean": -1945.05078125,
"KL/std": 1548.585693359375,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 758.5699462890625,
"fcm_dpo/q_t": 0.49979206919670105,
"grad_norm": 0.1228935718536377,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -0.2601107954978943,
"logits/rejected": -0.2672087550163269,
"logps/chosen": -1240.388671875,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -2040.1671142578125,
"loss": 1.3855,
"margin_dpo/margin_mean": 758.5700073242188,
"margin_dpo/margin_std": 1593.4921875,
"step": 479
},
{
"KL/chosen_KL_mean": -1442.3372802734375,
"KL/mean": -1559.3134765625,
"KL/rejected_KL_mean": -1676.2896728515625,
"KL/std": 1411.744873046875,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 233.95236206054688,
"fcm_dpo/q_t": 0.4999358654022217,
"grad_norm": 0.17090091109275818,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -0.22087758779525757,
"logits/rejected": -0.2172781080007553,
"logps/chosen": -1501.02001953125,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -1759.22216796875,
"loss": 1.386,
"margin_dpo/margin_mean": 233.95237731933594,
"margin_dpo/margin_std": 1820.455810546875,
"step": 480
},
{
"KL/chosen_KL_mean": -1078.3800048828125,
"KL/mean": -1499.417236328125,
"KL/rejected_KL_mean": -1920.454833984375,
"KL/std": 1449.1815185546875,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 842.0747680664062,
"fcm_dpo/q_t": 0.4997692108154297,
"grad_norm": 0.13816991448402405,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -0.2306959182024002,
"logits/rejected": -0.24468478560447693,
"logps/chosen": -1133.34423828125,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -2012.875244140625,
"loss": 1.3854,
"margin_dpo/margin_mean": 842.0747680664062,
"margin_dpo/margin_std": 1748.59130859375,
"step": 481
},
{
"KL/chosen_KL_mean": -1496.91259765625,
"KL/mean": -1609.91015625,
"KL/rejected_KL_mean": -1722.907958984375,
"KL/std": 1626.68359375,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 225.99520874023438,
"fcm_dpo/q_t": 0.49993807077407837,
"grad_norm": 0.11034970730543137,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -0.2795412540435791,
"logits/rejected": -0.2840754985809326,
"logps/chosen": -1564.466064453125,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -1810.4974365234375,
"loss": 1.386,
"margin_dpo/margin_mean": 225.99520874023438,
"margin_dpo/margin_std": 2063.758544921875,
"step": 482
},
{
"KL/chosen_KL_mean": -1195.4193115234375,
"KL/mean": -1492.416259765625,
"KL/rejected_KL_mean": -1789.4130859375,
"KL/std": 1201.998779296875,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 593.9937133789062,
"fcm_dpo/q_t": 0.499837189912796,
"grad_norm": 0.0831841230392456,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -0.28483182191848755,
"logits/rejected": -0.29176396131515503,
"logps/chosen": -1258.7491455078125,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -1885.2000732421875,
"loss": 1.3856,
"margin_dpo/margin_mean": 593.9937133789062,
"margin_dpo/margin_std": 1516.528564453125,
"step": 483
},
{
"KL/chosen_KL_mean": -1180.9385986328125,
"KL/mean": -1519.308349609375,
"KL/rejected_KL_mean": -1857.677978515625,
"KL/std": 1603.910888671875,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 676.7393798828125,
"fcm_dpo/q_t": 0.499814510345459,
"grad_norm": 0.10013484954833984,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -0.25293561816215515,
"logits/rejected": -0.2621699571609497,
"logps/chosen": -1240.07666015625,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -1942.0494384765625,
"loss": 1.3856,
"margin_dpo/margin_mean": 676.7393798828125,
"margin_dpo/margin_std": 2009.601318359375,
"step": 484
},
{
"KL/chosen_KL_mean": -1249.12060546875,
"KL/mean": -1599.3984375,
"KL/rejected_KL_mean": -1949.676513671875,
"KL/std": 1546.91796875,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 700.555908203125,
"fcm_dpo/q_t": 0.4998079836368561,
"grad_norm": 0.10648392140865326,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -0.27771198749542236,
"logits/rejected": -0.29167696833610535,
"logps/chosen": -1307.97021484375,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -2053.04052734375,
"loss": 1.3855,
"margin_dpo/margin_mean": 700.555908203125,
"margin_dpo/margin_std": 2090.70458984375,
"step": 485
},
{
"KL/chosen_KL_mean": -1269.178466796875,
"KL/mean": -1632.4638671875,
"KL/rejected_KL_mean": -1995.749267578125,
"KL/std": 1473.224853515625,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 726.5706787109375,
"fcm_dpo/q_t": 0.4998008608818054,
"grad_norm": 0.10227925330400467,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -0.2663297653198242,
"logits/rejected": -0.29365283250808716,
"logps/chosen": -1324.43798828125,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -2087.888671875,
"loss": 1.3855,
"margin_dpo/margin_mean": 726.5706787109375,
"margin_dpo/margin_std": 1893.989501953125,
"step": 486
},
{
"KL/chosen_KL_mean": -1264.476806640625,
"KL/mean": -1589.610107421875,
"KL/rejected_KL_mean": -1914.743408203125,
"KL/std": 1264.77880859375,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 650.2666015625,
"fcm_dpo/q_t": 0.4998217821121216,
"grad_norm": 0.11868096888065338,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -0.266316294670105,
"logits/rejected": -0.2759333848953247,
"logps/chosen": -1317.5400390625,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -2007.162109375,
"loss": 1.3856,
"margin_dpo/margin_mean": 650.2665405273438,
"margin_dpo/margin_std": 1406.77490234375,
"step": 487
},
{
"KL/chosen_KL_mean": -947.0422973632812,
"KL/mean": -1332.60205078125,
"KL/rejected_KL_mean": -1718.16162109375,
"KL/std": 1295.9998779296875,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 771.1192626953125,
"fcm_dpo/q_t": 0.49978864192962646,
"grad_norm": 0.08489668369293213,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -0.2604157328605652,
"logits/rejected": -0.2639409303665161,
"logps/chosen": -999.2705078125,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -1802.168212890625,
"loss": 1.3854,
"margin_dpo/margin_mean": 771.1192626953125,
"margin_dpo/margin_std": 1547.508056640625,
"step": 488
},
{
"KL/chosen_KL_mean": -1105.8419189453125,
"KL/mean": -1353.78662109375,
"KL/rejected_KL_mean": -1601.7314453125,
"KL/std": 1445.2835693359375,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 495.88958740234375,
"fcm_dpo/q_t": 0.4998641014099121,
"grad_norm": 0.14254043996334076,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -0.3298376500606537,
"logits/rejected": -0.333371639251709,
"logps/chosen": -1161.83154296875,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -1681.129638671875,
"loss": 1.3858,
"margin_dpo/margin_mean": 495.8895568847656,
"margin_dpo/margin_std": 1753.814453125,
"step": 489
},
{
"KL/chosen_KL_mean": -1167.674072265625,
"KL/mean": -1577.68701171875,
"KL/rejected_KL_mean": -1987.699951171875,
"KL/std": 1686.5123291015625,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 820.026123046875,
"fcm_dpo/q_t": 0.499775230884552,
"grad_norm": 0.14442802965641022,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -0.3567619323730469,
"logits/rejected": -0.3724161982536316,
"logps/chosen": -1220.0404052734375,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -2098.109130859375,
"loss": 1.3854,
"margin_dpo/margin_mean": 820.026123046875,
"margin_dpo/margin_std": 2067.93212890625,
"step": 490
},
{
"KL/chosen_KL_mean": -1103.107177734375,
"KL/mean": -1224.5498046875,
"KL/rejected_KL_mean": -1345.99267578125,
"KL/std": 1070.229736328125,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 242.8854217529297,
"fcm_dpo/q_t": 0.4999334216117859,
"grad_norm": 0.10099928826093674,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -0.24923652410507202,
"logits/rejected": -0.2605559825897217,
"logps/chosen": -1163.223388671875,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -1419.265380859375,
"loss": 1.386,
"margin_dpo/margin_mean": 242.88543701171875,
"margin_dpo/margin_std": 1138.2882080078125,
"step": 491
},
{
"KL/chosen_KL_mean": -1155.7711181640625,
"KL/mean": -1256.9951171875,
"KL/rejected_KL_mean": -1358.218994140625,
"KL/std": 1135.314208984375,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 202.44784545898438,
"fcm_dpo/q_t": 0.4999445080757141,
"grad_norm": 0.10892591625452042,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -0.211615651845932,
"logits/rejected": -0.21193912625312805,
"logps/chosen": -1209.765380859375,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -1430.878662109375,
"loss": 1.3861,
"margin_dpo/margin_mean": 202.44786071777344,
"margin_dpo/margin_std": 1114.23291015625,
"step": 492
},
{
"KL/chosen_KL_mean": -1369.488525390625,
"KL/mean": -1456.96533203125,
"KL/rejected_KL_mean": -1544.44189453125,
"KL/std": 1230.5186767578125,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 174.95330810546875,
"fcm_dpo/q_t": 0.4999520182609558,
"grad_norm": 0.23652927577495575,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -0.269443154335022,
"logits/rejected": -0.26523542404174805,
"logps/chosen": -1444.98583984375,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -1631.764892578125,
"loss": 1.3861,
"margin_dpo/margin_mean": 174.95333862304688,
"margin_dpo/margin_std": 1543.530029296875,
"step": 493
},
{
"KL/chosen_KL_mean": -821.7962036132812,
"KL/mean": -1201.152587890625,
"KL/rejected_KL_mean": -1580.509033203125,
"KL/std": 1113.7763671875,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 758.7128295898438,
"fcm_dpo/q_t": 0.49979203939437866,
"grad_norm": 0.18678583204746246,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -0.27484673261642456,
"logits/rejected": -0.29044800996780396,
"logps/chosen": -863.1554565429688,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -1666.600341796875,
"loss": 1.3855,
"margin_dpo/margin_mean": 758.7128295898438,
"margin_dpo/margin_std": 1463.646484375,
"step": 494
},
{
"KL/chosen_KL_mean": -1247.958740234375,
"KL/mean": -1488.2684326171875,
"KL/rejected_KL_mean": -1728.578125,
"KL/std": 1216.287109375,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 480.6193542480469,
"fcm_dpo/q_t": 0.499868243932724,
"grad_norm": 0.10272727906703949,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -0.2950224280357361,
"logits/rejected": -0.3044213652610779,
"logps/chosen": -1311.493896484375,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -1820.0025634765625,
"loss": 1.3858,
"margin_dpo/margin_mean": 480.619384765625,
"margin_dpo/margin_std": 1789.05224609375,
"step": 495
},
{
"KL/chosen_KL_mean": -1368.56884765625,
"KL/mean": -1406.943115234375,
"KL/rejected_KL_mean": -1445.317626953125,
"KL/std": 1098.4658203125,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 76.74879455566406,
"fcm_dpo/q_t": 0.4999789595603943,
"grad_norm": 0.19332145154476166,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -0.22733855247497559,
"logits/rejected": -0.23675402998924255,
"logps/chosen": -1441.1607666015625,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -1529.64697265625,
"loss": 1.3862,
"margin_dpo/margin_mean": 76.74880981445312,
"margin_dpo/margin_std": 1383.187255859375,
"step": 496
},
{
"KL/chosen_KL_mean": -1017.3779296875,
"KL/mean": -1118.634765625,
"KL/rejected_KL_mean": -1219.8914794921875,
"KL/std": 858.4376831054688,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 202.51368713378906,
"fcm_dpo/q_t": 0.49994444847106934,
"grad_norm": 0.11383572220802307,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -0.23537415266036987,
"logits/rejected": -0.23635084927082062,
"logps/chosen": -1075.971923828125,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -1296.179931640625,
"loss": 1.3861,
"margin_dpo/margin_mean": 202.51368713378906,
"margin_dpo/margin_std": 849.7100830078125,
"step": 497
},
{
"KL/chosen_KL_mean": -1325.987060546875,
"KL/mean": -1531.426025390625,
"KL/rejected_KL_mean": -1736.864990234375,
"KL/std": 1285.070068359375,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 410.8779602050781,
"fcm_dpo/q_t": 0.4998873770236969,
"grad_norm": 0.14165054261684418,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -0.26703372597694397,
"logits/rejected": -0.2662290334701538,
"logps/chosen": -1397.192626953125,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -1820.8231201171875,
"loss": 1.3858,
"margin_dpo/margin_mean": 410.8779602050781,
"margin_dpo/margin_std": 1588.934814453125,
"step": 498
},
{
"KL/chosen_KL_mean": -987.0281982421875,
"KL/mean": -1476.96337890625,
"KL/rejected_KL_mean": -1966.8984375,
"KL/std": 1366.307373046875,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 979.8701171875,
"fcm_dpo/q_t": 0.4997314214706421,
"grad_norm": 0.11465054005384445,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -0.2148178517818451,
"logits/rejected": -0.22484102845191956,
"logps/chosen": -1038.283447265625,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -2067.97705078125,
"loss": 1.3852,
"margin_dpo/margin_mean": 979.8701171875,
"margin_dpo/margin_std": 1764.344970703125,
"step": 499
},
{
"KL/chosen_KL_mean": -1238.251708984375,
"KL/mean": -1586.513671875,
"KL/rejected_KL_mean": -1934.7757568359375,
"KL/std": 1321.060791015625,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 696.5239868164062,
"fcm_dpo/q_t": 0.4998090863227844,
"grad_norm": 0.0977897122502327,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -0.2564852237701416,
"logits/rejected": -0.2703487277030945,
"logps/chosen": -1295.279296875,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -2028.7099609375,
"loss": 1.3855,
"margin_dpo/margin_mean": 696.5239868164062,
"margin_dpo/margin_std": 1756.97802734375,
"step": 500
},
{
"KL/chosen_KL_mean": -1067.537109375,
"KL/mean": -1295.0264892578125,
"KL/rejected_KL_mean": -1522.515869140625,
"KL/std": 1159.325927734375,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 454.978759765625,
"fcm_dpo/q_t": 0.4998753070831299,
"grad_norm": 0.0951857939362526,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -0.25967469811439514,
"logits/rejected": -0.25465235114097595,
"logps/chosen": -1121.8966064453125,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -1602.672607421875,
"loss": 1.3858,
"margin_dpo/margin_mean": 454.978759765625,
"margin_dpo/margin_std": 1693.336669921875,
"step": 501
},
{
"KL/chosen_KL_mean": -1118.3916015625,
"KL/mean": -1272.55224609375,
"KL/rejected_KL_mean": -1426.713134765625,
"KL/std": 1068.8515625,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 308.3215026855469,
"fcm_dpo/q_t": 0.4999154806137085,
"grad_norm": 0.20658744871616364,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -0.2730405032634735,
"logits/rejected": -0.2703663110733032,
"logps/chosen": -1185.9921875,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -1509.661865234375,
"loss": 1.386,
"margin_dpo/margin_mean": 308.32147216796875,
"margin_dpo/margin_std": 1074.383544921875,
"step": 502
},
{
"KL/chosen_KL_mean": -966.2994384765625,
"KL/mean": -1260.075439453125,
"KL/rejected_KL_mean": -1553.8515625,
"KL/std": 1145.9617919921875,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 587.552001953125,
"fcm_dpo/q_t": 0.4998389482498169,
"grad_norm": 0.11802306771278381,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -0.23120424151420593,
"logits/rejected": -0.2398468554019928,
"logps/chosen": -1021.3778686523438,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -1636.35693359375,
"loss": 1.3857,
"margin_dpo/margin_mean": 587.552001953125,
"margin_dpo/margin_std": 1356.695068359375,
"step": 503
},
{
"KL/chosen_KL_mean": -1249.80419921875,
"KL/mean": -1683.2874755859375,
"KL/rejected_KL_mean": -2116.770751953125,
"KL/std": 1505.3199462890625,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 866.9666137695312,
"fcm_dpo/q_t": 0.4997623562812805,
"grad_norm": 0.16962231695652008,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -0.28023457527160645,
"logits/rejected": -0.29524755477905273,
"logps/chosen": -1309.7698974609375,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -2220.532958984375,
"loss": 1.3853,
"margin_dpo/margin_mean": 866.9666137695312,
"margin_dpo/margin_std": 1842.38818359375,
"step": 504
},
{
"KL/chosen_KL_mean": -1363.998779296875,
"KL/mean": -1380.1290283203125,
"KL/rejected_KL_mean": -1396.25927734375,
"KL/std": 1373.6171875,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 32.26036071777344,
"fcm_dpo/q_t": 0.49999117851257324,
"grad_norm": 0.16270355880260468,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -0.3044007420539856,
"logits/rejected": -0.3048001229763031,
"logps/chosen": -1440.15380859375,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -1484.8446044921875,
"loss": 1.3863,
"margin_dpo/margin_mean": 32.26038360595703,
"margin_dpo/margin_std": 1598.21826171875,
"step": 505
},
{
"KL/chosen_KL_mean": -901.4547729492188,
"KL/mean": -1252.69677734375,
"KL/rejected_KL_mean": -1603.938720703125,
"KL/std": 1069.479248046875,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 702.4840087890625,
"fcm_dpo/q_t": 0.4998074471950531,
"grad_norm": 0.07248572260141373,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -0.2846633791923523,
"logits/rejected": -0.29492151737213135,
"logps/chosen": -950.415283203125,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -1682.353759765625,
"loss": 1.3855,
"margin_dpo/margin_mean": 702.4840087890625,
"margin_dpo/margin_std": 1198.2059326171875,
"step": 506
},
{
"KL/chosen_KL_mean": -1269.53466796875,
"KL/mean": -1508.1639404296875,
"KL/rejected_KL_mean": -1746.793212890625,
"KL/std": 1457.87353515625,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 477.2585754394531,
"fcm_dpo/q_t": 0.4998691976070404,
"grad_norm": 0.07682963460683823,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -0.26889654994010925,
"logits/rejected": -0.2736053466796875,
"logps/chosen": -1336.336181640625,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -1842.1661376953125,
"loss": 1.3858,
"margin_dpo/margin_mean": 477.258544921875,
"margin_dpo/margin_std": 1696.332763671875,
"step": 507
},
{
"KL/chosen_KL_mean": -1198.90625,
"KL/mean": -1422.8087158203125,
"KL/rejected_KL_mean": -1646.7113037109375,
"KL/std": 1172.433837890625,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 447.80511474609375,
"fcm_dpo/q_t": 0.49987727403640747,
"grad_norm": 0.11619852483272552,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -0.27682003378868103,
"logits/rejected": -0.29388418793678284,
"logps/chosen": -1270.209716796875,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -1742.3388671875,
"loss": 1.3858,
"margin_dpo/margin_mean": 447.80511474609375,
"margin_dpo/margin_std": 1444.593994140625,
"step": 508
},
{
"KL/chosen_KL_mean": -972.7108764648438,
"KL/mean": -1096.7337646484375,
"KL/rejected_KL_mean": -1220.7567138671875,
"KL/std": 1112.60400390625,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 248.04592895507812,
"fcm_dpo/q_t": 0.4999319911003113,
"grad_norm": 0.1427529752254486,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -0.2824665904045105,
"logits/rejected": -0.2912002205848694,
"logps/chosen": -1036.52978515625,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -1304.01318359375,
"loss": 1.386,
"margin_dpo/margin_mean": 248.04592895507812,
"margin_dpo/margin_std": 1467.73828125,
"step": 509
},
{
"KL/chosen_KL_mean": -1051.2216796875,
"KL/mean": -1357.23681640625,
"KL/rejected_KL_mean": -1663.2518310546875,
"KL/std": 1066.6669921875,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 612.0302734375,
"fcm_dpo/q_t": 0.4998322129249573,
"grad_norm": 0.13177689909934998,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -0.24248462915420532,
"logits/rejected": -0.2578527629375458,
"logps/chosen": -1103.10009765625,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -1766.01708984375,
"loss": 1.3856,
"margin_dpo/margin_mean": 612.0302734375,
"margin_dpo/margin_std": 1055.28466796875,
"step": 510
},
{
"KL/chosen_KL_mean": -1168.831298828125,
"KL/mean": -1416.7269287109375,
"KL/rejected_KL_mean": -1664.62255859375,
"KL/std": 1328.281982421875,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 495.7911376953125,
"fcm_dpo/q_t": 0.4998641014099121,
"grad_norm": 0.08437525480985641,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -0.2679316997528076,
"logits/rejected": -0.2680022120475769,
"logps/chosen": -1229.0694580078125,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -1757.479248046875,
"loss": 1.3858,
"margin_dpo/margin_mean": 495.79119873046875,
"margin_dpo/margin_std": 1695.87744140625,
"step": 511
},
{
"KL/chosen_KL_mean": -957.021484375,
"KL/mean": -1312.630126953125,
"KL/rejected_KL_mean": -1668.23876953125,
"KL/std": 1370.5361328125,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 711.2174072265625,
"fcm_dpo/q_t": 0.4998050630092621,
"grad_norm": 0.11381080746650696,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -0.2461620569229126,
"logits/rejected": -0.2682782709598541,
"logps/chosen": -1011.9268798828125,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -1750.11474609375,
"loss": 1.3855,
"margin_dpo/margin_mean": 711.2174682617188,
"margin_dpo/margin_std": 1575.836181640625,
"step": 512
},
{
"KL/chosen_KL_mean": -1210.6767578125,
"KL/mean": -1353.77685546875,
"KL/rejected_KL_mean": -1496.876953125,
"KL/std": 1319.700927734375,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 286.2001953125,
"fcm_dpo/q_t": 0.4999215602874756,
"grad_norm": 0.08147355914115906,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -0.22810330986976624,
"logits/rejected": -0.23092329502105713,
"logps/chosen": -1275.59326171875,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -1572.9393310546875,
"loss": 1.386,
"margin_dpo/margin_mean": 286.2001647949219,
"margin_dpo/margin_std": 1764.772216796875,
"step": 513
},
{
"KL/chosen_KL_mean": -1492.766845703125,
"KL/mean": -1600.861083984375,
"KL/rejected_KL_mean": -1708.955322265625,
"KL/std": 1616.942626953125,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 216.18853759765625,
"fcm_dpo/q_t": 0.49994075298309326,
"grad_norm": 0.1525028944015503,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -0.2767585515975952,
"logits/rejected": -0.27932238578796387,
"logps/chosen": -1566.996337890625,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -1787.90087890625,
"loss": 1.3861,
"margin_dpo/margin_mean": 216.18850708007812,
"margin_dpo/margin_std": 1974.9534912109375,
"step": 514
},
{
"KL/chosen_KL_mean": -965.72412109375,
"KL/mean": -1177.3330078125,
"KL/rejected_KL_mean": -1388.9420166015625,
"KL/std": 1055.767578125,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 423.2179260253906,
"fcm_dpo/q_t": 0.4998840093612671,
"grad_norm": 0.07925697416067123,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -0.2743576169013977,
"logits/rejected": -0.28066229820251465,
"logps/chosen": -1016.1256713867188,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -1476.039794921875,
"loss": 1.3858,
"margin_dpo/margin_mean": 423.2179260253906,
"margin_dpo/margin_std": 1284.13916015625,
"step": 515
},
{
"KL/chosen_KL_mean": -1195.279541015625,
"KL/mean": -1572.9501953125,
"KL/rejected_KL_mean": -1950.620849609375,
"KL/std": 1871.8084716796875,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 755.3414306640625,
"fcm_dpo/q_t": 0.4997929632663727,
"grad_norm": 0.1906149983406067,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -0.2554052472114563,
"logits/rejected": -0.2716732621192932,
"logps/chosen": -1259.9290771484375,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -2062.34326171875,
"loss": 1.3855,
"margin_dpo/margin_mean": 755.3414306640625,
"margin_dpo/margin_std": 2260.271484375,
"step": 516
},
{
"KL/chosen_KL_mean": -1145.0179443359375,
"KL/mean": -1422.536865234375,
"KL/rejected_KL_mean": -1700.0556640625,
"KL/std": 1352.418212890625,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 555.037841796875,
"fcm_dpo/q_t": 0.4998478889465332,
"grad_norm": 0.09951838850975037,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -0.28132858872413635,
"logits/rejected": -0.2950979471206665,
"logps/chosen": -1205.9315185546875,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -1789.1387939453125,
"loss": 1.3857,
"margin_dpo/margin_mean": 555.037841796875,
"margin_dpo/margin_std": 1767.117431640625,
"step": 517
},
{
"KL/chosen_KL_mean": -1181.3603515625,
"KL/mean": -1440.18896484375,
"KL/rejected_KL_mean": -1699.017578125,
"KL/std": 1185.616943359375,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 517.6571655273438,
"fcm_dpo/q_t": 0.4998581111431122,
"grad_norm": 0.09385734796524048,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -0.27268654108047485,
"logits/rejected": -0.28850501775741577,
"logps/chosen": -1238.816162109375,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -1784.3302001953125,
"loss": 1.3857,
"margin_dpo/margin_mean": 517.6571655273438,
"margin_dpo/margin_std": 1383.952880859375,
"step": 518
},
{
"KL/chosen_KL_mean": -1131.969970703125,
"KL/mean": -1446.619873046875,
"KL/rejected_KL_mean": -1761.27001953125,
"KL/std": 1455.6708984375,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 629.3001708984375,
"fcm_dpo/q_t": 0.49982750415802,
"grad_norm": 0.12801778316497803,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -0.3059653043746948,
"logits/rejected": -0.3031119108200073,
"logps/chosen": -1206.033203125,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -1865.7142333984375,
"loss": 1.3856,
"margin_dpo/margin_mean": 629.3001708984375,
"margin_dpo/margin_std": 1972.010986328125,
"step": 519
},
{
"KL/chosen_KL_mean": -1056.5577392578125,
"KL/mean": -1340.5191650390625,
"KL/rejected_KL_mean": -1624.4805908203125,
"KL/std": 1151.4697265625,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 567.9228515625,
"fcm_dpo/q_t": 0.49984434247016907,
"grad_norm": 0.11603305488824844,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -0.23705577850341797,
"logits/rejected": -0.24325022101402283,
"logps/chosen": -1126.857666015625,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -1724.4619140625,
"loss": 1.3857,
"margin_dpo/margin_mean": 567.9227905273438,
"margin_dpo/margin_std": 1425.7099609375,
"step": 520
},
{
"KL/chosen_KL_mean": -1010.5167846679688,
"KL/mean": -1221.34521484375,
"KL/rejected_KL_mean": -1432.173583984375,
"KL/std": 1044.7950439453125,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 421.65692138671875,
"fcm_dpo/q_t": 0.4998844265937805,
"grad_norm": 0.0845324769616127,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -0.26425081491470337,
"logits/rejected": -0.2640856206417084,
"logps/chosen": -1068.65966796875,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -1515.454345703125,
"loss": 1.3858,
"margin_dpo/margin_mean": 421.65692138671875,
"margin_dpo/margin_std": 1294.17919921875,
"step": 521
},
{
"KL/chosen_KL_mean": -1076.2818603515625,
"KL/mean": -1360.576416015625,
"KL/rejected_KL_mean": -1644.87109375,
"KL/std": 1606.474853515625,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 568.589111328125,
"fcm_dpo/q_t": 0.49984413385391235,
"grad_norm": 0.07642640173435211,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -0.2520676851272583,
"logits/rejected": -0.2678247392177582,
"logps/chosen": -1122.82958984375,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -1710.885009765625,
"loss": 1.3857,
"margin_dpo/margin_mean": 568.589111328125,
"margin_dpo/margin_std": 2015.031982421875,
"step": 522
},
{
"KL/chosen_KL_mean": -1033.318359375,
"KL/mean": -1335.800537109375,
"KL/rejected_KL_mean": -1638.28271484375,
"KL/std": 1357.097412109375,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 604.9642944335938,
"fcm_dpo/q_t": 0.49983417987823486,
"grad_norm": 0.10172777622938156,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -0.2722175717353821,
"logits/rejected": -0.2761549651622772,
"logps/chosen": -1095.087890625,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -1722.0440673828125,
"loss": 1.3856,
"margin_dpo/margin_mean": 604.96435546875,
"margin_dpo/margin_std": 1610.768798828125,
"step": 523
},
{
"KL/chosen_KL_mean": -1311.5902099609375,
"KL/mean": -1617.090576171875,
"KL/rejected_KL_mean": -1922.5909423828125,
"KL/std": 1613.0634765625,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 611.0007934570312,
"fcm_dpo/q_t": 0.49983254075050354,
"grad_norm": 0.1079510748386383,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -0.32919204235076904,
"logits/rejected": -0.31630319356918335,
"logps/chosen": -1389.6622314453125,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -2003.8929443359375,
"loss": 1.3856,
"margin_dpo/margin_mean": 611.000732421875,
"margin_dpo/margin_std": 1673.618896484375,
"step": 524
},
{
"KL/chosen_KL_mean": -979.3692626953125,
"KL/mean": -1385.7138671875,
"KL/rejected_KL_mean": -1792.058349609375,
"KL/std": 1283.711669921875,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 812.6890869140625,
"fcm_dpo/q_t": 0.49977725744247437,
"grad_norm": 0.18908537924289703,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -0.26024335622787476,
"logits/rejected": -0.2741534113883972,
"logps/chosen": -1030.1971435546875,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -1892.111328125,
"loss": 1.3854,
"margin_dpo/margin_mean": 812.6890869140625,
"margin_dpo/margin_std": 1620.133056640625,
"step": 525
},
{
"KL/chosen_KL_mean": -1037.4547119140625,
"KL/mean": -1374.19921875,
"KL/rejected_KL_mean": -1710.94384765625,
"KL/std": 1307.3369140625,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 673.4891357421875,
"fcm_dpo/q_t": 0.4998154044151306,
"grad_norm": 0.09858764708042145,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -0.28175538778305054,
"logits/rejected": -0.2941705584526062,
"logps/chosen": -1100.6219482421875,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -1797.253173828125,
"loss": 1.3856,
"margin_dpo/margin_mean": 673.4891357421875,
"margin_dpo/margin_std": 1447.4334716796875,
"step": 526
},
{
"KL/chosen_KL_mean": -1263.2935791015625,
"KL/mean": -1372.807861328125,
"KL/rejected_KL_mean": -1482.322265625,
"KL/std": 1163.5068359375,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 219.02862548828125,
"fcm_dpo/q_t": 0.4999399781227112,
"grad_norm": 0.1761639416217804,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -0.23617637157440186,
"logits/rejected": -0.23162142932415009,
"logps/chosen": -1334.4222412109375,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -1560.664794921875,
"loss": 1.3861,
"margin_dpo/margin_mean": 219.02862548828125,
"margin_dpo/margin_std": 1113.69873046875,
"step": 527
},
{
"KL/chosen_KL_mean": -1018.8012084960938,
"KL/mean": -1362.28564453125,
"KL/rejected_KL_mean": -1705.7701416015625,
"KL/std": 1300.2275390625,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 686.9689331054688,
"fcm_dpo/q_t": 0.49981170892715454,
"grad_norm": 0.08465294539928436,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -0.2715555429458618,
"logits/rejected": -0.263353168964386,
"logps/chosen": -1086.890625,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -1799.68017578125,
"loss": 1.3855,
"margin_dpo/margin_mean": 686.9688720703125,
"margin_dpo/margin_std": 1761.4932861328125,
"step": 528
},
{
"KL/chosen_KL_mean": -1227.3046875,
"KL/mean": -1537.86181640625,
"KL/rejected_KL_mean": -1848.4189453125,
"KL/std": 1296.86279296875,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 621.1142578125,
"fcm_dpo/q_t": 0.4998297691345215,
"grad_norm": 0.08333950489759445,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -0.2381981611251831,
"logits/rejected": -0.25011301040649414,
"logps/chosen": -1282.879638671875,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -1937.6279296875,
"loss": 1.3856,
"margin_dpo/margin_mean": 621.1142578125,
"margin_dpo/margin_std": 1707.134765625,
"step": 529
},
{
"KL/chosen_KL_mean": -909.7897338867188,
"KL/mean": -1290.269775390625,
"KL/rejected_KL_mean": -1670.75,
"KL/std": 1348.552734375,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 760.960205078125,
"fcm_dpo/q_t": 0.4997914433479309,
"grad_norm": 0.1400674730539322,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -0.2535117566585541,
"logits/rejected": -0.25747549533843994,
"logps/chosen": -957.39111328125,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -1758.034423828125,
"loss": 1.3855,
"margin_dpo/margin_mean": 760.960205078125,
"margin_dpo/margin_std": 1795.02294921875,
"step": 530
},
{
"KL/chosen_KL_mean": -895.466552734375,
"KL/mean": -1198.396484375,
"KL/rejected_KL_mean": -1501.32666015625,
"KL/std": 1112.87744140625,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 605.860107421875,
"fcm_dpo/q_t": 0.49983394145965576,
"grad_norm": 0.08130745589733124,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -0.28837865591049194,
"logits/rejected": -0.3007437586784363,
"logps/chosen": -950.712646484375,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -1571.9326171875,
"loss": 1.3856,
"margin_dpo/margin_mean": 605.860107421875,
"margin_dpo/margin_std": 1083.08837890625,
"step": 531
},
{
"KL/chosen_KL_mean": -1241.084228515625,
"KL/mean": -1509.7344970703125,
"KL/rejected_KL_mean": -1778.384765625,
"KL/std": 1285.7984619140625,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 537.300537109375,
"fcm_dpo/q_t": 0.49985271692276,
"grad_norm": 0.17090633511543274,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -0.2878720164299011,
"logits/rejected": -0.2896765470504761,
"logps/chosen": -1311.3702392578125,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -1864.97607421875,
"loss": 1.3857,
"margin_dpo/margin_mean": 537.300537109375,
"margin_dpo/margin_std": 1761.650390625,
"step": 532
},
{
"KL/chosen_KL_mean": -1169.46240234375,
"KL/mean": -1342.16552734375,
"KL/rejected_KL_mean": -1514.868408203125,
"KL/std": 1102.200439453125,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 345.4060363769531,
"fcm_dpo/q_t": 0.4999053180217743,
"grad_norm": 0.08745371550321579,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -0.23491737246513367,
"logits/rejected": -0.2443659007549286,
"logps/chosen": -1234.317626953125,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -1591.45654296875,
"loss": 1.3859,
"margin_dpo/margin_mean": 345.406005859375,
"margin_dpo/margin_std": 1247.744873046875,
"step": 533
},
{
"KL/chosen_KL_mean": -1153.016845703125,
"KL/mean": -1350.9849853515625,
"KL/rejected_KL_mean": -1548.953125,
"KL/std": 1091.08642578125,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 395.936279296875,
"fcm_dpo/q_t": 0.499891459941864,
"grad_norm": 0.06997805833816528,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -0.285087913274765,
"logits/rejected": -0.2840203046798706,
"logps/chosen": -1213.13623046875,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -1627.49658203125,
"loss": 1.3859,
"margin_dpo/margin_mean": 395.9362487792969,
"margin_dpo/margin_std": 1245.4730224609375,
"step": 534
},
{
"KL/chosen_KL_mean": -931.0291748046875,
"KL/mean": -1277.57470703125,
"KL/rejected_KL_mean": -1624.1201171875,
"KL/std": 1066.3153076171875,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 693.091064453125,
"fcm_dpo/q_t": 0.49981003999710083,
"grad_norm": 0.2384890913963318,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -0.25958138704299927,
"logits/rejected": -0.2646050751209259,
"logps/chosen": -985.359375,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -1720.4278564453125,
"loss": 1.3855,
"margin_dpo/margin_mean": 693.091064453125,
"margin_dpo/margin_std": 1384.616943359375,
"step": 535
},
{
"KL/chosen_KL_mean": -901.54736328125,
"KL/mean": -1342.12353515625,
"KL/rejected_KL_mean": -1782.69970703125,
"KL/std": 1309.0029296875,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 881.1522216796875,
"fcm_dpo/q_t": 0.4997584819793701,
"grad_norm": 0.08399941027164459,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -0.25612354278564453,
"logits/rejected": -0.26890262961387634,
"logps/chosen": -948.6279296875,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -1871.797607421875,
"loss": 1.3853,
"margin_dpo/margin_mean": 881.1522216796875,
"margin_dpo/margin_std": 1578.1298828125,
"step": 536
},
{
"KL/chosen_KL_mean": -964.5877685546875,
"KL/mean": -1165.6494140625,
"KL/rejected_KL_mean": -1366.7109375,
"KL/std": 1214.634033203125,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 402.1231994628906,
"fcm_dpo/q_t": 0.4998897910118103,
"grad_norm": 0.12264511734247208,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -0.24898353219032288,
"logits/rejected": -0.2636551558971405,
"logps/chosen": -1022.335205078125,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -1437.1492919921875,
"loss": 1.3859,
"margin_dpo/margin_mean": 402.1231689453125,
"margin_dpo/margin_std": 1610.4888916015625,
"step": 537
},
{
"KL/chosen_KL_mean": -1419.4970703125,
"KL/mean": -1645.308349609375,
"KL/rejected_KL_mean": -1871.11962890625,
"KL/std": 1241.21337890625,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 451.6224670410156,
"fcm_dpo/q_t": 0.4998762011528015,
"grad_norm": 0.15423403680324554,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -0.27448660135269165,
"logits/rejected": -0.27430930733680725,
"logps/chosen": -1485.9130859375,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -1955.34765625,
"loss": 1.3858,
"margin_dpo/margin_mean": 451.6224670410156,
"margin_dpo/margin_std": 1483.188232421875,
"step": 538
},
{
"KL/chosen_KL_mean": -1247.627197265625,
"KL/mean": -1601.21533203125,
"KL/rejected_KL_mean": -1954.803466796875,
"KL/std": 1497.35498046875,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 707.17626953125,
"fcm_dpo/q_t": 0.49980616569519043,
"grad_norm": 0.08800466358661652,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -0.23933789134025574,
"logits/rejected": -0.23772892355918884,
"logps/chosen": -1306.1201171875,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -2046.657470703125,
"loss": 1.3855,
"margin_dpo/margin_mean": 707.17626953125,
"margin_dpo/margin_std": 1571.142822265625,
"step": 539
},
{
"KL/chosen_KL_mean": -1132.70263671875,
"KL/mean": -1627.818603515625,
"KL/rejected_KL_mean": -2122.934326171875,
"KL/std": 1450.533935546875,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 990.231689453125,
"fcm_dpo/q_t": 0.49972859025001526,
"grad_norm": 0.2339281588792801,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -0.2771497368812561,
"logits/rejected": -0.28398674726486206,
"logps/chosen": -1196.1851806640625,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -2239.3642578125,
"loss": 1.3852,
"margin_dpo/margin_mean": 990.2315673828125,
"margin_dpo/margin_std": 1892.115966796875,
"step": 540
},
{
"KL/chosen_KL_mean": -1705.358154296875,
"KL/mean": -1753.754638671875,
"KL/rejected_KL_mean": -1802.151123046875,
"KL/std": 1488.664794921875,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 96.79295349121094,
"fcm_dpo/q_t": 0.49997344613075256,
"grad_norm": 0.18836979568004608,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -0.27575767040252686,
"logits/rejected": -0.28435438871383667,
"logps/chosen": -1783.6385498046875,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -1905.4248046875,
"loss": 1.3862,
"margin_dpo/margin_mean": 96.79295349121094,
"margin_dpo/margin_std": 1649.4097900390625,
"step": 541
},
{
"KL/chosen_KL_mean": -1150.528076171875,
"KL/mean": -1601.9080810546875,
"KL/rejected_KL_mean": -2053.2880859375,
"KL/std": 1511.0010986328125,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 902.7598876953125,
"fcm_dpo/q_t": 0.49975258111953735,
"grad_norm": 0.09656842052936554,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -0.2542586326599121,
"logits/rejected": -0.258800745010376,
"logps/chosen": -1208.01318359375,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -2149.76318359375,
"loss": 1.3853,
"margin_dpo/margin_mean": 902.7598876953125,
"margin_dpo/margin_std": 1906.48046875,
"step": 542
},
{
"KL/chosen_KL_mean": -1388.8468017578125,
"KL/mean": -1585.422607421875,
"KL/rejected_KL_mean": -1781.9981689453125,
"KL/std": 1386.4052734375,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 393.1514587402344,
"fcm_dpo/q_t": 0.4998922348022461,
"grad_norm": 0.08593297749757767,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -0.2811450660228729,
"logits/rejected": -0.2933238744735718,
"logps/chosen": -1449.46435546875,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -1887.59716796875,
"loss": 1.3859,
"margin_dpo/margin_mean": 393.15142822265625,
"margin_dpo/margin_std": 1973.71875,
"step": 543
},
{
"KL/chosen_KL_mean": -1308.500244140625,
"KL/mean": -1758.7490234375,
"KL/rejected_KL_mean": -2208.997802734375,
"KL/std": 1845.3236083984375,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 900.497802734375,
"fcm_dpo/q_t": 0.4997531771659851,
"grad_norm": 0.11313273012638092,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -0.2731946110725403,
"logits/rejected": -0.2808229923248291,
"logps/chosen": -1368.1424560546875,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -2309.95263671875,
"loss": 1.3853,
"margin_dpo/margin_mean": 900.497802734375,
"margin_dpo/margin_std": 2468.87841796875,
"step": 544
},
{
"KL/chosen_KL_mean": -1315.6982421875,
"KL/mean": -1654.6534423828125,
"KL/rejected_KL_mean": -1993.6083984375,
"KL/std": 1559.415771484375,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 677.9102783203125,
"fcm_dpo/q_t": 0.4998142123222351,
"grad_norm": 0.1363622099161148,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -0.2993026077747345,
"logits/rejected": -0.3052732050418854,
"logps/chosen": -1383.346923828125,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -2089.5166015625,
"loss": 1.3856,
"margin_dpo/margin_mean": 677.9102783203125,
"margin_dpo/margin_std": 1948.7215576171875,
"step": 545
},
{
"KL/chosen_KL_mean": -1233.961669921875,
"KL/mean": -1423.4217529296875,
"KL/rejected_KL_mean": -1612.8819580078125,
"KL/std": 1190.663818359375,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 378.920166015625,
"fcm_dpo/q_t": 0.4998961389064789,
"grad_norm": 0.1315159946680069,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -0.2829248607158661,
"logits/rejected": -0.2874290645122528,
"logps/chosen": -1284.7059326171875,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -1694.7481689453125,
"loss": 1.3859,
"margin_dpo/margin_mean": 378.920166015625,
"margin_dpo/margin_std": 1452.426025390625,
"step": 546
},
{
"KL/chosen_KL_mean": -1516.5628662109375,
"KL/mean": -1795.2900390625,
"KL/rejected_KL_mean": -2074.01708984375,
"KL/std": 2049.671142578125,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 557.4541625976562,
"fcm_dpo/q_t": 0.4998472034931183,
"grad_norm": 0.10258148610591888,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -0.30326375365257263,
"logits/rejected": -0.30700263381004333,
"logps/chosen": -1590.2506103515625,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -2164.778564453125,
"loss": 1.3857,
"margin_dpo/margin_mean": 557.4541625976562,
"margin_dpo/margin_std": 2793.952392578125,
"step": 547
},
{
"KL/chosen_KL_mean": -1279.252197265625,
"KL/mean": -1552.047607421875,
"KL/rejected_KL_mean": -1824.84326171875,
"KL/std": 1413.7266845703125,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 545.5911865234375,
"fcm_dpo/q_t": 0.49985045194625854,
"grad_norm": 0.08995510637760162,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -0.2966746985912323,
"logits/rejected": -0.2986268401145935,
"logps/chosen": -1344.49853515625,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -1918.9613037109375,
"loss": 1.3857,
"margin_dpo/margin_mean": 545.5911865234375,
"margin_dpo/margin_std": 1728.4644775390625,
"step": 548
},
{
"KL/chosen_KL_mean": -1257.45849609375,
"KL/mean": -1397.7392578125,
"KL/rejected_KL_mean": -1538.02001953125,
"KL/std": 1288.01708984375,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 280.5616455078125,
"fcm_dpo/q_t": 0.49992311000823975,
"grad_norm": 0.0936603769659996,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -0.23646308481693268,
"logits/rejected": -0.23927268385887146,
"logps/chosen": -1306.6708984375,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -1611.930419921875,
"loss": 1.386,
"margin_dpo/margin_mean": 280.5616455078125,
"margin_dpo/margin_std": 1482.34765625,
"step": 549
},
{
"KL/chosen_KL_mean": -1198.25732421875,
"KL/mean": -1502.1494140625,
"KL/rejected_KL_mean": -1806.04150390625,
"KL/std": 1327.75634765625,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 607.7839965820312,
"fcm_dpo/q_t": 0.4998334050178528,
"grad_norm": 0.12111429870128632,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -0.2754247784614563,
"logits/rejected": -0.2830020785331726,
"logps/chosen": -1255.0643310546875,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -1901.167236328125,
"loss": 1.3856,
"margin_dpo/margin_mean": 607.783935546875,
"margin_dpo/margin_std": 1616.297119140625,
"step": 550
},
{
"KL/chosen_KL_mean": -1037.71484375,
"KL/mean": -1604.3447265625,
"KL/rejected_KL_mean": -2170.974609375,
"KL/std": 1566.04296875,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1133.2598876953125,
"fcm_dpo/q_t": 0.49968937039375305,
"grad_norm": 0.18915165960788727,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -0.24009448289871216,
"logits/rejected": -0.24743011593818665,
"logps/chosen": -1096.821044921875,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -2282.6474609375,
"loss": 1.3851,
"margin_dpo/margin_mean": 1133.259765625,
"margin_dpo/margin_std": 1756.314453125,
"step": 551
},
{
"KL/chosen_KL_mean": -1126.0601806640625,
"KL/mean": -1655.011962890625,
"KL/rejected_KL_mean": -2183.963623046875,
"KL/std": 1489.4439697265625,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1057.9033203125,
"fcm_dpo/q_t": 0.4997100532054901,
"grad_norm": 0.170448437333107,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -0.27598607540130615,
"logits/rejected": -0.279513955116272,
"logps/chosen": -1188.414794921875,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -2288.52587890625,
"loss": 1.3851,
"margin_dpo/margin_mean": 1057.9033203125,
"margin_dpo/margin_std": 1938.4671630859375,
"step": 552
},
{
"KL/chosen_KL_mean": -1235.739990234375,
"KL/mean": -1500.985595703125,
"KL/rejected_KL_mean": -1766.2314453125,
"KL/std": 1410.8720703125,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 530.491455078125,
"fcm_dpo/q_t": 0.49985459446907043,
"grad_norm": 0.0851590484380722,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -0.307264506816864,
"logits/rejected": -0.3080984950065613,
"logps/chosen": -1303.998779296875,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -1864.32861328125,
"loss": 1.3857,
"margin_dpo/margin_mean": 530.4915161132812,
"margin_dpo/margin_std": 1780.082763671875,
"step": 553
},
{
"KL/chosen_KL_mean": -1218.116455078125,
"KL/mean": -1475.4810791015625,
"KL/rejected_KL_mean": -1732.845703125,
"KL/std": 1494.07275390625,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 514.72900390625,
"fcm_dpo/q_t": 0.49985891580581665,
"grad_norm": 0.17466723918914795,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -0.29812121391296387,
"logits/rejected": -0.3056314289569855,
"logps/chosen": -1286.064208984375,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -1822.62841796875,
"loss": 1.3857,
"margin_dpo/margin_mean": 514.7290649414062,
"margin_dpo/margin_std": 2061.585205078125,
"step": 554
},
{
"KL/chosen_KL_mean": -1154.8209228515625,
"KL/mean": -1718.331298828125,
"KL/rejected_KL_mean": -2281.841796875,
"KL/std": 1818.7333984375,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1127.020751953125,
"fcm_dpo/q_t": 0.49969106912612915,
"grad_norm": 0.10861152410507202,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -0.28964054584503174,
"logits/rejected": -0.30473101139068604,
"logps/chosen": -1208.1513671875,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -2390.321044921875,
"loss": 1.3851,
"margin_dpo/margin_mean": 1127.020751953125,
"margin_dpo/margin_std": 2308.85205078125,
"step": 555
},
{
"KL/chosen_KL_mean": -1159.304443359375,
"KL/mean": -1518.688232421875,
"KL/rejected_KL_mean": -1878.072021484375,
"KL/std": 1266.625244140625,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 718.7675170898438,
"fcm_dpo/q_t": 0.49980297684669495,
"grad_norm": 0.2766326069831848,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -0.292389452457428,
"logits/rejected": -0.3039926290512085,
"logps/chosen": -1217.948974609375,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -1979.4124755859375,
"loss": 1.3855,
"margin_dpo/margin_mean": 718.767578125,
"margin_dpo/margin_std": 1265.7459716796875,
"step": 556
},
{
"KL/chosen_KL_mean": -1447.0789794921875,
"KL/mean": -1821.1226806640625,
"KL/rejected_KL_mean": -2195.16650390625,
"KL/std": 1652.57763671875,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 748.08740234375,
"fcm_dpo/q_t": 0.49979496002197266,
"grad_norm": 0.14727355539798737,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -0.3024740219116211,
"logits/rejected": -0.3093288242816925,
"logps/chosen": -1514.919677734375,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -2305.10595703125,
"loss": 1.3855,
"margin_dpo/margin_mean": 748.0873413085938,
"margin_dpo/margin_std": 1915.6552734375,
"step": 557
},
{
"KL/chosen_KL_mean": -1248.1954345703125,
"KL/mean": -1703.244140625,
"KL/rejected_KL_mean": -2158.29296875,
"KL/std": 1753.9251708984375,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 910.097412109375,
"fcm_dpo/q_t": 0.499750554561615,
"grad_norm": 0.15762417018413544,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -0.29016417264938354,
"logits/rejected": -0.310586541891098,
"logps/chosen": -1310.563720703125,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -2260.453857421875,
"loss": 1.3853,
"margin_dpo/margin_mean": 910.097412109375,
"margin_dpo/margin_std": 2112.92626953125,
"step": 558
},
{
"KL/chosen_KL_mean": -1379.48486328125,
"KL/mean": -1745.984619140625,
"KL/rejected_KL_mean": -2112.484375,
"KL/std": 1507.8521728515625,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 732.9996337890625,
"fcm_dpo/q_t": 0.49979907274246216,
"grad_norm": 0.19343577325344086,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -0.2922292947769165,
"logits/rejected": -0.2874986529350281,
"logps/chosen": -1440.237060546875,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -2205.9267578125,
"loss": 1.3855,
"margin_dpo/margin_mean": 732.9995727539062,
"margin_dpo/margin_std": 2046.0732421875,
"step": 559
},
{
"KL/chosen_KL_mean": -1155.489990234375,
"KL/mean": -1382.8680419921875,
"KL/rejected_KL_mean": -1610.24609375,
"KL/std": 1237.19091796875,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 454.7560729980469,
"fcm_dpo/q_t": 0.49987536668777466,
"grad_norm": 0.12281012535095215,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -0.3528594374656677,
"logits/rejected": -0.3549266755580902,
"logps/chosen": -1213.5938720703125,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -1690.2373046875,
"loss": 1.3858,
"margin_dpo/margin_mean": 454.7560729980469,
"margin_dpo/margin_std": 1460.5318603515625,
"step": 560
},
{
"KL/chosen_KL_mean": -1324.06201171875,
"KL/mean": -1605.061767578125,
"KL/rejected_KL_mean": -1886.061279296875,
"KL/std": 1419.917236328125,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 561.9992065429688,
"fcm_dpo/q_t": 0.499845951795578,
"grad_norm": 0.09632235765457153,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -0.30480241775512695,
"logits/rejected": -0.3091276288032532,
"logps/chosen": -1374.9749755859375,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -1985.1298828125,
"loss": 1.3857,
"margin_dpo/margin_mean": 561.999267578125,
"margin_dpo/margin_std": 1919.3912353515625,
"step": 561
},
{
"KL/chosen_KL_mean": -1076.00439453125,
"KL/mean": -1480.02001953125,
"KL/rejected_KL_mean": -1884.035400390625,
"KL/std": 1383.9652099609375,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 808.031005859375,
"fcm_dpo/q_t": 0.49977853894233704,
"grad_norm": 0.11567293107509613,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -0.2903629541397095,
"logits/rejected": -0.3054039776325226,
"logps/chosen": -1135.4688720703125,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -1980.578125,
"loss": 1.3854,
"margin_dpo/margin_mean": 808.031005859375,
"margin_dpo/margin_std": 1623.262939453125,
"step": 562
},
{
"KL/chosen_KL_mean": -1435.2509765625,
"KL/mean": -1691.4266357421875,
"KL/rejected_KL_mean": -1947.60205078125,
"KL/std": 1680.6656494140625,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 512.3511352539062,
"fcm_dpo/q_t": 0.4998595714569092,
"grad_norm": 0.2434445023536682,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -0.2510477304458618,
"logits/rejected": -0.2518584430217743,
"logps/chosen": -1495.859130859375,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -2042.1697998046875,
"loss": 1.3857,
"margin_dpo/margin_mean": 512.3511352539062,
"margin_dpo/margin_std": 1910.9169921875,
"step": 563
},
{
"KL/chosen_KL_mean": -1244.1939697265625,
"KL/mean": -1480.053955078125,
"KL/rejected_KL_mean": -1715.9140625,
"KL/std": 1373.150634765625,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 471.72003173828125,
"fcm_dpo/q_t": 0.4998707175254822,
"grad_norm": 0.11106614768505096,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -0.28053054213523865,
"logits/rejected": -0.276623010635376,
"logps/chosen": -1318.46240234375,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -1809.1522216796875,
"loss": 1.3858,
"margin_dpo/margin_mean": 471.72003173828125,
"margin_dpo/margin_std": 1663.8760986328125,
"step": 564
},
{
"KL/chosen_KL_mean": -1467.5367431640625,
"KL/mean": -1689.719482421875,
"KL/rejected_KL_mean": -1911.90234375,
"KL/std": 1842.6632080078125,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 444.36553955078125,
"fcm_dpo/q_t": 0.4998781979084015,
"grad_norm": 0.12787476181983948,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -0.3117861747741699,
"logits/rejected": -0.3122204542160034,
"logps/chosen": -1536.556640625,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -1997.6812744140625,
"loss": 1.3858,
"margin_dpo/margin_mean": 444.36553955078125,
"margin_dpo/margin_std": 2245.18408203125,
"step": 565
},
{
"KL/chosen_KL_mean": -1446.294677734375,
"KL/mean": -1779.049072265625,
"KL/rejected_KL_mean": -2111.803466796875,
"KL/std": 1603.380126953125,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 665.5087890625,
"fcm_dpo/q_t": 0.4998175799846649,
"grad_norm": 0.12405520677566528,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -0.30484163761138916,
"logits/rejected": -0.3086811304092407,
"logps/chosen": -1512.8399658203125,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -2215.6728515625,
"loss": 1.3856,
"margin_dpo/margin_mean": 665.5087890625,
"margin_dpo/margin_std": 2031.6650390625,
"step": 566
},
{
"KL/chosen_KL_mean": -1235.8284912109375,
"KL/mean": -1397.942626953125,
"KL/rejected_KL_mean": -1560.056884765625,
"KL/std": 1143.8087158203125,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 324.22845458984375,
"fcm_dpo/q_t": 0.4999111294746399,
"grad_norm": 0.15335722267627716,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -0.23657140135765076,
"logits/rejected": -0.24908195436000824,
"logps/chosen": -1288.686767578125,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -1645.427978515625,
"loss": 1.3859,
"margin_dpo/margin_mean": 324.22845458984375,
"margin_dpo/margin_std": 1360.3614501953125,
"step": 567
},
{
"KL/chosen_KL_mean": -988.34228515625,
"KL/mean": -1452.588623046875,
"KL/rejected_KL_mean": -1916.8349609375,
"KL/std": 1454.534423828125,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 928.4924926757812,
"fcm_dpo/q_t": 0.49974551796913147,
"grad_norm": 0.1366208791732788,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -0.28976213932037354,
"logits/rejected": -0.30281949043273926,
"logps/chosen": -1033.53466796875,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -2005.92724609375,
"loss": 1.3853,
"margin_dpo/margin_mean": 928.4924926757812,
"margin_dpo/margin_std": 1870.345947265625,
"step": 568
},
{
"KL/chosen_KL_mean": -1346.852294921875,
"KL/mean": -1611.91015625,
"KL/rejected_KL_mean": -1876.9678955078125,
"KL/std": 1544.3018798828125,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 530.1155395507812,
"fcm_dpo/q_t": 0.4998546838760376,
"grad_norm": 0.11353439837694168,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -0.282316118478775,
"logits/rejected": -0.28839677572250366,
"logps/chosen": -1410.5728759765625,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -1956.071044921875,
"loss": 1.3857,
"margin_dpo/margin_mean": 530.1155395507812,
"margin_dpo/margin_std": 1794.29150390625,
"step": 569
},
{
"KL/chosen_KL_mean": -1434.75341796875,
"KL/mean": -1581.2052001953125,
"KL/rejected_KL_mean": -1727.656982421875,
"KL/std": 1449.1363525390625,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 292.9036865234375,
"fcm_dpo/q_t": 0.49991971254348755,
"grad_norm": 0.19029207527637482,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -0.3203706443309784,
"logits/rejected": -0.32249945402145386,
"logps/chosen": -1496.367919921875,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -1809.7989501953125,
"loss": 1.386,
"margin_dpo/margin_mean": 292.9036865234375,
"margin_dpo/margin_std": 1891.564697265625,
"step": 570
},
{
"KL/chosen_KL_mean": -1393.2899169921875,
"KL/mean": -1646.490478515625,
"KL/rejected_KL_mean": -1899.691162109375,
"KL/std": 1575.805908203125,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 506.4012756347656,
"fcm_dpo/q_t": 0.4998612105846405,
"grad_norm": 0.10252400487661362,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -0.31373995542526245,
"logits/rejected": -0.3259819746017456,
"logps/chosen": -1446.343994140625,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -1991.02783203125,
"loss": 1.3857,
"margin_dpo/margin_mean": 506.4012145996094,
"margin_dpo/margin_std": 1877.108154296875,
"step": 571
},
{
"KL/chosen_KL_mean": -1139.670166015625,
"KL/mean": -1465.0963134765625,
"KL/rejected_KL_mean": -1790.5224609375,
"KL/std": 1222.9151611328125,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 650.8524169921875,
"fcm_dpo/q_t": 0.49982160329818726,
"grad_norm": 0.06481507420539856,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -0.29127341508865356,
"logits/rejected": -0.2989095449447632,
"logps/chosen": -1188.12939453125,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -1874.07958984375,
"loss": 1.3856,
"margin_dpo/margin_mean": 650.8524169921875,
"margin_dpo/margin_std": 1491.64501953125,
"step": 572
},
{
"KL/chosen_KL_mean": -1286.8150634765625,
"KL/mean": -1555.65185546875,
"KL/rejected_KL_mean": -1824.4886474609375,
"KL/std": 1397.0576171875,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 537.6735229492188,
"fcm_dpo/q_t": 0.49985259771347046,
"grad_norm": 0.19167454540729523,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -0.2431725263595581,
"logits/rejected": -0.24603477120399475,
"logps/chosen": -1338.437744140625,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -1909.8135986328125,
"loss": 1.3857,
"margin_dpo/margin_mean": 537.6735229492188,
"margin_dpo/margin_std": 1513.1845703125,
"step": 573
},
{
"KL/chosen_KL_mean": -1268.6588134765625,
"KL/mean": -1637.5677490234375,
"KL/rejected_KL_mean": -2006.4765625,
"KL/std": 1344.135009765625,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 737.81787109375,
"fcm_dpo/q_t": 0.4997977614402771,
"grad_norm": 0.1747097373008728,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -0.3217272460460663,
"logits/rejected": -0.3388410806655884,
"logps/chosen": -1319.7032470703125,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -2099.283203125,
"loss": 1.3855,
"margin_dpo/margin_mean": 737.81787109375,
"margin_dpo/margin_std": 1608.3687744140625,
"step": 574
},
{
"KL/chosen_KL_mean": -1481.532470703125,
"KL/mean": -1745.35205078125,
"KL/rejected_KL_mean": -2009.171630859375,
"KL/std": 1655.3306884765625,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 527.63916015625,
"fcm_dpo/q_t": 0.4998553693294525,
"grad_norm": 0.14064446091651917,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -0.28930941224098206,
"logits/rejected": -0.30165034532546997,
"logps/chosen": -1553.322509765625,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -2104.557861328125,
"loss": 1.3857,
"margin_dpo/margin_mean": 527.63916015625,
"margin_dpo/margin_std": 1978.1063232421875,
"step": 575
},
{
"KL/chosen_KL_mean": -1153.835205078125,
"KL/mean": -1575.923095703125,
"KL/rejected_KL_mean": -1998.0111083984375,
"KL/std": 1582.68212890625,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 844.176025390625,
"fcm_dpo/q_t": 0.49976861476898193,
"grad_norm": 0.17970313131809235,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -0.26422959566116333,
"logits/rejected": -0.2707493007183075,
"logps/chosen": -1208.09814453125,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -2098.765380859375,
"loss": 1.3854,
"margin_dpo/margin_mean": 844.176025390625,
"margin_dpo/margin_std": 1967.03369140625,
"step": 576
},
{
"KL/chosen_KL_mean": -1261.4288330078125,
"KL/mean": -1436.2958984375,
"KL/rejected_KL_mean": -1611.1629638671875,
"KL/std": 1171.484130859375,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 349.734130859375,
"fcm_dpo/q_t": 0.49990415573120117,
"grad_norm": 0.11516524851322174,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -0.28592342138290405,
"logits/rejected": -0.2935359477996826,
"logps/chosen": -1323.338623046875,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -1695.233642578125,
"loss": 1.3859,
"margin_dpo/margin_mean": 349.7341003417969,
"margin_dpo/margin_std": 1406.8392333984375,
"step": 577
},
{
"KL/chosen_KL_mean": -1049.0771484375,
"KL/mean": -1431.76318359375,
"KL/rejected_KL_mean": -1814.4490966796875,
"KL/std": 1241.43701171875,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 765.3720703125,
"fcm_dpo/q_t": 0.4997902512550354,
"grad_norm": 0.06985733658075333,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -0.26539209485054016,
"logits/rejected": -0.2741260528564453,
"logps/chosen": -1098.3408203125,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -1897.8853759765625,
"loss": 1.3855,
"margin_dpo/margin_mean": 765.3720092773438,
"margin_dpo/margin_std": 1238.185546875,
"step": 578
},
{
"KL/chosen_KL_mean": -1310.3199462890625,
"KL/mean": -1541.328857421875,
"KL/rejected_KL_mean": -1772.337890625,
"KL/std": 1645.46044921875,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 462.01800537109375,
"fcm_dpo/q_t": 0.4998733401298523,
"grad_norm": 0.11409077048301697,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -0.2609277367591858,
"logits/rejected": -0.27032503485679626,
"logps/chosen": -1369.205810546875,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -1867.12548828125,
"loss": 1.3858,
"margin_dpo/margin_mean": 462.0180358886719,
"margin_dpo/margin_std": 2106.6123046875,
"step": 579
},
{
"KL/chosen_KL_mean": -962.358154296875,
"KL/mean": -1297.83984375,
"KL/rejected_KL_mean": -1633.3214111328125,
"KL/std": 1324.714599609375,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 670.9632568359375,
"fcm_dpo/q_t": 0.49981608986854553,
"grad_norm": 0.15588979423046112,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -0.29815828800201416,
"logits/rejected": -0.31025731563568115,
"logps/chosen": -1011.06494140625,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -1715.079833984375,
"loss": 1.3856,
"margin_dpo/margin_mean": 670.9632568359375,
"margin_dpo/margin_std": 1515.060302734375,
"step": 580
},
{
"KL/chosen_KL_mean": -1320.83349609375,
"KL/mean": -1543.870361328125,
"KL/rejected_KL_mean": -1766.9071044921875,
"KL/std": 1370.167724609375,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 446.07366943359375,
"fcm_dpo/q_t": 0.4998777508735657,
"grad_norm": 0.11037159711122513,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -0.2854025065898895,
"logits/rejected": -0.29709920287132263,
"logps/chosen": -1383.055908203125,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -1858.642822265625,
"loss": 1.3858,
"margin_dpo/margin_mean": 446.0736999511719,
"margin_dpo/margin_std": 1553.092041015625,
"step": 581
},
{
"KL/chosen_KL_mean": -1259.357421875,
"KL/mean": -1655.957275390625,
"KL/rejected_KL_mean": -2052.55712890625,
"KL/std": 1327.726318359375,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 793.1998901367188,
"fcm_dpo/q_t": 0.4997825622558594,
"grad_norm": 0.182276651263237,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -0.29098963737487793,
"logits/rejected": -0.3058258593082428,
"logps/chosen": -1317.5860595703125,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -2162.626953125,
"loss": 1.3854,
"margin_dpo/margin_mean": 793.1998901367188,
"margin_dpo/margin_std": 1530.111083984375,
"step": 582
},
{
"KL/chosen_KL_mean": -1210.8515625,
"KL/mean": -1471.690185546875,
"KL/rejected_KL_mean": -1732.52880859375,
"KL/std": 1479.544677734375,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 521.67724609375,
"fcm_dpo/q_t": 0.49985700845718384,
"grad_norm": 0.2394990772008896,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -0.32131147384643555,
"logits/rejected": -0.3250162601470947,
"logps/chosen": -1267.7144775390625,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -1820.932861328125,
"loss": 1.3857,
"margin_dpo/margin_mean": 521.67724609375,
"margin_dpo/margin_std": 1832.6014404296875,
"step": 583
},
{
"KL/chosen_KL_mean": -1345.5399169921875,
"KL/mean": -1634.996826171875,
"KL/rejected_KL_mean": -1924.45361328125,
"KL/std": 1520.7857666015625,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 578.91357421875,
"fcm_dpo/q_t": 0.4998413324356079,
"grad_norm": 0.2904651463031769,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -0.3031461238861084,
"logits/rejected": -0.315969854593277,
"logps/chosen": -1402.440673828125,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -2022.089599609375,
"loss": 1.3857,
"margin_dpo/margin_mean": 578.91357421875,
"margin_dpo/margin_std": 1762.9036865234375,
"step": 584
},
{
"KL/chosen_KL_mean": -1292.40576171875,
"KL/mean": -1650.23583984375,
"KL/rejected_KL_mean": -2008.06591796875,
"KL/std": 1508.56494140625,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 715.66015625,
"fcm_dpo/q_t": 0.4998038411140442,
"grad_norm": 0.07096313685178757,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -0.3163740932941437,
"logits/rejected": -0.32364410161972046,
"logps/chosen": -1353.379638671875,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -2092.2353515625,
"loss": 1.3855,
"margin_dpo/margin_mean": 715.66015625,
"margin_dpo/margin_std": 2054.603515625,
"step": 585
},
{
"KL/chosen_KL_mean": -1451.108642578125,
"KL/mean": -1723.1895751953125,
"KL/rejected_KL_mean": -1995.2705078125,
"KL/std": 1733.5499267578125,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 544.1619262695312,
"fcm_dpo/q_t": 0.4998508393764496,
"grad_norm": 0.13893093168735504,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -0.26731252670288086,
"logits/rejected": -0.2813112139701843,
"logps/chosen": -1507.9642333984375,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -2087.0732421875,
"loss": 1.3857,
"margin_dpo/margin_mean": 544.1619262695312,
"margin_dpo/margin_std": 1499.9649658203125,
"step": 586
},
{
"KL/chosen_KL_mean": -989.1253662109375,
"KL/mean": -1301.080810546875,
"KL/rejected_KL_mean": -1613.0361328125,
"KL/std": 1182.3687744140625,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 623.9107666015625,
"fcm_dpo/q_t": 0.4998289942741394,
"grad_norm": 0.19780661165714264,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -0.26986804604530334,
"logits/rejected": -0.2820490598678589,
"logps/chosen": -1033.81689453125,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -1695.659912109375,
"loss": 1.3856,
"margin_dpo/margin_mean": 623.9107666015625,
"margin_dpo/margin_std": 1396.97021484375,
"step": 587
},
{
"KL/chosen_KL_mean": -1144.2742919921875,
"KL/mean": -1511.224609375,
"KL/rejected_KL_mean": -1878.175048828125,
"KL/std": 1298.37890625,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 733.900634765625,
"fcm_dpo/q_t": 0.49979886412620544,
"grad_norm": 0.20571593940258026,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -0.27409303188323975,
"logits/rejected": -0.28234797716140747,
"logps/chosen": -1194.5692138671875,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -1985.544921875,
"loss": 1.3855,
"margin_dpo/margin_mean": 733.900634765625,
"margin_dpo/margin_std": 1562.85205078125,
"step": 588
},
{
"KL/chosen_KL_mean": -1428.042724609375,
"KL/mean": -1843.700439453125,
"KL/rejected_KL_mean": -2259.358154296875,
"KL/std": 1713.4039306640625,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 831.3154296875,
"fcm_dpo/q_t": 0.4997721314430237,
"grad_norm": 0.11402349174022675,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -0.3039804995059967,
"logits/rejected": -0.31366050243377686,
"logps/chosen": -1487.97265625,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -2371.013671875,
"loss": 1.3854,
"margin_dpo/margin_mean": 831.3154296875,
"margin_dpo/margin_std": 2030.774658203125,
"step": 589
},
{
"KL/chosen_KL_mean": -1006.583251953125,
"KL/mean": -1440.79833984375,
"KL/rejected_KL_mean": -1875.013427734375,
"KL/std": 1557.4278564453125,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 868.43017578125,
"fcm_dpo/q_t": 0.4997619688510895,
"grad_norm": 0.09915481507778168,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -0.29607412219047546,
"logits/rejected": -0.30719584226608276,
"logps/chosen": -1062.39306640625,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -1981.076171875,
"loss": 1.3853,
"margin_dpo/margin_mean": 868.4302368164062,
"margin_dpo/margin_std": 1991.411865234375,
"step": 590
},
{
"KL/chosen_KL_mean": -1081.7724609375,
"KL/mean": -1400.282470703125,
"KL/rejected_KL_mean": -1718.79248046875,
"KL/std": 1268.409912109375,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 637.02001953125,
"fcm_dpo/q_t": 0.4998254179954529,
"grad_norm": 0.149546280503273,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -0.2419218122959137,
"logits/rejected": -0.2442915141582489,
"logps/chosen": -1138.0130615234375,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -1802.5787353515625,
"loss": 1.3856,
"margin_dpo/margin_mean": 637.02001953125,
"margin_dpo/margin_std": 1643.7451171875,
"step": 591
},
{
"KL/chosen_KL_mean": -1236.419189453125,
"KL/mean": -1495.9189453125,
"KL/rejected_KL_mean": -1755.418701171875,
"KL/std": 1230.9473876953125,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 518.9995727539062,
"fcm_dpo/q_t": 0.49985772371292114,
"grad_norm": 0.11462758481502533,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -0.2386205792427063,
"logits/rejected": -0.24258291721343994,
"logps/chosen": -1284.359375,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -1831.151611328125,
"loss": 1.3857,
"margin_dpo/margin_mean": 518.9995727539062,
"margin_dpo/margin_std": 1722.8271484375,
"step": 592
},
{
"KL/chosen_KL_mean": -1129.862548828125,
"KL/mean": -1286.4898681640625,
"KL/rejected_KL_mean": -1443.1171875,
"KL/std": 1180.02734375,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 313.25439453125,
"fcm_dpo/q_t": 0.49991413950920105,
"grad_norm": 0.07443266361951828,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -0.296904981136322,
"logits/rejected": -0.302301287651062,
"logps/chosen": -1178.5533447265625,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -1504.025146484375,
"loss": 1.386,
"margin_dpo/margin_mean": 313.2544250488281,
"margin_dpo/margin_std": 1417.1861572265625,
"step": 593
},
{
"KL/chosen_KL_mean": -1156.621337890625,
"KL/mean": -1437.635986328125,
"KL/rejected_KL_mean": -1718.6505126953125,
"KL/std": 1289.333984375,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 562.0292358398438,
"fcm_dpo/q_t": 0.499845951795578,
"grad_norm": 0.08804459124803543,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -0.29046431183815,
"logits/rejected": -0.291781485080719,
"logps/chosen": -1211.55615234375,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -1804.750244140625,
"loss": 1.3857,
"margin_dpo/margin_mean": 562.029296875,
"margin_dpo/margin_std": 1550.49609375,
"step": 594
},
{
"KL/chosen_KL_mean": -1090.362060546875,
"KL/mean": -1483.697021484375,
"KL/rejected_KL_mean": -1877.0322265625,
"KL/std": 1451.2752685546875,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 786.6702880859375,
"fcm_dpo/q_t": 0.49978435039520264,
"grad_norm": 0.18253269791603088,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -0.2718271017074585,
"logits/rejected": -0.2887038588523865,
"logps/chosen": -1139.782470703125,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -1957.65966796875,
"loss": 1.3854,
"margin_dpo/margin_mean": 786.6702880859375,
"margin_dpo/margin_std": 1885.255859375,
"step": 595
},
{
"KL/chosen_KL_mean": -1452.7657470703125,
"KL/mean": -1550.644287109375,
"KL/rejected_KL_mean": -1648.522705078125,
"KL/std": 1658.8240966796875,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 195.75689697265625,
"fcm_dpo/q_t": 0.4999462962150574,
"grad_norm": 0.1237877607345581,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -0.2950517535209656,
"logits/rejected": -0.30836156010627747,
"logps/chosen": -1512.557373046875,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -1728.61376953125,
"loss": 1.3861,
"margin_dpo/margin_mean": 195.75692749023438,
"margin_dpo/margin_std": 2088.86572265625,
"step": 596
},
{
"KL/chosen_KL_mean": -1269.8262939453125,
"KL/mean": -1677.495849609375,
"KL/rejected_KL_mean": -2085.16552734375,
"KL/std": 1558.5076904296875,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 815.3389892578125,
"fcm_dpo/q_t": 0.49977654218673706,
"grad_norm": 0.09684523195028305,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -0.2993488013744354,
"logits/rejected": -0.3121323585510254,
"logps/chosen": -1327.087158203125,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -2185.859130859375,
"loss": 1.3854,
"margin_dpo/margin_mean": 815.3390502929688,
"margin_dpo/margin_std": 1795.6142578125,
"step": 597
},
{
"KL/chosen_KL_mean": -1115.9447021484375,
"KL/mean": -1398.647216796875,
"KL/rejected_KL_mean": -1681.349853515625,
"KL/std": 1223.042236328125,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 565.4051513671875,
"fcm_dpo/q_t": 0.499845027923584,
"grad_norm": 0.0884322002530098,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -0.25602036714553833,
"logits/rejected": -0.26312920451164246,
"logps/chosen": -1168.463134765625,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -1770.793701171875,
"loss": 1.3857,
"margin_dpo/margin_mean": 565.4051513671875,
"margin_dpo/margin_std": 1462.629150390625,
"step": 598
},
{
"KL/chosen_KL_mean": -1233.416015625,
"KL/mean": -1432.468017578125,
"KL/rejected_KL_mean": -1631.52001953125,
"KL/std": 1322.410888671875,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 398.10406494140625,
"fcm_dpo/q_t": 0.49989089369773865,
"grad_norm": 0.11296947300434113,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -0.30364251136779785,
"logits/rejected": -0.3106527626514435,
"logps/chosen": -1283.21875,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -1714.49853515625,
"loss": 1.3859,
"margin_dpo/margin_mean": 398.10406494140625,
"margin_dpo/margin_std": 1483.496337890625,
"step": 599
},
{
"KL/chosen_KL_mean": -1541.261962890625,
"KL/mean": -1724.763427734375,
"KL/rejected_KL_mean": -1908.2646484375,
"KL/std": 1566.6689453125,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 367.0028991699219,
"fcm_dpo/q_t": 0.49989941716194153,
"grad_norm": 0.14191482961177826,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -0.28465044498443604,
"logits/rejected": -0.2864704430103302,
"logps/chosen": -1607.69677734375,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -1993.72119140625,
"loss": 1.3859,
"margin_dpo/margin_mean": 367.0029296875,
"margin_dpo/margin_std": 2167.3740234375,
"step": 600
},
{
"KL/chosen_KL_mean": -1384.5076904296875,
"KL/mean": -1658.9830322265625,
"KL/rejected_KL_mean": -1933.458251953125,
"KL/std": 1353.0380859375,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 548.95068359375,
"fcm_dpo/q_t": 0.4998495280742645,
"grad_norm": 0.10512831807136536,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -0.27232688665390015,
"logits/rejected": -0.28348982334136963,
"logps/chosen": -1443.641357421875,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -2028.1492919921875,
"loss": 1.3857,
"margin_dpo/margin_mean": 548.95068359375,
"margin_dpo/margin_std": 1670.189208984375,
"step": 601
},
{
"KL/chosen_KL_mean": -983.9776000976562,
"KL/mean": -1467.59375,
"KL/rejected_KL_mean": -1951.2099609375,
"KL/std": 1416.9248046875,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 967.2324829101562,
"fcm_dpo/q_t": 0.49973487854003906,
"grad_norm": 0.20267777144908905,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -0.3050517141819,
"logits/rejected": -0.31041401624679565,
"logps/chosen": -1032.5711669921875,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -2038.878662109375,
"loss": 1.3852,
"margin_dpo/margin_mean": 967.2324829101562,
"margin_dpo/margin_std": 1628.935302734375,
"step": 602
},
{
"KL/chosen_KL_mean": -1451.09375,
"KL/mean": -1751.2628173828125,
"KL/rejected_KL_mean": -2051.431884765625,
"KL/std": 1832.94677734375,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 600.338134765625,
"fcm_dpo/q_t": 0.49983546137809753,
"grad_norm": 0.17601169645786285,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -0.317105770111084,
"logits/rejected": -0.32393383979797363,
"logps/chosen": -1521.50830078125,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -2151.75732421875,
"loss": 1.3856,
"margin_dpo/margin_mean": 600.338134765625,
"margin_dpo/margin_std": 2273.0205078125,
"step": 603
},
{
"KL/chosen_KL_mean": -1229.3902587890625,
"KL/mean": -1712.8712158203125,
"KL/rejected_KL_mean": -2196.35205078125,
"KL/std": 1597.14794921875,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 966.9619140625,
"fcm_dpo/q_t": 0.49973493814468384,
"grad_norm": 0.16109979152679443,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -0.2505277395248413,
"logits/rejected": -0.2674684226512909,
"logps/chosen": -1275.848388671875,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -2288.20654296875,
"loss": 1.3852,
"margin_dpo/margin_mean": 966.9619750976562,
"margin_dpo/margin_std": 2260.287109375,
"step": 604
},
{
"KL/chosen_KL_mean": -1268.0721435546875,
"KL/mean": -1561.545654296875,
"KL/rejected_KL_mean": -1855.019287109375,
"KL/std": 1494.844482421875,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 586.9471435546875,
"fcm_dpo/q_t": 0.4998391270637512,
"grad_norm": 0.11086463928222656,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -0.32396382093429565,
"logits/rejected": -0.3408808708190918,
"logps/chosen": -1334.321533203125,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -1957.32421875,
"loss": 1.3857,
"margin_dpo/margin_mean": 586.9471435546875,
"margin_dpo/margin_std": 2043.394287109375,
"step": 605
},
{
"KL/chosen_KL_mean": -1332.7421875,
"KL/mean": -1664.57763671875,
"KL/rejected_KL_mean": -1996.4130859375,
"KL/std": 1885.167236328125,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 663.6708984375,
"fcm_dpo/q_t": 0.4998180866241455,
"grad_norm": 0.10756733268499374,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -0.26073816418647766,
"logits/rejected": -0.2791990637779236,
"logps/chosen": -1387.561279296875,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -2094.784423828125,
"loss": 1.3856,
"margin_dpo/margin_mean": 663.6708984375,
"margin_dpo/margin_std": 2427.927734375,
"step": 606
},
{
"KL/chosen_KL_mean": -1345.4205322265625,
"KL/mean": -1563.27392578125,
"KL/rejected_KL_mean": -1781.12744140625,
"KL/std": 1508.8651123046875,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 435.70703125,
"fcm_dpo/q_t": 0.4998805820941925,
"grad_norm": 0.14810404181480408,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -0.26724961400032043,
"logits/rejected": -0.26801979541778564,
"logps/chosen": -1403.5045166015625,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -1860.904541015625,
"loss": 1.3858,
"margin_dpo/margin_mean": 435.70703125,
"margin_dpo/margin_std": 1512.35107421875,
"step": 607
},
{
"KL/chosen_KL_mean": -1220.277099609375,
"KL/mean": -1495.2237548828125,
"KL/rejected_KL_mean": -1770.17041015625,
"KL/std": 1245.552734375,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 549.8931274414062,
"fcm_dpo/q_t": 0.49984925985336304,
"grad_norm": 0.1457754224538803,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -0.30721715092658997,
"logits/rejected": -0.30426025390625,
"logps/chosen": -1277.72802734375,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -1864.9437255859375,
"loss": 1.3857,
"margin_dpo/margin_mean": 549.8931274414062,
"margin_dpo/margin_std": 1515.70947265625,
"step": 608
},
{
"KL/chosen_KL_mean": -1087.919189453125,
"KL/mean": -1466.904052734375,
"KL/rejected_KL_mean": -1845.888671875,
"KL/std": 1240.9404296875,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 757.969482421875,
"fcm_dpo/q_t": 0.4997922480106354,
"grad_norm": 0.11522836238145828,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -0.2894991636276245,
"logits/rejected": -0.2997610569000244,
"logps/chosen": -1146.724609375,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -1934.70458984375,
"loss": 1.3855,
"margin_dpo/margin_mean": 757.969482421875,
"margin_dpo/margin_std": 1586.349609375,
"step": 609
},
{
"KL/chosen_KL_mean": -1374.5,
"KL/mean": -1497.664794921875,
"KL/rejected_KL_mean": -1620.8294677734375,
"KL/std": 1488.536865234375,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 246.32940673828125,
"fcm_dpo/q_t": 0.49993249773979187,
"grad_norm": 0.23700614273548126,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -0.30130696296691895,
"logits/rejected": -0.30593734979629517,
"logps/chosen": -1440.195068359375,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -1704.23486328125,
"loss": 1.386,
"margin_dpo/margin_mean": 246.32940673828125,
"margin_dpo/margin_std": 2019.56494140625,
"step": 610
},
{
"KL/chosen_KL_mean": -1254.2606201171875,
"KL/mean": -1662.910888671875,
"KL/rejected_KL_mean": -2071.561279296875,
"KL/std": 1563.9205322265625,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 817.3005981445312,
"fcm_dpo/q_t": 0.4997760057449341,
"grad_norm": 0.11909603327512741,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -0.2602178752422333,
"logits/rejected": -0.27245327830314636,
"logps/chosen": -1306.860107421875,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -2157.89208984375,
"loss": 1.3854,
"margin_dpo/margin_mean": 817.3007202148438,
"margin_dpo/margin_std": 1820.92919921875,
"step": 611
},
{
"KL/chosen_KL_mean": -1383.036376953125,
"KL/mean": -1637.382568359375,
"KL/rejected_KL_mean": -1891.7288818359375,
"KL/std": 1444.4013671875,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 508.692626953125,
"fcm_dpo/q_t": 0.49986058473587036,
"grad_norm": 0.17006807029247284,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -0.24827291071414948,
"logits/rejected": -0.2793204188346863,
"logps/chosen": -1442.3599853515625,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -1980.041259765625,
"loss": 1.3857,
"margin_dpo/margin_mean": 508.692626953125,
"margin_dpo/margin_std": 1928.337890625,
"step": 612
},
{
"KL/chosen_KL_mean": -1221.9931640625,
"KL/mean": -1589.656005859375,
"KL/rejected_KL_mean": -1957.319091796875,
"KL/std": 1439.966796875,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 735.325927734375,
"fcm_dpo/q_t": 0.4997984766960144,
"grad_norm": 0.09561502188444138,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -0.2727387547492981,
"logits/rejected": -0.2924768030643463,
"logps/chosen": -1281.72314453125,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -2062.4267578125,
"loss": 1.3855,
"margin_dpo/margin_mean": 735.325927734375,
"margin_dpo/margin_std": 1807.414306640625,
"step": 613
},
{
"KL/chosen_KL_mean": -1286.597900390625,
"KL/mean": -1722.857177734375,
"KL/rejected_KL_mean": -2159.116455078125,
"KL/std": 1653.336669921875,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 872.5186157226562,
"fcm_dpo/q_t": 0.49976086616516113,
"grad_norm": 0.130734384059906,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -0.28130537271499634,
"logits/rejected": -0.2899243235588074,
"logps/chosen": -1339.536865234375,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -2263.7958984375,
"loss": 1.3853,
"margin_dpo/margin_mean": 872.5186157226562,
"margin_dpo/margin_std": 2277.9794921875,
"step": 614
},
{
"KL/chosen_KL_mean": -1210.6558837890625,
"KL/mean": -1517.5098876953125,
"KL/rejected_KL_mean": -1824.3638916015625,
"KL/std": 1424.183837890625,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 613.7081298828125,
"fcm_dpo/q_t": 0.49983179569244385,
"grad_norm": 0.09694825112819672,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -0.306599497795105,
"logits/rejected": -0.31541186571121216,
"logps/chosen": -1276.47314453125,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -1919.5413818359375,
"loss": 1.3856,
"margin_dpo/margin_mean": 613.7081298828125,
"margin_dpo/margin_std": 1839.60302734375,
"step": 615
},
{
"KL/chosen_KL_mean": -1449.919189453125,
"KL/mean": -1705.6136474609375,
"KL/rejected_KL_mean": -1961.30810546875,
"KL/std": 1484.240966796875,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 511.3887939453125,
"fcm_dpo/q_t": 0.49985983967781067,
"grad_norm": 0.09458575397729874,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -0.2769748270511627,
"logits/rejected": -0.2750539779663086,
"logps/chosen": -1515.0521240234375,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -2036.008544921875,
"loss": 1.3857,
"margin_dpo/margin_mean": 511.3887939453125,
"margin_dpo/margin_std": 1792.7286376953125,
"step": 616
},
{
"KL/chosen_KL_mean": -1687.5743408203125,
"KL/mean": -1679.172119140625,
"KL/rejected_KL_mean": -1670.7698974609375,
"KL/std": 1601.2689208984375,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -16.80441665649414,
"fcm_dpo/q_t": 0.5000046491622925,
"grad_norm": 0.2592397928237915,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -0.28151631355285645,
"logits/rejected": -0.28539106249809265,
"logps/chosen": -1750.579833984375,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -1735.0040283203125,
"loss": 1.3863,
"margin_dpo/margin_mean": -16.804431915283203,
"margin_dpo/margin_std": 2027.412109375,
"step": 617
},
{
"KL/chosen_KL_mean": -1634.56396484375,
"KL/mean": -1991.8416748046875,
"KL/rejected_KL_mean": -2349.119140625,
"KL/std": 1661.743408203125,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 714.5552978515625,
"fcm_dpo/q_t": 0.49980413913726807,
"grad_norm": 0.1074250191450119,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -0.29117417335510254,
"logits/rejected": -0.2950536608695984,
"logps/chosen": -1701.665283203125,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -2441.272705078125,
"loss": 1.3855,
"margin_dpo/margin_mean": 714.5552978515625,
"margin_dpo/margin_std": 1959.51904296875,
"step": 618
},
{
"KL/chosen_KL_mean": -1394.2530517578125,
"KL/mean": -1653.03515625,
"KL/rejected_KL_mean": -1911.817138671875,
"KL/std": 1647.781982421875,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 517.564208984375,
"fcm_dpo/q_t": 0.49985814094543457,
"grad_norm": 0.1203947588801384,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -0.29787400364875793,
"logits/rejected": -0.30747509002685547,
"logps/chosen": -1450.2313232421875,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -2005.002685546875,
"loss": 1.3857,
"margin_dpo/margin_mean": 517.5641479492188,
"margin_dpo/margin_std": 2044.474853515625,
"step": 619
},
{
"KL/chosen_KL_mean": -1305.902099609375,
"KL/mean": -1547.302734375,
"KL/rejected_KL_mean": -1788.7034912109375,
"KL/std": 1333.70654296875,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 482.80145263671875,
"fcm_dpo/q_t": 0.49986767768859863,
"grad_norm": 0.10311917960643768,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -0.2799570858478546,
"logits/rejected": -0.2871643304824829,
"logps/chosen": -1365.6995849609375,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -1867.1142578125,
"loss": 1.3858,
"margin_dpo/margin_mean": 482.8013916015625,
"margin_dpo/margin_std": 1826.294189453125,
"step": 620
},
{
"KL/chosen_KL_mean": -1232.763427734375,
"KL/mean": -1668.678955078125,
"KL/rejected_KL_mean": -2104.59423828125,
"KL/std": 1500.2493896484375,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 871.8309936523438,
"fcm_dpo/q_t": 0.49976104497909546,
"grad_norm": 0.08543728291988373,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -0.26303690671920776,
"logits/rejected": -0.27306389808654785,
"logps/chosen": -1286.697265625,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -2192.9638671875,
"loss": 1.3853,
"margin_dpo/margin_mean": 871.8309326171875,
"margin_dpo/margin_std": 1943.66357421875,
"step": 621
},
{
"KL/chosen_KL_mean": -1230.0145263671875,
"KL/mean": -1463.860107421875,
"KL/rejected_KL_mean": -1697.70556640625,
"KL/std": 1287.034423828125,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 467.6911926269531,
"fcm_dpo/q_t": 0.49987179040908813,
"grad_norm": 0.12527480721473694,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -0.2892225980758667,
"logits/rejected": -0.29653114080429077,
"logps/chosen": -1290.30029296875,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -1783.224365234375,
"loss": 1.3858,
"margin_dpo/margin_mean": 467.6911926269531,
"margin_dpo/margin_std": 1662.3623046875,
"step": 622
},
{
"KL/chosen_KL_mean": -1547.281494140625,
"KL/mean": -1695.7080078125,
"KL/rejected_KL_mean": -1844.134765625,
"KL/std": 1741.5623779296875,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 296.853271484375,
"fcm_dpo/q_t": 0.4999186396598816,
"grad_norm": 0.21695291996002197,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -0.2584553360939026,
"logits/rejected": -0.25353342294692993,
"logps/chosen": -1611.4383544921875,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -1929.2177734375,
"loss": 1.386,
"margin_dpo/margin_mean": 296.8532409667969,
"margin_dpo/margin_std": 2120.8935546875,
"step": 623
},
{
"KL/chosen_KL_mean": -1372.0699462890625,
"KL/mean": -1760.053955078125,
"KL/rejected_KL_mean": -2148.037841796875,
"KL/std": 1600.662353515625,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 775.9679565429688,
"fcm_dpo/q_t": 0.4997873306274414,
"grad_norm": 0.13158676028251648,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -0.30938100814819336,
"logits/rejected": -0.3142179846763611,
"logps/chosen": -1443.988525390625,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -2245.169921875,
"loss": 1.3854,
"margin_dpo/margin_mean": 775.9679565429688,
"margin_dpo/margin_std": 2050.025390625,
"step": 624
},
{
"KL/chosen_KL_mean": -1321.643798828125,
"KL/mean": -1664.8505859375,
"KL/rejected_KL_mean": -2008.057373046875,
"KL/std": 1380.1881103515625,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 686.41357421875,
"fcm_dpo/q_t": 0.49981188774108887,
"grad_norm": 0.08273319900035858,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -0.2660679519176483,
"logits/rejected": -0.26351404190063477,
"logps/chosen": -1379.98583984375,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -2094.147705078125,
"loss": 1.3855,
"margin_dpo/margin_mean": 686.41357421875,
"margin_dpo/margin_std": 1431.6141357421875,
"step": 625
},
{
"KL/chosen_KL_mean": -1492.2509765625,
"KL/mean": -1669.962890625,
"KL/rejected_KL_mean": -1847.6748046875,
"KL/std": 1588.1826171875,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 355.42376708984375,
"fcm_dpo/q_t": 0.49990254640579224,
"grad_norm": 0.11186686158180237,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -0.298173189163208,
"logits/rejected": -0.3013492822647095,
"logps/chosen": -1567.363525390625,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -1946.863525390625,
"loss": 1.3859,
"margin_dpo/margin_mean": 355.4237365722656,
"margin_dpo/margin_std": 1804.6630859375,
"step": 626
},
{
"KL/chosen_KL_mean": -888.7783203125,
"KL/mean": -1339.638916015625,
"KL/rejected_KL_mean": -1790.4996337890625,
"KL/std": 1510.39453125,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 901.7212524414062,
"fcm_dpo/q_t": 0.49975284934043884,
"grad_norm": 0.09694371372461319,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -0.27288323640823364,
"logits/rejected": -0.2847945988178253,
"logps/chosen": -936.5214233398438,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -1897.254150390625,
"loss": 1.3853,
"margin_dpo/margin_mean": 901.7213134765625,
"margin_dpo/margin_std": 1993.0135498046875,
"step": 627
},
{
"KL/chosen_KL_mean": -1294.9599609375,
"KL/mean": -1728.273193359375,
"KL/rejected_KL_mean": -2161.586181640625,
"KL/std": 1660.6864013671875,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 866.6260986328125,
"fcm_dpo/q_t": 0.49976247549057007,
"grad_norm": 0.12284790724515915,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -0.2784745693206787,
"logits/rejected": -0.2820804715156555,
"logps/chosen": -1355.14306640625,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -2263.140869140625,
"loss": 1.3853,
"margin_dpo/margin_mean": 866.6260986328125,
"margin_dpo/margin_std": 2102.763671875,
"step": 628
},
{
"KL/chosen_KL_mean": -1483.812255859375,
"KL/mean": -1790.922607421875,
"KL/rejected_KL_mean": -2098.032958984375,
"KL/std": 1915.09033203125,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 614.2205810546875,
"fcm_dpo/q_t": 0.4998316466808319,
"grad_norm": 0.12040552496910095,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -0.23021405935287476,
"logits/rejected": -0.23377496004104614,
"logps/chosen": -1548.02587890625,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -2189.6865234375,
"loss": 1.3856,
"margin_dpo/margin_mean": 614.2206420898438,
"margin_dpo/margin_std": 2356.16259765625,
"step": 629
},
{
"KL/chosen_KL_mean": -1243.9722900390625,
"KL/mean": -1250.4580078125,
"KL/rejected_KL_mean": -1256.94384765625,
"KL/std": 1320.2613525390625,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 12.971527099609375,
"fcm_dpo/q_t": 0.4999964237213135,
"grad_norm": 0.1596960425376892,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -0.30787450075149536,
"logits/rejected": -0.30197858810424805,
"logps/chosen": -1303.2633056640625,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -1340.5421142578125,
"loss": 1.3863,
"margin_dpo/margin_mean": 12.971527099609375,
"margin_dpo/margin_std": 1790.94384765625,
"step": 630
},
{
"KL/chosen_KL_mean": -1294.0426025390625,
"KL/mean": -1551.5064697265625,
"KL/rejected_KL_mean": -1808.97021484375,
"KL/std": 1244.8095703125,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 514.9276733398438,
"fcm_dpo/q_t": 0.4998588562011719,
"grad_norm": 0.11925707757472992,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -0.24693992733955383,
"logits/rejected": -0.26080799102783203,
"logps/chosen": -1353.4962158203125,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -1889.921875,
"loss": 1.3857,
"margin_dpo/margin_mean": 514.927734375,
"margin_dpo/margin_std": 1695.441162109375,
"step": 631
},
{
"KL/chosen_KL_mean": -1142.801513671875,
"KL/mean": -1368.833984375,
"KL/rejected_KL_mean": -1594.8663330078125,
"KL/std": 1164.1339111328125,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 452.06488037109375,
"fcm_dpo/q_t": 0.49987608194351196,
"grad_norm": 0.08552023023366928,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -0.25456511974334717,
"logits/rejected": -0.2616008222103119,
"logps/chosen": -1204.153076171875,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -1681.0264892578125,
"loss": 1.3858,
"margin_dpo/margin_mean": 452.06488037109375,
"margin_dpo/margin_std": 1714.6295166015625,
"step": 632
},
{
"KL/chosen_KL_mean": -1356.769287109375,
"KL/mean": -1573.962158203125,
"KL/rejected_KL_mean": -1791.1549072265625,
"KL/std": 1136.525390625,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 434.38580322265625,
"fcm_dpo/q_t": 0.49988090991973877,
"grad_norm": 0.10118667781352997,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -0.26894307136535645,
"logits/rejected": -0.2839438021183014,
"logps/chosen": -1414.04736328125,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -1882.7388916015625,
"loss": 1.3858,
"margin_dpo/margin_mean": 434.38580322265625,
"margin_dpo/margin_std": 1371.88671875,
"step": 633
},
{
"KL/chosen_KL_mean": -1407.193359375,
"KL/mean": -1582.7567138671875,
"KL/rejected_KL_mean": -1758.320068359375,
"KL/std": 1496.943359375,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 351.12677001953125,
"fcm_dpo/q_t": 0.49990376830101013,
"grad_norm": 0.10061313956975937,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -0.312860369682312,
"logits/rejected": -0.32211965322494507,
"logps/chosen": -1473.812255859375,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -1865.44580078125,
"loss": 1.3859,
"margin_dpo/margin_mean": 351.1267395019531,
"margin_dpo/margin_std": 1739.1158447265625,
"step": 634
},
{
"KL/chosen_KL_mean": -1539.84619140625,
"KL/mean": -1678.4853515625,
"KL/rejected_KL_mean": -1817.12451171875,
"KL/std": 1806.3402099609375,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 277.2781982421875,
"fcm_dpo/q_t": 0.4999240040779114,
"grad_norm": 0.15843528509140015,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -0.3303324580192566,
"logits/rejected": -0.31194010376930237,
"logps/chosen": -1613.8017578125,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -1899.625,
"loss": 1.386,
"margin_dpo/margin_mean": 277.2781982421875,
"margin_dpo/margin_std": 1970.5750732421875,
"step": 635
},
{
"KL/chosen_KL_mean": -1336.476318359375,
"KL/mean": -1476.48828125,
"KL/rejected_KL_mean": -1616.5,
"KL/std": 1605.052734375,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 280.0237731933594,
"fcm_dpo/q_t": 0.4999232590198517,
"grad_norm": 0.10239730030298233,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -0.3119809031486511,
"logits/rejected": -0.3075520992279053,
"logps/chosen": -1396.105224609375,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -1698.4788818359375,
"loss": 1.386,
"margin_dpo/margin_mean": 280.0237731933594,
"margin_dpo/margin_std": 2099.6357421875,
"step": 636
},
{
"KL/chosen_KL_mean": -1203.1727294921875,
"KL/mean": -1576.09716796875,
"KL/rejected_KL_mean": -1949.0216064453125,
"KL/std": 1357.664306640625,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 745.848876953125,
"fcm_dpo/q_t": 0.4997955560684204,
"grad_norm": 0.10817310959100723,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -0.2595457136631012,
"logits/rejected": -0.27550387382507324,
"logps/chosen": -1252.825439453125,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -2047.4267578125,
"loss": 1.3855,
"margin_dpo/margin_mean": 745.848876953125,
"margin_dpo/margin_std": 1527.508056640625,
"step": 637
},
{
"KL/chosen_KL_mean": -1121.654052734375,
"KL/mean": -1419.8997802734375,
"KL/rejected_KL_mean": -1718.1455078125,
"KL/std": 1313.08056640625,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 596.4912719726562,
"fcm_dpo/q_t": 0.4998365044593811,
"grad_norm": 0.0910566970705986,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -0.28878459334373474,
"logits/rejected": -0.29849740862846375,
"logps/chosen": -1179.810791015625,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -1797.4468994140625,
"loss": 1.3856,
"margin_dpo/margin_mean": 596.4913330078125,
"margin_dpo/margin_std": 1834.4677734375,
"step": 638
},
{
"KL/chosen_KL_mean": -1723.419189453125,
"KL/mean": -1606.199951171875,
"KL/rejected_KL_mean": -1488.980712890625,
"KL/std": 1386.6025390625,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -234.43856811523438,
"fcm_dpo/q_t": 0.5000642538070679,
"grad_norm": 0.28992533683776855,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -0.2773544192314148,
"logits/rejected": -0.27241623401641846,
"logps/chosen": -1795.742431640625,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -1563.255615234375,
"loss": 1.3866,
"margin_dpo/margin_mean": -234.4385223388672,
"margin_dpo/margin_std": 1711.879638671875,
"step": 639
},
{
"KL/chosen_KL_mean": -1145.3177490234375,
"KL/mean": -1647.8726806640625,
"KL/rejected_KL_mean": -2150.427490234375,
"KL/std": 1739.093505859375,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1005.10986328125,
"fcm_dpo/q_t": 0.49972450733184814,
"grad_norm": 0.08556817471981049,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -0.2978939414024353,
"logits/rejected": -0.31190764904022217,
"logps/chosen": -1201.4521484375,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -2259.02783203125,
"loss": 1.3852,
"margin_dpo/margin_mean": 1005.10986328125,
"margin_dpo/margin_std": 2191.3310546875,
"step": 640
},
{
"KL/chosen_KL_mean": -1309.43408203125,
"KL/mean": -1710.717529296875,
"KL/rejected_KL_mean": -2112.0009765625,
"KL/std": 1404.828857421875,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 802.56689453125,
"fcm_dpo/q_t": 0.4997800290584564,
"grad_norm": 0.11998488008975983,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -0.23249101638793945,
"logits/rejected": -0.23721981048583984,
"logps/chosen": -1374.430908203125,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -2198.9931640625,
"loss": 1.3854,
"margin_dpo/margin_mean": 802.56689453125,
"margin_dpo/margin_std": 1926.6470947265625,
"step": 641
},
{
"KL/chosen_KL_mean": -1485.472900390625,
"KL/mean": -1839.117919921875,
"KL/rejected_KL_mean": -2192.7626953125,
"KL/std": 1906.6669921875,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 707.289794921875,
"fcm_dpo/q_t": 0.49980613589286804,
"grad_norm": 0.1159994825720787,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -0.2811075448989868,
"logits/rejected": -0.293199360370636,
"logps/chosen": -1551.1622314453125,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -2303.0048828125,
"loss": 1.3855,
"margin_dpo/margin_mean": 707.289794921875,
"margin_dpo/margin_std": 2484.1240234375,
"step": 642
},
{
"KL/chosen_KL_mean": -1354.5499267578125,
"KL/mean": -1558.93896484375,
"KL/rejected_KL_mean": -1763.3280029296875,
"KL/std": 1380.723876953125,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 408.7780456542969,
"fcm_dpo/q_t": 0.49988794326782227,
"grad_norm": 0.1577404886484146,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -0.24811697006225586,
"logits/rejected": -0.2620430588722229,
"logps/chosen": -1406.5,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -1850.79638671875,
"loss": 1.3858,
"margin_dpo/margin_mean": 408.778076171875,
"margin_dpo/margin_std": 1750.3544921875,
"step": 643
},
{
"KL/chosen_KL_mean": -1188.603759765625,
"KL/mean": -1472.237548828125,
"KL/rejected_KL_mean": -1755.87158203125,
"KL/std": 1589.1102294921875,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 567.2678833007812,
"fcm_dpo/q_t": 0.499844491481781,
"grad_norm": 0.11378132551908493,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -0.28449851274490356,
"logits/rejected": -0.28798413276672363,
"logps/chosen": -1247.621337890625,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -1843.00830078125,
"loss": 1.3857,
"margin_dpo/margin_mean": 567.2678833007812,
"margin_dpo/margin_std": 1886.039306640625,
"step": 644
},
{
"KL/chosen_KL_mean": -1202.0780029296875,
"KL/mean": -1344.8292236328125,
"KL/rejected_KL_mean": -1487.580322265625,
"KL/std": 1167.88427734375,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 285.50225830078125,
"fcm_dpo/q_t": 0.4999217391014099,
"grad_norm": 0.09203989803791046,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -0.27387818694114685,
"logits/rejected": -0.28450918197631836,
"logps/chosen": -1257.9541015625,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -1585.361083984375,
"loss": 1.386,
"margin_dpo/margin_mean": 285.50225830078125,
"margin_dpo/margin_std": 1691.6541748046875,
"step": 645
},
{
"KL/chosen_KL_mean": -1182.0609130859375,
"KL/mean": -1340.3609619140625,
"KL/rejected_KL_mean": -1498.6610107421875,
"KL/std": 1275.49169921875,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 316.60015869140625,
"fcm_dpo/q_t": 0.49991321563720703,
"grad_norm": 0.1333022117614746,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -0.2691464424133301,
"logits/rejected": -0.2712402939796448,
"logps/chosen": -1243.336669921875,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -1576.1668701171875,
"loss": 1.3859,
"margin_dpo/margin_mean": 316.6001892089844,
"margin_dpo/margin_std": 1448.23486328125,
"step": 646
},
{
"KL/chosen_KL_mean": -1000.0733032226562,
"KL/mean": -1245.947998046875,
"KL/rejected_KL_mean": -1491.822509765625,
"KL/std": 1033.54150390625,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 491.749267578125,
"fcm_dpo/q_t": 0.49986523389816284,
"grad_norm": 0.15133433043956757,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -0.29532963037490845,
"logits/rejected": -0.3036816120147705,
"logps/chosen": -1054.92578125,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -1585.342041015625,
"loss": 1.3858,
"margin_dpo/margin_mean": 491.74920654296875,
"margin_dpo/margin_std": 1186.4923095703125,
"step": 647
},
{
"KL/chosen_KL_mean": -1204.0748291015625,
"KL/mean": -1623.40478515625,
"KL/rejected_KL_mean": -2042.7344970703125,
"KL/std": 1525.134033203125,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 838.65966796875,
"fcm_dpo/q_t": 0.4997701048851013,
"grad_norm": 0.16558484733104706,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -0.26186037063598633,
"logits/rejected": -0.2704648971557617,
"logps/chosen": -1258.246337890625,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -2141.447265625,
"loss": 1.3854,
"margin_dpo/margin_mean": 838.65966796875,
"margin_dpo/margin_std": 1715.158203125,
"step": 648
},
{
"KL/chosen_KL_mean": -1204.949462890625,
"KL/mean": -1460.752685546875,
"KL/rejected_KL_mean": -1716.555908203125,
"KL/std": 1403.10546875,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 511.6065368652344,
"fcm_dpo/q_t": 0.4998597502708435,
"grad_norm": 0.08931561559438705,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -0.28651025891304016,
"logits/rejected": -0.287070095539093,
"logps/chosen": -1267.4296875,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -1796.633056640625,
"loss": 1.3857,
"margin_dpo/margin_mean": 511.6065368652344,
"margin_dpo/margin_std": 1865.652099609375,
"step": 649
},
{
"KL/chosen_KL_mean": -1396.656494140625,
"KL/mean": -1628.199951171875,
"KL/rejected_KL_mean": -1859.7435302734375,
"KL/std": 1600.5380859375,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 463.08709716796875,
"fcm_dpo/q_t": 0.4998730719089508,
"grad_norm": 0.07497455924749374,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -0.2688780128955841,
"logits/rejected": -0.2748976945877075,
"logps/chosen": -1452.749267578125,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -1958.0084228515625,
"loss": 1.3858,
"margin_dpo/margin_mean": 463.08709716796875,
"margin_dpo/margin_std": 2139.24658203125,
"step": 650
},
{
"KL/chosen_KL_mean": -969.076171875,
"KL/mean": -1413.8095703125,
"KL/rejected_KL_mean": -1858.5428466796875,
"KL/std": 1293.34228515625,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 889.4666748046875,
"fcm_dpo/q_t": 0.49975621700286865,
"grad_norm": 0.19173209369182587,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -0.24564704298973083,
"logits/rejected": -0.2613285779953003,
"logps/chosen": -1012.5015869140625,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -1958.500732421875,
"loss": 1.3853,
"margin_dpo/margin_mean": 889.4666748046875,
"margin_dpo/margin_std": 1415.29345703125,
"step": 651
},
{
"KL/chosen_KL_mean": -1144.052734375,
"KL/mean": -1469.6400146484375,
"KL/rejected_KL_mean": -1795.227294921875,
"KL/std": 1436.3759765625,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 651.1746826171875,
"fcm_dpo/q_t": 0.4998215436935425,
"grad_norm": 0.2549162805080414,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -0.2820361852645874,
"logits/rejected": -0.2880634665489197,
"logps/chosen": -1206.62939453125,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -1906.9951171875,
"loss": 1.3856,
"margin_dpo/margin_mean": 651.1746215820312,
"margin_dpo/margin_std": 1813.627197265625,
"step": 652
},
{
"KL/chosen_KL_mean": -1301.207763671875,
"KL/mean": -1669.75732421875,
"KL/rejected_KL_mean": -2038.306884765625,
"KL/std": 1641.0732421875,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 737.0989990234375,
"fcm_dpo/q_t": 0.4997979998588562,
"grad_norm": 0.15215131640434265,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -0.2845923900604248,
"logits/rejected": -0.295174241065979,
"logps/chosen": -1362.32080078125,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -2141.556396484375,
"loss": 1.3855,
"margin_dpo/margin_mean": 737.0989990234375,
"margin_dpo/margin_std": 1964.80078125,
"step": 653
},
{
"KL/chosen_KL_mean": -1242.6630859375,
"KL/mean": -1478.22998046875,
"KL/rejected_KL_mean": -1713.796875,
"KL/std": 1377.341796875,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 471.13385009765625,
"fcm_dpo/q_t": 0.4998708665370941,
"grad_norm": 0.11739946901798248,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -0.2545585632324219,
"logits/rejected": -0.2595174312591553,
"logps/chosen": -1304.391357421875,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -1812.57080078125,
"loss": 1.3858,
"margin_dpo/margin_mean": 471.13385009765625,
"margin_dpo/margin_std": 1693.6844482421875,
"step": 654
},
{
"KL/chosen_KL_mean": -1132.216552734375,
"KL/mean": -1500.26123046875,
"KL/rejected_KL_mean": -1868.305908203125,
"KL/std": 1507.53271484375,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 736.0894165039062,
"fcm_dpo/q_t": 0.4997982382774353,
"grad_norm": 0.24736790359020233,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -0.26983144879341125,
"logits/rejected": -0.2772262692451477,
"logps/chosen": -1181.793212890625,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -1966.59765625,
"loss": 1.3855,
"margin_dpo/margin_mean": 736.0894165039062,
"margin_dpo/margin_std": 1864.055908203125,
"step": 655
},
{
"KL/chosen_KL_mean": -1159.69091796875,
"KL/mean": -1784.111572265625,
"KL/rejected_KL_mean": -2408.5322265625,
"KL/std": 1622.1591796875,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1248.8411865234375,
"fcm_dpo/q_t": 0.4996577203273773,
"grad_norm": 0.3061199188232422,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -0.2754252552986145,
"logits/rejected": -0.2866843342781067,
"logps/chosen": -1212.240478515625,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -2522.206787109375,
"loss": 1.3849,
"margin_dpo/margin_mean": 1248.8411865234375,
"margin_dpo/margin_std": 1730.600830078125,
"step": 656
},
{
"KL/chosen_KL_mean": -1232.510986328125,
"KL/mean": -1685.77783203125,
"KL/rejected_KL_mean": -2139.04443359375,
"KL/std": 1773.5999755859375,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 906.533203125,
"fcm_dpo/q_t": 0.4997515082359314,
"grad_norm": 0.09144622087478638,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -0.2707776427268982,
"logits/rejected": -0.29089421033859253,
"logps/chosen": -1279.211669921875,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -2236.959228515625,
"loss": 1.3853,
"margin_dpo/margin_mean": 906.533203125,
"margin_dpo/margin_std": 1992.08984375,
"step": 657
},
{
"KL/chosen_KL_mean": -1293.7935791015625,
"KL/mean": -1548.7041015625,
"KL/rejected_KL_mean": -1803.614501953125,
"KL/std": 1294.736572265625,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 509.8209228515625,
"fcm_dpo/q_t": 0.4998602569103241,
"grad_norm": 0.13024263083934784,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -0.22683298587799072,
"logits/rejected": -0.23709672689437866,
"logps/chosen": -1354.7518310546875,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -1899.553955078125,
"loss": 1.3857,
"margin_dpo/margin_mean": 509.82086181640625,
"margin_dpo/margin_std": 1722.4686279296875,
"step": 658
},
{
"KL/chosen_KL_mean": -1185.252197265625,
"KL/mean": -1470.142578125,
"KL/rejected_KL_mean": -1755.032958984375,
"KL/std": 1306.4189453125,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 569.7805786132812,
"fcm_dpo/q_t": 0.4998438358306885,
"grad_norm": 0.08449683338403702,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -0.30121439695358276,
"logits/rejected": -0.29801568388938904,
"logps/chosen": -1261.9952392578125,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -1842.50390625,
"loss": 1.3857,
"margin_dpo/margin_mean": 569.7805786132812,
"margin_dpo/margin_std": 1671.21728515625,
"step": 659
},
{
"KL/chosen_KL_mean": -1273.4119873046875,
"KL/mean": -1517.3409423828125,
"KL/rejected_KL_mean": -1761.269775390625,
"KL/std": 1257.9273681640625,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 487.85797119140625,
"fcm_dpo/q_t": 0.4998662769794464,
"grad_norm": 0.15911172330379486,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -0.246282696723938,
"logits/rejected": -0.2528143525123596,
"logps/chosen": -1332.4599609375,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -1837.22998046875,
"loss": 1.3858,
"margin_dpo/margin_mean": 487.8579406738281,
"margin_dpo/margin_std": 1523.2557373046875,
"step": 660
},
{
"KL/chosen_KL_mean": -1017.4349365234375,
"KL/mean": -1504.7825927734375,
"KL/rejected_KL_mean": -1992.1302490234375,
"KL/std": 2107.52685546875,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 974.6953735351562,
"fcm_dpo/q_t": 0.4997328221797943,
"grad_norm": 0.09211470186710358,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -0.29294365644454956,
"logits/rejected": -0.3058052062988281,
"logps/chosen": -1068.10888671875,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -2078.135986328125,
"loss": 1.3852,
"margin_dpo/margin_mean": 974.6953735351562,
"margin_dpo/margin_std": 2413.55224609375,
"step": 661
},
{
"KL/chosen_KL_mean": -1166.93701171875,
"KL/mean": -1460.897705078125,
"KL/rejected_KL_mean": -1754.8583984375,
"KL/std": 1245.419189453125,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 587.92138671875,
"fcm_dpo/q_t": 0.49983882904052734,
"grad_norm": 0.09338005632162094,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -0.301297128200531,
"logits/rejected": -0.2998049855232239,
"logps/chosen": -1236.1981201171875,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -1843.914306640625,
"loss": 1.3857,
"margin_dpo/margin_mean": 587.92138671875,
"margin_dpo/margin_std": 1546.5623779296875,
"step": 662
},
{
"KL/chosen_KL_mean": -1115.79345703125,
"KL/mean": -1571.531005859375,
"KL/rejected_KL_mean": -2027.2685546875,
"KL/std": 1618.50732421875,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 911.47509765625,
"fcm_dpo/q_t": 0.49975016713142395,
"grad_norm": 0.11960398405790329,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -0.33110612630844116,
"logits/rejected": -0.3475431799888611,
"logps/chosen": -1180.67236328125,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -2141.19384765625,
"loss": 1.3853,
"margin_dpo/margin_mean": 911.47509765625,
"margin_dpo/margin_std": 2104.358154296875,
"step": 663
},
{
"KL/chosen_KL_mean": -1458.552978515625,
"KL/mean": -1998.9429931640625,
"KL/rejected_KL_mean": -2539.3330078125,
"KL/std": 2229.985595703125,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1080.7802734375,
"fcm_dpo/q_t": 0.4997037649154663,
"grad_norm": 0.11196719855070114,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -0.2733624577522278,
"logits/rejected": -0.2783682346343994,
"logps/chosen": -1519.44140625,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -2644.85498046875,
"loss": 1.3851,
"margin_dpo/margin_mean": 1080.7801513671875,
"margin_dpo/margin_std": 2739.387939453125,
"step": 664
},
{
"KL/chosen_KL_mean": -1209.31884765625,
"KL/mean": -1550.11328125,
"KL/rejected_KL_mean": -1890.90771484375,
"KL/std": 1507.995361328125,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 681.5887451171875,
"fcm_dpo/q_t": 0.4998131990432739,
"grad_norm": 0.12111763656139374,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -0.19828855991363525,
"logits/rejected": -0.2052626609802246,
"logps/chosen": -1269.883056640625,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -1975.716552734375,
"loss": 1.3855,
"margin_dpo/margin_mean": 681.5887451171875,
"margin_dpo/margin_std": 1712.932373046875,
"step": 665
},
{
"KL/chosen_KL_mean": -1191.26708984375,
"KL/mean": -1576.1512451171875,
"KL/rejected_KL_mean": -1961.035400390625,
"KL/std": 1501.64208984375,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 769.7685546875,
"fcm_dpo/q_t": 0.4997889995574951,
"grad_norm": 0.10389173030853271,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -0.2726587951183319,
"logits/rejected": -0.27047300338745117,
"logps/chosen": -1255.68701171875,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -2056.92724609375,
"loss": 1.3855,
"margin_dpo/margin_mean": 769.7685546875,
"margin_dpo/margin_std": 1734.6624755859375,
"step": 666
},
{
"KL/chosen_KL_mean": -1549.532958984375,
"KL/mean": -1783.92333984375,
"KL/rejected_KL_mean": -2018.3138427734375,
"KL/std": 1649.5316162109375,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 468.78094482421875,
"fcm_dpo/q_t": 0.49987149238586426,
"grad_norm": 0.2005223035812378,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -0.289089560508728,
"logits/rejected": -0.286052405834198,
"logps/chosen": -1618.81005859375,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -2106.1494140625,
"loss": 1.3858,
"margin_dpo/margin_mean": 468.78094482421875,
"margin_dpo/margin_std": 1956.940185546875,
"step": 667
},
{
"KL/chosen_KL_mean": -1474.34619140625,
"KL/mean": -1627.808349609375,
"KL/rejected_KL_mean": -1781.2705078125,
"KL/std": 1383.871826171875,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 306.92425537109375,
"fcm_dpo/q_t": 0.4999158978462219,
"grad_norm": 0.3075452744960785,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -0.29430443048477173,
"logits/rejected": -0.30596286058425903,
"logps/chosen": -1546.9501953125,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -1885.009521484375,
"loss": 1.386,
"margin_dpo/margin_mean": 306.92425537109375,
"margin_dpo/margin_std": 1629.07177734375,
"step": 668
},
{
"KL/chosen_KL_mean": -1009.0479125976562,
"KL/mean": -1348.203857421875,
"KL/rejected_KL_mean": -1687.359619140625,
"KL/std": 1146.410888671875,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 678.311767578125,
"fcm_dpo/q_t": 0.49981409311294556,
"grad_norm": 0.06971733272075653,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -0.25400370359420776,
"logits/rejected": -0.2593185305595398,
"logps/chosen": -1055.164306640625,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -1765.283935546875,
"loss": 1.3856,
"margin_dpo/margin_mean": 678.311767578125,
"margin_dpo/margin_std": 1458.8255615234375,
"step": 669
},
{
"KL/chosen_KL_mean": -1203.6142578125,
"KL/mean": -1688.494384765625,
"KL/rejected_KL_mean": -2173.37451171875,
"KL/std": 1796.819091796875,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 969.760009765625,
"fcm_dpo/q_t": 0.49973419308662415,
"grad_norm": 0.1595345139503479,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -0.24518835544586182,
"logits/rejected": -0.2584257423877716,
"logps/chosen": -1265.9600830078125,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -2270.31494140625,
"loss": 1.3852,
"margin_dpo/margin_mean": 969.760009765625,
"margin_dpo/margin_std": 2220.41748046875,
"step": 670
},
{
"KL/chosen_KL_mean": -1205.3656005859375,
"KL/mean": -1510.864501953125,
"KL/rejected_KL_mean": -1816.3634033203125,
"KL/std": 1332.0640869140625,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 610.9979248046875,
"fcm_dpo/q_t": 0.49983251094818115,
"grad_norm": 0.06985840201377869,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -0.2608600854873657,
"logits/rejected": -0.27443787455558777,
"logps/chosen": -1253.36572265625,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -1900.1827392578125,
"loss": 1.3856,
"margin_dpo/margin_mean": 610.9979248046875,
"margin_dpo/margin_std": 1832.479736328125,
"step": 671
},
{
"KL/chosen_KL_mean": -1534.4940185546875,
"KL/mean": -1729.3173828125,
"KL/rejected_KL_mean": -1924.140625,
"KL/std": 1462.22021484375,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 389.6468505859375,
"fcm_dpo/q_t": 0.4998931884765625,
"grad_norm": 0.23390474915504456,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -0.2893441915512085,
"logits/rejected": -0.2950705885887146,
"logps/chosen": -1593.0772705078125,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -2017.2808837890625,
"loss": 1.3859,
"margin_dpo/margin_mean": 389.6468200683594,
"margin_dpo/margin_std": 1572.853515625,
"step": 672
},
{
"KL/chosen_KL_mean": -1143.24072265625,
"KL/mean": -1430.210693359375,
"KL/rejected_KL_mean": -1717.1806640625,
"KL/std": 1203.540771484375,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 573.93994140625,
"fcm_dpo/q_t": 0.49984270334243774,
"grad_norm": 0.09268736839294434,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -0.2500056326389313,
"logits/rejected": -0.25664058327674866,
"logps/chosen": -1189.9638671875,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -1802.476806640625,
"loss": 1.3857,
"margin_dpo/margin_mean": 573.93994140625,
"margin_dpo/margin_std": 1410.111083984375,
"step": 673
},
{
"KL/chosen_KL_mean": -984.7044067382812,
"KL/mean": -1303.4468994140625,
"KL/rejected_KL_mean": -1622.189453125,
"KL/std": 1034.3787841796875,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 637.4850463867188,
"fcm_dpo/q_t": 0.49982526898384094,
"grad_norm": 0.0636168122291565,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -0.24484899640083313,
"logits/rejected": -0.24942630529403687,
"logps/chosen": -1030.14990234375,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -1692.2353515625,
"loss": 1.3856,
"margin_dpo/margin_mean": 637.4850463867188,
"margin_dpo/margin_std": 1280.18359375,
"step": 674
},
{
"KL/chosen_KL_mean": -1180.5653076171875,
"KL/mean": -1411.843994140625,
"KL/rejected_KL_mean": -1643.12255859375,
"KL/std": 1438.281005859375,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 462.5572509765625,
"fcm_dpo/q_t": 0.49987322092056274,
"grad_norm": 0.12069758027791977,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -0.2583056390285492,
"logits/rejected": -0.26913708448410034,
"logps/chosen": -1224.7415771484375,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -1717.214599609375,
"loss": 1.3858,
"margin_dpo/margin_mean": 462.5572509765625,
"margin_dpo/margin_std": 1866.17919921875,
"step": 675
},
{
"KL/chosen_KL_mean": -1252.801025390625,
"KL/mean": -1582.269287109375,
"KL/rejected_KL_mean": -1911.7374267578125,
"KL/std": 1282.24658203125,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 658.9365234375,
"fcm_dpo/q_t": 0.49981939792633057,
"grad_norm": 0.0914895310997963,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -0.28977373242378235,
"logits/rejected": -0.3063029944896698,
"logps/chosen": -1324.199462890625,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -2000.09619140625,
"loss": 1.3856,
"margin_dpo/margin_mean": 658.9365234375,
"margin_dpo/margin_std": 1711.732666015625,
"step": 676
},
{
"KL/chosen_KL_mean": -1265.82275390625,
"KL/mean": -1535.177734375,
"KL/rejected_KL_mean": -1804.532958984375,
"KL/std": 1238.73095703125,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 538.710205078125,
"fcm_dpo/q_t": 0.49985232949256897,
"grad_norm": 0.1667097508907318,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -0.23336629569530487,
"logits/rejected": -0.24292391538619995,
"logps/chosen": -1322.35009765625,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -1882.7593994140625,
"loss": 1.3857,
"margin_dpo/margin_mean": 538.710205078125,
"margin_dpo/margin_std": 1765.0107421875,
"step": 677
},
{
"KL/chosen_KL_mean": -927.6842041015625,
"KL/mean": -1416.5367431640625,
"KL/rejected_KL_mean": -1905.38916015625,
"KL/std": 1505.28564453125,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 977.7049560546875,
"fcm_dpo/q_t": 0.49973201751708984,
"grad_norm": 0.19531825184822083,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -0.2596043646335602,
"logits/rejected": -0.2691473960876465,
"logps/chosen": -973.8187255859375,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -1985.9937744140625,
"loss": 1.3852,
"margin_dpo/margin_mean": 977.7049560546875,
"margin_dpo/margin_std": 1734.43798828125,
"step": 678
},
{
"KL/chosen_KL_mean": -1160.0855712890625,
"KL/mean": -1379.29541015625,
"KL/rejected_KL_mean": -1598.50537109375,
"KL/std": 1232.62939453125,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 438.4197998046875,
"fcm_dpo/q_t": 0.4998798370361328,
"grad_norm": 0.0700586810708046,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -0.26852041482925415,
"logits/rejected": -0.2719077169895172,
"logps/chosen": -1210.3804931640625,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -1675.103515625,
"loss": 1.3858,
"margin_dpo/margin_mean": 438.4197998046875,
"margin_dpo/margin_std": 1638.0318603515625,
"step": 679
},
{
"KL/chosen_KL_mean": -1294.1790771484375,
"KL/mean": -1602.512939453125,
"KL/rejected_KL_mean": -1910.8466796875,
"KL/std": 1501.509033203125,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 616.6676025390625,
"fcm_dpo/q_t": 0.499830961227417,
"grad_norm": 0.09760192036628723,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -0.28134095668792725,
"logits/rejected": -0.28399893641471863,
"logps/chosen": -1371.0947265625,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -2023.2314453125,
"loss": 1.3856,
"margin_dpo/margin_mean": 616.667724609375,
"margin_dpo/margin_std": 1909.36962890625,
"step": 680
},
{
"KL/chosen_KL_mean": -1360.1488037109375,
"KL/mean": -1591.733642578125,
"KL/rejected_KL_mean": -1823.318603515625,
"KL/std": 1528.53564453125,
"epoch": 1.0,
"fcm_dpo/beta": 1.0963610748149222e-06,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 463.16986083984375,
"fcm_dpo/q_t": 0.4998730719089508,
"grad_norm": 0.14217820763587952,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -0.27003633975982666,
"logits/rejected": -0.27188071608543396,
"logps/chosen": -1421.1060791015625,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -1911.8765869140625,
"loss": 1.3858,
"margin_dpo/margin_mean": 463.1697998046875,
"margin_dpo/margin_std": 2003.16552734375,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 1.356521034380763,
"train_runtime": 1742.8549,
"train_samples_per_second": 25.015,
"train_steps_per_second": 0.391
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink