9c454e4013
Model: jackf857/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.5-s_star-0.6 Source: Original Platform
13034 lines
481 KiB
JSON
13034 lines
481 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 200,
|
|
"global_step": 681,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0014684287812041115,
|
|
"fcm_dpo/beta": 0.10308968275785446,
|
|
"fcm_dpo/delta": 0.29979780316352844,
|
|
"fcm_dpo/margin": -0.02287006378173828,
|
|
"fcm_dpo/q_t": 0.5005706548690796,
|
|
"grad_norm": 86.183349609375,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.4974287748336792,
|
|
"logits/rejected": -0.43299180269241333,
|
|
"logps/chosen": -50.1435661315918,
|
|
"logps/ref_chosen": -50.14883804321289,
|
|
"logps/ref_rejected": -74.1280517578125,
|
|
"logps/rejected": -74.09991455078125,
|
|
"loss": 1.389,
|
|
"margin_dpo/margin_mean": -0.02287048101425171,
|
|
"margin_dpo/margin_std": 0.41920793056488037,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.002936857562408223,
|
|
"fcm_dpo/beta": 0.1061793640255928,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.06572261452674866,
|
|
"fcm_dpo/q_t": 0.5017440319061279,
|
|
"grad_norm": 76.63970184326172,
|
|
"learning_rate": 7.246376811594203e-09,
|
|
"logits/chosen": -0.49536412954330444,
|
|
"logits/rejected": -0.4594460427761078,
|
|
"logps/chosen": -52.65568923950195,
|
|
"logps/ref_chosen": -52.620704650878906,
|
|
"logps/ref_rejected": -75.30413818359375,
|
|
"logps/rejected": -75.27340698242188,
|
|
"loss": 1.3936,
|
|
"margin_dpo/margin_mean": -0.06572240591049194,
|
|
"margin_dpo/margin_std": 0.35048407316207886,
|
|
"step": 2
|
|
},
|
|
{
|
|
"epoch": 0.004405286343612335,
|
|
"fcm_dpo/beta": 0.10943304747343063,
|
|
"fcm_dpo/delta": 0.29740938544273376,
|
|
"fcm_dpo/margin": 0.009920269250869751,
|
|
"fcm_dpo/q_t": 0.4997372329235077,
|
|
"grad_norm": 77.7017822265625,
|
|
"learning_rate": 1.4492753623188406e-08,
|
|
"logits/chosen": -0.4817405045032501,
|
|
"logits/rejected": -0.4422529935836792,
|
|
"logps/chosen": -60.98210144042969,
|
|
"logps/ref_chosen": -60.981597900390625,
|
|
"logps/ref_rejected": -68.67259216308594,
|
|
"logps/rejected": -68.68302154541016,
|
|
"loss": 1.3855,
|
|
"margin_dpo/margin_mean": 0.009920388460159302,
|
|
"margin_dpo/margin_std": 0.3543340563774109,
|
|
"step": 3
|
|
},
|
|
{
|
|
"epoch": 0.005873715124816446,
|
|
"fcm_dpo/beta": 0.1161462590098381,
|
|
"fcm_dpo/delta": 0.29794785380363464,
|
|
"fcm_dpo/margin": -0.01907077431678772,
|
|
"fcm_dpo/q_t": 0.5005367994308472,
|
|
"grad_norm": 83.44183349609375,
|
|
"learning_rate": 2.1739130434782606e-08,
|
|
"logits/chosen": -0.46823394298553467,
|
|
"logits/rejected": -0.44061601161956787,
|
|
"logps/chosen": -56.77912139892578,
|
|
"logps/ref_chosen": -56.7677116394043,
|
|
"logps/ref_rejected": -86.64710998535156,
|
|
"logps/rejected": -86.63945007324219,
|
|
"loss": 1.3887,
|
|
"margin_dpo/margin_mean": -0.01907128095626831,
|
|
"margin_dpo/margin_std": 0.34393125772476196,
|
|
"step": 4
|
|
},
|
|
{
|
|
"epoch": 0.007342143906020558,
|
|
"fcm_dpo/beta": 0.13080811500549316,
|
|
"fcm_dpo/delta": 0.5949947834014893,
|
|
"fcm_dpo/margin": 0.04144471883773804,
|
|
"fcm_dpo/q_t": 0.4987483024597168,
|
|
"grad_norm": 116.84333038330078,
|
|
"learning_rate": 2.898550724637681e-08,
|
|
"logits/chosen": -0.5144734978675842,
|
|
"logits/rejected": -0.4707837700843811,
|
|
"logps/chosen": -53.85641860961914,
|
|
"logps/ref_chosen": -53.859375,
|
|
"logps/ref_rejected": -84.14918518066406,
|
|
"logps/rejected": -84.18767547607422,
|
|
"loss": 1.3815,
|
|
"margin_dpo/margin_mean": 0.041444987058639526,
|
|
"margin_dpo/margin_std": 0.32837140560150146,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.00881057268722467,
|
|
"fcm_dpo/beta": 0.13887986540794373,
|
|
"fcm_dpo/delta": 0.2996029257774353,
|
|
"fcm_dpo/margin": -0.049517929553985596,
|
|
"fcm_dpo/q_t": 0.5016666650772095,
|
|
"grad_norm": 127.5752182006836,
|
|
"learning_rate": 3.6231884057971014e-08,
|
|
"logits/chosen": -0.5032555460929871,
|
|
"logits/rejected": -0.4607737958431244,
|
|
"logps/chosen": -63.0675048828125,
|
|
"logps/ref_chosen": -63.007484436035156,
|
|
"logps/ref_rejected": -92.64534759521484,
|
|
"logps/rejected": -92.65585327148438,
|
|
"loss": 1.3938,
|
|
"margin_dpo/margin_mean": -0.04951813817024231,
|
|
"margin_dpo/margin_std": 0.41555964946746826,
|
|
"step": 6
|
|
},
|
|
{
|
|
"epoch": 0.010279001468428781,
|
|
"fcm_dpo/beta": 0.15183104574680328,
|
|
"fcm_dpo/delta": 0.29823803901672363,
|
|
"fcm_dpo/margin": -0.0031449496746063232,
|
|
"fcm_dpo/q_t": 0.5001453161239624,
|
|
"grad_norm": 124.75188446044922,
|
|
"learning_rate": 4.347826086956521e-08,
|
|
"logits/chosen": -0.4999721646308899,
|
|
"logits/rejected": -0.466085284948349,
|
|
"logps/chosen": -57.7862548828125,
|
|
"logps/ref_chosen": -57.774818420410156,
|
|
"logps/ref_rejected": -103.92059326171875,
|
|
"logps/rejected": -103.92887878417969,
|
|
"loss": 1.3877,
|
|
"margin_dpo/margin_mean": -0.0031441152095794678,
|
|
"margin_dpo/margin_std": 0.39584606885910034,
|
|
"step": 7
|
|
},
|
|
{
|
|
"epoch": 0.011747430249632892,
|
|
"fcm_dpo/beta": 0.1610293984413147,
|
|
"fcm_dpo/delta": 0.2940930426120758,
|
|
"fcm_dpo/margin": 0.014331787824630737,
|
|
"fcm_dpo/q_t": 0.4995083510875702,
|
|
"grad_norm": 126.80061340332031,
|
|
"learning_rate": 5.0724637681159424e-08,
|
|
"logits/chosen": -0.5133894085884094,
|
|
"logits/rejected": -0.4880775511264801,
|
|
"logps/chosen": -58.66931915283203,
|
|
"logps/ref_chosen": -58.716033935546875,
|
|
"logps/ref_rejected": -79.3114242553711,
|
|
"logps/rejected": -79.27903747558594,
|
|
"loss": 1.385,
|
|
"margin_dpo/margin_mean": 0.014332354068756104,
|
|
"margin_dpo/margin_std": 0.381694495677948,
|
|
"step": 8
|
|
},
|
|
{
|
|
"epoch": 0.013215859030837005,
|
|
"fcm_dpo/beta": 0.17621421813964844,
|
|
"fcm_dpo/delta": 0.5977947115898132,
|
|
"fcm_dpo/margin": 0.013306349515914917,
|
|
"fcm_dpo/q_t": 0.4994489848613739,
|
|
"grad_norm": 150.93214416503906,
|
|
"learning_rate": 5.797101449275362e-08,
|
|
"logits/chosen": -0.4874652028083801,
|
|
"logits/rejected": -0.44039708375930786,
|
|
"logps/chosen": -69.85830688476562,
|
|
"logps/ref_chosen": -69.8668441772461,
|
|
"logps/ref_rejected": -99.6026611328125,
|
|
"logps/rejected": -99.60743713378906,
|
|
"loss": 1.3852,
|
|
"margin_dpo/margin_mean": 0.013305962085723877,
|
|
"margin_dpo/margin_std": 0.3926519751548767,
|
|
"step": 9
|
|
},
|
|
{
|
|
"epoch": 0.014684287812041116,
|
|
"fcm_dpo/beta": 0.18701426684856415,
|
|
"fcm_dpo/delta": 0.2960095703601837,
|
|
"fcm_dpo/margin": 0.019543275237083435,
|
|
"fcm_dpo/q_t": 0.4991165101528168,
|
|
"grad_norm": 132.65455627441406,
|
|
"learning_rate": 6.521739130434782e-08,
|
|
"logits/chosen": -0.4857853651046753,
|
|
"logits/rejected": -0.4421781301498413,
|
|
"logps/chosen": -48.35417938232422,
|
|
"logps/ref_chosen": -48.35768508911133,
|
|
"logps/ref_rejected": -80.37206268310547,
|
|
"logps/rejected": -80.38810729980469,
|
|
"loss": 1.3838,
|
|
"margin_dpo/margin_mean": 0.01954329013824463,
|
|
"margin_dpo/margin_std": 0.37247973680496216,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.016152716593245228,
|
|
"fcm_dpo/beta": 0.20438191294670105,
|
|
"fcm_dpo/delta": 0.29821285605430603,
|
|
"fcm_dpo/margin": -0.0019693076610565186,
|
|
"fcm_dpo/q_t": 0.5001306533813477,
|
|
"grad_norm": 140.15432739257812,
|
|
"learning_rate": 7.246376811594203e-08,
|
|
"logits/chosen": -0.4591175615787506,
|
|
"logits/rejected": -0.43393969535827637,
|
|
"logps/chosen": -53.05950927734375,
|
|
"logps/ref_chosen": -53.01685333251953,
|
|
"logps/ref_rejected": -87.78038024902344,
|
|
"logps/rejected": -87.82106018066406,
|
|
"loss": 1.388,
|
|
"margin_dpo/margin_mean": -0.001968860626220703,
|
|
"margin_dpo/margin_std": 0.34557127952575684,
|
|
"step": 11
|
|
},
|
|
{
|
|
"epoch": 0.01762114537444934,
|
|
"fcm_dpo/beta": 0.2106446623802185,
|
|
"fcm_dpo/delta": 0.29740071296691895,
|
|
"fcm_dpo/margin": 0.00841417908668518,
|
|
"fcm_dpo/q_t": 0.4995623826980591,
|
|
"grad_norm": 188.97665405273438,
|
|
"learning_rate": 7.971014492753623e-08,
|
|
"logits/chosen": -0.5176132321357727,
|
|
"logits/rejected": -0.4807056188583374,
|
|
"logps/chosen": -61.83100128173828,
|
|
"logps/ref_chosen": -61.80543518066406,
|
|
"logps/ref_rejected": -104.8582763671875,
|
|
"logps/rejected": -104.89225769042969,
|
|
"loss": 1.3861,
|
|
"margin_dpo/margin_mean": 0.008413076400756836,
|
|
"margin_dpo/margin_std": 0.3932174742221832,
|
|
"step": 12
|
|
},
|
|
{
|
|
"epoch": 0.01908957415565345,
|
|
"fcm_dpo/beta": 0.2233951985836029,
|
|
"fcm_dpo/delta": 0.2904997766017914,
|
|
"fcm_dpo/margin": 0.003662973642349243,
|
|
"fcm_dpo/q_t": 0.4998163878917694,
|
|
"grad_norm": 176.16429138183594,
|
|
"learning_rate": 8.695652173913042e-08,
|
|
"logits/chosen": -0.4700925946235657,
|
|
"logits/rejected": -0.4411465525627136,
|
|
"logps/chosen": -64.25465393066406,
|
|
"logps/ref_chosen": -64.2603530883789,
|
|
"logps/ref_rejected": -87.20307922363281,
|
|
"logps/rejected": -87.20104217529297,
|
|
"loss": 1.387,
|
|
"margin_dpo/margin_mean": 0.0036628246307373047,
|
|
"margin_dpo/margin_std": 0.39085596799850464,
|
|
"step": 13
|
|
},
|
|
{
|
|
"epoch": 0.020558002936857563,
|
|
"fcm_dpo/beta": 0.24400313198566437,
|
|
"fcm_dpo/delta": 0.29805275797843933,
|
|
"fcm_dpo/margin": 0.005518615245819092,
|
|
"fcm_dpo/q_t": 0.499683141708374,
|
|
"grad_norm": 209.18350219726562,
|
|
"learning_rate": 9.420289855072464e-08,
|
|
"logits/chosen": -0.47941046953201294,
|
|
"logits/rejected": -0.4401160478591919,
|
|
"logps/chosen": -58.11064910888672,
|
|
"logps/ref_chosen": -58.11021041870117,
|
|
"logps/ref_rejected": -104.04708099365234,
|
|
"logps/rejected": -104.05303192138672,
|
|
"loss": 1.3883,
|
|
"margin_dpo/margin_mean": 0.005518555641174316,
|
|
"margin_dpo/margin_std": 0.47164684534072876,
|
|
"step": 14
|
|
},
|
|
{
|
|
"epoch": 0.022026431718061675,
|
|
"fcm_dpo/beta": 0.25842905044555664,
|
|
"fcm_dpo/delta": 0.2872004508972168,
|
|
"fcm_dpo/margin": 0.03296121954917908,
|
|
"fcm_dpo/q_t": 0.49806299805641174,
|
|
"grad_norm": 164.04367065429688,
|
|
"learning_rate": 1.0144927536231885e-07,
|
|
"logits/chosen": -0.5029046535491943,
|
|
"logits/rejected": -0.4847126603126526,
|
|
"logps/chosen": -56.98225402832031,
|
|
"logps/ref_chosen": -56.96691131591797,
|
|
"logps/ref_rejected": -80.80863952636719,
|
|
"logps/rejected": -80.85694885253906,
|
|
"loss": 1.3795,
|
|
"margin_dpo/margin_mean": 0.03296104073524475,
|
|
"margin_dpo/margin_std": 0.3130000829696655,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.023494860499265784,
|
|
"fcm_dpo/beta": 0.2661767303943634,
|
|
"fcm_dpo/delta": 0.2911548614501953,
|
|
"fcm_dpo/margin": 0.006061911582946777,
|
|
"fcm_dpo/q_t": 0.49960726499557495,
|
|
"grad_norm": 225.11155700683594,
|
|
"learning_rate": 1.0869565217391303e-07,
|
|
"logits/chosen": -0.5254543423652649,
|
|
"logits/rejected": -0.48440250754356384,
|
|
"logps/chosen": -61.74738693237305,
|
|
"logps/ref_chosen": -61.739891052246094,
|
|
"logps/ref_rejected": -84.36947631835938,
|
|
"logps/rejected": -84.38302612304688,
|
|
"loss": 1.3865,
|
|
"margin_dpo/margin_mean": 0.0060617029666900635,
|
|
"margin_dpo/margin_std": 0.35263463854789734,
|
|
"step": 16
|
|
},
|
|
{
|
|
"epoch": 0.024963289280469897,
|
|
"fcm_dpo/beta": 0.29960495233535767,
|
|
"fcm_dpo/delta": 0.5921929478645325,
|
|
"fcm_dpo/margin": 0.02691623568534851,
|
|
"fcm_dpo/q_t": 0.49804800748825073,
|
|
"grad_norm": 234.62332153320312,
|
|
"learning_rate": 1.1594202898550725e-07,
|
|
"logits/chosen": -0.4995841979980469,
|
|
"logits/rejected": -0.4622904658317566,
|
|
"logps/chosen": -67.72857666015625,
|
|
"logps/ref_chosen": -67.71033477783203,
|
|
"logps/ref_rejected": -85.37865447998047,
|
|
"logps/rejected": -85.42382049560547,
|
|
"loss": 1.3808,
|
|
"margin_dpo/margin_mean": 0.02691650390625,
|
|
"margin_dpo/margin_std": 0.35206741094589233,
|
|
"step": 17
|
|
},
|
|
{
|
|
"epoch": 0.02643171806167401,
|
|
"fcm_dpo/beta": 0.3366771340370178,
|
|
"fcm_dpo/delta": 0.5766884088516235,
|
|
"fcm_dpo/margin": 0.07164651155471802,
|
|
"fcm_dpo/q_t": 0.4941858947277069,
|
|
"grad_norm": 273.0143127441406,
|
|
"learning_rate": 1.2318840579710146e-07,
|
|
"logits/chosen": -0.5020445585250854,
|
|
"logits/rejected": -0.44765499234199524,
|
|
"logps/chosen": -47.74727249145508,
|
|
"logps/ref_chosen": -47.7394905090332,
|
|
"logps/ref_rejected": -75.4722900390625,
|
|
"logps/rejected": -75.55171966552734,
|
|
"loss": 1.3652,
|
|
"margin_dpo/margin_mean": 0.07164624333381653,
|
|
"margin_dpo/margin_std": 0.3414478302001953,
|
|
"step": 18
|
|
},
|
|
{
|
|
"epoch": 0.027900146842878122,
|
|
"fcm_dpo/beta": 0.37680280208587646,
|
|
"fcm_dpo/delta": 0.5566200017929077,
|
|
"fcm_dpo/margin": 0.11977455019950867,
|
|
"fcm_dpo/q_t": 0.4892091155052185,
|
|
"grad_norm": 273.0553894042969,
|
|
"learning_rate": 1.3043478260869563e-07,
|
|
"logits/chosen": -0.4712415337562561,
|
|
"logits/rejected": -0.41980695724487305,
|
|
"logps/chosen": -70.16747283935547,
|
|
"logps/ref_chosen": -70.20536041259766,
|
|
"logps/ref_rejected": -89.7575912475586,
|
|
"logps/rejected": -89.83948516845703,
|
|
"loss": 1.3457,
|
|
"margin_dpo/margin_mean": 0.11977550387382507,
|
|
"margin_dpo/margin_std": 0.3553928732872009,
|
|
"step": 19
|
|
},
|
|
{
|
|
"epoch": 0.02936857562408223,
|
|
"fcm_dpo/beta": 0.39811867475509644,
|
|
"fcm_dpo/delta": 0.2839474380016327,
|
|
"fcm_dpo/margin": 0.03722500801086426,
|
|
"fcm_dpo/q_t": 0.49641382694244385,
|
|
"grad_norm": 294.3759460449219,
|
|
"learning_rate": 1.3768115942028986e-07,
|
|
"logits/chosen": -0.5450824499130249,
|
|
"logits/rejected": -0.48803430795669556,
|
|
"logps/chosen": -50.81207275390625,
|
|
"logps/ref_chosen": -50.80324172973633,
|
|
"logps/ref_rejected": -78.82334899902344,
|
|
"logps/rejected": -78.86940002441406,
|
|
"loss": 1.3748,
|
|
"margin_dpo/margin_mean": 0.037224918603897095,
|
|
"margin_dpo/margin_std": 0.3072945773601532,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.030837004405286344,
|
|
"fcm_dpo/beta": 0.443250834941864,
|
|
"fcm_dpo/delta": 0.5273383259773254,
|
|
"fcm_dpo/margin": 0.1729957014322281,
|
|
"fcm_dpo/q_t": 0.48192811012268066,
|
|
"grad_norm": 327.8818054199219,
|
|
"learning_rate": 1.4492753623188405e-07,
|
|
"logits/chosen": -0.517810046672821,
|
|
"logits/rejected": -0.49564769864082336,
|
|
"logps/chosen": -50.032989501953125,
|
|
"logps/ref_chosen": -50.063018798828125,
|
|
"logps/ref_rejected": -77.86878967285156,
|
|
"logps/rejected": -78.01175689697266,
|
|
"loss": 1.3156,
|
|
"margin_dpo/margin_mean": 0.1729956418275833,
|
|
"margin_dpo/margin_std": 0.3008769452571869,
|
|
"step": 21
|
|
},
|
|
{
|
|
"epoch": 0.032305433186490456,
|
|
"fcm_dpo/beta": 0.49217575788497925,
|
|
"fcm_dpo/delta": 0.4990707039833069,
|
|
"fcm_dpo/margin": 0.21139326691627502,
|
|
"fcm_dpo/q_t": 0.4752381443977356,
|
|
"grad_norm": 389.9123229980469,
|
|
"learning_rate": 1.5217391304347825e-07,
|
|
"logits/chosen": -0.48431238532066345,
|
|
"logits/rejected": -0.4406638741493225,
|
|
"logps/chosen": -58.97994613647461,
|
|
"logps/ref_chosen": -59.05763626098633,
|
|
"logps/ref_rejected": -97.50466918945312,
|
|
"logps/rejected": -97.63837432861328,
|
|
"loss": 1.2977,
|
|
"margin_dpo/margin_mean": 0.21139344573020935,
|
|
"margin_dpo/margin_std": 0.46386557817459106,
|
|
"step": 22
|
|
},
|
|
{
|
|
"epoch": 0.033773861967694566,
|
|
"fcm_dpo/beta": 0.5401284694671631,
|
|
"fcm_dpo/delta": 0.4757743179798126,
|
|
"fcm_dpo/margin": 0.24120861291885376,
|
|
"fcm_dpo/q_t": 0.469452828168869,
|
|
"grad_norm": 389.5323791503906,
|
|
"learning_rate": 1.5942028985507245e-07,
|
|
"logits/chosen": -0.4874040186405182,
|
|
"logits/rejected": -0.4647292494773865,
|
|
"logps/chosen": -60.02528381347656,
|
|
"logps/ref_chosen": -60.07769775390625,
|
|
"logps/ref_rejected": -81.13955688476562,
|
|
"logps/rejected": -81.3283462524414,
|
|
"loss": 1.2741,
|
|
"margin_dpo/margin_mean": 0.24120870232582092,
|
|
"margin_dpo/margin_std": 0.418229341506958,
|
|
"step": 23
|
|
},
|
|
{
|
|
"epoch": 0.03524229074889868,
|
|
"fcm_dpo/beta": 0.5957721471786499,
|
|
"fcm_dpo/delta": 0.4795808792114258,
|
|
"fcm_dpo/margin": 0.20983095467090607,
|
|
"fcm_dpo/q_t": 0.47032594680786133,
|
|
"grad_norm": 471.896728515625,
|
|
"learning_rate": 1.6666666666666665e-07,
|
|
"logits/chosen": -0.5221496820449829,
|
|
"logits/rejected": -0.5054002404212952,
|
|
"logps/chosen": -44.25273895263672,
|
|
"logps/ref_chosen": -44.29103469848633,
|
|
"logps/ref_rejected": -99.12521362304688,
|
|
"logps/rejected": -99.2967529296875,
|
|
"loss": 1.2789,
|
|
"margin_dpo/margin_mean": 0.20983177423477173,
|
|
"margin_dpo/margin_std": 0.3987899422645569,
|
|
"step": 24
|
|
},
|
|
{
|
|
"epoch": 0.03671071953010279,
|
|
"fcm_dpo/beta": 0.6493321657180786,
|
|
"fcm_dpo/delta": 0.4309549033641815,
|
|
"fcm_dpo/margin": 0.27194589376449585,
|
|
"fcm_dpo/q_t": 0.4587538540363312,
|
|
"grad_norm": 428.5508117675781,
|
|
"learning_rate": 1.7391304347826085e-07,
|
|
"logits/chosen": -0.4805753827095032,
|
|
"logits/rejected": -0.4500219225883484,
|
|
"logps/chosen": -52.49953842163086,
|
|
"logps/ref_chosen": -52.537052154541016,
|
|
"logps/ref_rejected": -89.34219360351562,
|
|
"logps/rejected": -89.57662200927734,
|
|
"loss": 1.2366,
|
|
"margin_dpo/margin_mean": 0.27194616198539734,
|
|
"margin_dpo/margin_std": 0.4171033203601837,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.0381791483113069,
|
|
"fcm_dpo/beta": 0.6972817182540894,
|
|
"fcm_dpo/delta": 0.3234151601791382,
|
|
"fcm_dpo/margin": 0.4091455340385437,
|
|
"fcm_dpo/q_t": 0.4343709349632263,
|
|
"grad_norm": 487.4859619140625,
|
|
"learning_rate": 1.8115942028985507e-07,
|
|
"logits/chosen": -0.5440469980239868,
|
|
"logits/rejected": -0.5126461386680603,
|
|
"logps/chosen": -53.82080078125,
|
|
"logps/ref_chosen": -53.92280578613281,
|
|
"logps/ref_rejected": -103.35971069335938,
|
|
"logps/rejected": -103.66685485839844,
|
|
"loss": 1.1593,
|
|
"margin_dpo/margin_mean": 0.4091459810733795,
|
|
"margin_dpo/margin_std": 0.5674811601638794,
|
|
"step": 26
|
|
},
|
|
{
|
|
"epoch": 0.039647577092511016,
|
|
"fcm_dpo/beta": 0.7315391302108765,
|
|
"fcm_dpo/delta": 0.22802981734275818,
|
|
"fcm_dpo/margin": 0.5205895304679871,
|
|
"fcm_dpo/q_t": 0.41203832626342773,
|
|
"grad_norm": 530.424560546875,
|
|
"learning_rate": 1.8840579710144927e-07,
|
|
"logits/chosen": -0.5132520198822021,
|
|
"logits/rejected": -0.47679802775382996,
|
|
"logps/chosen": -42.7576904296875,
|
|
"logps/ref_chosen": -42.898529052734375,
|
|
"logps/ref_rejected": -98.72419738769531,
|
|
"logps/rejected": -99.10395812988281,
|
|
"loss": 1.0773,
|
|
"margin_dpo/margin_mean": 0.5205891132354736,
|
|
"margin_dpo/margin_std": 0.5357187986373901,
|
|
"step": 27
|
|
},
|
|
{
|
|
"epoch": 0.041116005873715125,
|
|
"fcm_dpo/beta": 0.7871224284172058,
|
|
"fcm_dpo/delta": 0.3837660551071167,
|
|
"fcm_dpo/margin": 0.28373512625694275,
|
|
"fcm_dpo/q_t": 0.44818979501724243,
|
|
"grad_norm": 503.4446105957031,
|
|
"learning_rate": 1.9565217391304347e-07,
|
|
"logits/chosen": -0.5058661699295044,
|
|
"logits/rejected": -0.4506160616874695,
|
|
"logps/chosen": -60.58196258544922,
|
|
"logps/ref_chosen": -60.55650329589844,
|
|
"logps/ref_rejected": -91.40111541748047,
|
|
"logps/rejected": -91.71031188964844,
|
|
"loss": 1.2172,
|
|
"margin_dpo/margin_mean": 0.2837350368499756,
|
|
"margin_dpo/margin_std": 0.5103665590286255,
|
|
"step": 28
|
|
},
|
|
{
|
|
"epoch": 0.042584434654919234,
|
|
"fcm_dpo/beta": 0.8140842914581299,
|
|
"fcm_dpo/delta": 0.12859481573104858,
|
|
"fcm_dpo/margin": 0.5867732763290405,
|
|
"fcm_dpo/q_t": 0.39105674624443054,
|
|
"grad_norm": 521.9832763671875,
|
|
"learning_rate": 2.028985507246377e-07,
|
|
"logits/chosen": -0.5453035831451416,
|
|
"logits/rejected": -0.498327374458313,
|
|
"logps/chosen": -57.687744140625,
|
|
"logps/ref_chosen": -57.80778503417969,
|
|
"logps/ref_rejected": -97.39434814453125,
|
|
"logps/rejected": -97.86107635498047,
|
|
"loss": 1.0177,
|
|
"margin_dpo/margin_mean": 0.5867743492126465,
|
|
"margin_dpo/margin_std": 0.5909589529037476,
|
|
"step": 29
|
|
},
|
|
{
|
|
"epoch": 0.04405286343612335,
|
|
"fcm_dpo/beta": 0.8216707706451416,
|
|
"fcm_dpo/delta": 0.0340820774435997,
|
|
"fcm_dpo/margin": 0.690904438495636,
|
|
"fcm_dpo/q_t": 0.3710797429084778,
|
|
"grad_norm": 468.0931396484375,
|
|
"learning_rate": 2.1014492753623187e-07,
|
|
"logits/chosen": -0.4391070306301117,
|
|
"logits/rejected": -0.40617626905441284,
|
|
"logps/chosen": -52.45497131347656,
|
|
"logps/ref_chosen": -52.577369689941406,
|
|
"logps/ref_rejected": -98.48920440673828,
|
|
"logps/rejected": -99.05770874023438,
|
|
"loss": 0.9603,
|
|
"margin_dpo/margin_mean": 0.6909043788909912,
|
|
"margin_dpo/margin_std": 0.6301661729812622,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.04552129221732746,
|
|
"fcm_dpo/beta": 0.8536430597305298,
|
|
"fcm_dpo/delta": 0.21446718275547028,
|
|
"fcm_dpo/margin": 0.46136218309402466,
|
|
"fcm_dpo/q_t": 0.4132009744644165,
|
|
"grad_norm": 441.92120361328125,
|
|
"learning_rate": 2.1739130434782607e-07,
|
|
"logits/chosen": -0.5114755034446716,
|
|
"logits/rejected": -0.46733975410461426,
|
|
"logps/chosen": -63.724403381347656,
|
|
"logps/ref_chosen": -63.806922912597656,
|
|
"logps/ref_rejected": -72.89400482177734,
|
|
"logps/rejected": -73.2728500366211,
|
|
"loss": 1.0973,
|
|
"margin_dpo/margin_mean": 0.46136197447776794,
|
|
"margin_dpo/margin_std": 0.6410406827926636,
|
|
"step": 31
|
|
},
|
|
{
|
|
"epoch": 0.04698972099853157,
|
|
"fcm_dpo/beta": 0.8558568954467773,
|
|
"fcm_dpo/delta": -0.04837347939610481,
|
|
"fcm_dpo/margin": 0.7538948655128479,
|
|
"fcm_dpo/q_t": 0.363816499710083,
|
|
"grad_norm": 431.97833251953125,
|
|
"learning_rate": 2.2463768115942027e-07,
|
|
"logits/chosen": -0.5344721078872681,
|
|
"logits/rejected": -0.4945007562637329,
|
|
"logps/chosen": -62.547889709472656,
|
|
"logps/ref_chosen": -62.739524841308594,
|
|
"logps/ref_rejected": -89.3175048828125,
|
|
"logps/rejected": -89.8797607421875,
|
|
"loss": 0.95,
|
|
"margin_dpo/margin_mean": 0.7538947463035583,
|
|
"margin_dpo/margin_std": 0.8735177516937256,
|
|
"step": 32
|
|
},
|
|
{
|
|
"epoch": 0.048458149779735685,
|
|
"fcm_dpo/beta": 0.8686736822128296,
|
|
"fcm_dpo/delta": 0.07413993775844574,
|
|
"fcm_dpo/margin": 0.608627438545227,
|
|
"fcm_dpo/q_t": 0.3796751797199249,
|
|
"grad_norm": 434.5819091796875,
|
|
"learning_rate": 2.318840579710145e-07,
|
|
"logits/chosen": -0.48133665323257446,
|
|
"logits/rejected": -0.4549505412578583,
|
|
"logps/chosen": -53.15459442138672,
|
|
"logps/ref_chosen": -53.26097106933594,
|
|
"logps/ref_rejected": -87.8851318359375,
|
|
"logps/rejected": -88.38739013671875,
|
|
"loss": 0.9889,
|
|
"margin_dpo/margin_mean": 0.6086278557777405,
|
|
"margin_dpo/margin_std": 0.5917040109634399,
|
|
"step": 33
|
|
},
|
|
{
|
|
"epoch": 0.049926578560939794,
|
|
"fcm_dpo/beta": 0.8607486486434937,
|
|
"fcm_dpo/delta": -0.029908865690231323,
|
|
"fcm_dpo/margin": 0.7293862104415894,
|
|
"fcm_dpo/q_t": 0.3666630983352661,
|
|
"grad_norm": 413.73388671875,
|
|
"learning_rate": 2.391304347826087e-07,
|
|
"logits/chosen": -0.5056343078613281,
|
|
"logits/rejected": -0.4880971908569336,
|
|
"logps/chosen": -50.74631118774414,
|
|
"logps/ref_chosen": -50.81732940673828,
|
|
"logps/ref_rejected": -101.92184448242188,
|
|
"logps/rejected": -102.58021545410156,
|
|
"loss": 0.9715,
|
|
"margin_dpo/margin_mean": 0.7293854355812073,
|
|
"margin_dpo/margin_std": 0.8646777272224426,
|
|
"step": 34
|
|
},
|
|
{
|
|
"epoch": 0.0513950073421439,
|
|
"fcm_dpo/beta": 0.8129470348358154,
|
|
"fcm_dpo/delta": -0.3546205759048462,
|
|
"fcm_dpo/margin": 1.1309483051300049,
|
|
"fcm_dpo/q_t": 0.3066979646682739,
|
|
"grad_norm": 335.1808166503906,
|
|
"learning_rate": 2.463768115942029e-07,
|
|
"logits/chosen": -0.5237250328063965,
|
|
"logits/rejected": -0.48680445551872253,
|
|
"logps/chosen": -50.88426208496094,
|
|
"logps/ref_chosen": -51.02449035644531,
|
|
"logps/ref_rejected": -106.82443237304688,
|
|
"logps/rejected": -107.81513977050781,
|
|
"loss": 0.7836,
|
|
"margin_dpo/margin_mean": 1.1309483051300049,
|
|
"margin_dpo/margin_std": 1.0144567489624023,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.05286343612334802,
|
|
"fcm_dpo/beta": 0.7814772129058838,
|
|
"fcm_dpo/delta": -0.24869973957538605,
|
|
"fcm_dpo/margin": 1.0601816177368164,
|
|
"fcm_dpo/q_t": 0.3314136862754822,
|
|
"grad_norm": 286.6738586425781,
|
|
"learning_rate": 2.536231884057971e-07,
|
|
"logits/chosen": -0.5335814952850342,
|
|
"logits/rejected": -0.4959707260131836,
|
|
"logps/chosen": -51.95500946044922,
|
|
"logps/ref_chosen": -51.991493225097656,
|
|
"logps/ref_rejected": -86.0406265258789,
|
|
"logps/rejected": -87.06432342529297,
|
|
"loss": 0.8938,
|
|
"margin_dpo/margin_mean": 1.0601813793182373,
|
|
"margin_dpo/margin_std": 1.1594077348709106,
|
|
"step": 36
|
|
},
|
|
{
|
|
"epoch": 0.05433186490455213,
|
|
"fcm_dpo/beta": 0.7387670278549194,
|
|
"fcm_dpo/delta": -0.1298440545797348,
|
|
"fcm_dpo/margin": 0.9678917527198792,
|
|
"fcm_dpo/q_t": 0.35818910598754883,
|
|
"grad_norm": 304.4512939453125,
|
|
"learning_rate": 2.6086956521739126e-07,
|
|
"logits/chosen": -0.5167920589447021,
|
|
"logits/rejected": -0.4735545217990875,
|
|
"logps/chosen": -62.80792999267578,
|
|
"logps/ref_chosen": -62.807106018066406,
|
|
"logps/ref_rejected": -77.89507293701172,
|
|
"logps/rejected": -78.86378479003906,
|
|
"loss": 0.982,
|
|
"margin_dpo/margin_mean": 0.9678912162780762,
|
|
"margin_dpo/margin_std": 1.311614751815796,
|
|
"step": 37
|
|
},
|
|
{
|
|
"epoch": 0.055800293685756244,
|
|
"fcm_dpo/beta": 0.7068374156951904,
|
|
"fcm_dpo/delta": -0.37072476744651794,
|
|
"fcm_dpo/margin": 1.3253750801086426,
|
|
"fcm_dpo/q_t": 0.32258689403533936,
|
|
"grad_norm": 279.52642822265625,
|
|
"learning_rate": 2.681159420289855e-07,
|
|
"logits/chosen": -0.5299612879753113,
|
|
"logits/rejected": -0.49737080931663513,
|
|
"logps/chosen": -48.24340057373047,
|
|
"logps/ref_chosen": -48.39051818847656,
|
|
"logps/ref_rejected": -97.91244506835938,
|
|
"logps/rejected": -99.0906982421875,
|
|
"loss": 0.8506,
|
|
"margin_dpo/margin_mean": 1.3253748416900635,
|
|
"margin_dpo/margin_std": 1.5660966634750366,
|
|
"step": 38
|
|
},
|
|
{
|
|
"epoch": 0.05726872246696035,
|
|
"fcm_dpo/beta": 0.6403067111968994,
|
|
"fcm_dpo/delta": -0.4625703692436218,
|
|
"fcm_dpo/margin": 1.5791468620300293,
|
|
"fcm_dpo/q_t": 0.2878713607788086,
|
|
"grad_norm": 240.9384765625,
|
|
"learning_rate": 2.753623188405797e-07,
|
|
"logits/chosen": -0.5889618396759033,
|
|
"logits/rejected": -0.5511986613273621,
|
|
"logps/chosen": -50.64854431152344,
|
|
"logps/ref_chosen": -50.75047302246094,
|
|
"logps/ref_rejected": -78.56951141357422,
|
|
"logps/rejected": -80.0467300415039,
|
|
"loss": 0.7516,
|
|
"margin_dpo/margin_mean": 1.5791466236114502,
|
|
"margin_dpo/margin_std": 1.3109757900238037,
|
|
"step": 39
|
|
},
|
|
{
|
|
"epoch": 0.05873715124816446,
|
|
"fcm_dpo/beta": 0.6059188842773438,
|
|
"fcm_dpo/delta": -0.3013249933719635,
|
|
"fcm_dpo/margin": 1.4455113410949707,
|
|
"fcm_dpo/q_t": 0.3245118260383606,
|
|
"grad_norm": 211.55166625976562,
|
|
"learning_rate": 2.8260869565217386e-07,
|
|
"logits/chosen": -0.5337985754013062,
|
|
"logits/rejected": -0.5050651431083679,
|
|
"logps/chosen": -57.77853012084961,
|
|
"logps/ref_chosen": -57.985069274902344,
|
|
"logps/ref_rejected": -74.3000717163086,
|
|
"logps/rejected": -75.53904724121094,
|
|
"loss": 0.8502,
|
|
"margin_dpo/margin_mean": 1.445512294769287,
|
|
"margin_dpo/margin_std": 1.521177053451538,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.06020558002936858,
|
|
"fcm_dpo/beta": 0.5612444877624512,
|
|
"fcm_dpo/delta": -0.3904670178890228,
|
|
"fcm_dpo/margin": 1.6995972394943237,
|
|
"fcm_dpo/q_t": 0.31129536032676697,
|
|
"grad_norm": 220.2853546142578,
|
|
"learning_rate": 2.898550724637681e-07,
|
|
"logits/chosen": -0.5450239777565002,
|
|
"logits/rejected": -0.5087831020355225,
|
|
"logps/chosen": -62.68452835083008,
|
|
"logps/ref_chosen": -62.69581604003906,
|
|
"logps/ref_rejected": -97.02352905273438,
|
|
"logps/rejected": -98.71183776855469,
|
|
"loss": 0.8303,
|
|
"margin_dpo/margin_mean": 1.6995975971221924,
|
|
"margin_dpo/margin_std": 1.7764086723327637,
|
|
"step": 41
|
|
},
|
|
{
|
|
"epoch": 0.06167400881057269,
|
|
"fcm_dpo/beta": 0.4963580369949341,
|
|
"fcm_dpo/delta": -0.6783915758132935,
|
|
"fcm_dpo/margin": 2.4108240604400635,
|
|
"fcm_dpo/q_t": 0.2684577703475952,
|
|
"grad_norm": 179.24830627441406,
|
|
"learning_rate": 2.971014492753623e-07,
|
|
"logits/chosen": -0.5304057598114014,
|
|
"logits/rejected": -0.48365843296051025,
|
|
"logps/chosen": -58.75749588012695,
|
|
"logps/ref_chosen": -58.966426849365234,
|
|
"logps/ref_rejected": -109.90837097167969,
|
|
"logps/rejected": -112.11026000976562,
|
|
"loss": 0.6984,
|
|
"margin_dpo/margin_mean": 2.4108238220214844,
|
|
"margin_dpo/margin_std": 2.168299436569214,
|
|
"step": 42
|
|
},
|
|
{
|
|
"epoch": 0.0631424375917768,
|
|
"fcm_dpo/beta": 0.44308847188949585,
|
|
"fcm_dpo/delta": -0.48201635479927063,
|
|
"fcm_dpo/margin": 2.3211944103240967,
|
|
"fcm_dpo/q_t": 0.2817467451095581,
|
|
"grad_norm": 169.86190795898438,
|
|
"learning_rate": 3.043478260869565e-07,
|
|
"logits/chosen": -0.5538966059684753,
|
|
"logits/rejected": -0.5297778844833374,
|
|
"logps/chosen": -53.645652770996094,
|
|
"logps/ref_chosen": -54.15599822998047,
|
|
"logps/ref_rejected": -96.48019409179688,
|
|
"logps/rejected": -98.29104614257812,
|
|
"loss": 0.7188,
|
|
"margin_dpo/margin_mean": 2.3211941719055176,
|
|
"margin_dpo/margin_std": 1.7973530292510986,
|
|
"step": 43
|
|
},
|
|
{
|
|
"epoch": 0.06461086637298091,
|
|
"fcm_dpo/beta": 0.4017961323261261,
|
|
"fcm_dpo/delta": -0.49928557872772217,
|
|
"fcm_dpo/margin": 2.597970485687256,
|
|
"fcm_dpo/q_t": 0.2795543372631073,
|
|
"grad_norm": 164.8052215576172,
|
|
"learning_rate": 3.115942028985507e-07,
|
|
"logits/chosen": -0.4567152261734009,
|
|
"logits/rejected": -0.43692851066589355,
|
|
"logps/chosen": -49.864402770996094,
|
|
"logps/ref_chosen": -50.07849884033203,
|
|
"logps/ref_rejected": -108.78376007080078,
|
|
"logps/rejected": -111.16763305664062,
|
|
"loss": 0.713,
|
|
"margin_dpo/margin_mean": 2.597970962524414,
|
|
"margin_dpo/margin_std": 1.9955940246582031,
|
|
"step": 44
|
|
},
|
|
{
|
|
"epoch": 0.06607929515418502,
|
|
"fcm_dpo/beta": 0.3842885196208954,
|
|
"fcm_dpo/delta": -0.18539269268512726,
|
|
"fcm_dpo/margin": 2.0074782371520996,
|
|
"fcm_dpo/q_t": 0.3449398875236511,
|
|
"grad_norm": 142.70497131347656,
|
|
"learning_rate": 3.188405797101449e-07,
|
|
"logits/chosen": -0.4787759780883789,
|
|
"logits/rejected": -0.46609732508659363,
|
|
"logps/chosen": -48.27836227416992,
|
|
"logps/ref_chosen": -48.4149284362793,
|
|
"logps/ref_rejected": -77.93643188476562,
|
|
"logps/rejected": -79.80734252929688,
|
|
"loss": 0.9085,
|
|
"margin_dpo/margin_mean": 2.0074782371520996,
|
|
"margin_dpo/margin_std": 2.3263068199157715,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.06754772393538913,
|
|
"fcm_dpo/beta": 0.3586476445198059,
|
|
"fcm_dpo/delta": -0.3937702775001526,
|
|
"fcm_dpo/margin": 2.6675519943237305,
|
|
"fcm_dpo/q_t": 0.31235700845718384,
|
|
"grad_norm": 143.95310974121094,
|
|
"learning_rate": 3.260869565217391e-07,
|
|
"logits/chosen": -0.5191431045532227,
|
|
"logits/rejected": -0.46773868799209595,
|
|
"logps/chosen": -55.786354064941406,
|
|
"logps/ref_chosen": -55.999427795410156,
|
|
"logps/ref_rejected": -95.652587890625,
|
|
"logps/rejected": -98.10707092285156,
|
|
"loss": 0.8258,
|
|
"margin_dpo/margin_mean": 2.6675515174865723,
|
|
"margin_dpo/margin_std": 2.8206050395965576,
|
|
"step": 46
|
|
},
|
|
{
|
|
"epoch": 0.06901615271659324,
|
|
"fcm_dpo/beta": 0.33721715211868286,
|
|
"fcm_dpo/delta": -0.31360626220703125,
|
|
"fcm_dpo/margin": 2.6299524307250977,
|
|
"fcm_dpo/q_t": 0.31478771567344666,
|
|
"grad_norm": 143.6584930419922,
|
|
"learning_rate": 3.333333333333333e-07,
|
|
"logits/chosen": -0.5552619695663452,
|
|
"logits/rejected": -0.5014165639877319,
|
|
"logps/chosen": -57.52300262451172,
|
|
"logps/ref_chosen": -57.92607879638672,
|
|
"logps/ref_rejected": -94.67920684814453,
|
|
"logps/rejected": -96.90608215332031,
|
|
"loss": 0.8165,
|
|
"margin_dpo/margin_mean": 2.6299524307250977,
|
|
"margin_dpo/margin_std": 2.3889541625976562,
|
|
"step": 47
|
|
},
|
|
{
|
|
"epoch": 0.07048458149779736,
|
|
"fcm_dpo/beta": 0.31545713543891907,
|
|
"fcm_dpo/delta": -0.23579037189483643,
|
|
"fcm_dpo/margin": 2.579418659210205,
|
|
"fcm_dpo/q_t": 0.3256019353866577,
|
|
"grad_norm": 148.15615844726562,
|
|
"learning_rate": 3.4057971014492755e-07,
|
|
"logits/chosen": -0.584140956401825,
|
|
"logits/rejected": -0.5260648131370544,
|
|
"logps/chosen": -57.080162048339844,
|
|
"logps/ref_chosen": -57.188072204589844,
|
|
"logps/ref_rejected": -88.0166015625,
|
|
"logps/rejected": -90.48811340332031,
|
|
"loss": 0.8611,
|
|
"margin_dpo/margin_mean": 2.5794191360473633,
|
|
"margin_dpo/margin_std": 2.3878588676452637,
|
|
"step": 48
|
|
},
|
|
{
|
|
"epoch": 0.07195301027900147,
|
|
"fcm_dpo/beta": 0.30197674036026,
|
|
"fcm_dpo/delta": -0.30794399976730347,
|
|
"fcm_dpo/margin": 2.9200515747070312,
|
|
"fcm_dpo/q_t": 0.31857985258102417,
|
|
"grad_norm": 391.1684265136719,
|
|
"learning_rate": 3.478260869565217e-07,
|
|
"logits/chosen": -0.5332294702529907,
|
|
"logits/rejected": -0.47411999106407166,
|
|
"logps/chosen": -61.26190948486328,
|
|
"logps/ref_chosen": -61.685272216796875,
|
|
"logps/ref_rejected": -83.76747131347656,
|
|
"logps/rejected": -86.26417541503906,
|
|
"loss": 0.8336,
|
|
"margin_dpo/margin_mean": 2.9200520515441895,
|
|
"margin_dpo/margin_std": 2.897745132446289,
|
|
"step": 49
|
|
},
|
|
{
|
|
"epoch": 0.07342143906020558,
|
|
"fcm_dpo/beta": 0.2793174982070923,
|
|
"fcm_dpo/delta": -0.3477798402309418,
|
|
"fcm_dpo/margin": 3.2729971408843994,
|
|
"fcm_dpo/q_t": 0.3099723756313324,
|
|
"grad_norm": 119.61611938476562,
|
|
"learning_rate": 3.5507246376811595e-07,
|
|
"logits/chosen": -0.5257272124290466,
|
|
"logits/rejected": -0.4897651672363281,
|
|
"logps/chosen": -58.62849426269531,
|
|
"logps/ref_chosen": -58.72413635253906,
|
|
"logps/ref_rejected": -96.35814666748047,
|
|
"logps/rejected": -99.5354995727539,
|
|
"loss": 0.8143,
|
|
"margin_dpo/margin_mean": 3.2729973793029785,
|
|
"margin_dpo/margin_std": 3.099050521850586,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.07488986784140969,
|
|
"fcm_dpo/beta": 0.2618306875228882,
|
|
"fcm_dpo/delta": -0.2724575996398926,
|
|
"fcm_dpo/margin": 3.226135730743408,
|
|
"fcm_dpo/q_t": 0.337465763092041,
|
|
"grad_norm": 97.82896423339844,
|
|
"learning_rate": 3.6231884057971015e-07,
|
|
"logits/chosen": -0.5134993195533752,
|
|
"logits/rejected": -0.48083585500717163,
|
|
"logps/chosen": -61.43326950073242,
|
|
"logps/ref_chosen": -61.3736686706543,
|
|
"logps/ref_rejected": -76.00199890136719,
|
|
"logps/rejected": -79.28773498535156,
|
|
"loss": 0.9065,
|
|
"margin_dpo/margin_mean": 3.2261362075805664,
|
|
"margin_dpo/margin_std": 3.9684386253356934,
|
|
"step": 51
|
|
},
|
|
{
|
|
"epoch": 0.0763582966226138,
|
|
"fcm_dpo/beta": 0.2358231097459793,
|
|
"fcm_dpo/delta": -0.6680800914764404,
|
|
"fcm_dpo/margin": 5.0320281982421875,
|
|
"fcm_dpo/q_t": 0.2603490352630615,
|
|
"grad_norm": 94.4676513671875,
|
|
"learning_rate": 3.695652173913043e-07,
|
|
"logits/chosen": -0.557870090007782,
|
|
"logits/rejected": -0.5032240748405457,
|
|
"logps/chosen": -51.786834716796875,
|
|
"logps/ref_chosen": -52.33735656738281,
|
|
"logps/ref_rejected": -79.97391510009766,
|
|
"logps/rejected": -84.45541381835938,
|
|
"loss": 0.6746,
|
|
"margin_dpo/margin_mean": 5.0320281982421875,
|
|
"margin_dpo/margin_std": 4.007593154907227,
|
|
"step": 52
|
|
},
|
|
{
|
|
"epoch": 0.07782672540381791,
|
|
"fcm_dpo/beta": 0.21381568908691406,
|
|
"fcm_dpo/delta": -0.5255783796310425,
|
|
"fcm_dpo/margin": 5.014308452606201,
|
|
"fcm_dpo/q_t": 0.298156201839447,
|
|
"grad_norm": 91.9140853881836,
|
|
"learning_rate": 3.7681159420289855e-07,
|
|
"logits/chosen": -0.6126323342323303,
|
|
"logits/rejected": -0.5912652015686035,
|
|
"logps/chosen": -53.282958984375,
|
|
"logps/ref_chosen": -53.31465148925781,
|
|
"logps/ref_rejected": -91.78359985351562,
|
|
"logps/rejected": -96.7662124633789,
|
|
"loss": 0.7792,
|
|
"margin_dpo/margin_mean": 5.014307975769043,
|
|
"margin_dpo/margin_std": 4.957146644592285,
|
|
"step": 53
|
|
},
|
|
{
|
|
"epoch": 0.07929515418502203,
|
|
"fcm_dpo/beta": 0.1964169442653656,
|
|
"fcm_dpo/delta": -0.3215777575969696,
|
|
"fcm_dpo/margin": 4.548583984375,
|
|
"fcm_dpo/q_t": 0.3135607838630676,
|
|
"grad_norm": 79.06949615478516,
|
|
"learning_rate": 3.8405797101449274e-07,
|
|
"logits/chosen": -0.5691754817962646,
|
|
"logits/rejected": -0.5145428776741028,
|
|
"logps/chosen": -50.683597564697266,
|
|
"logps/ref_chosen": -50.68865966796875,
|
|
"logps/ref_rejected": -91.71539306640625,
|
|
"logps/rejected": -96.2589111328125,
|
|
"loss": 0.8046,
|
|
"margin_dpo/margin_mean": 4.548584461212158,
|
|
"margin_dpo/margin_std": 4.153512954711914,
|
|
"step": 54
|
|
},
|
|
{
|
|
"epoch": 0.08076358296622614,
|
|
"fcm_dpo/beta": 0.1820051074028015,
|
|
"fcm_dpo/delta": -0.4129885137081146,
|
|
"fcm_dpo/margin": 5.350862979888916,
|
|
"fcm_dpo/q_t": 0.31068581342697144,
|
|
"grad_norm": 79.86705780029297,
|
|
"learning_rate": 3.9130434782608694e-07,
|
|
"logits/chosen": -0.6421248912811279,
|
|
"logits/rejected": -0.5801297426223755,
|
|
"logps/chosen": -63.01799774169922,
|
|
"logps/ref_chosen": -62.615234375,
|
|
"logps/ref_rejected": -88.99349975585938,
|
|
"logps/rejected": -94.74713134765625,
|
|
"loss": 0.8358,
|
|
"margin_dpo/margin_mean": 5.350862503051758,
|
|
"margin_dpo/margin_std": 5.820072650909424,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.08223201174743025,
|
|
"fcm_dpo/beta": 0.17025861144065857,
|
|
"fcm_dpo/delta": -0.2861342430114746,
|
|
"fcm_dpo/margin": 5.0614013671875,
|
|
"fcm_dpo/q_t": 0.32851046323776245,
|
|
"grad_norm": 65.21794891357422,
|
|
"learning_rate": 3.9855072463768114e-07,
|
|
"logits/chosen": -0.6073439717292786,
|
|
"logits/rejected": -0.5643370151519775,
|
|
"logps/chosen": -58.113746643066406,
|
|
"logps/ref_chosen": -57.9327278137207,
|
|
"logps/ref_rejected": -94.1744384765625,
|
|
"logps/rejected": -99.41685485839844,
|
|
"loss": 0.8886,
|
|
"margin_dpo/margin_mean": 5.061400890350342,
|
|
"margin_dpo/margin_std": 5.794804573059082,
|
|
"step": 56
|
|
},
|
|
{
|
|
"epoch": 0.08370044052863436,
|
|
"fcm_dpo/beta": 0.16040679812431335,
|
|
"fcm_dpo/delta": -0.334885835647583,
|
|
"fcm_dpo/margin": 5.6461501121521,
|
|
"fcm_dpo/q_t": 0.3076401352882385,
|
|
"grad_norm": 72.25611877441406,
|
|
"learning_rate": 4.057971014492754e-07,
|
|
"logits/chosen": -0.5476347208023071,
|
|
"logits/rejected": -0.5185646414756775,
|
|
"logps/chosen": -70.79481506347656,
|
|
"logps/ref_chosen": -70.49528503417969,
|
|
"logps/ref_rejected": -95.56546020507812,
|
|
"logps/rejected": -101.51113891601562,
|
|
"loss": 0.806,
|
|
"margin_dpo/margin_mean": 5.646149635314941,
|
|
"margin_dpo/margin_std": 4.98412561416626,
|
|
"step": 57
|
|
},
|
|
{
|
|
"epoch": 0.08516886930983847,
|
|
"fcm_dpo/beta": 0.14835116267204285,
|
|
"fcm_dpo/delta": -0.41062241792678833,
|
|
"fcm_dpo/margin": 6.553308963775635,
|
|
"fcm_dpo/q_t": 0.30739498138427734,
|
|
"grad_norm": 73.45458984375,
|
|
"learning_rate": 4.1304347826086954e-07,
|
|
"logits/chosen": -0.5891647338867188,
|
|
"logits/rejected": -0.513205885887146,
|
|
"logps/chosen": -62.415225982666016,
|
|
"logps/ref_chosen": -62.13294219970703,
|
|
"logps/ref_rejected": -84.61729431152344,
|
|
"logps/rejected": -91.45289611816406,
|
|
"loss": 0.8144,
|
|
"margin_dpo/margin_mean": 6.553309440612793,
|
|
"margin_dpo/margin_std": 6.610756874084473,
|
|
"step": 58
|
|
},
|
|
{
|
|
"epoch": 0.08663729809104258,
|
|
"fcm_dpo/beta": 0.13525643944740295,
|
|
"fcm_dpo/delta": -0.35778504610061646,
|
|
"fcm_dpo/margin": 6.788931846618652,
|
|
"fcm_dpo/q_t": 0.3129827380180359,
|
|
"grad_norm": 67.0660171508789,
|
|
"learning_rate": 4.2028985507246374e-07,
|
|
"logits/chosen": -0.6016139984130859,
|
|
"logits/rejected": -0.558463454246521,
|
|
"logps/chosen": -52.71605682373047,
|
|
"logps/ref_chosen": -51.932525634765625,
|
|
"logps/ref_rejected": -88.88520050048828,
|
|
"logps/rejected": -96.45765686035156,
|
|
"loss": 0.8265,
|
|
"margin_dpo/margin_mean": 6.7889323234558105,
|
|
"margin_dpo/margin_std": 6.713343620300293,
|
|
"step": 59
|
|
},
|
|
{
|
|
"epoch": 0.0881057268722467,
|
|
"fcm_dpo/beta": 0.1309395432472229,
|
|
"fcm_dpo/delta": -0.13870461285114288,
|
|
"fcm_dpo/margin": 5.555472373962402,
|
|
"fcm_dpo/q_t": 0.3427530527114868,
|
|
"grad_norm": 79.94298553466797,
|
|
"learning_rate": 4.2753623188405794e-07,
|
|
"logits/chosen": -0.5748249888420105,
|
|
"logits/rejected": -0.512750506401062,
|
|
"logps/chosen": -62.536293029785156,
|
|
"logps/ref_chosen": -60.94218826293945,
|
|
"logps/ref_rejected": -85.39340209960938,
|
|
"logps/rejected": -92.54298400878906,
|
|
"loss": 0.9199,
|
|
"margin_dpo/margin_mean": 5.555472373962402,
|
|
"margin_dpo/margin_std": 6.041206359863281,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.08957415565345081,
|
|
"fcm_dpo/beta": 0.12486925721168518,
|
|
"fcm_dpo/delta": -0.2642156481742859,
|
|
"fcm_dpo/margin": 6.724519729614258,
|
|
"fcm_dpo/q_t": 0.3386513590812683,
|
|
"grad_norm": 61.240787506103516,
|
|
"learning_rate": 4.3478260869565214e-07,
|
|
"logits/chosen": -0.5803092122077942,
|
|
"logits/rejected": -0.5446271896362305,
|
|
"logps/chosen": -61.23955535888672,
|
|
"logps/ref_chosen": -60.633522033691406,
|
|
"logps/ref_rejected": -89.85249328613281,
|
|
"logps/rejected": -97.18305206298828,
|
|
"loss": 0.9087,
|
|
"margin_dpo/margin_mean": 6.724519729614258,
|
|
"margin_dpo/margin_std": 8.698123931884766,
|
|
"step": 61
|
|
},
|
|
{
|
|
"epoch": 0.09104258443465492,
|
|
"fcm_dpo/beta": 0.12395919859409332,
|
|
"fcm_dpo/delta": -0.035701360553503036,
|
|
"fcm_dpo/margin": 5.107054233551025,
|
|
"fcm_dpo/q_t": 0.3664395213127136,
|
|
"grad_norm": 62.39984893798828,
|
|
"learning_rate": 4.420289855072464e-07,
|
|
"logits/chosen": -0.5827194452285767,
|
|
"logits/rejected": -0.547878086566925,
|
|
"logps/chosen": -57.02729034423828,
|
|
"logps/ref_chosen": -56.15077209472656,
|
|
"logps/ref_rejected": -75.56619262695312,
|
|
"logps/rejected": -81.54976654052734,
|
|
"loss": 0.9761,
|
|
"margin_dpo/margin_mean": 5.107054710388184,
|
|
"margin_dpo/margin_std": 6.175351619720459,
|
|
"step": 62
|
|
},
|
|
{
|
|
"epoch": 0.09251101321585903,
|
|
"fcm_dpo/beta": 0.11800666153430939,
|
|
"fcm_dpo/delta": -0.23345595598220825,
|
|
"fcm_dpo/margin": 6.8794989585876465,
|
|
"fcm_dpo/q_t": 0.3267975449562073,
|
|
"grad_norm": 60.61161422729492,
|
|
"learning_rate": 4.4927536231884053e-07,
|
|
"logits/chosen": -0.5791250467300415,
|
|
"logits/rejected": -0.5316081643104553,
|
|
"logps/chosen": -74.61734008789062,
|
|
"logps/ref_chosen": -73.14739227294922,
|
|
"logps/ref_rejected": -97.61006164550781,
|
|
"logps/rejected": -105.95950317382812,
|
|
"loss": 0.8674,
|
|
"margin_dpo/margin_mean": 6.879498481750488,
|
|
"margin_dpo/margin_std": 6.8499274253845215,
|
|
"step": 63
|
|
},
|
|
{
|
|
"epoch": 0.09397944199706314,
|
|
"fcm_dpo/beta": 0.11056903749704361,
|
|
"fcm_dpo/delta": -0.3205702304840088,
|
|
"fcm_dpo/margin": 8.005148887634277,
|
|
"fcm_dpo/q_t": 0.32072484493255615,
|
|
"grad_norm": 56.07626724243164,
|
|
"learning_rate": 4.5652173913043473e-07,
|
|
"logits/chosen": -0.5584267377853394,
|
|
"logits/rejected": -0.5256708264350891,
|
|
"logps/chosen": -54.30480194091797,
|
|
"logps/ref_chosen": -53.998600006103516,
|
|
"logps/ref_rejected": -93.53019714355469,
|
|
"logps/rejected": -101.841552734375,
|
|
"loss": 0.8584,
|
|
"margin_dpo/margin_mean": 8.005149841308594,
|
|
"margin_dpo/margin_std": 8.640867233276367,
|
|
"step": 64
|
|
},
|
|
{
|
|
"epoch": 0.09544787077826726,
|
|
"fcm_dpo/beta": 0.10528378188610077,
|
|
"fcm_dpo/delta": -0.26119303703308105,
|
|
"fcm_dpo/margin": 7.948797225952148,
|
|
"fcm_dpo/q_t": 0.32479870319366455,
|
|
"grad_norm": 56.537654876708984,
|
|
"learning_rate": 4.63768115942029e-07,
|
|
"logits/chosen": -0.6330504417419434,
|
|
"logits/rejected": -0.6177892684936523,
|
|
"logps/chosen": -66.75464630126953,
|
|
"logps/ref_chosen": -64.83599853515625,
|
|
"logps/ref_rejected": -109.94645690917969,
|
|
"logps/rejected": -119.81390380859375,
|
|
"loss": 0.8793,
|
|
"margin_dpo/margin_mean": 7.948797225952148,
|
|
"margin_dpo/margin_std": 8.451780319213867,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.09691629955947137,
|
|
"fcm_dpo/beta": 0.10108963400125504,
|
|
"fcm_dpo/delta": -0.16579298675060272,
|
|
"fcm_dpo/margin": 7.422573089599609,
|
|
"fcm_dpo/q_t": 0.34628963470458984,
|
|
"grad_norm": 50.0311164855957,
|
|
"learning_rate": 4.7101449275362313e-07,
|
|
"logits/chosen": -0.6193344593048096,
|
|
"logits/rejected": -0.5846483707427979,
|
|
"logps/chosen": -53.2528076171875,
|
|
"logps/ref_chosen": -51.44352722167969,
|
|
"logps/ref_rejected": -75.63629913330078,
|
|
"logps/rejected": -84.86815643310547,
|
|
"loss": 0.9152,
|
|
"margin_dpo/margin_mean": 7.422572135925293,
|
|
"margin_dpo/margin_std": 8.596309661865234,
|
|
"step": 66
|
|
},
|
|
{
|
|
"epoch": 0.09838472834067548,
|
|
"fcm_dpo/beta": 0.09882417321205139,
|
|
"fcm_dpo/delta": -0.17645135521888733,
|
|
"fcm_dpo/margin": 7.723736763000488,
|
|
"fcm_dpo/q_t": 0.3419727683067322,
|
|
"grad_norm": 51.04230499267578,
|
|
"learning_rate": 4.782608695652174e-07,
|
|
"logits/chosen": -0.5773788690567017,
|
|
"logits/rejected": -0.5347466468811035,
|
|
"logps/chosen": -60.845420837402344,
|
|
"logps/ref_chosen": -59.34080505371094,
|
|
"logps/ref_rejected": -72.78728485107422,
|
|
"logps/rejected": -82.01564025878906,
|
|
"loss": 0.9005,
|
|
"margin_dpo/margin_mean": 7.72373628616333,
|
|
"margin_dpo/margin_std": 8.540781021118164,
|
|
"step": 67
|
|
},
|
|
{
|
|
"epoch": 0.09985315712187959,
|
|
"fcm_dpo/beta": 0.09611781686544418,
|
|
"fcm_dpo/delta": -0.1308629810810089,
|
|
"fcm_dpo/margin": 7.5075178146362305,
|
|
"fcm_dpo/q_t": 0.34419840574264526,
|
|
"grad_norm": 48.844181060791016,
|
|
"learning_rate": 4.855072463768116e-07,
|
|
"logits/chosen": -0.6071466207504272,
|
|
"logits/rejected": -0.5482945442199707,
|
|
"logps/chosen": -66.77816772460938,
|
|
"logps/ref_chosen": -65.2058334350586,
|
|
"logps/ref_rejected": -77.20724487304688,
|
|
"logps/rejected": -86.28710174560547,
|
|
"loss": 0.8922,
|
|
"margin_dpo/margin_mean": 7.507518291473389,
|
|
"margin_dpo/margin_std": 7.301938056945801,
|
|
"step": 68
|
|
},
|
|
{
|
|
"epoch": 0.1013215859030837,
|
|
"fcm_dpo/beta": 0.09169034659862518,
|
|
"fcm_dpo/delta": -0.20954345166683197,
|
|
"fcm_dpo/margin": 8.629631042480469,
|
|
"fcm_dpo/q_t": 0.3316349387168884,
|
|
"grad_norm": 50.881256103515625,
|
|
"learning_rate": 4.927536231884058e-07,
|
|
"logits/chosen": -0.5818198323249817,
|
|
"logits/rejected": -0.557062029838562,
|
|
"logps/chosen": -61.803890228271484,
|
|
"logps/ref_chosen": -59.81924057006836,
|
|
"logps/ref_rejected": -103.38886260986328,
|
|
"logps/rejected": -114.00314331054688,
|
|
"loss": 0.8551,
|
|
"margin_dpo/margin_mean": 8.629631042480469,
|
|
"margin_dpo/margin_std": 8.243066787719727,
|
|
"step": 69
|
|
},
|
|
{
|
|
"epoch": 0.1027900146842878,
|
|
"fcm_dpo/beta": 0.08815528452396393,
|
|
"fcm_dpo/delta": -0.25355854630470276,
|
|
"fcm_dpo/margin": 9.44918155670166,
|
|
"fcm_dpo/q_t": 0.3277716040611267,
|
|
"grad_norm": 51.42112350463867,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": -0.6246175765991211,
|
|
"logits/rejected": -0.5891268253326416,
|
|
"logps/chosen": -64.80339050292969,
|
|
"logps/ref_chosen": -61.930641174316406,
|
|
"logps/ref_rejected": -91.06078338623047,
|
|
"logps/rejected": -103.38272094726562,
|
|
"loss": 0.8569,
|
|
"margin_dpo/margin_mean": 9.449182510375977,
|
|
"margin_dpo/margin_std": 9.51607894897461,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.10425844346549193,
|
|
"fcm_dpo/beta": 0.08285696059465408,
|
|
"fcm_dpo/delta": -0.30268311500549316,
|
|
"fcm_dpo/margin": 10.571916580200195,
|
|
"fcm_dpo/q_t": 0.3174167275428772,
|
|
"grad_norm": 45.99234390258789,
|
|
"learning_rate": 4.999967061337492e-07,
|
|
"logits/chosen": -0.6398484110832214,
|
|
"logits/rejected": -0.5968210101127625,
|
|
"logps/chosen": -64.16336822509766,
|
|
"logps/ref_chosen": -61.750335693359375,
|
|
"logps/ref_rejected": -97.33662414550781,
|
|
"logps/rejected": -110.32157135009766,
|
|
"loss": 0.8213,
|
|
"margin_dpo/margin_mean": 10.571917533874512,
|
|
"margin_dpo/margin_std": 9.830376625061035,
|
|
"step": 71
|
|
},
|
|
{
|
|
"epoch": 0.10572687224669604,
|
|
"fcm_dpo/beta": 0.07784741371870041,
|
|
"fcm_dpo/delta": -0.2931217849254608,
|
|
"fcm_dpo/margin": 11.129253387451172,
|
|
"fcm_dpo/q_t": 0.31700342893600464,
|
|
"grad_norm": 48.5871696472168,
|
|
"learning_rate": 4.999868246217933e-07,
|
|
"logits/chosen": -0.6470623016357422,
|
|
"logits/rejected": -0.6127582788467407,
|
|
"logps/chosen": -68.55953216552734,
|
|
"logps/ref_chosen": -66.05341339111328,
|
|
"logps/ref_rejected": -95.2869873046875,
|
|
"logps/rejected": -108.92236328125,
|
|
"loss": 0.8263,
|
|
"margin_dpo/margin_mean": 11.129253387451172,
|
|
"margin_dpo/margin_std": 10.417036056518555,
|
|
"step": 72
|
|
},
|
|
{
|
|
"epoch": 0.10719530102790015,
|
|
"fcm_dpo/beta": 0.07408735156059265,
|
|
"fcm_dpo/delta": -0.2234322428703308,
|
|
"fcm_dpo/margin": 10.851455688476562,
|
|
"fcm_dpo/q_t": 0.3458544611930847,
|
|
"grad_norm": 45.48418045043945,
|
|
"learning_rate": 4.999703557245192e-07,
|
|
"logits/chosen": -0.6490052938461304,
|
|
"logits/rejected": -0.6046302914619446,
|
|
"logps/chosen": -70.05899047851562,
|
|
"logps/ref_chosen": -66.25627136230469,
|
|
"logps/ref_rejected": -90.45613098144531,
|
|
"logps/rejected": -105.11030578613281,
|
|
"loss": 0.953,
|
|
"margin_dpo/margin_mean": 10.851455688476562,
|
|
"margin_dpo/margin_std": 14.541223526000977,
|
|
"step": 73
|
|
},
|
|
{
|
|
"epoch": 0.10866372980910426,
|
|
"fcm_dpo/beta": 0.07084085047245026,
|
|
"fcm_dpo/delta": -0.22987580299377441,
|
|
"fcm_dpo/margin": 11.434507369995117,
|
|
"fcm_dpo/q_t": 0.3413064479827881,
|
|
"grad_norm": 47.909542083740234,
|
|
"learning_rate": 4.999472998758977e-07,
|
|
"logits/chosen": -0.6333368420600891,
|
|
"logits/rejected": -0.6184351444244385,
|
|
"logps/chosen": -57.71470260620117,
|
|
"logps/ref_chosen": -53.42488098144531,
|
|
"logps/ref_rejected": -95.94693756103516,
|
|
"logps/rejected": -111.6712646484375,
|
|
"loss": 0.9336,
|
|
"margin_dpo/margin_mean": 11.4345064163208,
|
|
"margin_dpo/margin_std": 15.739431381225586,
|
|
"step": 74
|
|
},
|
|
{
|
|
"epoch": 0.11013215859030837,
|
|
"fcm_dpo/beta": 0.06567974388599396,
|
|
"fcm_dpo/delta": -0.5119467377662659,
|
|
"fcm_dpo/margin": 16.138355255126953,
|
|
"fcm_dpo/q_t": 0.291500985622406,
|
|
"grad_norm": 41.545692443847656,
|
|
"learning_rate": 4.999176576834721e-07,
|
|
"logits/chosen": -0.6396368741989136,
|
|
"logits/rejected": -0.6279274225234985,
|
|
"logps/chosen": -55.962501525878906,
|
|
"logps/ref_chosen": -51.861663818359375,
|
|
"logps/ref_rejected": -111.25398254394531,
|
|
"logps/rejected": -131.4931640625,
|
|
"loss": 0.7651,
|
|
"margin_dpo/margin_mean": 16.138355255126953,
|
|
"margin_dpo/margin_std": 15.203086853027344,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.11160058737151249,
|
|
"fcm_dpo/beta": 0.062492579221725464,
|
|
"fcm_dpo/delta": -0.09926581382751465,
|
|
"fcm_dpo/margin": 11.07880973815918,
|
|
"fcm_dpo/q_t": 0.34875011444091797,
|
|
"grad_norm": 41.94511795043945,
|
|
"learning_rate": 4.998814299283415e-07,
|
|
"logits/chosen": -0.7100133895874023,
|
|
"logits/rejected": -0.6680034399032593,
|
|
"logps/chosen": -58.421531677246094,
|
|
"logps/ref_chosen": -53.26603698730469,
|
|
"logps/ref_rejected": -78.21662902832031,
|
|
"logps/rejected": -94.45092010498047,
|
|
"loss": 0.9298,
|
|
"margin_dpo/margin_mean": 11.078810691833496,
|
|
"margin_dpo/margin_std": 12.21889877319336,
|
|
"step": 76
|
|
},
|
|
{
|
|
"epoch": 0.1130690161527166,
|
|
"fcm_dpo/beta": 0.05767790973186493,
|
|
"fcm_dpo/delta": -0.45353737473487854,
|
|
"fcm_dpo/margin": 17.38974380493164,
|
|
"fcm_dpo/q_t": 0.2953811585903168,
|
|
"grad_norm": 43.50868606567383,
|
|
"learning_rate": 4.998386175651409e-07,
|
|
"logits/chosen": -0.6637353897094727,
|
|
"logits/rejected": -0.6201448440551758,
|
|
"logps/chosen": -62.73433303833008,
|
|
"logps/ref_chosen": -58.0966796875,
|
|
"logps/ref_rejected": -93.77361297607422,
|
|
"logps/rejected": -115.80101013183594,
|
|
"loss": 0.7985,
|
|
"margin_dpo/margin_mean": 17.38974380493164,
|
|
"margin_dpo/margin_std": 16.823715209960938,
|
|
"step": 77
|
|
},
|
|
{
|
|
"epoch": 0.1145374449339207,
|
|
"fcm_dpo/beta": 0.0555085651576519,
|
|
"fcm_dpo/delta": -0.18481989204883575,
|
|
"fcm_dpo/margin": 13.888815879821777,
|
|
"fcm_dpo/q_t": 0.3378889262676239,
|
|
"grad_norm": 39.630531311035156,
|
|
"learning_rate": 4.997892217220159e-07,
|
|
"logits/chosen": -0.6263148784637451,
|
|
"logits/rejected": -0.5966655015945435,
|
|
"logps/chosen": -60.46125030517578,
|
|
"logps/ref_chosen": -55.61378479003906,
|
|
"logps/ref_rejected": -84.93436431884766,
|
|
"logps/rejected": -103.67064666748047,
|
|
"loss": 0.8902,
|
|
"margin_dpo/margin_mean": 13.888814926147461,
|
|
"margin_dpo/margin_std": 14.465457916259766,
|
|
"step": 78
|
|
},
|
|
{
|
|
"epoch": 0.11600587371512482,
|
|
"fcm_dpo/beta": 0.05275849997997284,
|
|
"fcm_dpo/delta": -0.24378372728824615,
|
|
"fcm_dpo/margin": 15.608173370361328,
|
|
"fcm_dpo/q_t": 0.3392306864261627,
|
|
"grad_norm": 33.420074462890625,
|
|
"learning_rate": 4.997332437005931e-07,
|
|
"logits/chosen": -0.6634904146194458,
|
|
"logits/rejected": -0.6324459314346313,
|
|
"logps/chosen": -60.526432037353516,
|
|
"logps/ref_chosen": -55.45048522949219,
|
|
"logps/ref_rejected": -87.64756774902344,
|
|
"logps/rejected": -108.33168029785156,
|
|
"loss": 0.9079,
|
|
"margin_dpo/margin_mean": 15.608173370361328,
|
|
"margin_dpo/margin_std": 18.54351043701172,
|
|
"step": 79
|
|
},
|
|
{
|
|
"epoch": 0.11747430249632893,
|
|
"fcm_dpo/beta": 0.05068628117442131,
|
|
"fcm_dpo/delta": -0.12069609016180038,
|
|
"fcm_dpo/margin": 13.985533714294434,
|
|
"fcm_dpo/q_t": 0.3596472144126892,
|
|
"grad_norm": 37.298439025878906,
|
|
"learning_rate": 4.996706849759452e-07,
|
|
"logits/chosen": -0.7097140550613403,
|
|
"logits/rejected": -0.6619396805763245,
|
|
"logps/chosen": -66.05055236816406,
|
|
"logps/ref_chosen": -58.519290924072266,
|
|
"logps/ref_rejected": -87.54750061035156,
|
|
"logps/rejected": -109.06429290771484,
|
|
"loss": 0.9679,
|
|
"margin_dpo/margin_mean": 13.98553466796875,
|
|
"margin_dpo/margin_std": 17.724082946777344,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.11894273127753303,
|
|
"fcm_dpo/beta": 0.047711554914712906,
|
|
"fcm_dpo/delta": -0.3803252577781677,
|
|
"fcm_dpo/margin": 19.70598602294922,
|
|
"fcm_dpo/q_t": 0.31643375754356384,
|
|
"grad_norm": 38.47947692871094,
|
|
"learning_rate": 4.996015471965529e-07,
|
|
"logits/chosen": -0.6981919407844543,
|
|
"logits/rejected": -0.6639083623886108,
|
|
"logps/chosen": -73.06269836425781,
|
|
"logps/ref_chosen": -66.44886779785156,
|
|
"logps/ref_rejected": -129.66270446777344,
|
|
"logps/rejected": -155.98251342773438,
|
|
"loss": 0.8483,
|
|
"margin_dpo/margin_mean": 19.70598602294922,
|
|
"margin_dpo/margin_std": 22.23188018798828,
|
|
"step": 81
|
|
},
|
|
{
|
|
"epoch": 0.12041116005873716,
|
|
"fcm_dpo/beta": 0.04673580825328827,
|
|
"fcm_dpo/delta": -0.15118417143821716,
|
|
"fcm_dpo/margin": 15.803937911987305,
|
|
"fcm_dpo/q_t": 0.3553317189216614,
|
|
"grad_norm": 42.93745422363281,
|
|
"learning_rate": 4.995258321842611e-07,
|
|
"logits/chosen": -0.6455098390579224,
|
|
"logits/rejected": -0.6292978525161743,
|
|
"logps/chosen": -61.03137969970703,
|
|
"logps/ref_chosen": -52.232383728027344,
|
|
"logps/ref_rejected": -90.74325561523438,
|
|
"logps/rejected": -115.34617614746094,
|
|
"loss": 1.018,
|
|
"margin_dpo/margin_mean": 15.803937911987305,
|
|
"margin_dpo/margin_std": 23.58461570739746,
|
|
"step": 82
|
|
},
|
|
{
|
|
"epoch": 0.12187958883994127,
|
|
"fcm_dpo/beta": 0.04404474422335625,
|
|
"fcm_dpo/delta": -0.23587247729301453,
|
|
"fcm_dpo/margin": 18.518720626831055,
|
|
"fcm_dpo/q_t": 0.33574339747428894,
|
|
"grad_norm": 40.53355407714844,
|
|
"learning_rate": 4.994435419342304e-07,
|
|
"logits/chosen": -0.6932875514030457,
|
|
"logits/rejected": -0.6512936353683472,
|
|
"logps/chosen": -64.86161804199219,
|
|
"logps/ref_chosen": -55.82738494873047,
|
|
"logps/ref_rejected": -103.71589660644531,
|
|
"logps/rejected": -131.26885986328125,
|
|
"loss": 0.9031,
|
|
"margin_dpo/margin_mean": 18.518720626831055,
|
|
"margin_dpo/margin_std": 21.3782901763916,
|
|
"step": 83
|
|
},
|
|
{
|
|
"epoch": 0.12334801762114538,
|
|
"fcm_dpo/beta": 0.042480941861867905,
|
|
"fcm_dpo/delta": -0.12278434634208679,
|
|
"fcm_dpo/margin": 16.74671173095703,
|
|
"fcm_dpo/q_t": 0.34350255131721497,
|
|
"grad_norm": 34.17844772338867,
|
|
"learning_rate": 4.993546786148857e-07,
|
|
"logits/chosen": -0.6624174118041992,
|
|
"logits/rejected": -0.624229907989502,
|
|
"logps/chosen": -75.19337463378906,
|
|
"logps/ref_chosen": -67.1761703491211,
|
|
"logps/ref_rejected": -87.29859924316406,
|
|
"logps/rejected": -112.06253051757812,
|
|
"loss": 0.9041,
|
|
"margin_dpo/margin_mean": 16.74671173095703,
|
|
"margin_dpo/margin_std": 16.270580291748047,
|
|
"step": 84
|
|
},
|
|
{
|
|
"epoch": 0.12481644640234948,
|
|
"fcm_dpo/beta": 0.042263105511665344,
|
|
"fcm_dpo/delta": -0.1225767433643341,
|
|
"fcm_dpo/margin": 16.87063217163086,
|
|
"fcm_dpo/q_t": 0.3529721796512604,
|
|
"grad_norm": 36.07400131225586,
|
|
"learning_rate": 4.992592445678582e-07,
|
|
"logits/chosen": -0.645004391670227,
|
|
"logits/rejected": -0.6133354902267456,
|
|
"logps/chosen": -67.26490783691406,
|
|
"logps/ref_chosen": -58.4066162109375,
|
|
"logps/ref_rejected": -78.63880157470703,
|
|
"logps/rejected": -104.36772155761719,
|
|
"loss": 0.9467,
|
|
"margin_dpo/margin_mean": 16.87063217163086,
|
|
"margin_dpo/margin_std": 19.575843811035156,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.1262848751835536,
|
|
"fcm_dpo/beta": 0.04085496813058853,
|
|
"fcm_dpo/delta": -0.15022027492523193,
|
|
"fcm_dpo/margin": 18.096515655517578,
|
|
"fcm_dpo/q_t": 0.3664831817150116,
|
|
"grad_norm": 40.77984619140625,
|
|
"learning_rate": 4.991572423079235e-07,
|
|
"logits/chosen": -0.6773054599761963,
|
|
"logits/rejected": -0.6616036891937256,
|
|
"logps/chosen": -67.63337707519531,
|
|
"logps/ref_chosen": -56.13746643066406,
|
|
"logps/ref_rejected": -88.12165069580078,
|
|
"logps/rejected": -117.71407318115234,
|
|
"loss": 1.0372,
|
|
"margin_dpo/margin_mean": 18.096515655517578,
|
|
"margin_dpo/margin_std": 29.443721771240234,
|
|
"step": 86
|
|
},
|
|
{
|
|
"epoch": 0.1277533039647577,
|
|
"fcm_dpo/beta": 0.03880458325147629,
|
|
"fcm_dpo/delta": -0.21359695494174957,
|
|
"fcm_dpo/margin": 20.468475341796875,
|
|
"fcm_dpo/q_t": 0.3394929766654968,
|
|
"grad_norm": 33.87302780151367,
|
|
"learning_rate": 4.990486745229364e-07,
|
|
"logits/chosen": -0.7122533917427063,
|
|
"logits/rejected": -0.6807093620300293,
|
|
"logps/chosen": -67.1455078125,
|
|
"logps/ref_chosen": -55.63609313964844,
|
|
"logps/ref_rejected": -95.46757507324219,
|
|
"logps/rejected": -127.44546508789062,
|
|
"loss": 0.943,
|
|
"margin_dpo/margin_mean": 20.468475341796875,
|
|
"margin_dpo/margin_std": 25.73678207397461,
|
|
"step": 87
|
|
},
|
|
{
|
|
"epoch": 0.12922173274596183,
|
|
"fcm_dpo/beta": 0.038395434617996216,
|
|
"fcm_dpo/delta": 0.0005285218358039856,
|
|
"fcm_dpo/margin": 15.591540336608887,
|
|
"fcm_dpo/q_t": 0.3785722851753235,
|
|
"grad_norm": 36.46826171875,
|
|
"learning_rate": 4.989335440737586e-07,
|
|
"logits/chosen": -0.6723873615264893,
|
|
"logits/rejected": -0.6606534719467163,
|
|
"logps/chosen": -88.06733703613281,
|
|
"logps/ref_chosen": -73.67115020751953,
|
|
"logps/ref_rejected": -106.70849609375,
|
|
"logps/rejected": -136.69622802734375,
|
|
"loss": 1.0542,
|
|
"margin_dpo/margin_mean": 15.59153938293457,
|
|
"margin_dpo/margin_std": 22.799100875854492,
|
|
"step": 88
|
|
},
|
|
{
|
|
"epoch": 0.13069016152716592,
|
|
"fcm_dpo/beta": 0.03851180896162987,
|
|
"fcm_dpo/delta": -0.06854115426540375,
|
|
"fcm_dpo/margin": 17.231027603149414,
|
|
"fcm_dpo/q_t": 0.36006081104278564,
|
|
"grad_norm": 31.71763038635254,
|
|
"learning_rate": 4.988118539941847e-07,
|
|
"logits/chosen": -0.7132691144943237,
|
|
"logits/rejected": -0.6763544082641602,
|
|
"logps/chosen": -69.2666015625,
|
|
"logps/ref_chosen": -60.624916076660156,
|
|
"logps/ref_rejected": -82.08354949951172,
|
|
"logps/rejected": -107.95626831054688,
|
|
"loss": 0.9577,
|
|
"margin_dpo/margin_mean": 17.231027603149414,
|
|
"margin_dpo/margin_std": 20.84075927734375,
|
|
"step": 89
|
|
},
|
|
{
|
|
"epoch": 0.13215859030837004,
|
|
"fcm_dpo/beta": 0.037114374339580536,
|
|
"fcm_dpo/delta": -0.26081135869026184,
|
|
"fcm_dpo/margin": 22.61049461364746,
|
|
"fcm_dpo/q_t": 0.34644296765327454,
|
|
"grad_norm": 35.28056716918945,
|
|
"learning_rate": 4.986836074908615e-07,
|
|
"logits/chosen": -0.6307741403579712,
|
|
"logits/rejected": -0.6357797384262085,
|
|
"logps/chosen": -64.71424102783203,
|
|
"logps/ref_chosen": -53.285308837890625,
|
|
"logps/ref_rejected": -111.54470825195312,
|
|
"logps/rejected": -145.58413696289062,
|
|
"loss": 0.9619,
|
|
"margin_dpo/margin_mean": 22.610496520996094,
|
|
"margin_dpo/margin_std": 31.246440887451172,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.13362701908957417,
|
|
"fcm_dpo/beta": 0.035702183842659,
|
|
"fcm_dpo/delta": -0.12644991278648376,
|
|
"fcm_dpo/margin": 20.09602928161621,
|
|
"fcm_dpo/q_t": 0.3567592203617096,
|
|
"grad_norm": 33.433372497558594,
|
|
"learning_rate": 4.985488079432037e-07,
|
|
"logits/chosen": -0.6847516894340515,
|
|
"logits/rejected": -0.6470792889595032,
|
|
"logps/chosen": -73.15221405029297,
|
|
"logps/ref_chosen": -61.802955627441406,
|
|
"logps/ref_rejected": -87.87395477294922,
|
|
"logps/rejected": -119.31924438476562,
|
|
"loss": 0.9724,
|
|
"margin_dpo/margin_mean": 20.096027374267578,
|
|
"margin_dpo/margin_std": 26.328693389892578,
|
|
"step": 91
|
|
},
|
|
{
|
|
"epoch": 0.13509544787077826,
|
|
"fcm_dpo/beta": 0.03499949723482132,
|
|
"fcm_dpo/delta": -0.10029930621385574,
|
|
"fcm_dpo/margin": 19.803434371948242,
|
|
"fcm_dpo/q_t": 0.36030054092407227,
|
|
"grad_norm": 30.455114364624023,
|
|
"learning_rate": 4.984074589033043e-07,
|
|
"logits/chosen": -0.7354650497436523,
|
|
"logits/rejected": -0.7112290859222412,
|
|
"logps/chosen": -62.292938232421875,
|
|
"logps/ref_chosen": -51.640769958496094,
|
|
"logps/ref_rejected": -77.88117980957031,
|
|
"logps/rejected": -108.3367919921875,
|
|
"loss": 0.9797,
|
|
"margin_dpo/margin_mean": 19.803434371948242,
|
|
"margin_dpo/margin_std": 25.99898910522461,
|
|
"step": 92
|
|
},
|
|
{
|
|
"epoch": 0.13656387665198239,
|
|
"fcm_dpo/beta": 0.03412316367030144,
|
|
"fcm_dpo/delta": -0.08976341784000397,
|
|
"fcm_dpo/margin": 20.03380584716797,
|
|
"fcm_dpo/q_t": 0.3562747538089752,
|
|
"grad_norm": 30.054187774658203,
|
|
"learning_rate": 4.982595640958425e-07,
|
|
"logits/chosen": -0.7533690333366394,
|
|
"logits/rejected": -0.6998313069343567,
|
|
"logps/chosen": -64.41513061523438,
|
|
"logps/ref_chosen": -52.529239654541016,
|
|
"logps/ref_rejected": -77.16075134277344,
|
|
"logps/rejected": -109.0804443359375,
|
|
"loss": 0.9377,
|
|
"margin_dpo/margin_mean": 20.03380584716797,
|
|
"margin_dpo/margin_std": 23.324481964111328,
|
|
"step": 93
|
|
},
|
|
{
|
|
"epoch": 0.13803230543318648,
|
|
"fcm_dpo/beta": 0.0329783670604229,
|
|
"fcm_dpo/delta": -0.1528889387845993,
|
|
"fcm_dpo/margin": 22.42888641357422,
|
|
"fcm_dpo/q_t": 0.3440149128437042,
|
|
"grad_norm": 30.21152114868164,
|
|
"learning_rate": 4.98105127417984e-07,
|
|
"logits/chosen": -0.6753390431404114,
|
|
"logits/rejected": -0.6542376279830933,
|
|
"logps/chosen": -74.20423889160156,
|
|
"logps/ref_chosen": -61.22261047363281,
|
|
"logps/ref_rejected": -99.59902954101562,
|
|
"logps/rejected": -135.00955200195312,
|
|
"loss": 0.9066,
|
|
"margin_dpo/margin_mean": 22.42888641357422,
|
|
"margin_dpo/margin_std": 23.697879791259766,
|
|
"step": 94
|
|
},
|
|
{
|
|
"epoch": 0.1395007342143906,
|
|
"fcm_dpo/beta": 0.03280120715498924,
|
|
"fcm_dpo/delta": -0.006751883774995804,
|
|
"fcm_dpo/margin": 18.477895736694336,
|
|
"fcm_dpo/q_t": 0.36756014823913574,
|
|
"grad_norm": 28.341552734375,
|
|
"learning_rate": 4.979441529392784e-07,
|
|
"logits/chosen": -0.732563853263855,
|
|
"logits/rejected": -0.701493501663208,
|
|
"logps/chosen": -64.28013610839844,
|
|
"logps/ref_chosen": -52.523643493652344,
|
|
"logps/ref_rejected": -75.8803482055664,
|
|
"logps/rejected": -106.11474609375,
|
|
"loss": 0.981,
|
|
"margin_dpo/margin_mean": 18.477895736694336,
|
|
"margin_dpo/margin_std": 21.757883071899414,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.14096916299559473,
|
|
"fcm_dpo/beta": 0.03166916221380234,
|
|
"fcm_dpo/delta": -0.20488545298576355,
|
|
"fcm_dpo/margin": 24.822622299194336,
|
|
"fcm_dpo/q_t": 0.3381555676460266,
|
|
"grad_norm": 29.113483428955078,
|
|
"learning_rate": 4.977766449015534e-07,
|
|
"logits/chosen": -0.7252618670463562,
|
|
"logits/rejected": -0.6913504600524902,
|
|
"logps/chosen": -73.15214538574219,
|
|
"logps/ref_chosen": -62.15697479248047,
|
|
"logps/ref_rejected": -96.59601593017578,
|
|
"logps/rejected": -132.413818359375,
|
|
"loss": 0.888,
|
|
"margin_dpo/margin_mean": 24.82262420654297,
|
|
"margin_dpo/margin_std": 27.286436080932617,
|
|
"step": 96
|
|
},
|
|
{
|
|
"epoch": 0.14243759177679882,
|
|
"fcm_dpo/beta": 0.03180006146430969,
|
|
"fcm_dpo/delta": -0.026856690645217896,
|
|
"fcm_dpo/margin": 19.596725463867188,
|
|
"fcm_dpo/q_t": 0.36258357763290405,
|
|
"grad_norm": 30.518535614013672,
|
|
"learning_rate": 4.976026077188012e-07,
|
|
"logits/chosen": -0.6421018838882446,
|
|
"logits/rejected": -0.591498851776123,
|
|
"logps/chosen": -66.89000701904297,
|
|
"logps/ref_chosen": -54.646366119384766,
|
|
"logps/ref_rejected": -76.96475219726562,
|
|
"logps/rejected": -108.80511474609375,
|
|
"loss": 0.9602,
|
|
"margin_dpo/margin_mean": 19.596725463867188,
|
|
"margin_dpo/margin_std": 20.48422622680664,
|
|
"step": 97
|
|
},
|
|
{
|
|
"epoch": 0.14390602055800295,
|
|
"fcm_dpo/beta": 0.03093864396214485,
|
|
"fcm_dpo/delta": -0.09095153957605362,
|
|
"fcm_dpo/margin": 22.12537384033203,
|
|
"fcm_dpo/q_t": 0.35298076272010803,
|
|
"grad_norm": 31.037092208862305,
|
|
"learning_rate": 4.974220459770639e-07,
|
|
"logits/chosen": -0.7037702798843384,
|
|
"logits/rejected": -0.684203028678894,
|
|
"logps/chosen": -80.09527587890625,
|
|
"logps/ref_chosen": -65.25862884521484,
|
|
"logps/ref_rejected": -96.5274887084961,
|
|
"logps/rejected": -133.48951721191406,
|
|
"loss": 0.9689,
|
|
"margin_dpo/margin_mean": 22.12537384033203,
|
|
"margin_dpo/margin_std": 27.224925994873047,
|
|
"step": 98
|
|
},
|
|
{
|
|
"epoch": 0.14537444933920704,
|
|
"fcm_dpo/beta": 0.029884759336709976,
|
|
"fcm_dpo/delta": -0.19739775359630585,
|
|
"fcm_dpo/margin": 26.147565841674805,
|
|
"fcm_dpo/q_t": 0.3427448868751526,
|
|
"grad_norm": 27.304397583007812,
|
|
"learning_rate": 4.972349644343108e-07,
|
|
"logits/chosen": -0.694936990737915,
|
|
"logits/rejected": -0.6911288499832153,
|
|
"logps/chosen": -57.74766159057617,
|
|
"logps/ref_chosen": -45.638484954833984,
|
|
"logps/ref_rejected": -86.43793487548828,
|
|
"logps/rejected": -124.69467163085938,
|
|
"loss": 0.9019,
|
|
"margin_dpo/margin_mean": 26.147567749023438,
|
|
"margin_dpo/margin_std": 30.577857971191406,
|
|
"step": 99
|
|
},
|
|
{
|
|
"epoch": 0.14684287812041116,
|
|
"fcm_dpo/beta": 0.0302484929561615,
|
|
"fcm_dpo/delta": 0.1044018343091011,
|
|
"fcm_dpo/margin": 16.52963638305664,
|
|
"fcm_dpo/q_t": 0.3935774266719818,
|
|
"grad_norm": 31.379667282104492,
|
|
"learning_rate": 4.970413680203148e-07,
|
|
"logits/chosen": -0.6751364469528198,
|
|
"logits/rejected": -0.6300145983695984,
|
|
"logps/chosen": -70.98727416992188,
|
|
"logps/ref_chosen": -57.59397888183594,
|
|
"logps/ref_rejected": -74.06021118164062,
|
|
"logps/rejected": -103.98313903808594,
|
|
"loss": 1.0889,
|
|
"margin_dpo/margin_mean": 16.529634475708008,
|
|
"margin_dpo/margin_std": 25.98699188232422,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.14831130690161526,
|
|
"fcm_dpo/beta": 0.030119918286800385,
|
|
"fcm_dpo/delta": 0.012738943099975586,
|
|
"fcm_dpo/margin": 19.492538452148438,
|
|
"fcm_dpo/q_t": 0.382551908493042,
|
|
"grad_norm": 31.488115310668945,
|
|
"learning_rate": 4.968412618365215e-07,
|
|
"logits/chosen": -0.7016730308532715,
|
|
"logits/rejected": -0.6665774583816528,
|
|
"logps/chosen": -77.9628677368164,
|
|
"logps/ref_chosen": -61.64885330200195,
|
|
"logps/ref_rejected": -83.18968200683594,
|
|
"logps/rejected": -118.99624633789062,
|
|
"loss": 1.0464,
|
|
"margin_dpo/margin_mean": 19.492538452148438,
|
|
"margin_dpo/margin_std": 29.35474395751953,
|
|
"step": 101
|
|
},
|
|
{
|
|
"epoch": 0.14977973568281938,
|
|
"fcm_dpo/beta": 0.03130955249071121,
|
|
"fcm_dpo/delta": 0.14849652349948883,
|
|
"fcm_dpo/margin": 14.546876907348633,
|
|
"fcm_dpo/q_t": 0.40587127208709717,
|
|
"grad_norm": 37.16018295288086,
|
|
"learning_rate": 4.966346511559149e-07,
|
|
"logits/chosen": -0.6912135481834412,
|
|
"logits/rejected": -0.6412276029586792,
|
|
"logps/chosen": -82.47193145751953,
|
|
"logps/ref_chosen": -64.0788803100586,
|
|
"logps/ref_rejected": -68.18707275390625,
|
|
"logps/rejected": -101.12701416015625,
|
|
"loss": 1.142,
|
|
"margin_dpo/margin_mean": 14.546875,
|
|
"margin_dpo/margin_std": 26.82202911376953,
|
|
"step": 102
|
|
},
|
|
{
|
|
"epoch": 0.1512481644640235,
|
|
"fcm_dpo/beta": 0.030011659488081932,
|
|
"fcm_dpo/delta": -0.2606973648071289,
|
|
"fcm_dpo/margin": 27.926231384277344,
|
|
"fcm_dpo/q_t": 0.3282929062843323,
|
|
"grad_norm": 28.890342712402344,
|
|
"learning_rate": 4.964215414228785e-07,
|
|
"logits/chosen": -0.6866222620010376,
|
|
"logits/rejected": -0.6497406959533691,
|
|
"logps/chosen": -74.54656982421875,
|
|
"logps/ref_chosen": -61.299278259277344,
|
|
"logps/ref_rejected": -93.57270812988281,
|
|
"logps/rejected": -134.74624633789062,
|
|
"loss": 0.8619,
|
|
"margin_dpo/margin_mean": 27.926231384277344,
|
|
"margin_dpo/margin_std": 29.437318801879883,
|
|
"step": 103
|
|
},
|
|
{
|
|
"epoch": 0.1527165932452276,
|
|
"fcm_dpo/beta": 0.028996918350458145,
|
|
"fcm_dpo/delta": -0.1786722093820572,
|
|
"fcm_dpo/margin": 26.393293380737305,
|
|
"fcm_dpo/q_t": 0.3505978584289551,
|
|
"grad_norm": 29.279157638549805,
|
|
"learning_rate": 4.96201938253052e-07,
|
|
"logits/chosen": -0.7208267450332642,
|
|
"logits/rejected": -0.683314859867096,
|
|
"logps/chosen": -69.56233215332031,
|
|
"logps/ref_chosen": -54.372772216796875,
|
|
"logps/ref_rejected": -89.5647201538086,
|
|
"logps/rejected": -131.14756774902344,
|
|
"loss": 0.9468,
|
|
"margin_dpo/margin_mean": 26.393293380737305,
|
|
"margin_dpo/margin_std": 33.748008728027344,
|
|
"step": 104
|
|
},
|
|
{
|
|
"epoch": 0.15418502202643172,
|
|
"fcm_dpo/beta": 0.026863520964980125,
|
|
"fcm_dpo/delta": -0.43884339928627014,
|
|
"fcm_dpo/margin": 37.05156707763672,
|
|
"fcm_dpo/q_t": 0.28996556997299194,
|
|
"grad_norm": 27.160659790039062,
|
|
"learning_rate": 4.959758474331832e-07,
|
|
"logits/chosen": -0.7008599638938904,
|
|
"logits/rejected": -0.6714988946914673,
|
|
"logps/chosen": -68.3555679321289,
|
|
"logps/ref_chosen": -54.638946533203125,
|
|
"logps/ref_rejected": -97.97351837158203,
|
|
"logps/rejected": -148.74169921875,
|
|
"loss": 0.7501,
|
|
"margin_dpo/margin_mean": 37.05156707763672,
|
|
"margin_dpo/margin_std": 30.501293182373047,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.15565345080763582,
|
|
"fcm_dpo/beta": 0.026156390085816383,
|
|
"fcm_dpo/delta": -0.01053478755056858,
|
|
"fcm_dpo/margin": 23.31535530090332,
|
|
"fcm_dpo/q_t": 0.3665970265865326,
|
|
"grad_norm": 27.37348747253418,
|
|
"learning_rate": 4.957432749209755e-07,
|
|
"logits/chosen": -0.650317370891571,
|
|
"logits/rejected": -0.6077761650085449,
|
|
"logps/chosen": -70.25245666503906,
|
|
"logps/ref_chosen": -54.83289337158203,
|
|
"logps/ref_rejected": -85.22461700439453,
|
|
"logps/rejected": -123.95954895019531,
|
|
"loss": 0.9688,
|
|
"margin_dpo/margin_mean": 23.315357208251953,
|
|
"margin_dpo/margin_std": 26.133378982543945,
|
|
"step": 106
|
|
},
|
|
{
|
|
"epoch": 0.15712187958883994,
|
|
"fcm_dpo/beta": 0.025723904371261597,
|
|
"fcm_dpo/delta": -0.0867917537689209,
|
|
"fcm_dpo/margin": 26.438936233520508,
|
|
"fcm_dpo/q_t": 0.3539083003997803,
|
|
"grad_norm": 26.59319496154785,
|
|
"learning_rate": 4.955042268449307e-07,
|
|
"logits/chosen": -0.7173153162002563,
|
|
"logits/rejected": -0.6675245761871338,
|
|
"logps/chosen": -88.4300537109375,
|
|
"logps/ref_chosen": -69.70780944824219,
|
|
"logps/ref_rejected": -94.73950958251953,
|
|
"logps/rejected": -139.90069580078125,
|
|
"loss": 0.9479,
|
|
"margin_dpo/margin_mean": 26.438936233520508,
|
|
"margin_dpo/margin_std": 29.977489471435547,
|
|
"step": 107
|
|
},
|
|
{
|
|
"epoch": 0.15859030837004406,
|
|
"fcm_dpo/beta": 0.025118088349699974,
|
|
"fcm_dpo/delta": -0.1593261957168579,
|
|
"fcm_dpo/margin": 29.753965377807617,
|
|
"fcm_dpo/q_t": 0.3548673987388611,
|
|
"grad_norm": 27.372953414916992,
|
|
"learning_rate": 4.952587095041881e-07,
|
|
"logits/chosen": -0.6825627088546753,
|
|
"logits/rejected": -0.6495173573493958,
|
|
"logps/chosen": -73.43077087402344,
|
|
"logps/ref_chosen": -56.0098876953125,
|
|
"logps/ref_rejected": -95.79601287841797,
|
|
"logps/rejected": -142.97085571289062,
|
|
"loss": 0.9717,
|
|
"margin_dpo/margin_mean": 29.75396728515625,
|
|
"margin_dpo/margin_std": 39.8942756652832,
|
|
"step": 108
|
|
},
|
|
{
|
|
"epoch": 0.16005873715124816,
|
|
"fcm_dpo/beta": 0.02407240867614746,
|
|
"fcm_dpo/delta": -0.20850417017936707,
|
|
"fcm_dpo/margin": 32.8599853515625,
|
|
"fcm_dpo/q_t": 0.33347180485725403,
|
|
"grad_norm": 26.96596908569336,
|
|
"learning_rate": 4.95006729368358e-07,
|
|
"logits/chosen": -0.6184978485107422,
|
|
"logits/rejected": -0.5932357311248779,
|
|
"logps/chosen": -78.72866821289062,
|
|
"logps/ref_chosen": -62.88549041748047,
|
|
"logps/ref_rejected": -98.68573760986328,
|
|
"logps/rejected": -147.38890075683594,
|
|
"loss": 0.8832,
|
|
"margin_dpo/margin_mean": 32.8599853515625,
|
|
"margin_dpo/margin_std": 33.96752166748047,
|
|
"step": 109
|
|
},
|
|
{
|
|
"epoch": 0.16152716593245228,
|
|
"fcm_dpo/beta": 0.023202266544103622,
|
|
"fcm_dpo/delta": -0.10694625228643417,
|
|
"fcm_dpo/margin": 29.92901611328125,
|
|
"fcm_dpo/q_t": 0.3550639748573303,
|
|
"grad_norm": 24.584754943847656,
|
|
"learning_rate": 4.947482930773511e-07,
|
|
"logits/chosen": -0.6574271321296692,
|
|
"logits/rejected": -0.6152883768081665,
|
|
"logps/chosen": -74.33702087402344,
|
|
"logps/ref_chosen": -58.753684997558594,
|
|
"logps/ref_rejected": -79.75001525878906,
|
|
"logps/rejected": -125.26237487792969,
|
|
"loss": 0.9612,
|
|
"margin_dpo/margin_mean": 29.92901611328125,
|
|
"margin_dpo/margin_std": 35.73463439941406,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.16299559471365638,
|
|
"fcm_dpo/beta": 0.022714221850037575,
|
|
"fcm_dpo/delta": -0.15530934929847717,
|
|
"fcm_dpo/margin": 32.667232513427734,
|
|
"fcm_dpo/q_t": 0.347843736410141,
|
|
"grad_norm": 27.264070510864258,
|
|
"learning_rate": 4.944834074412042e-07,
|
|
"logits/chosen": -0.6808880567550659,
|
|
"logits/rejected": -0.6555056571960449,
|
|
"logps/chosen": -87.53121948242188,
|
|
"logps/ref_chosen": -68.62410736083984,
|
|
"logps/ref_rejected": -98.42886352539062,
|
|
"logps/rejected": -150.00320434570312,
|
|
"loss": 0.9492,
|
|
"margin_dpo/margin_mean": 32.667232513427734,
|
|
"margin_dpo/margin_std": 39.969459533691406,
|
|
"step": 111
|
|
},
|
|
{
|
|
"epoch": 0.1644640234948605,
|
|
"fcm_dpo/beta": 0.022934526205062866,
|
|
"fcm_dpo/delta": 0.11087077856063843,
|
|
"fcm_dpo/margin": 21.566303253173828,
|
|
"fcm_dpo/q_t": 0.39232131838798523,
|
|
"grad_norm": 25.337993621826172,
|
|
"learning_rate": 4.942120794399002e-07,
|
|
"logits/chosen": -0.6598977446556091,
|
|
"logits/rejected": -0.613295316696167,
|
|
"logps/chosen": -67.96663665771484,
|
|
"logps/ref_chosen": -50.24964141845703,
|
|
"logps/ref_rejected": -64.77442932128906,
|
|
"logps/rejected": -104.05772399902344,
|
|
"loss": 1.0543,
|
|
"margin_dpo/margin_mean": 21.566301345825195,
|
|
"margin_dpo/margin_std": 29.049222946166992,
|
|
"step": 112
|
|
},
|
|
{
|
|
"epoch": 0.16593245227606462,
|
|
"fcm_dpo/beta": 0.023325610905885696,
|
|
"fcm_dpo/delta": 0.040511354804039,
|
|
"fcm_dpo/margin": 24.083799362182617,
|
|
"fcm_dpo/q_t": 0.37566256523132324,
|
|
"grad_norm": 25.780298233032227,
|
|
"learning_rate": 4.939343162231841e-07,
|
|
"logits/chosen": -0.647671639919281,
|
|
"logits/rejected": -0.5998808741569519,
|
|
"logps/chosen": -87.71575927734375,
|
|
"logps/ref_chosen": -66.71295166015625,
|
|
"logps/ref_rejected": -77.96870422363281,
|
|
"logps/rejected": -123.0553207397461,
|
|
"loss": 0.9905,
|
|
"margin_dpo/margin_mean": 24.083797454833984,
|
|
"margin_dpo/margin_std": 27.016586303710938,
|
|
"step": 113
|
|
},
|
|
{
|
|
"epoch": 0.16740088105726872,
|
|
"fcm_dpo/beta": 0.02235432155430317,
|
|
"fcm_dpo/delta": -0.24365481734275818,
|
|
"fcm_dpo/margin": 36.693424224853516,
|
|
"fcm_dpo/q_t": 0.33775386214256287,
|
|
"grad_norm": 25.40207862854004,
|
|
"learning_rate": 4.936501251103751e-07,
|
|
"logits/chosen": -0.6867313385009766,
|
|
"logits/rejected": -0.6511885523796082,
|
|
"logps/chosen": -77.00772094726562,
|
|
"logps/ref_chosen": -57.78507995605469,
|
|
"logps/ref_rejected": -87.10966491699219,
|
|
"logps/rejected": -143.02572631835938,
|
|
"loss": 0.8963,
|
|
"margin_dpo/margin_mean": 36.693424224853516,
|
|
"margin_dpo/margin_std": 44.51530456542969,
|
|
"step": 114
|
|
},
|
|
{
|
|
"epoch": 0.16886930983847284,
|
|
"fcm_dpo/beta": 0.022399384528398514,
|
|
"fcm_dpo/delta": 0.014765216037631035,
|
|
"fcm_dpo/margin": 26.151029586791992,
|
|
"fcm_dpo/q_t": 0.38476622104644775,
|
|
"grad_norm": 38.612613677978516,
|
|
"learning_rate": 4.933595135901732e-07,
|
|
"logits/chosen": -0.6795483827590942,
|
|
"logits/rejected": -0.6489361524581909,
|
|
"logps/chosen": -91.45716094970703,
|
|
"logps/ref_chosen": -65.5826416015625,
|
|
"logps/ref_rejected": -98.56552124023438,
|
|
"logps/rejected": -150.591064453125,
|
|
"loss": 1.0837,
|
|
"margin_dpo/margin_mean": 26.15102767944336,
|
|
"margin_dpo/margin_std": 44.87915802001953,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.17033773861967694,
|
|
"fcm_dpo/beta": 0.02236928418278694,
|
|
"fcm_dpo/delta": -0.02009018138051033,
|
|
"fcm_dpo/margin": 27.650157928466797,
|
|
"fcm_dpo/q_t": 0.3669181764125824,
|
|
"grad_norm": 28.57147979736328,
|
|
"learning_rate": 4.930624893204624e-07,
|
|
"logits/chosen": -0.6698657274246216,
|
|
"logits/rejected": -0.6501831412315369,
|
|
"logps/chosen": -70.91404724121094,
|
|
"logps/ref_chosen": -51.40031433105469,
|
|
"logps/ref_rejected": -80.5218505859375,
|
|
"logps/rejected": -127.68574523925781,
|
|
"loss": 0.9627,
|
|
"margin_dpo/margin_mean": 27.650157928466797,
|
|
"margin_dpo/margin_std": 31.316654205322266,
|
|
"step": 116
|
|
},
|
|
{
|
|
"epoch": 0.17180616740088106,
|
|
"fcm_dpo/beta": 0.022425107657909393,
|
|
"fcm_dpo/delta": 0.05522657185792923,
|
|
"fcm_dpo/margin": 24.42792510986328,
|
|
"fcm_dpo/q_t": 0.3833027184009552,
|
|
"grad_norm": 33.80198287963867,
|
|
"learning_rate": 4.927590601281083e-07,
|
|
"logits/chosen": -0.6356416940689087,
|
|
"logits/rejected": -0.594796895980835,
|
|
"logps/chosen": -93.27058410644531,
|
|
"logps/ref_chosen": -69.29840850830078,
|
|
"logps/ref_rejected": -66.583984375,
|
|
"logps/rejected": -114.98409271240234,
|
|
"loss": 1.0417,
|
|
"margin_dpo/margin_mean": 24.427927017211914,
|
|
"margin_dpo/margin_std": 34.87114715576172,
|
|
"step": 117
|
|
},
|
|
{
|
|
"epoch": 0.17327459618208516,
|
|
"fcm_dpo/beta": 0.022451553493738174,
|
|
"fcm_dpo/delta": -0.012901581823825836,
|
|
"fcm_dpo/margin": 27.263755798339844,
|
|
"fcm_dpo/q_t": 0.36932849884033203,
|
|
"grad_norm": 27.19830894470215,
|
|
"learning_rate": 4.924492340087524e-07,
|
|
"logits/chosen": -0.6901913285255432,
|
|
"logits/rejected": -0.6656088829040527,
|
|
"logps/chosen": -76.18692779541016,
|
|
"logps/ref_chosen": -55.6409797668457,
|
|
"logps/ref_rejected": -75.66905975341797,
|
|
"logps/rejected": -123.478759765625,
|
|
"loss": 0.9741,
|
|
"margin_dpo/margin_mean": 27.263755798339844,
|
|
"margin_dpo/margin_std": 32.05895233154297,
|
|
"step": 118
|
|
},
|
|
{
|
|
"epoch": 0.17474302496328928,
|
|
"fcm_dpo/beta": 0.02224084362387657,
|
|
"fcm_dpo/delta": -0.010049469769001007,
|
|
"fcm_dpo/margin": 27.355573654174805,
|
|
"fcm_dpo/q_t": 0.37495267391204834,
|
|
"grad_norm": 29.576772689819336,
|
|
"learning_rate": 4.92133019126601e-07,
|
|
"logits/chosen": -0.6743392944335938,
|
|
"logits/rejected": -0.6560527682304382,
|
|
"logps/chosen": -100.41120910644531,
|
|
"logps/ref_chosen": -73.51019287109375,
|
|
"logps/ref_rejected": -102.977294921875,
|
|
"logps/rejected": -157.23387145996094,
|
|
"loss": 1.0134,
|
|
"margin_dpo/margin_mean": 27.355573654174805,
|
|
"margin_dpo/margin_std": 37.5633430480957,
|
|
"step": 119
|
|
},
|
|
{
|
|
"epoch": 0.1762114537444934,
|
|
"fcm_dpo/beta": 0.02168981544673443,
|
|
"fcm_dpo/delta": -0.2090209424495697,
|
|
"fcm_dpo/margin": 36.529258728027344,
|
|
"fcm_dpo/q_t": 0.3369954228401184,
|
|
"grad_norm": 27.668405532836914,
|
|
"learning_rate": 4.918104238142103e-07,
|
|
"logits/chosen": -0.6739026308059692,
|
|
"logits/rejected": -0.6356397867202759,
|
|
"logps/chosen": -105.38459777832031,
|
|
"logps/ref_chosen": -76.78083801269531,
|
|
"logps/ref_rejected": -108.02374267578125,
|
|
"logps/rejected": -173.15676879882812,
|
|
"loss": 0.9002,
|
|
"margin_dpo/margin_mean": 36.529258728027344,
|
|
"margin_dpo/margin_std": 40.862762451171875,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.1776798825256975,
|
|
"fcm_dpo/beta": 0.020538385957479477,
|
|
"fcm_dpo/delta": -0.2580873370170593,
|
|
"fcm_dpo/margin": 40.62395477294922,
|
|
"fcm_dpo/q_t": 0.33450600504875183,
|
|
"grad_norm": 27.22615623474121,
|
|
"learning_rate": 4.91481456572267e-07,
|
|
"logits/chosen": -0.6262691020965576,
|
|
"logits/rejected": -0.6172099113464355,
|
|
"logps/chosen": -89.005126953125,
|
|
"logps/ref_chosen": -61.789894104003906,
|
|
"logps/ref_rejected": -109.99456787109375,
|
|
"logps/rejected": -177.83375549316406,
|
|
"loss": 0.9049,
|
|
"margin_dpo/margin_mean": 40.623958587646484,
|
|
"margin_dpo/margin_std": 47.878326416015625,
|
|
"step": 121
|
|
},
|
|
{
|
|
"epoch": 0.17914831130690162,
|
|
"fcm_dpo/beta": 0.01918221451342106,
|
|
"fcm_dpo/delta": -0.4117882251739502,
|
|
"fcm_dpo/margin": 50.65019989013672,
|
|
"fcm_dpo/q_t": 0.29864490032196045,
|
|
"grad_norm": 28.19658660888672,
|
|
"learning_rate": 4.911461260693638e-07,
|
|
"logits/chosen": -0.6220955848693848,
|
|
"logits/rejected": -0.6330730319023132,
|
|
"logps/chosen": -71.84508514404297,
|
|
"logps/ref_chosen": -46.9022102355957,
|
|
"logps/ref_rejected": -106.71418762207031,
|
|
"logps/rejected": -182.30726623535156,
|
|
"loss": 0.7772,
|
|
"margin_dpo/margin_mean": 50.65019989013672,
|
|
"margin_dpo/margin_std": 44.97718048095703,
|
|
"step": 122
|
|
},
|
|
{
|
|
"epoch": 0.18061674008810572,
|
|
"fcm_dpo/beta": 0.018608931452035904,
|
|
"fcm_dpo/delta": -0.027324281632900238,
|
|
"fcm_dpo/margin": 33.586708068847656,
|
|
"fcm_dpo/q_t": 0.374822199344635,
|
|
"grad_norm": 26.818113327026367,
|
|
"learning_rate": 4.908044411417711e-07,
|
|
"logits/chosen": -0.612564206123352,
|
|
"logits/rejected": -0.5866974592208862,
|
|
"logps/chosen": -88.52189636230469,
|
|
"logps/ref_chosen": -61.33863830566406,
|
|
"logps/ref_rejected": -87.775390625,
|
|
"logps/rejected": -148.5453643798828,
|
|
"loss": 1.0431,
|
|
"margin_dpo/margin_mean": 33.586708068847656,
|
|
"margin_dpo/margin_std": 51.27476501464844,
|
|
"step": 123
|
|
},
|
|
{
|
|
"epoch": 0.18208516886930984,
|
|
"fcm_dpo/beta": 0.017872150987386703,
|
|
"fcm_dpo/delta": -0.27416178584098816,
|
|
"fcm_dpo/margin": 47.546592712402344,
|
|
"fcm_dpo/q_t": 0.34260714054107666,
|
|
"grad_norm": 28.516874313354492,
|
|
"learning_rate": 4.904564107932048e-07,
|
|
"logits/chosen": -0.6066104173660278,
|
|
"logits/rejected": -0.6043754816055298,
|
|
"logps/chosen": -101.30255126953125,
|
|
"logps/ref_chosen": -71.44833374023438,
|
|
"logps/ref_rejected": -117.58056640625,
|
|
"logps/rejected": -194.98138427734375,
|
|
"loss": 0.9423,
|
|
"margin_dpo/margin_mean": 47.546592712402344,
|
|
"margin_dpo/margin_std": 64.18199157714844,
|
|
"step": 124
|
|
},
|
|
{
|
|
"epoch": 0.18355359765051396,
|
|
"fcm_dpo/beta": 0.017282985150814056,
|
|
"fcm_dpo/delta": -0.17058980464935303,
|
|
"fcm_dpo/margin": 43.85457229614258,
|
|
"fcm_dpo/q_t": 0.3455929458141327,
|
|
"grad_norm": 24.23063087463379,
|
|
"learning_rate": 4.90102044194588e-07,
|
|
"logits/chosen": -0.5887873768806458,
|
|
"logits/rejected": -0.5830005407333374,
|
|
"logps/chosen": -74.79535675048828,
|
|
"logps/ref_chosen": -50.136940002441406,
|
|
"logps/ref_rejected": -83.98861694335938,
|
|
"logps/rejected": -152.50160217285156,
|
|
"loss": 0.9197,
|
|
"margin_dpo/margin_mean": 43.85457229614258,
|
|
"margin_dpo/margin_std": 50.49039077758789,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.18502202643171806,
|
|
"fcm_dpo/beta": 0.016843030229210854,
|
|
"fcm_dpo/delta": -0.08583886921405792,
|
|
"fcm_dpo/margin": 40.36263656616211,
|
|
"fcm_dpo/q_t": 0.35755184292793274,
|
|
"grad_norm": 26.442380905151367,
|
|
"learning_rate": 4.897413506838102e-07,
|
|
"logits/chosen": -0.5875859260559082,
|
|
"logits/rejected": -0.5705457925796509,
|
|
"logps/chosen": -82.85122680664062,
|
|
"logps/ref_chosen": -55.66706848144531,
|
|
"logps/ref_rejected": -98.1297607421875,
|
|
"logps/rejected": -165.67654418945312,
|
|
"loss": 0.9522,
|
|
"margin_dpo/margin_mean": 40.36263656616211,
|
|
"margin_dpo/margin_std": 47.85084533691406,
|
|
"step": 126
|
|
},
|
|
{
|
|
"epoch": 0.18649045521292218,
|
|
"fcm_dpo/beta": 0.0169706791639328,
|
|
"fcm_dpo/delta": 0.061831459403038025,
|
|
"fcm_dpo/margin": 31.902957916259766,
|
|
"fcm_dpo/q_t": 0.38108593225479126,
|
|
"grad_norm": 26.799118041992188,
|
|
"learning_rate": 4.89374339765481e-07,
|
|
"logits/chosen": -0.576623797416687,
|
|
"logits/rejected": -0.5499337911605835,
|
|
"logps/chosen": -82.35685729980469,
|
|
"logps/ref_chosen": -56.55467987060547,
|
|
"logps/ref_rejected": -76.7957763671875,
|
|
"logps/rejected": -134.50091552734375,
|
|
"loss": 1.0288,
|
|
"margin_dpo/margin_mean": 31.902957916259766,
|
|
"margin_dpo/margin_std": 41.576568603515625,
|
|
"step": 127
|
|
},
|
|
{
|
|
"epoch": 0.18795888399412627,
|
|
"fcm_dpo/beta": 0.017191898077726364,
|
|
"fcm_dpo/delta": 0.03887506574392319,
|
|
"fcm_dpo/margin": 32.724117279052734,
|
|
"fcm_dpo/q_t": 0.3826625347137451,
|
|
"grad_norm": 38.53647994995117,
|
|
"learning_rate": 4.890010211106795e-07,
|
|
"logits/chosen": -0.5986074209213257,
|
|
"logits/rejected": -0.5656411051750183,
|
|
"logps/chosen": -86.5602035522461,
|
|
"logps/ref_chosen": -58.12095642089844,
|
|
"logps/ref_rejected": -76.43896484375,
|
|
"logps/rejected": -137.60232543945312,
|
|
"loss": 1.0607,
|
|
"margin_dpo/margin_mean": 32.72411346435547,
|
|
"margin_dpo/margin_std": 49.83234405517578,
|
|
"step": 128
|
|
},
|
|
{
|
|
"epoch": 0.1894273127753304,
|
|
"fcm_dpo/beta": 0.017184410244226456,
|
|
"fcm_dpo/delta": 0.005901549011468887,
|
|
"fcm_dpo/margin": 34.58758544921875,
|
|
"fcm_dpo/q_t": 0.3824169635772705,
|
|
"grad_norm": 28.598745346069336,
|
|
"learning_rate": 4.88621404556699e-07,
|
|
"logits/chosen": -0.6040947437286377,
|
|
"logits/rejected": -0.5859910845756531,
|
|
"logps/chosen": -102.27650451660156,
|
|
"logps/ref_chosen": -66.91637420654297,
|
|
"logps/ref_rejected": -96.6422119140625,
|
|
"logps/rejected": -166.58993530273438,
|
|
"loss": 1.0624,
|
|
"margin_dpo/margin_mean": 34.58758544921875,
|
|
"margin_dpo/margin_std": 55.73461151123047,
|
|
"step": 129
|
|
},
|
|
{
|
|
"epoch": 0.19089574155653452,
|
|
"fcm_dpo/beta": 0.016697879880666733,
|
|
"fcm_dpo/delta": -0.2792219817638397,
|
|
"fcm_dpo/margin": 51.21843338012695,
|
|
"fcm_dpo/q_t": 0.3288246989250183,
|
|
"grad_norm": 22.612878799438477,
|
|
"learning_rate": 4.882355001067891e-07,
|
|
"logits/chosen": -0.5792099237442017,
|
|
"logits/rejected": -0.5733057260513306,
|
|
"logps/chosen": -71.08717346191406,
|
|
"logps/ref_chosen": -44.66685104370117,
|
|
"logps/ref_rejected": -82.78165435791016,
|
|
"logps/rejected": -160.42041015625,
|
|
"loss": 0.8943,
|
|
"margin_dpo/margin_mean": 51.21843338012695,
|
|
"margin_dpo/margin_std": 56.39968490600586,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.19236417033773862,
|
|
"fcm_dpo/beta": 0.015755565837025642,
|
|
"fcm_dpo/delta": -0.18772238492965698,
|
|
"fcm_dpo/margin": 49.05485916137695,
|
|
"fcm_dpo/q_t": 0.3336232304573059,
|
|
"grad_norm": 26.170124053955078,
|
|
"learning_rate": 4.878433179298909e-07,
|
|
"logits/chosen": -0.5605712532997131,
|
|
"logits/rejected": -0.564369261264801,
|
|
"logps/chosen": -66.87252044677734,
|
|
"logps/ref_chosen": -44.924591064453125,
|
|
"logps/ref_rejected": -88.44401550292969,
|
|
"logps/rejected": -159.44680786132812,
|
|
"loss": 0.8818,
|
|
"margin_dpo/margin_mean": 49.05485916137695,
|
|
"margin_dpo/margin_std": 49.06263732910156,
|
|
"step": 131
|
|
},
|
|
{
|
|
"epoch": 0.19383259911894274,
|
|
"fcm_dpo/beta": 0.015453611500561237,
|
|
"fcm_dpo/delta": -0.05859919637441635,
|
|
"fcm_dpo/margin": 42.35327911376953,
|
|
"fcm_dpo/q_t": 0.36758387088775635,
|
|
"grad_norm": 25.90082359313965,
|
|
"learning_rate": 4.874448683603694e-07,
|
|
"logits/chosen": -0.6056761741638184,
|
|
"logits/rejected": -0.5935735702514648,
|
|
"logps/chosen": -89.5750732421875,
|
|
"logps/ref_chosen": -59.00108337402344,
|
|
"logps/ref_rejected": -87.89215087890625,
|
|
"logps/rejected": -160.81942749023438,
|
|
"loss": 0.9903,
|
|
"margin_dpo/margin_mean": 42.35327911376953,
|
|
"margin_dpo/margin_std": 57.95384979248047,
|
|
"step": 132
|
|
},
|
|
{
|
|
"epoch": 0.19530102790014683,
|
|
"fcm_dpo/beta": 0.01545734517276287,
|
|
"fcm_dpo/delta": 0.0167352557182312,
|
|
"fcm_dpo/margin": 37.79522705078125,
|
|
"fcm_dpo/q_t": 0.38009342551231384,
|
|
"grad_norm": 27.229202270507812,
|
|
"learning_rate": 4.870401618977415e-07,
|
|
"logits/chosen": -0.5968517065048218,
|
|
"logits/rejected": -0.579937219619751,
|
|
"logps/chosen": -102.51626586914062,
|
|
"logps/ref_chosen": -66.60449981689453,
|
|
"logps/ref_rejected": -96.33355712890625,
|
|
"logps/rejected": -170.04055786132812,
|
|
"loss": 1.0233,
|
|
"margin_dpo/margin_mean": 37.795230865478516,
|
|
"margin_dpo/margin_std": 53.25979995727539,
|
|
"step": 133
|
|
},
|
|
{
|
|
"epoch": 0.19676945668135096,
|
|
"fcm_dpo/beta": 0.01544961053878069,
|
|
"fcm_dpo/delta": -0.046767231076955795,
|
|
"fcm_dpo/margin": 41.66034698486328,
|
|
"fcm_dpo/q_t": 0.3632383346557617,
|
|
"grad_norm": 24.29633903503418,
|
|
"learning_rate": 4.866292092063986e-07,
|
|
"logits/chosen": -0.562139630317688,
|
|
"logits/rejected": -0.5372625589370728,
|
|
"logps/chosen": -80.47048950195312,
|
|
"logps/ref_chosen": -52.06925582885742,
|
|
"logps/ref_rejected": -87.6545181274414,
|
|
"logps/rejected": -157.71609497070312,
|
|
"loss": 0.9512,
|
|
"margin_dpo/margin_mean": 41.66035079956055,
|
|
"margin_dpo/margin_std": 47.288726806640625,
|
|
"step": 134
|
|
},
|
|
{
|
|
"epoch": 0.19823788546255505,
|
|
"fcm_dpo/beta": 0.014866790734231472,
|
|
"fcm_dpo/delta": -0.21857857704162598,
|
|
"fcm_dpo/margin": 53.88473892211914,
|
|
"fcm_dpo/q_t": 0.3398352265357971,
|
|
"grad_norm": 25.403532028198242,
|
|
"learning_rate": 4.862120211153265e-07,
|
|
"logits/chosen": -0.6001534461975098,
|
|
"logits/rejected": -0.627263069152832,
|
|
"logps/chosen": -82.99242401123047,
|
|
"logps/ref_chosen": -50.353858947753906,
|
|
"logps/ref_rejected": -115.97975158691406,
|
|
"logps/rejected": -202.5030517578125,
|
|
"loss": 0.9102,
|
|
"margin_dpo/margin_mean": 53.88473892211914,
|
|
"margin_dpo/margin_std": 62.925498962402344,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.19970631424375918,
|
|
"fcm_dpo/beta": 0.014702379703521729,
|
|
"fcm_dpo/delta": 0.05534332990646362,
|
|
"fcm_dpo/margin": 37.198089599609375,
|
|
"fcm_dpo/q_t": 0.3941587805747986,
|
|
"grad_norm": 27.017377853393555,
|
|
"learning_rate": 4.857886086178193e-07,
|
|
"logits/chosen": -0.5803062915802002,
|
|
"logits/rejected": -0.5640683174133301,
|
|
"logps/chosen": -103.38967895507812,
|
|
"logps/ref_chosen": -65.072509765625,
|
|
"logps/ref_rejected": -96.32122802734375,
|
|
"logps/rejected": -171.83648681640625,
|
|
"loss": 1.0679,
|
|
"margin_dpo/margin_mean": 37.198089599609375,
|
|
"margin_dpo/margin_std": 59.631011962890625,
|
|
"step": 136
|
|
},
|
|
{
|
|
"epoch": 0.2011747430249633,
|
|
"fcm_dpo/beta": 0.014293761923909187,
|
|
"fcm_dpo/delta": -0.24502620100975037,
|
|
"fcm_dpo/margin": 57.65531921386719,
|
|
"fcm_dpo/q_t": 0.3446000814437866,
|
|
"grad_norm": 23.374046325683594,
|
|
"learning_rate": 4.853589828711902e-07,
|
|
"logits/chosen": -0.5490569472312927,
|
|
"logits/rejected": -0.5711492300033569,
|
|
"logps/chosen": -87.5726089477539,
|
|
"logps/ref_chosen": -48.759117126464844,
|
|
"logps/ref_rejected": -113.86376953125,
|
|
"logps/rejected": -210.33258056640625,
|
|
"loss": 0.9452,
|
|
"margin_dpo/margin_mean": 57.65531921386719,
|
|
"margin_dpo/margin_std": 76.925048828125,
|
|
"step": 137
|
|
},
|
|
{
|
|
"epoch": 0.2026431718061674,
|
|
"fcm_dpo/beta": 0.014147358015179634,
|
|
"fcm_dpo/delta": -0.0190581101924181,
|
|
"fcm_dpo/margin": 43.66788864135742,
|
|
"fcm_dpo/q_t": 0.3662746548652649,
|
|
"grad_norm": 26.790586471557617,
|
|
"learning_rate": 4.849231551964771e-07,
|
|
"logits/chosen": -0.5593977570533752,
|
|
"logits/rejected": -0.5456332564353943,
|
|
"logps/chosen": -101.01687622070312,
|
|
"logps/ref_chosen": -60.519649505615234,
|
|
"logps/ref_rejected": -93.19694519042969,
|
|
"logps/rejected": -177.362060546875,
|
|
"loss": 0.9758,
|
|
"margin_dpo/margin_mean": 43.667884826660156,
|
|
"margin_dpo/margin_std": 51.35163879394531,
|
|
"step": 138
|
|
},
|
|
{
|
|
"epoch": 0.20411160058737152,
|
|
"fcm_dpo/beta": 0.01379378791898489,
|
|
"fcm_dpo/delta": -0.16102235019207,
|
|
"fcm_dpo/margin": 54.313804626464844,
|
|
"fcm_dpo/q_t": 0.34360840916633606,
|
|
"grad_norm": 22.88560676574707,
|
|
"learning_rate": 4.844811370781446e-07,
|
|
"logits/chosen": -0.5527390241622925,
|
|
"logits/rejected": -0.5373610258102417,
|
|
"logps/chosen": -79.2032470703125,
|
|
"logps/ref_chosen": -46.89138412475586,
|
|
"logps/ref_rejected": -79.72798156738281,
|
|
"logps/rejected": -166.3536376953125,
|
|
"loss": 0.906,
|
|
"margin_dpo/margin_mean": 54.313804626464844,
|
|
"margin_dpo/margin_std": 59.772552490234375,
|
|
"step": 139
|
|
},
|
|
{
|
|
"epoch": 0.2055800293685756,
|
|
"fcm_dpo/beta": 0.013468625023961067,
|
|
"fcm_dpo/delta": -0.0790109634399414,
|
|
"fcm_dpo/margin": 50.007423400878906,
|
|
"fcm_dpo/q_t": 0.36189499497413635,
|
|
"grad_norm": 24.64659309387207,
|
|
"learning_rate": 4.840329401637809e-07,
|
|
"logits/chosen": -0.5458736419677734,
|
|
"logits/rejected": -0.5282671451568604,
|
|
"logps/chosen": -98.45331573486328,
|
|
"logps/ref_chosen": -58.97471618652344,
|
|
"logps/ref_rejected": -83.28410339355469,
|
|
"logps/rejected": -172.77012634277344,
|
|
"loss": 0.9788,
|
|
"margin_dpo/margin_mean": 50.007423400878906,
|
|
"margin_dpo/margin_std": 64.80023193359375,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20704845814977973,
|
|
"fcm_dpo/beta": 0.013357013463973999,
|
|
"fcm_dpo/delta": -0.04534909874200821,
|
|
"fcm_dpo/margin": 48.098968505859375,
|
|
"fcm_dpo/q_t": 0.3626646399497986,
|
|
"grad_norm": 28.8856258392334,
|
|
"learning_rate": 4.83578576263792e-07,
|
|
"logits/chosen": -0.5459233522415161,
|
|
"logits/rejected": -0.5275824069976807,
|
|
"logps/chosen": -116.89370727539062,
|
|
"logps/ref_chosen": -75.07566833496094,
|
|
"logps/ref_rejected": -98.1922607421875,
|
|
"logps/rejected": -188.10928344726562,
|
|
"loss": 0.9921,
|
|
"margin_dpo/margin_mean": 48.098968505859375,
|
|
"margin_dpo/margin_std": 62.48729705810547,
|
|
"step": 141
|
|
},
|
|
{
|
|
"epoch": 0.20851688693098386,
|
|
"fcm_dpo/beta": 0.01312628760933876,
|
|
"fcm_dpo/delta": -0.13432356715202332,
|
|
"fcm_dpo/margin": 55.20783233642578,
|
|
"fcm_dpo/q_t": 0.35684800148010254,
|
|
"grad_norm": 28.54540252685547,
|
|
"learning_rate": 4.83118057351089e-07,
|
|
"logits/chosen": -0.5353626608848572,
|
|
"logits/rejected": -0.5301727056503296,
|
|
"logps/chosen": -103.46934509277344,
|
|
"logps/ref_chosen": -58.027931213378906,
|
|
"logps/ref_rejected": -94.58222961425781,
|
|
"logps/rejected": -195.23147583007812,
|
|
"loss": 0.9917,
|
|
"margin_dpo/margin_mean": 55.20783233642578,
|
|
"margin_dpo/margin_std": 75.06365203857422,
|
|
"step": 142
|
|
},
|
|
{
|
|
"epoch": 0.20998531571218795,
|
|
"fcm_dpo/beta": 0.013182537630200386,
|
|
"fcm_dpo/delta": 0.14479340612888336,
|
|
"fcm_dpo/margin": 35.0554313659668,
|
|
"fcm_dpo/q_t": 0.4067147374153137,
|
|
"grad_norm": 31.684499740600586,
|
|
"learning_rate": 4.826513955607734e-07,
|
|
"logits/chosen": -0.5001394152641296,
|
|
"logits/rejected": -0.48260974884033203,
|
|
"logps/chosen": -107.36073303222656,
|
|
"logps/ref_chosen": -57.59645080566406,
|
|
"logps/ref_rejected": -78.99957275390625,
|
|
"logps/rejected": -163.81927490234375,
|
|
"loss": 1.1328,
|
|
"margin_dpo/margin_mean": 35.05542755126953,
|
|
"margin_dpo/margin_std": 64.47492980957031,
|
|
"step": 143
|
|
},
|
|
{
|
|
"epoch": 0.21145374449339208,
|
|
"fcm_dpo/beta": 0.0133123230189085,
|
|
"fcm_dpo/delta": -0.005581766366958618,
|
|
"fcm_dpo/margin": 45.4619255065918,
|
|
"fcm_dpo/q_t": 0.3696960508823395,
|
|
"grad_norm": 24.83319854736328,
|
|
"learning_rate": 4.821786031898176e-07,
|
|
"logits/chosen": -0.5548040866851807,
|
|
"logits/rejected": -0.5310955047607422,
|
|
"logps/chosen": -100.95870971679688,
|
|
"logps/ref_chosen": -59.90636444091797,
|
|
"logps/ref_rejected": -82.00025939941406,
|
|
"logps/rejected": -168.51454162597656,
|
|
"loss": 0.9842,
|
|
"margin_dpo/margin_mean": 45.46192932128906,
|
|
"margin_dpo/margin_std": 54.452301025390625,
|
|
"step": 144
|
|
},
|
|
{
|
|
"epoch": 0.21292217327459617,
|
|
"fcm_dpo/beta": 0.013245116919279099,
|
|
"fcm_dpo/delta": -0.03191668167710304,
|
|
"fcm_dpo/margin": 47.55194854736328,
|
|
"fcm_dpo/q_t": 0.3665177524089813,
|
|
"grad_norm": 29.93503189086914,
|
|
"learning_rate": 4.816996926967401e-07,
|
|
"logits/chosen": -0.5175589323043823,
|
|
"logits/rejected": -0.48851436376571655,
|
|
"logps/chosen": -98.98822021484375,
|
|
"logps/ref_chosen": -56.60066604614258,
|
|
"logps/ref_rejected": -77.86631774902344,
|
|
"logps/rejected": -167.80581665039062,
|
|
"loss": 0.9815,
|
|
"margin_dpo/margin_mean": 47.55194854736328,
|
|
"margin_dpo/margin_std": 58.7222900390625,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.2143906020558003,
|
|
"fcm_dpo/beta": 0.013389000669121742,
|
|
"fcm_dpo/delta": 0.0990617573261261,
|
|
"fcm_dpo/margin": 37.79475021362305,
|
|
"fcm_dpo/q_t": 0.3926282227039337,
|
|
"grad_norm": 36.985809326171875,
|
|
"learning_rate": 4.812146767012779e-07,
|
|
"logits/chosen": -0.5096883177757263,
|
|
"logits/rejected": -0.4698811173439026,
|
|
"logps/chosen": -124.57284545898438,
|
|
"logps/ref_chosen": -66.00045013427734,
|
|
"logps/ref_rejected": -81.70278930664062,
|
|
"logps/rejected": -178.0699462890625,
|
|
"loss": 1.1025,
|
|
"margin_dpo/margin_mean": 37.79475021362305,
|
|
"margin_dpo/margin_std": 62.888580322265625,
|
|
"step": 146
|
|
},
|
|
{
|
|
"epoch": 0.21585903083700442,
|
|
"fcm_dpo/beta": 0.013290293514728546,
|
|
"fcm_dpo/delta": -0.0870274156332016,
|
|
"fcm_dpo/margin": 51.21372985839844,
|
|
"fcm_dpo/q_t": 0.36140725016593933,
|
|
"grad_norm": 26.684223175048828,
|
|
"learning_rate": 4.807235679840536e-07,
|
|
"logits/chosen": -0.5257313251495361,
|
|
"logits/rejected": -0.49477505683898926,
|
|
"logps/chosen": -96.1661605834961,
|
|
"logps/ref_chosen": -53.405487060546875,
|
|
"logps/ref_rejected": -71.39060974121094,
|
|
"logps/rejected": -165.36502075195312,
|
|
"loss": 0.9801,
|
|
"margin_dpo/margin_mean": 51.21372985839844,
|
|
"margin_dpo/margin_std": 67.14807891845703,
|
|
"step": 147
|
|
},
|
|
{
|
|
"epoch": 0.2173274596182085,
|
|
"fcm_dpo/beta": 0.01351265050470829,
|
|
"fcm_dpo/delta": 0.03601977974176407,
|
|
"fcm_dpo/margin": 41.63152313232422,
|
|
"fcm_dpo/q_t": 0.3840796947479248,
|
|
"grad_norm": 26.11017608642578,
|
|
"learning_rate": 4.802263794862384e-07,
|
|
"logits/chosen": -0.5603736639022827,
|
|
"logits/rejected": -0.5478102564811707,
|
|
"logps/chosen": -106.5193862915039,
|
|
"logps/ref_chosen": -64.93708038330078,
|
|
"logps/ref_rejected": -103.09384155273438,
|
|
"logps/rejected": -186.30767822265625,
|
|
"loss": 1.0332,
|
|
"margin_dpo/margin_mean": 41.63152313232422,
|
|
"margin_dpo/margin_std": 55.343299865722656,
|
|
"step": 148
|
|
},
|
|
{
|
|
"epoch": 0.21879588839941264,
|
|
"fcm_dpo/beta": 0.013042960315942764,
|
|
"fcm_dpo/delta": -0.0962439477443695,
|
|
"fcm_dpo/margin": 52.64115905761719,
|
|
"fcm_dpo/q_t": 0.35281527042388916,
|
|
"grad_norm": 24.8011417388916,
|
|
"learning_rate": 4.797231243092118e-07,
|
|
"logits/chosen": -0.590922474861145,
|
|
"logits/rejected": -0.5712438225746155,
|
|
"logps/chosen": -99.46907043457031,
|
|
"logps/ref_chosen": -58.47376251220703,
|
|
"logps/ref_rejected": -99.31474304199219,
|
|
"logps/rejected": -192.95120239257812,
|
|
"loss": 0.9376,
|
|
"margin_dpo/margin_mean": 52.64115905761719,
|
|
"margin_dpo/margin_std": 57.46501922607422,
|
|
"step": 149
|
|
},
|
|
{
|
|
"epoch": 0.22026431718061673,
|
|
"fcm_dpo/beta": 0.012883552350103855,
|
|
"fcm_dpo/delta": -0.049887098371982574,
|
|
"fcm_dpo/margin": 49.994728088378906,
|
|
"fcm_dpo/q_t": 0.37187010049819946,
|
|
"grad_norm": 24.784076690673828,
|
|
"learning_rate": 4.792138157142157e-07,
|
|
"logits/chosen": -0.557221531867981,
|
|
"logits/rejected": -0.5565592050552368,
|
|
"logps/chosen": -82.40767669677734,
|
|
"logps/ref_chosen": -45.705810546875,
|
|
"logps/ref_rejected": -83.34759521484375,
|
|
"logps/rejected": -170.044189453125,
|
|
"loss": 0.9916,
|
|
"margin_dpo/margin_mean": 49.994728088378906,
|
|
"margin_dpo/margin_std": 68.09803771972656,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.22173274596182085,
|
|
"fcm_dpo/beta": 0.012838941067457199,
|
|
"fcm_dpo/delta": -0.08010067045688629,
|
|
"fcm_dpo/margin": 52.55384063720703,
|
|
"fcm_dpo/q_t": 0.35465726256370544,
|
|
"grad_norm": 29.252914428710938,
|
|
"learning_rate": 4.786984671220053e-07,
|
|
"logits/chosen": -0.6052649021148682,
|
|
"logits/rejected": -0.569302499294281,
|
|
"logps/chosen": -114.0267333984375,
|
|
"logps/ref_chosen": -70.57083129882812,
|
|
"logps/ref_rejected": -100.46382141113281,
|
|
"logps/rejected": -196.47357177734375,
|
|
"loss": 0.9399,
|
|
"margin_dpo/margin_mean": 52.55384063720703,
|
|
"margin_dpo/margin_std": 58.15677261352539,
|
|
"step": 151
|
|
},
|
|
{
|
|
"epoch": 0.22320117474302498,
|
|
"fcm_dpo/beta": 0.012523850426077843,
|
|
"fcm_dpo/delta": -0.16435271501541138,
|
|
"fcm_dpo/margin": 60.06843185424805,
|
|
"fcm_dpo/q_t": 0.34519827365875244,
|
|
"grad_norm": 23.900650024414062,
|
|
"learning_rate": 4.78177092112495e-07,
|
|
"logits/chosen": -0.588593602180481,
|
|
"logits/rejected": -0.5805606842041016,
|
|
"logps/chosen": -98.1216049194336,
|
|
"logps/ref_chosen": -60.16438674926758,
|
|
"logps/ref_rejected": -106.14045715332031,
|
|
"logps/rejected": -204.16610717773438,
|
|
"loss": 0.918,
|
|
"margin_dpo/margin_mean": 60.06843185424805,
|
|
"margin_dpo/margin_std": 66.94328308105469,
|
|
"step": 152
|
|
},
|
|
{
|
|
"epoch": 0.22466960352422907,
|
|
"fcm_dpo/beta": 0.012331589125096798,
|
|
"fcm_dpo/delta": -0.03408785164356232,
|
|
"fcm_dpo/margin": 51.24233627319336,
|
|
"fcm_dpo/q_t": 0.37102195620536804,
|
|
"grad_norm": 21.188068389892578,
|
|
"learning_rate": 4.776497044244016e-07,
|
|
"logits/chosen": -0.576537013053894,
|
|
"logits/rejected": -0.5661931037902832,
|
|
"logps/chosen": -94.53947448730469,
|
|
"logps/ref_chosen": -56.315277099609375,
|
|
"logps/ref_rejected": -85.65583801269531,
|
|
"logps/rejected": -175.12237548828125,
|
|
"loss": 1.0004,
|
|
"margin_dpo/margin_mean": 51.24233627319336,
|
|
"margin_dpo/margin_std": 70.23153686523438,
|
|
"step": 153
|
|
},
|
|
{
|
|
"epoch": 0.2261380323054332,
|
|
"fcm_dpo/beta": 0.012307348661124706,
|
|
"fcm_dpo/delta": -0.028648383915424347,
|
|
"fcm_dpo/margin": 50.88260269165039,
|
|
"fcm_dpo/q_t": 0.37250685691833496,
|
|
"grad_norm": 26.30771255493164,
|
|
"learning_rate": 4.771163179548808e-07,
|
|
"logits/chosen": -0.5932099223136902,
|
|
"logits/rejected": -0.5894089937210083,
|
|
"logps/chosen": -107.8296127319336,
|
|
"logps/ref_chosen": -62.74256896972656,
|
|
"logps/ref_rejected": -104.24420166015625,
|
|
"logps/rejected": -200.21383666992188,
|
|
"loss": 1.0289,
|
|
"margin_dpo/margin_mean": 50.88260269165039,
|
|
"margin_dpo/margin_std": 72.57513427734375,
|
|
"step": 154
|
|
},
|
|
{
|
|
"epoch": 0.2276064610866373,
|
|
"fcm_dpo/beta": 0.012188572436571121,
|
|
"fcm_dpo/delta": -0.041290249675512314,
|
|
"fcm_dpo/margin": 52.38173294067383,
|
|
"fcm_dpo/q_t": 0.3654828369617462,
|
|
"grad_norm": 24.021343231201172,
|
|
"learning_rate": 4.7657694675916247e-07,
|
|
"logits/chosen": -0.6054894924163818,
|
|
"logits/rejected": -0.581174373626709,
|
|
"logps/chosen": -101.20684814453125,
|
|
"logps/ref_chosen": -60.65318298339844,
|
|
"logps/ref_rejected": -77.49220275878906,
|
|
"logps/rejected": -170.42758178710938,
|
|
"loss": 0.9842,
|
|
"margin_dpo/margin_mean": 52.38173294067383,
|
|
"margin_dpo/margin_std": 66.93553161621094,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.2290748898678414,
|
|
"fcm_dpo/beta": 0.012476801872253418,
|
|
"fcm_dpo/delta": 0.20647725462913513,
|
|
"fcm_dpo/margin": 32.21952819824219,
|
|
"fcm_dpo/q_t": 0.41537606716156006,
|
|
"grad_norm": 38.838287353515625,
|
|
"learning_rate": 4.7603160505017893e-07,
|
|
"logits/chosen": -0.531362771987915,
|
|
"logits/rejected": -0.5084357857704163,
|
|
"logps/chosen": -126.44786071777344,
|
|
"logps/ref_chosen": -69.49188232421875,
|
|
"logps/ref_rejected": -77.16929626464844,
|
|
"logps/rejected": -166.3448028564453,
|
|
"loss": 1.2212,
|
|
"margin_dpo/margin_mean": 32.21952819824219,
|
|
"margin_dpo/margin_std": 76.02751159667969,
|
|
"step": 156
|
|
},
|
|
{
|
|
"epoch": 0.2305433186490455,
|
|
"fcm_dpo/beta": 0.012192957103252411,
|
|
"fcm_dpo/delta": -0.18633562326431274,
|
|
"fcm_dpo/margin": 63.14281463623047,
|
|
"fcm_dpo/q_t": 0.33702823519706726,
|
|
"grad_norm": 27.627710342407227,
|
|
"learning_rate": 4.7548030719819154e-07,
|
|
"logits/chosen": -0.5252609252929688,
|
|
"logits/rejected": -0.5241682529449463,
|
|
"logps/chosen": -113.81539916992188,
|
|
"logps/ref_chosen": -61.368438720703125,
|
|
"logps/ref_rejected": -107.64636993408203,
|
|
"logps/rejected": -223.23614501953125,
|
|
"loss": 0.9149,
|
|
"margin_dpo/margin_mean": 63.14281463623047,
|
|
"margin_dpo/margin_std": 69.51899719238281,
|
|
"step": 157
|
|
},
|
|
{
|
|
"epoch": 0.23201174743024963,
|
|
"fcm_dpo/beta": 0.011821776628494263,
|
|
"fcm_dpo/delta": -0.1891898512840271,
|
|
"fcm_dpo/margin": 65.51558685302734,
|
|
"fcm_dpo/q_t": 0.35599738359451294,
|
|
"grad_norm": 23.29702377319336,
|
|
"learning_rate": 4.7492306773041136e-07,
|
|
"logits/chosen": -0.5330479145050049,
|
|
"logits/rejected": -0.5438076257705688,
|
|
"logps/chosen": -108.17401123046875,
|
|
"logps/ref_chosen": -57.612918853759766,
|
|
"logps/ref_rejected": -113.6946792602539,
|
|
"logps/rejected": -229.7713623046875,
|
|
"loss": 0.9692,
|
|
"margin_dpo/margin_mean": 65.51559448242188,
|
|
"margin_dpo/margin_std": 92.29185485839844,
|
|
"step": 158
|
|
},
|
|
{
|
|
"epoch": 0.23348017621145375,
|
|
"fcm_dpo/beta": 0.011839054524898529,
|
|
"fcm_dpo/delta": 0.03908521309494972,
|
|
"fcm_dpo/margin": 47.499595642089844,
|
|
"fcm_dpo/q_t": 0.38455915451049805,
|
|
"grad_norm": 30.979955673217773,
|
|
"learning_rate": 4.743599013306165e-07,
|
|
"logits/chosen": -0.5508168935775757,
|
|
"logits/rejected": -0.5111355781555176,
|
|
"logps/chosen": -139.3695526123047,
|
|
"logps/ref_chosen": -81.56034851074219,
|
|
"logps/ref_rejected": -88.89871215820312,
|
|
"logps/rejected": -194.20751953125,
|
|
"loss": 1.0618,
|
|
"margin_dpo/margin_mean": 47.499595642089844,
|
|
"margin_dpo/margin_std": 73.20787811279297,
|
|
"step": 159
|
|
},
|
|
{
|
|
"epoch": 0.23494860499265785,
|
|
"fcm_dpo/beta": 0.011559359729290009,
|
|
"fcm_dpo/delta": -0.1065782979130745,
|
|
"fcm_dpo/margin": 60.404293060302734,
|
|
"fcm_dpo/q_t": 0.365913450717926,
|
|
"grad_norm": 25.069011688232422,
|
|
"learning_rate": 4.737908228387656e-07,
|
|
"logits/chosen": -0.5436577796936035,
|
|
"logits/rejected": -0.5286530256271362,
|
|
"logps/chosen": -123.36406707763672,
|
|
"logps/ref_chosen": -65.73088073730469,
|
|
"logps/ref_rejected": -97.21781921386719,
|
|
"logps/rejected": -215.25531005859375,
|
|
"loss": 1.0081,
|
|
"margin_dpo/margin_mean": 60.404296875,
|
|
"margin_dpo/margin_std": 88.11878967285156,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23641703377386197,
|
|
"fcm_dpo/beta": 0.01151471957564354,
|
|
"fcm_dpo/delta": -0.025925662368535995,
|
|
"fcm_dpo/margin": 54.21880340576172,
|
|
"fcm_dpo/q_t": 0.3725304901599884,
|
|
"grad_norm": 25.731287002563477,
|
|
"learning_rate": 4.7321584725060594e-07,
|
|
"logits/chosen": -0.5345560312271118,
|
|
"logits/rejected": -0.5234242677688599,
|
|
"logps/chosen": -102.83772277832031,
|
|
"logps/ref_chosen": -52.43647003173828,
|
|
"logps/ref_rejected": -83.43095397949219,
|
|
"logps/rejected": -188.05099487304688,
|
|
"loss": 1.0083,
|
|
"margin_dpo/margin_mean": 54.21880340576172,
|
|
"margin_dpo/margin_std": 74.05145263671875,
|
|
"step": 161
|
|
},
|
|
{
|
|
"epoch": 0.23788546255506607,
|
|
"fcm_dpo/beta": 0.011299725621938705,
|
|
"fcm_dpo/delta": -0.03639139235019684,
|
|
"fcm_dpo/margin": 55.859214782714844,
|
|
"fcm_dpo/q_t": 0.37161916494369507,
|
|
"grad_norm": 23.54488182067871,
|
|
"learning_rate": 4.7263498971727905e-07,
|
|
"logits/chosen": -0.5820114612579346,
|
|
"logits/rejected": -0.5577338933944702,
|
|
"logps/chosen": -109.61683654785156,
|
|
"logps/ref_chosen": -62.6105842590332,
|
|
"logps/ref_rejected": -89.39057922363281,
|
|
"logps/rejected": -192.25604248046875,
|
|
"loss": 1.0073,
|
|
"margin_dpo/margin_mean": 55.859214782714844,
|
|
"margin_dpo/margin_std": 76.0927734375,
|
|
"step": 162
|
|
},
|
|
{
|
|
"epoch": 0.2393538913362702,
|
|
"fcm_dpo/beta": 0.01139851938933134,
|
|
"fcm_dpo/delta": -0.009991994127631187,
|
|
"fcm_dpo/margin": 53.46131134033203,
|
|
"fcm_dpo/q_t": 0.374041885137558,
|
|
"grad_norm": 23.255294799804688,
|
|
"learning_rate": 4.720482655449212e-07,
|
|
"logits/chosen": -0.5350059270858765,
|
|
"logits/rejected": -0.510658323764801,
|
|
"logps/chosen": -106.6119384765625,
|
|
"logps/ref_chosen": -55.021629333496094,
|
|
"logps/ref_rejected": -75.418212890625,
|
|
"logps/rejected": -180.4698486328125,
|
|
"loss": 1.0059,
|
|
"margin_dpo/margin_mean": 53.46131134033203,
|
|
"margin_dpo/margin_std": 71.67129516601562,
|
|
"step": 163
|
|
},
|
|
{
|
|
"epoch": 0.24082232011747431,
|
|
"fcm_dpo/beta": 0.011088410392403603,
|
|
"fcm_dpo/delta": -0.1139398068189621,
|
|
"fcm_dpo/margin": 63.41717529296875,
|
|
"fcm_dpo/q_t": 0.3512231409549713,
|
|
"grad_norm": 25.359941482543945,
|
|
"learning_rate": 4.714556901942599e-07,
|
|
"logits/chosen": -0.5142285823822021,
|
|
"logits/rejected": -0.4906259775161743,
|
|
"logps/chosen": -104.08917236328125,
|
|
"logps/ref_chosen": -55.64066696166992,
|
|
"logps/ref_rejected": -79.66463470458984,
|
|
"logps/rejected": -191.5303192138672,
|
|
"loss": 0.9321,
|
|
"margin_dpo/margin_mean": 63.41717529296875,
|
|
"margin_dpo/margin_std": 71.19700622558594,
|
|
"step": 164
|
|
},
|
|
{
|
|
"epoch": 0.2422907488986784,
|
|
"fcm_dpo/beta": 0.011331463232636452,
|
|
"fcm_dpo/delta": 0.1354885995388031,
|
|
"fcm_dpo/margin": 41.572532653808594,
|
|
"fcm_dpo/q_t": 0.39925825595855713,
|
|
"grad_norm": 34.06428527832031,
|
|
"learning_rate": 4.708572792802069e-07,
|
|
"logits/chosen": -0.5556213855743408,
|
|
"logits/rejected": -0.5192099213600159,
|
|
"logps/chosen": -114.64573669433594,
|
|
"logps/ref_chosen": -61.310691833496094,
|
|
"logps/ref_rejected": -73.67060852050781,
|
|
"logps/rejected": -168.57818603515625,
|
|
"loss": 1.094,
|
|
"margin_dpo/margin_mean": 41.572532653808594,
|
|
"margin_dpo/margin_std": 65.32736206054688,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.24375917767988253,
|
|
"fcm_dpo/beta": 0.010963313281536102,
|
|
"fcm_dpo/delta": -0.2387712448835373,
|
|
"fcm_dpo/margin": 74.51507568359375,
|
|
"fcm_dpo/q_t": 0.3456820547580719,
|
|
"grad_norm": 21.995134353637695,
|
|
"learning_rate": 4.702530485714461e-07,
|
|
"logits/chosen": -0.49838924407958984,
|
|
"logits/rejected": -0.5080593824386597,
|
|
"logps/chosen": -99.90086364746094,
|
|
"logps/ref_chosen": -50.98360061645508,
|
|
"logps/ref_rejected": -98.09512329101562,
|
|
"logps/rejected": -221.5274658203125,
|
|
"loss": 0.9183,
|
|
"margin_dpo/margin_mean": 74.51507568359375,
|
|
"margin_dpo/margin_std": 95.35562896728516,
|
|
"step": 166
|
|
},
|
|
{
|
|
"epoch": 0.24522760646108663,
|
|
"fcm_dpo/beta": 0.010487915948033333,
|
|
"fcm_dpo/delta": -0.2514822483062744,
|
|
"fcm_dpo/margin": 79.16014862060547,
|
|
"fcm_dpo/q_t": 0.3255418539047241,
|
|
"grad_norm": 22.585697174072266,
|
|
"learning_rate": 4.6964301399001877e-07,
|
|
"logits/chosen": -0.4470037817955017,
|
|
"logits/rejected": -0.4476546347141266,
|
|
"logps/chosen": -99.91099548339844,
|
|
"logps/ref_chosen": -50.424095153808594,
|
|
"logps/ref_rejected": -96.03042602539062,
|
|
"logps/rejected": -224.677490234375,
|
|
"loss": 0.8589,
|
|
"margin_dpo/margin_mean": 79.16014862060547,
|
|
"margin_dpo/margin_std": 78.89952087402344,
|
|
"step": 167
|
|
},
|
|
{
|
|
"epoch": 0.24669603524229075,
|
|
"fcm_dpo/beta": 0.010302037000656128,
|
|
"fcm_dpo/delta": -0.02494889497756958,
|
|
"fcm_dpo/margin": 60.4947395324707,
|
|
"fcm_dpo/q_t": 0.36779898405075073,
|
|
"grad_norm": 23.62080192565918,
|
|
"learning_rate": 4.690271916109034e-07,
|
|
"logits/chosen": -0.4938944876194,
|
|
"logits/rejected": -0.4785892963409424,
|
|
"logps/chosen": -104.25721740722656,
|
|
"logps/ref_chosen": -49.462825775146484,
|
|
"logps/ref_rejected": -75.30855560302734,
|
|
"logps/rejected": -190.59768676757812,
|
|
"loss": 0.9727,
|
|
"margin_dpo/margin_mean": 60.49474334716797,
|
|
"margin_dpo/margin_std": 72.37049865722656,
|
|
"step": 168
|
|
},
|
|
{
|
|
"epoch": 0.24816446402349487,
|
|
"fcm_dpo/beta": 0.010289417579770088,
|
|
"fcm_dpo/delta": 0.049492061138153076,
|
|
"fcm_dpo/margin": 53.629798889160156,
|
|
"fcm_dpo/q_t": 0.3923034071922302,
|
|
"grad_norm": 26.301158905029297,
|
|
"learning_rate": 4.6840559766159235e-07,
|
|
"logits/chosen": -0.4954657554626465,
|
|
"logits/rejected": -0.4744875431060791,
|
|
"logps/chosen": -114.426513671875,
|
|
"logps/ref_chosen": -59.803443908691406,
|
|
"logps/ref_rejected": -83.34574890136719,
|
|
"logps/rejected": -191.59860229492188,
|
|
"loss": 1.0873,
|
|
"margin_dpo/margin_mean": 53.629798889160156,
|
|
"margin_dpo/margin_std": 91.4170913696289,
|
|
"step": 169
|
|
},
|
|
{
|
|
"epoch": 0.24963289280469897,
|
|
"fcm_dpo/beta": 0.010223541408777237,
|
|
"fcm_dpo/delta": -0.07429465651512146,
|
|
"fcm_dpo/margin": 65.30518341064453,
|
|
"fcm_dpo/q_t": 0.35696014761924744,
|
|
"grad_norm": 24.374975204467773,
|
|
"learning_rate": 4.6777824852166437e-07,
|
|
"logits/chosen": -0.4555085003376007,
|
|
"logits/rejected": -0.4390765428543091,
|
|
"logps/chosen": -99.52091979980469,
|
|
"logps/ref_chosen": -49.471771240234375,
|
|
"logps/ref_rejected": -75.91734313964844,
|
|
"logps/rejected": -191.27166748046875,
|
|
"loss": 0.958,
|
|
"margin_dpo/margin_mean": 65.30518341064453,
|
|
"margin_dpo/margin_std": 75.40479278564453,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.2511013215859031,
|
|
"fcm_dpo/beta": 0.010339688509702682,
|
|
"fcm_dpo/delta": 0.05246017128229141,
|
|
"fcm_dpo/margin": 53.23592758178711,
|
|
"fcm_dpo/q_t": 0.39151298999786377,
|
|
"grad_norm": 31.557279586791992,
|
|
"learning_rate": 4.6714516072235273e-07,
|
|
"logits/chosen": -0.5044229030609131,
|
|
"logits/rejected": -0.47634661197662354,
|
|
"logps/chosen": -157.2343292236328,
|
|
"logps/ref_chosen": -84.49931335449219,
|
|
"logps/ref_rejected": -109.38209533691406,
|
|
"logps/rejected": -235.35305786132812,
|
|
"loss": 1.0862,
|
|
"margin_dpo/margin_mean": 53.23592758178711,
|
|
"margin_dpo/margin_std": 91.4041748046875,
|
|
"step": 171
|
|
},
|
|
{
|
|
"epoch": 0.2525697503671072,
|
|
"fcm_dpo/beta": 0.010385138913989067,
|
|
"fcm_dpo/delta": 0.024514978751540184,
|
|
"fcm_dpo/margin": 55.545555114746094,
|
|
"fcm_dpo/q_t": 0.3802782893180847,
|
|
"grad_norm": 29.966943740844727,
|
|
"learning_rate": 4.6650635094610966e-07,
|
|
"logits/chosen": -0.5027618408203125,
|
|
"logits/rejected": -0.47274085879325867,
|
|
"logps/chosen": -136.3382568359375,
|
|
"logps/ref_chosen": -68.65391540527344,
|
|
"logps/ref_rejected": -85.43667602539062,
|
|
"logps/rejected": -208.66656494140625,
|
|
"loss": 1.0408,
|
|
"margin_dpo/margin_mean": 55.54555892944336,
|
|
"margin_dpo/margin_std": 81.37893676757812,
|
|
"step": 172
|
|
},
|
|
{
|
|
"epoch": 0.2540381791483113,
|
|
"fcm_dpo/beta": 0.010488756000995636,
|
|
"fcm_dpo/delta": 0.0158163383603096,
|
|
"fcm_dpo/margin": 55.749046325683594,
|
|
"fcm_dpo/q_t": 0.3747754693031311,
|
|
"grad_norm": 29.66188621520996,
|
|
"learning_rate": 4.6586183602616687e-07,
|
|
"logits/chosen": -0.5061242580413818,
|
|
"logits/rejected": -0.4692162275314331,
|
|
"logps/chosen": -123.67233276367188,
|
|
"logps/ref_chosen": -63.050880432128906,
|
|
"logps/ref_rejected": -78.68392181396484,
|
|
"logps/rejected": -195.054443359375,
|
|
"loss": 0.9973,
|
|
"margin_dpo/margin_mean": 55.74905014038086,
|
|
"margin_dpo/margin_std": 69.4229736328125,
|
|
"step": 173
|
|
},
|
|
{
|
|
"epoch": 0.2555066079295154,
|
|
"fcm_dpo/beta": 0.01044470351189375,
|
|
"fcm_dpo/delta": -0.0565309077501297,
|
|
"fcm_dpo/margin": 62.44271469116211,
|
|
"fcm_dpo/q_t": 0.3679395318031311,
|
|
"grad_norm": 31.072416305541992,
|
|
"learning_rate": 4.652116329460919e-07,
|
|
"logits/chosen": -0.42711180448532104,
|
|
"logits/rejected": -0.44361674785614014,
|
|
"logps/chosen": -113.0875015258789,
|
|
"logps/ref_chosen": -53.36296844482422,
|
|
"logps/ref_rejected": -101.91120910644531,
|
|
"logps/rejected": -224.07846069335938,
|
|
"loss": 0.9957,
|
|
"margin_dpo/margin_mean": 62.44270706176758,
|
|
"margin_dpo/margin_std": 83.03677368164062,
|
|
"step": 174
|
|
},
|
|
{
|
|
"epoch": 0.25697503671071953,
|
|
"fcm_dpo/beta": 0.009896190837025642,
|
|
"fcm_dpo/delta": -0.282113254070282,
|
|
"fcm_dpo/margin": 86.63121032714844,
|
|
"fcm_dpo/q_t": 0.32039257884025574,
|
|
"grad_norm": 29.841856002807617,
|
|
"learning_rate": 4.645557588393406e-07,
|
|
"logits/chosen": -0.39522379636764526,
|
|
"logits/rejected": -0.38075965642929077,
|
|
"logps/chosen": -103.10018920898438,
|
|
"logps/ref_chosen": -45.417762756347656,
|
|
"logps/ref_rejected": -89.50579833984375,
|
|
"logps/rejected": -233.8194580078125,
|
|
"loss": 0.835,
|
|
"margin_dpo/margin_mean": 86.63121032714844,
|
|
"margin_dpo/margin_std": 82.89295196533203,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.25844346549192365,
|
|
"fcm_dpo/beta": 0.009663419798016548,
|
|
"fcm_dpo/delta": -0.09544707089662552,
|
|
"fcm_dpo/margin": 71.29586791992188,
|
|
"fcm_dpo/q_t": 0.3592815101146698,
|
|
"grad_norm": 28.798011779785156,
|
|
"learning_rate": 4.638942309888058e-07,
|
|
"logits/chosen": -0.39833515882492065,
|
|
"logits/rejected": -0.41467368602752686,
|
|
"logps/chosen": -110.1692123413086,
|
|
"logps/ref_chosen": -50.452842712402344,
|
|
"logps/ref_rejected": -95.5589599609375,
|
|
"logps/rejected": -226.57119750976562,
|
|
"loss": 0.9511,
|
|
"margin_dpo/margin_mean": 71.29586791992188,
|
|
"margin_dpo/margin_std": 86.2072525024414,
|
|
"step": 176
|
|
},
|
|
{
|
|
"epoch": 0.2599118942731278,
|
|
"fcm_dpo/beta": 0.009497301653027534,
|
|
"fcm_dpo/delta": -0.09194030612707138,
|
|
"fcm_dpo/margin": 72.19395446777344,
|
|
"fcm_dpo/q_t": 0.35893410444259644,
|
|
"grad_norm": 32.332763671875,
|
|
"learning_rate": 4.6322706682636137e-07,
|
|
"logits/chosen": -0.4460568428039551,
|
|
"logits/rejected": -0.4327634572982788,
|
|
"logps/chosen": -128.06985473632812,
|
|
"logps/ref_chosen": -61.216468811035156,
|
|
"logps/ref_rejected": -95.89378356933594,
|
|
"logps/rejected": -234.94113159179688,
|
|
"loss": 0.9502,
|
|
"margin_dpo/margin_mean": 72.19395446777344,
|
|
"margin_dpo/margin_std": 87.10002136230469,
|
|
"step": 177
|
|
},
|
|
{
|
|
"epoch": 0.26138032305433184,
|
|
"fcm_dpo/beta": 0.009019111283123493,
|
|
"fcm_dpo/delta": -0.24799524247646332,
|
|
"fcm_dpo/margin": 91.52853393554688,
|
|
"fcm_dpo/q_t": 0.3365201950073242,
|
|
"grad_norm": 31.862321853637695,
|
|
"learning_rate": 4.6255428393240354e-07,
|
|
"logits/chosen": -0.3157181143760681,
|
|
"logits/rejected": -0.31319284439086914,
|
|
"logps/chosen": -133.81484985351562,
|
|
"logps/ref_chosen": -58.26478958129883,
|
|
"logps/ref_rejected": -105.3653335571289,
|
|
"logps/rejected": -272.44390869140625,
|
|
"loss": 0.8982,
|
|
"margin_dpo/margin_mean": 91.52853393554688,
|
|
"margin_dpo/margin_std": 108.31314086914062,
|
|
"step": 178
|
|
},
|
|
{
|
|
"epoch": 0.26284875183553597,
|
|
"fcm_dpo/beta": 0.008883455768227577,
|
|
"fcm_dpo/delta": -0.01396089792251587,
|
|
"fcm_dpo/margin": 68.92755889892578,
|
|
"fcm_dpo/q_t": 0.37395423650741577,
|
|
"grad_norm": 51.664894104003906,
|
|
"learning_rate": 4.6187590003538724e-07,
|
|
"logits/chosen": -0.3555856943130493,
|
|
"logits/rejected": -0.35472238063812256,
|
|
"logps/chosen": -136.07827758789062,
|
|
"logps/ref_chosen": -61.05832290649414,
|
|
"logps/ref_rejected": -90.52782440185547,
|
|
"logps/rejected": -234.475341796875,
|
|
"loss": 1.0389,
|
|
"margin_dpo/margin_mean": 68.92755889892578,
|
|
"margin_dpo/margin_std": 100.45736694335938,
|
|
"step": 179
|
|
},
|
|
{
|
|
"epoch": 0.2643171806167401,
|
|
"fcm_dpo/beta": 0.008733205497264862,
|
|
"fcm_dpo/delta": -0.13286282122135162,
|
|
"fcm_dpo/margin": 82.76274108886719,
|
|
"fcm_dpo/q_t": 0.34625113010406494,
|
|
"grad_norm": 34.209205627441406,
|
|
"learning_rate": 4.611919330113591e-07,
|
|
"logits/chosen": -0.39520972967147827,
|
|
"logits/rejected": -0.3850780725479126,
|
|
"logps/chosen": -116.90325927734375,
|
|
"logps/ref_chosen": -54.34272003173828,
|
|
"logps/ref_rejected": -98.21183776855469,
|
|
"logps/rejected": -243.5351104736328,
|
|
"loss": 0.9237,
|
|
"margin_dpo/margin_mean": 82.76274108886719,
|
|
"margin_dpo/margin_std": 91.62629699707031,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.2657856093979442,
|
|
"fcm_dpo/beta": 0.008904640562832355,
|
|
"fcm_dpo/delta": 0.14196932315826416,
|
|
"fcm_dpo/margin": 52.12580871582031,
|
|
"fcm_dpo/q_t": 0.4002281725406647,
|
|
"grad_norm": 29.242713928222656,
|
|
"learning_rate": 4.605024008834863e-07,
|
|
"logits/chosen": -0.4259217381477356,
|
|
"logits/rejected": -0.3925449252128601,
|
|
"logps/chosen": -108.0588607788086,
|
|
"logps/ref_chosen": -55.000457763671875,
|
|
"logps/ref_rejected": -61.656166076660156,
|
|
"logps/rejected": -166.8403778076172,
|
|
"loss": 1.0943,
|
|
"margin_dpo/margin_mean": 52.12580871582031,
|
|
"margin_dpo/margin_std": 82.18958282470703,
|
|
"step": 181
|
|
},
|
|
{
|
|
"epoch": 0.26725403817914833,
|
|
"fcm_dpo/beta": 0.008686801418662071,
|
|
"fcm_dpo/delta": -0.16562983393669128,
|
|
"fcm_dpo/margin": 86.55928802490234,
|
|
"fcm_dpo/q_t": 0.3413204550743103,
|
|
"grad_norm": 27.281949996948242,
|
|
"learning_rate": 4.598073218215817e-07,
|
|
"logits/chosen": -0.3973362445831299,
|
|
"logits/rejected": -0.4009121060371399,
|
|
"logps/chosen": -91.49468231201172,
|
|
"logps/ref_chosen": -41.107852935791016,
|
|
"logps/ref_rejected": -89.5215835571289,
|
|
"logps/rejected": -226.4676971435547,
|
|
"loss": 0.9226,
|
|
"margin_dpo/margin_mean": 86.55928802490234,
|
|
"margin_dpo/margin_std": 97.51657104492188,
|
|
"step": 182
|
|
},
|
|
{
|
|
"epoch": 0.2687224669603524,
|
|
"fcm_dpo/beta": 0.008756536990404129,
|
|
"fcm_dpo/delta": 0.15965726971626282,
|
|
"fcm_dpo/margin": 50.974124908447266,
|
|
"fcm_dpo/q_t": 0.40270137786865234,
|
|
"grad_norm": 26.794544219970703,
|
|
"learning_rate": 4.5910671414162484e-07,
|
|
"logits/chosen": -0.4323645234107971,
|
|
"logits/rejected": -0.41012632846832275,
|
|
"logps/chosen": -132.4599151611328,
|
|
"logps/ref_chosen": -57.52456283569336,
|
|
"logps/ref_rejected": -75.97572326660156,
|
|
"logps/rejected": -201.8852081298828,
|
|
"loss": 1.0928,
|
|
"margin_dpo/margin_mean": 50.97412872314453,
|
|
"margin_dpo/margin_std": 70.7024917602539,
|
|
"step": 183
|
|
},
|
|
{
|
|
"epoch": 0.2701908957415565,
|
|
"fcm_dpo/beta": 0.009184681810438633,
|
|
"fcm_dpo/delta": 0.13521449267864227,
|
|
"fcm_dpo/margin": 51.08043670654297,
|
|
"fcm_dpo/q_t": 0.40079188346862793,
|
|
"grad_norm": 24.206710815429688,
|
|
"learning_rate": 4.5840059630527985e-07,
|
|
"logits/chosen": -0.4838387966156006,
|
|
"logits/rejected": -0.4702536165714264,
|
|
"logps/chosen": -119.1988296508789,
|
|
"logps/ref_chosen": -58.544952392578125,
|
|
"logps/ref_rejected": -76.63406372070312,
|
|
"logps/rejected": -188.36837768554688,
|
|
"loss": 1.083,
|
|
"margin_dpo/margin_mean": 51.080440521240234,
|
|
"margin_dpo/margin_std": 76.5762939453125,
|
|
"step": 184
|
|
},
|
|
{
|
|
"epoch": 0.27165932452276065,
|
|
"fcm_dpo/beta": 0.00946805626153946,
|
|
"fcm_dpo/delta": 0.21153177320957184,
|
|
"fcm_dpo/margin": 41.93297576904297,
|
|
"fcm_dpo/q_t": 0.41936683654785156,
|
|
"grad_norm": 26.689727783203125,
|
|
"learning_rate": 4.5768898691940836e-07,
|
|
"logits/chosen": -0.4485119581222534,
|
|
"logits/rejected": -0.4117010831832886,
|
|
"logps/chosen": -128.1175079345703,
|
|
"logps/ref_chosen": -62.025848388671875,
|
|
"logps/ref_rejected": -73.7625961303711,
|
|
"logps/rejected": -181.7872314453125,
|
|
"loss": 1.1579,
|
|
"margin_dpo/margin_mean": 41.93297576904297,
|
|
"margin_dpo/margin_std": 82.34846496582031,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.27312775330396477,
|
|
"fcm_dpo/beta": 0.009380245581269264,
|
|
"fcm_dpo/delta": -0.12999294698238373,
|
|
"fcm_dpo/margin": 76.79740905761719,
|
|
"fcm_dpo/q_t": 0.3472128212451935,
|
|
"grad_norm": 49.98681640625,
|
|
"learning_rate": 4.5697190473557947e-07,
|
|
"logits/chosen": -0.4748345613479614,
|
|
"logits/rejected": -0.43923431634902954,
|
|
"logps/chosen": -130.9991455078125,
|
|
"logps/ref_chosen": -69.35346984863281,
|
|
"logps/ref_rejected": -88.07244873046875,
|
|
"logps/rejected": -226.51553344726562,
|
|
"loss": 0.9126,
|
|
"margin_dpo/margin_mean": 76.79740905761719,
|
|
"margin_dpo/margin_std": 82.54960632324219,
|
|
"step": 186
|
|
},
|
|
{
|
|
"epoch": 0.2745961820851689,
|
|
"fcm_dpo/beta": 0.00942191667854786,
|
|
"fcm_dpo/delta": 0.014476820826530457,
|
|
"fcm_dpo/margin": 62.11200714111328,
|
|
"fcm_dpo/q_t": 0.3749355673789978,
|
|
"grad_norm": 33.28017807006836,
|
|
"learning_rate": 4.5624936864957555e-07,
|
|
"logits/chosen": -0.4199049770832062,
|
|
"logits/rejected": -0.41446995735168457,
|
|
"logps/chosen": -114.2715072631836,
|
|
"logps/ref_chosen": -52.7564582824707,
|
|
"logps/ref_rejected": -81.96910095214844,
|
|
"logps/rejected": -205.59616088867188,
|
|
"loss": 1.0036,
|
|
"margin_dpo/margin_mean": 62.11200714111328,
|
|
"margin_dpo/margin_std": 77.76611328125,
|
|
"step": 187
|
|
},
|
|
{
|
|
"epoch": 0.27606461086637296,
|
|
"fcm_dpo/beta": 0.009187942370772362,
|
|
"fcm_dpo/delta": -0.12695354223251343,
|
|
"fcm_dpo/margin": 78.13199615478516,
|
|
"fcm_dpo/q_t": 0.35078778862953186,
|
|
"grad_norm": 41.620758056640625,
|
|
"learning_rate": 4.5552139770089454e-07,
|
|
"logits/chosen": -0.4144795536994934,
|
|
"logits/rejected": -0.4158819019794464,
|
|
"logps/chosen": -107.33932495117188,
|
|
"logps/ref_chosen": -49.415489196777344,
|
|
"logps/ref_rejected": -89.54043579101562,
|
|
"logps/rejected": -225.59625244140625,
|
|
"loss": 0.9298,
|
|
"margin_dpo/margin_mean": 78.13198852539062,
|
|
"margin_dpo/margin_std": 89.57806396484375,
|
|
"step": 188
|
|
},
|
|
{
|
|
"epoch": 0.2775330396475771,
|
|
"fcm_dpo/beta": 0.00912393070757389,
|
|
"fcm_dpo/delta": -0.008689474314451218,
|
|
"fcm_dpo/margin": 66.65364074707031,
|
|
"fcm_dpo/q_t": 0.37750983238220215,
|
|
"grad_norm": 41.65003967285156,
|
|
"learning_rate": 4.5478801107224794e-07,
|
|
"logits/chosen": -0.4112040400505066,
|
|
"logits/rejected": -0.38720983266830444,
|
|
"logps/chosen": -123.0237045288086,
|
|
"logps/ref_chosen": -52.39896011352539,
|
|
"logps/ref_rejected": -72.16735076904297,
|
|
"logps/rejected": -209.44573974609375,
|
|
"loss": 1.028,
|
|
"margin_dpo/margin_mean": 66.65364074707031,
|
|
"margin_dpo/margin_std": 99.14986419677734,
|
|
"step": 189
|
|
},
|
|
{
|
|
"epoch": 0.2790014684287812,
|
|
"fcm_dpo/beta": 0.009091001003980637,
|
|
"fcm_dpo/delta": -0.09938493371009827,
|
|
"fcm_dpo/margin": 75.96780395507812,
|
|
"fcm_dpo/q_t": 0.361076295375824,
|
|
"grad_norm": 30.051088333129883,
|
|
"learning_rate": 4.5404922808905543e-07,
|
|
"logits/chosen": -0.43357422947883606,
|
|
"logits/rejected": -0.4192691147327423,
|
|
"logps/chosen": -141.0237579345703,
|
|
"logps/ref_chosen": -64.68305969238281,
|
|
"logps/ref_rejected": -102.55052185058594,
|
|
"logps/rejected": -254.85903930664062,
|
|
"loss": 0.9991,
|
|
"margin_dpo/margin_mean": 75.96780395507812,
|
|
"margin_dpo/margin_std": 101.82373809814453,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.28046989720998533,
|
|
"fcm_dpo/beta": 0.008413515985012054,
|
|
"fcm_dpo/delta": -0.3635314106941223,
|
|
"fcm_dpo/margin": 110.20065307617188,
|
|
"fcm_dpo/q_t": 0.3166946768760681,
|
|
"grad_norm": 25.67074203491211,
|
|
"learning_rate": 4.5330506821893565e-07,
|
|
"logits/chosen": -0.35954105854034424,
|
|
"logits/rejected": -0.33676308393478394,
|
|
"logps/chosen": -138.35675048828125,
|
|
"logps/ref_chosen": -68.65887451171875,
|
|
"logps/ref_rejected": -110.1396713256836,
|
|
"logps/rejected": -290.0382080078125,
|
|
"loss": 0.8336,
|
|
"margin_dpo/margin_mean": 110.20065307617188,
|
|
"margin_dpo/margin_std": 117.54473876953125,
|
|
"step": 191
|
|
},
|
|
{
|
|
"epoch": 0.28193832599118945,
|
|
"fcm_dpo/beta": 0.008292671293020248,
|
|
"fcm_dpo/delta": -0.028325140476226807,
|
|
"fcm_dpo/margin": 75.54448699951172,
|
|
"fcm_dpo/q_t": 0.37382009625434875,
|
|
"grad_norm": 34.30679702758789,
|
|
"learning_rate": 4.5255555107119336e-07,
|
|
"logits/chosen": -0.37166693806648254,
|
|
"logits/rejected": -0.37043970823287964,
|
|
"logps/chosen": -160.6333770751953,
|
|
"logps/ref_chosen": -69.72691345214844,
|
|
"logps/ref_rejected": -103.32135009765625,
|
|
"logps/rejected": -269.77227783203125,
|
|
"loss": 1.0202,
|
|
"margin_dpo/margin_mean": 75.54447937011719,
|
|
"margin_dpo/margin_std": 108.79905700683594,
|
|
"step": 192
|
|
},
|
|
{
|
|
"epoch": 0.2834067547723935,
|
|
"fcm_dpo/beta": 0.008594648912549019,
|
|
"fcm_dpo/delta": 0.2289629727602005,
|
|
"fcm_dpo/margin": 43.941375732421875,
|
|
"fcm_dpo/q_t": 0.4161950647830963,
|
|
"grad_norm": 38.5386848449707,
|
|
"learning_rate": 4.5180069639630236e-07,
|
|
"logits/chosen": -0.37603700160980225,
|
|
"logits/rejected": -0.35951176285743713,
|
|
"logps/chosen": -151.70782470703125,
|
|
"logps/ref_chosen": -60.19049835205078,
|
|
"logps/ref_rejected": -76.40755462646484,
|
|
"logps/rejected": -211.86627197265625,
|
|
"loss": 1.2035,
|
|
"margin_dpo/margin_mean": 43.941375732421875,
|
|
"margin_dpo/margin_std": 97.33943176269531,
|
|
"step": 193
|
|
},
|
|
{
|
|
"epoch": 0.28487518355359764,
|
|
"fcm_dpo/beta": 0.008597111329436302,
|
|
"fcm_dpo/delta": -0.016570590436458588,
|
|
"fcm_dpo/margin": 71.57522583007812,
|
|
"fcm_dpo/q_t": 0.36463361978530884,
|
|
"grad_norm": 24.157445907592773,
|
|
"learning_rate": 4.510405240853854e-07,
|
|
"logits/chosen": -0.28482919931411743,
|
|
"logits/rejected": -0.2655167579650879,
|
|
"logps/chosen": -93.96827697753906,
|
|
"logps/ref_chosen": -37.84037399291992,
|
|
"logps/ref_rejected": -60.684783935546875,
|
|
"logps/rejected": -188.38790893554688,
|
|
"loss": 0.9556,
|
|
"margin_dpo/margin_mean": 71.57522583007812,
|
|
"margin_dpo/margin_std": 75.2847671508789,
|
|
"step": 194
|
|
},
|
|
{
|
|
"epoch": 0.28634361233480177,
|
|
"fcm_dpo/beta": 0.008543387055397034,
|
|
"fcm_dpo/delta": -0.04339263588190079,
|
|
"fcm_dpo/margin": 74.96640014648438,
|
|
"fcm_dpo/q_t": 0.361470490694046,
|
|
"grad_norm": 31.8635311126709,
|
|
"learning_rate": 4.5027505416968985e-07,
|
|
"logits/chosen": -0.3245890140533447,
|
|
"logits/rejected": -0.3413041830062866,
|
|
"logps/chosen": -144.32359313964844,
|
|
"logps/ref_chosen": -54.891571044921875,
|
|
"logps/ref_rejected": -96.77095794677734,
|
|
"logps/rejected": -261.1694030761719,
|
|
"loss": 0.9508,
|
|
"margin_dpo/margin_mean": 74.96640014648438,
|
|
"margin_dpo/margin_std": 82.75733947753906,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.2878120411160059,
|
|
"fcm_dpo/beta": 0.008355004712939262,
|
|
"fcm_dpo/delta": -0.11748833954334259,
|
|
"fcm_dpo/margin": 84.77229309082031,
|
|
"fcm_dpo/q_t": 0.352541983127594,
|
|
"grad_norm": 28.523649215698242,
|
|
"learning_rate": 4.495043068200599e-07,
|
|
"logits/chosen": -0.3666142225265503,
|
|
"logits/rejected": -0.34821170568466187,
|
|
"logps/chosen": -121.3379898071289,
|
|
"logps/ref_chosen": -53.245243072509766,
|
|
"logps/ref_rejected": -76.05294799804688,
|
|
"logps/rejected": -228.91798400878906,
|
|
"loss": 0.9446,
|
|
"margin_dpo/margin_mean": 84.77229309082031,
|
|
"margin_dpo/margin_std": 98.6243896484375,
|
|
"step": 196
|
|
},
|
|
{
|
|
"epoch": 0.28928046989721,
|
|
"fcm_dpo/beta": 0.00841631181538105,
|
|
"fcm_dpo/delta": 0.04314948618412018,
|
|
"fcm_dpo/margin": 66.4241714477539,
|
|
"fcm_dpo/q_t": 0.3789867162704468,
|
|
"grad_norm": 36.85988235473633,
|
|
"learning_rate": 4.4872830234640493e-07,
|
|
"logits/chosen": -0.34630054235458374,
|
|
"logits/rejected": -0.33658844232559204,
|
|
"logps/chosen": -127.1322250366211,
|
|
"logps/ref_chosen": -60.42033386230469,
|
|
"logps/ref_rejected": -77.20890808105469,
|
|
"logps/rejected": -210.344970703125,
|
|
"loss": 1.0009,
|
|
"margin_dpo/margin_mean": 66.4241714477539,
|
|
"margin_dpo/margin_std": 79.97479248046875,
|
|
"step": 197
|
|
},
|
|
{
|
|
"epoch": 0.2907488986784141,
|
|
"fcm_dpo/beta": 0.008350390940904617,
|
|
"fcm_dpo/delta": -0.068976491689682,
|
|
"fcm_dpo/margin": 79.5688705444336,
|
|
"fcm_dpo/q_t": 0.3639752268791199,
|
|
"grad_norm": 31.69214630126953,
|
|
"learning_rate": 4.479470611971645e-07,
|
|
"logits/chosen": -0.36272114515304565,
|
|
"logits/rejected": -0.36416107416152954,
|
|
"logps/chosen": -129.2322998046875,
|
|
"logps/ref_chosen": -55.03618621826172,
|
|
"logps/ref_rejected": -97.24325561523438,
|
|
"logps/rejected": -251.00823974609375,
|
|
"loss": 0.9641,
|
|
"margin_dpo/margin_mean": 79.5688705444336,
|
|
"margin_dpo/margin_std": 100.84001922607422,
|
|
"step": 198
|
|
},
|
|
{
|
|
"epoch": 0.2922173274596182,
|
|
"fcm_dpo/beta": 0.008206663653254509,
|
|
"fcm_dpo/delta": -0.027660571038722992,
|
|
"fcm_dpo/margin": 76.12931060791016,
|
|
"fcm_dpo/q_t": 0.3673062324523926,
|
|
"grad_norm": 35.12301254272461,
|
|
"learning_rate": 4.471606039587695e-07,
|
|
"logits/chosen": -0.3977452516555786,
|
|
"logits/rejected": -0.38206660747528076,
|
|
"logps/chosen": -128.30673217773438,
|
|
"logps/ref_chosen": -56.828826904296875,
|
|
"logps/ref_rejected": -84.64820861816406,
|
|
"logps/rejected": -232.25543212890625,
|
|
"loss": 0.9921,
|
|
"margin_dpo/margin_mean": 76.12931060791016,
|
|
"margin_dpo/margin_std": 95.28636169433594,
|
|
"step": 199
|
|
},
|
|
{
|
|
"epoch": 0.2936857562408223,
|
|
"fcm_dpo/beta": 0.008154574781656265,
|
|
"fcm_dpo/delta": -0.05656225234270096,
|
|
"fcm_dpo/margin": 80.00682830810547,
|
|
"fcm_dpo/q_t": 0.3651021122932434,
|
|
"grad_norm": 30.504737854003906,
|
|
"learning_rate": 4.4636895135509966e-07,
|
|
"logits/chosen": -0.3527507185935974,
|
|
"logits/rejected": -0.3269976079463959,
|
|
"logps/chosen": -120.8394775390625,
|
|
"logps/ref_chosen": -53.06706237792969,
|
|
"logps/ref_rejected": -80.60843658447266,
|
|
"logps/rejected": -228.38766479492188,
|
|
"loss": 0.9914,
|
|
"margin_dpo/margin_mean": 80.00682830810547,
|
|
"margin_dpo/margin_std": 106.2452621459961,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2936857562408223,
|
|
"eval_fcm_dpo/beta": 0.00816048588603735,
|
|
"eval_logits/chosen": -0.4719357490539551,
|
|
"eval_logits/rejected": -0.45578357577323914,
|
|
"eval_logps/chosen": -172.25338745117188,
|
|
"eval_logps/ref_chosen": -79.05104064941406,
|
|
"eval_logps/ref_rejected": -86.79793548583984,
|
|
"eval_logps/rejected": -225.0161590576172,
|
|
"eval_loss": 0.6008790135383606,
|
|
"eval_margin_dpo/margin_mean": 45.01588439941406,
|
|
"eval_margin_dpo/margin_std": 98.482421875,
|
|
"eval_runtime": 39.3122,
|
|
"eval_samples_per_second": 59.498,
|
|
"eval_steps_per_second": 1.882,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.29515418502202645,
|
|
"fcm_dpo/beta": 0.008100075647234917,
|
|
"fcm_dpo/delta": -0.0565091148018837,
|
|
"fcm_dpo/margin": 80.59832000732422,
|
|
"fcm_dpo/q_t": 0.3638731837272644,
|
|
"grad_norm": 30.036048889160156,
|
|
"learning_rate": 4.455721242469372e-07,
|
|
"logits/chosen": -0.4626866281032562,
|
|
"logits/rejected": -0.4563639163970947,
|
|
"logps/chosen": -146.43814086914062,
|
|
"logps/ref_chosen": -75.4022216796875,
|
|
"logps/ref_rejected": -114.80821990966797,
|
|
"logps/rejected": -266.44244384765625,
|
|
"loss": 0.9815,
|
|
"margin_dpo/margin_mean": 80.59831237792969,
|
|
"margin_dpo/margin_std": 103.2762222290039,
|
|
"step": 201
|
|
},
|
|
{
|
|
"epoch": 0.2966226138032305,
|
|
"fcm_dpo/beta": 0.00825195387005806,
|
|
"fcm_dpo/delta": 0.13680000603199005,
|
|
"fcm_dpo/margin": 56.866981506347656,
|
|
"fcm_dpo/q_t": 0.40256237983703613,
|
|
"grad_norm": 26.29439353942871,
|
|
"learning_rate": 4.4477014363141755e-07,
|
|
"logits/chosen": -0.3789522051811218,
|
|
"logits/rejected": -0.3867080807685852,
|
|
"logps/chosen": -124.50201416015625,
|
|
"logps/ref_chosen": -50.101318359375,
|
|
"logps/ref_rejected": -86.98503112792969,
|
|
"logps/rejected": -218.25270080566406,
|
|
"loss": 1.1171,
|
|
"margin_dpo/margin_mean": 56.866981506347656,
|
|
"margin_dpo/margin_std": 99.0474853515625,
|
|
"step": 202
|
|
},
|
|
{
|
|
"epoch": 0.29809104258443464,
|
|
"fcm_dpo/beta": 0.008316997438669205,
|
|
"fcm_dpo/delta": 0.0254436694085598,
|
|
"fcm_dpo/margin": 69.2591781616211,
|
|
"fcm_dpo/q_t": 0.3762577772140503,
|
|
"grad_norm": 31.99873924255371,
|
|
"learning_rate": 4.439630306414758e-07,
|
|
"logits/chosen": -0.4376029968261719,
|
|
"logits/rejected": -0.4240657389163971,
|
|
"logps/chosen": -134.08056640625,
|
|
"logps/ref_chosen": -60.60969543457031,
|
|
"logps/ref_rejected": -85.89596557617188,
|
|
"logps/rejected": -228.62603759765625,
|
|
"loss": 0.9972,
|
|
"margin_dpo/margin_mean": 69.2591781616211,
|
|
"margin_dpo/margin_std": 84.98655700683594,
|
|
"step": 203
|
|
},
|
|
{
|
|
"epoch": 0.29955947136563876,
|
|
"fcm_dpo/beta": 0.00838047917932272,
|
|
"fcm_dpo/delta": 0.04098369926214218,
|
|
"fcm_dpo/margin": 66.98109436035156,
|
|
"fcm_dpo/q_t": 0.3864881992340088,
|
|
"grad_norm": 29.694320678710938,
|
|
"learning_rate": 4.431508065452897e-07,
|
|
"logits/chosen": -0.47066575288772583,
|
|
"logits/rejected": -0.4281098246574402,
|
|
"logps/chosen": -162.51052856445312,
|
|
"logps/ref_chosen": -80.16496276855469,
|
|
"logps/ref_rejected": -87.69590759277344,
|
|
"logps/rejected": -237.0225830078125,
|
|
"loss": 1.0491,
|
|
"margin_dpo/margin_mean": 66.98109436035156,
|
|
"margin_dpo/margin_std": 100.66291046142578,
|
|
"step": 204
|
|
},
|
|
{
|
|
"epoch": 0.3010279001468429,
|
|
"fcm_dpo/beta": 0.008169597014784813,
|
|
"fcm_dpo/delta": -0.11707717180252075,
|
|
"fcm_dpo/margin": 86.33795166015625,
|
|
"fcm_dpo/q_t": 0.350330114364624,
|
|
"grad_norm": 24.040218353271484,
|
|
"learning_rate": 4.4233349274571974e-07,
|
|
"logits/chosen": -0.38784509897232056,
|
|
"logits/rejected": -0.35509926080703735,
|
|
"logps/chosen": -142.8284454345703,
|
|
"logps/ref_chosen": -59.384735107421875,
|
|
"logps/ref_rejected": -85.12505340576172,
|
|
"logps/rejected": -254.90672302246094,
|
|
"loss": 0.9583,
|
|
"margin_dpo/margin_mean": 86.33795166015625,
|
|
"margin_dpo/margin_std": 101.34950256347656,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.302496328928047,
|
|
"fcm_dpo/beta": 0.007971297949552536,
|
|
"fcm_dpo/delta": -0.16260316967964172,
|
|
"fcm_dpo/margin": 93.97215270996094,
|
|
"fcm_dpo/q_t": 0.33909979462623596,
|
|
"grad_norm": 34.61493682861328,
|
|
"learning_rate": 4.415111107797445e-07,
|
|
"logits/chosen": -0.3233247995376587,
|
|
"logits/rejected": -0.32719945907592773,
|
|
"logps/chosen": -125.01007843017578,
|
|
"logps/ref_chosen": -46.964500427246094,
|
|
"logps/ref_rejected": -98.9534912109375,
|
|
"logps/rejected": -270.9712219238281,
|
|
"loss": 0.9038,
|
|
"margin_dpo/margin_mean": 93.97215270996094,
|
|
"margin_dpo/margin_std": 98.51936340332031,
|
|
"step": 206
|
|
},
|
|
{
|
|
"epoch": 0.3039647577092511,
|
|
"fcm_dpo/beta": 0.00767237413674593,
|
|
"fcm_dpo/delta": -0.24557989835739136,
|
|
"fcm_dpo/margin": 107.59861755371094,
|
|
"fcm_dpo/q_t": 0.33342668414115906,
|
|
"grad_norm": 28.77750587463379,
|
|
"learning_rate": 4.4068368231789365e-07,
|
|
"logits/chosen": -0.40268674492836,
|
|
"logits/rejected": -0.37191277742385864,
|
|
"logps/chosen": -121.83184814453125,
|
|
"logps/ref_chosen": -56.05625915527344,
|
|
"logps/ref_rejected": -84.44779968261719,
|
|
"logps/rejected": -257.822021484375,
|
|
"loss": 0.8758,
|
|
"margin_dpo/margin_mean": 107.59861755371094,
|
|
"margin_dpo/margin_std": 117.77052307128906,
|
|
"step": 207
|
|
},
|
|
{
|
|
"epoch": 0.3054331864904552,
|
|
"fcm_dpo/beta": 0.007520717103034258,
|
|
"fcm_dpo/delta": -0.02344403602182865,
|
|
"fcm_dpo/margin": 82.68033599853516,
|
|
"fcm_dpo/q_t": 0.36858728528022766,
|
|
"grad_norm": 29.981359481811523,
|
|
"learning_rate": 4.398512291636768e-07,
|
|
"logits/chosen": -0.42680084705352783,
|
|
"logits/rejected": -0.4093056321144104,
|
|
"logps/chosen": -167.66326904296875,
|
|
"logps/ref_chosen": -67.06761169433594,
|
|
"logps/ref_rejected": -94.28689575195312,
|
|
"logps/rejected": -277.5628662109375,
|
|
"loss": 1.007,
|
|
"margin_dpo/margin_mean": 82.68034362792969,
|
|
"margin_dpo/margin_std": 110.93801879882812,
|
|
"step": 208
|
|
},
|
|
{
|
|
"epoch": 0.3069016152716593,
|
|
"fcm_dpo/beta": 0.0075776162557303905,
|
|
"fcm_dpo/delta": 0.04492693394422531,
|
|
"fcm_dpo/margin": 73.58404541015625,
|
|
"fcm_dpo/q_t": 0.3805572986602783,
|
|
"grad_norm": 32.00458908081055,
|
|
"learning_rate": 4.3901377325300857e-07,
|
|
"logits/chosen": -0.33511584997177124,
|
|
"logits/rejected": -0.3224318027496338,
|
|
"logps/chosen": -138.41427612304688,
|
|
"logps/ref_chosen": -56.18169403076172,
|
|
"logps/ref_rejected": -80.94152069091797,
|
|
"logps/rejected": -236.75814819335938,
|
|
"loss": 1.0361,
|
|
"margin_dpo/margin_mean": 73.58403778076172,
|
|
"margin_dpo/margin_std": 99.81734466552734,
|
|
"step": 209
|
|
},
|
|
{
|
|
"epoch": 0.30837004405286345,
|
|
"fcm_dpo/beta": 0.007613741792738438,
|
|
"fcm_dpo/delta": -0.010615115985274315,
|
|
"fcm_dpo/margin": 80.08528137207031,
|
|
"fcm_dpo/q_t": 0.3702564239501953,
|
|
"grad_norm": 22.40015411376953,
|
|
"learning_rate": 4.381713366536311e-07,
|
|
"logits/chosen": -0.3724694550037384,
|
|
"logits/rejected": -0.36007583141326904,
|
|
"logps/chosen": -120.55526733398438,
|
|
"logps/ref_chosen": -46.371822357177734,
|
|
"logps/ref_rejected": -76.68162536621094,
|
|
"logps/rejected": -230.95034790039062,
|
|
"loss": 0.984,
|
|
"margin_dpo/margin_mean": 80.08528137207031,
|
|
"margin_dpo/margin_std": 96.79129028320312,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.30983847283406757,
|
|
"fcm_dpo/beta": 0.0076542324386537075,
|
|
"fcm_dpo/delta": 0.0667615756392479,
|
|
"fcm_dpo/margin": 70.13720703125,
|
|
"fcm_dpo/q_t": 0.38842254877090454,
|
|
"grad_norm": 34.49944305419922,
|
|
"learning_rate": 4.373239415645323e-07,
|
|
"logits/chosen": -0.4161061644554138,
|
|
"logits/rejected": -0.37069594860076904,
|
|
"logps/chosen": -185.52003479003906,
|
|
"logps/ref_chosen": -78.93235778808594,
|
|
"logps/ref_rejected": -86.82098388671875,
|
|
"logps/rejected": -263.5458984375,
|
|
"loss": 1.0509,
|
|
"margin_dpo/margin_mean": 70.13721466064453,
|
|
"margin_dpo/margin_std": 100.4876937866211,
|
|
"step": 211
|
|
},
|
|
{
|
|
"epoch": 0.31130690161527164,
|
|
"fcm_dpo/beta": 0.007427205331623554,
|
|
"fcm_dpo/delta": -0.15895992517471313,
|
|
"fcm_dpo/margin": 100.10624694824219,
|
|
"fcm_dpo/q_t": 0.3454575836658478,
|
|
"grad_norm": 28.596296310424805,
|
|
"learning_rate": 4.3647161031536086e-07,
|
|
"logits/chosen": -0.3710683584213257,
|
|
"logits/rejected": -0.35650646686553955,
|
|
"logps/chosen": -147.09378051757812,
|
|
"logps/ref_chosen": -58.19701385498047,
|
|
"logps/ref_rejected": -103.05785369873047,
|
|
"logps/rejected": -292.06085205078125,
|
|
"loss": 0.9345,
|
|
"margin_dpo/margin_mean": 100.10624694824219,
|
|
"margin_dpo/margin_std": 112.30523681640625,
|
|
"step": 212
|
|
},
|
|
{
|
|
"epoch": 0.31277533039647576,
|
|
"fcm_dpo/beta": 0.007264791056513786,
|
|
"fcm_dpo/delta": -0.1239488273859024,
|
|
"fcm_dpo/margin": 98.2468490600586,
|
|
"fcm_dpo/q_t": 0.34874916076660156,
|
|
"grad_norm": 25.72871208190918,
|
|
"learning_rate": 4.3561436536583774e-07,
|
|
"logits/chosen": -0.41955453157424927,
|
|
"logits/rejected": -0.3891031742095947,
|
|
"logps/chosen": -149.52923583984375,
|
|
"logps/ref_chosen": -67.51271057128906,
|
|
"logps/ref_rejected": -93.91471862792969,
|
|
"logps/rejected": -274.1781005859375,
|
|
"loss": 0.9295,
|
|
"margin_dpo/margin_mean": 98.24685668945312,
|
|
"margin_dpo/margin_std": 108.88845825195312,
|
|
"step": 213
|
|
},
|
|
{
|
|
"epoch": 0.3142437591776799,
|
|
"fcm_dpo/beta": 0.007254892028868198,
|
|
"fcm_dpo/delta": -0.009785597212612629,
|
|
"fcm_dpo/margin": 83.96929168701172,
|
|
"fcm_dpo/q_t": 0.37200796604156494,
|
|
"grad_norm": 26.340700149536133,
|
|
"learning_rate": 4.3475222930516473e-07,
|
|
"logits/chosen": -0.3503088653087616,
|
|
"logits/rejected": -0.35779234766960144,
|
|
"logps/chosen": -118.85914611816406,
|
|
"logps/ref_chosen": -41.604888916015625,
|
|
"logps/ref_rejected": -77.51741027832031,
|
|
"logps/rejected": -238.740966796875,
|
|
"loss": 0.9962,
|
|
"margin_dpo/margin_mean": 83.96929931640625,
|
|
"margin_dpo/margin_std": 107.99652099609375,
|
|
"step": 214
|
|
},
|
|
{
|
|
"epoch": 0.315712187958884,
|
|
"fcm_dpo/beta": 0.007164125796407461,
|
|
"fcm_dpo/delta": -0.04974304512143135,
|
|
"fcm_dpo/margin": 90.17284393310547,
|
|
"fcm_dpo/q_t": 0.3607136309146881,
|
|
"grad_norm": 27.573640823364258,
|
|
"learning_rate": 4.3388522485142885e-07,
|
|
"logits/chosen": -0.3955865502357483,
|
|
"logits/rejected": -0.3844960331916809,
|
|
"logps/chosen": -139.45066833496094,
|
|
"logps/ref_chosen": -53.279266357421875,
|
|
"logps/ref_rejected": -89.96464538574219,
|
|
"logps/rejected": -266.30889892578125,
|
|
"loss": 0.9501,
|
|
"margin_dpo/margin_mean": 90.1728515625,
|
|
"margin_dpo/margin_std": 99.1728515625,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.31718061674008813,
|
|
"fcm_dpo/beta": 0.007151792757213116,
|
|
"fcm_dpo/delta": -0.04989144578576088,
|
|
"fcm_dpo/margin": 90.40126037597656,
|
|
"fcm_dpo/q_t": 0.36354175209999084,
|
|
"grad_norm": 33.67525863647461,
|
|
"learning_rate": 4.330133748510036e-07,
|
|
"logits/chosen": -0.3702380061149597,
|
|
"logits/rejected": -0.3506859540939331,
|
|
"logps/chosen": -137.24215698242188,
|
|
"logps/ref_chosen": -48.887794494628906,
|
|
"logps/ref_rejected": -77.19892883300781,
|
|
"logps/rejected": -255.95455932617188,
|
|
"loss": 0.9844,
|
|
"margin_dpo/margin_mean": 90.40126037597656,
|
|
"margin_dpo/margin_std": 113.781982421875,
|
|
"step": 216
|
|
},
|
|
{
|
|
"epoch": 0.3186490455212922,
|
|
"fcm_dpo/beta": 0.006948284804821014,
|
|
"fcm_dpo/delta": -0.1394684761762619,
|
|
"fcm_dpo/margin": 104.92652893066406,
|
|
"fcm_dpo/q_t": 0.34340259432792664,
|
|
"grad_norm": 25.96721649169922,
|
|
"learning_rate": 4.3213670227794757e-07,
|
|
"logits/chosen": -0.3827437162399292,
|
|
"logits/rejected": -0.3740556240081787,
|
|
"logps/chosen": -134.2442626953125,
|
|
"logps/ref_chosen": -49.845306396484375,
|
|
"logps/ref_rejected": -100.07832336425781,
|
|
"logps/rejected": -289.40380859375,
|
|
"loss": 0.897,
|
|
"margin_dpo/margin_mean": 104.92652893066406,
|
|
"margin_dpo/margin_std": 105.56159210205078,
|
|
"step": 217
|
|
},
|
|
{
|
|
"epoch": 0.3201174743024963,
|
|
"fcm_dpo/beta": 0.0069186268374323845,
|
|
"fcm_dpo/delta": 0.01962290331721306,
|
|
"fcm_dpo/margin": 84.04899597167969,
|
|
"fcm_dpo/q_t": 0.37689316272735596,
|
|
"grad_norm": 29.522064208984375,
|
|
"learning_rate": 4.3125523023339815e-07,
|
|
"logits/chosen": -0.4043159484863281,
|
|
"logits/rejected": -0.3937210440635681,
|
|
"logps/chosen": -144.95455932617188,
|
|
"logps/ref_chosen": -58.576683044433594,
|
|
"logps/ref_rejected": -87.84639739990234,
|
|
"logps/rejected": -258.2732849121094,
|
|
"loss": 1.008,
|
|
"margin_dpo/margin_mean": 84.04900360107422,
|
|
"margin_dpo/margin_std": 107.98806762695312,
|
|
"step": 218
|
|
},
|
|
{
|
|
"epoch": 0.32158590308370044,
|
|
"fcm_dpo/beta": 0.007075803354382515,
|
|
"fcm_dpo/delta": 0.09832102805376053,
|
|
"fcm_dpo/margin": 71.45269775390625,
|
|
"fcm_dpo/q_t": 0.3926897644996643,
|
|
"grad_norm": 25.61137580871582,
|
|
"learning_rate": 4.303689819449636e-07,
|
|
"logits/chosen": -0.4170767664909363,
|
|
"logits/rejected": -0.4031888246536255,
|
|
"logps/chosen": -152.66976928710938,
|
|
"logps/ref_chosen": -61.083858489990234,
|
|
"logps/ref_rejected": -85.83042907714844,
|
|
"logps/rejected": -248.86904907226562,
|
|
"loss": 1.0769,
|
|
"margin_dpo/margin_mean": 71.45269775390625,
|
|
"margin_dpo/margin_std": 110.11042785644531,
|
|
"step": 219
|
|
},
|
|
{
|
|
"epoch": 0.32305433186490456,
|
|
"fcm_dpo/beta": 0.007226555608212948,
|
|
"fcm_dpo/delta": 0.1684371680021286,
|
|
"fcm_dpo/margin": 60.78836441040039,
|
|
"fcm_dpo/q_t": 0.40072399377822876,
|
|
"grad_norm": 25.96778106689453,
|
|
"learning_rate": 4.2947798076611047e-07,
|
|
"logits/chosen": -0.40701720118522644,
|
|
"logits/rejected": -0.3823140263557434,
|
|
"logps/chosen": -173.3245849609375,
|
|
"logps/ref_chosen": -70.03128051757812,
|
|
"logps/ref_rejected": -87.68551635742188,
|
|
"logps/rejected": -251.76718139648438,
|
|
"loss": 1.0723,
|
|
"margin_dpo/margin_mean": 60.788368225097656,
|
|
"margin_dpo/margin_std": 78.86666870117188,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.3245227606461087,
|
|
"fcm_dpo/beta": 0.007043336518108845,
|
|
"fcm_dpo/delta": -0.2806098461151123,
|
|
"fcm_dpo/margin": 121.70394897460938,
|
|
"fcm_dpo/q_t": 0.3166281580924988,
|
|
"grad_norm": 29.857473373413086,
|
|
"learning_rate": 4.285822501755485e-07,
|
|
"logits/chosen": -0.36656078696250916,
|
|
"logits/rejected": -0.3734094500541687,
|
|
"logps/chosen": -148.55584716796875,
|
|
"logps/ref_chosen": -52.15470886230469,
|
|
"logps/ref_rejected": -106.46768188476562,
|
|
"logps/rejected": -324.57275390625,
|
|
"loss": 0.8286,
|
|
"margin_dpo/margin_mean": 121.70394897460938,
|
|
"margin_dpo/margin_std": 108.79747009277344,
|
|
"step": 221
|
|
},
|
|
{
|
|
"epoch": 0.32599118942731276,
|
|
"fcm_dpo/beta": 0.00688206497579813,
|
|
"fcm_dpo/delta": -0.058866240084171295,
|
|
"fcm_dpo/margin": 95.16747283935547,
|
|
"fcm_dpo/q_t": 0.3594551086425781,
|
|
"grad_norm": 27.359058380126953,
|
|
"learning_rate": 4.276818137766118e-07,
|
|
"logits/chosen": -0.4119839668273926,
|
|
"logits/rejected": -0.40599650144577026,
|
|
"logps/chosen": -164.09307861328125,
|
|
"logps/ref_chosen": -60.971099853515625,
|
|
"logps/ref_rejected": -100.00115203857422,
|
|
"logps/rejected": -298.29058837890625,
|
|
"loss": 0.9574,
|
|
"margin_dpo/margin_mean": 95.16747283935547,
|
|
"margin_dpo/margin_std": 109.49444580078125,
|
|
"step": 222
|
|
},
|
|
{
|
|
"epoch": 0.3274596182085169,
|
|
"fcm_dpo/beta": 0.006874423939734697,
|
|
"fcm_dpo/delta": 0.045850180089473724,
|
|
"fcm_dpo/margin": 80.98518371582031,
|
|
"fcm_dpo/q_t": 0.38237255811691284,
|
|
"grad_norm": 31.334300994873047,
|
|
"learning_rate": 4.2677669529663686e-07,
|
|
"logits/chosen": -0.3320853114128113,
|
|
"logits/rejected": -0.31920111179351807,
|
|
"logps/chosen": -163.82130432128906,
|
|
"logps/ref_chosen": -52.64057540893555,
|
|
"logps/ref_rejected": -82.82502746582031,
|
|
"logps/rejected": -274.9909362792969,
|
|
"loss": 1.0635,
|
|
"margin_dpo/margin_mean": 80.98518371582031,
|
|
"margin_dpo/margin_std": 123.70716857910156,
|
|
"step": 223
|
|
},
|
|
{
|
|
"epoch": 0.328928046989721,
|
|
"fcm_dpo/beta": 0.006771251559257507,
|
|
"fcm_dpo/delta": -0.05909021943807602,
|
|
"fcm_dpo/margin": 96.40702056884766,
|
|
"fcm_dpo/q_t": 0.3656306862831116,
|
|
"grad_norm": 32.546329498291016,
|
|
"learning_rate": 4.2586691858633747e-07,
|
|
"logits/chosen": -0.36080002784729004,
|
|
"logits/rejected": -0.3402705192565918,
|
|
"logps/chosen": -143.6527862548828,
|
|
"logps/ref_chosen": -48.59541320800781,
|
|
"logps/ref_rejected": -77.11648559570312,
|
|
"logps/rejected": -268.58087158203125,
|
|
"loss": 0.9886,
|
|
"margin_dpo/margin_mean": 96.40702056884766,
|
|
"margin_dpo/margin_std": 124.457275390625,
|
|
"step": 224
|
|
},
|
|
{
|
|
"epoch": 0.3303964757709251,
|
|
"fcm_dpo/beta": 0.006611201912164688,
|
|
"fcm_dpo/delta": -0.19814401865005493,
|
|
"fcm_dpo/margin": 118.33515930175781,
|
|
"fcm_dpo/q_t": 0.3383178114891052,
|
|
"grad_norm": 26.60202407836914,
|
|
"learning_rate": 4.249525076191759e-07,
|
|
"logits/chosen": -0.35223451256752014,
|
|
"logits/rejected": -0.33783799409866333,
|
|
"logps/chosen": -169.36512756347656,
|
|
"logps/ref_chosen": -58.000465393066406,
|
|
"logps/ref_rejected": -99.90291595458984,
|
|
"logps/rejected": -329.60272216796875,
|
|
"loss": 0.9074,
|
|
"margin_dpo/margin_mean": 118.33515930175781,
|
|
"margin_dpo/margin_std": 133.42843627929688,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.33186490455212925,
|
|
"fcm_dpo/beta": 0.006490599364042282,
|
|
"fcm_dpo/delta": 0.004586532711982727,
|
|
"fcm_dpo/margin": 91.46955871582031,
|
|
"fcm_dpo/q_t": 0.37888455390930176,
|
|
"grad_norm": 27.73929214477539,
|
|
"learning_rate": 4.2403348649073167e-07,
|
|
"logits/chosen": -0.389433890581131,
|
|
"logits/rejected": -0.3457120954990387,
|
|
"logps/chosen": -155.97816467285156,
|
|
"logps/ref_chosen": -58.898799896240234,
|
|
"logps/ref_rejected": -78.68775939941406,
|
|
"logps/rejected": -267.2366943359375,
|
|
"loss": 1.0272,
|
|
"margin_dpo/margin_mean": 91.46955871582031,
|
|
"margin_dpo/margin_std": 127.21261596679688,
|
|
"step": 226
|
|
},
|
|
{
|
|
"epoch": 0.3333333333333333,
|
|
"fcm_dpo/beta": 0.00636161956936121,
|
|
"fcm_dpo/delta": -0.16426458954811096,
|
|
"fcm_dpo/margin": 117.93624877929688,
|
|
"fcm_dpo/q_t": 0.34007027745246887,
|
|
"grad_norm": 27.874391555786133,
|
|
"learning_rate": 4.2310987941806615e-07,
|
|
"logits/chosen": -0.37095752358436584,
|
|
"logits/rejected": -0.35500913858413696,
|
|
"logps/chosen": -170.41358947753906,
|
|
"logps/ref_chosen": -59.072181701660156,
|
|
"logps/ref_rejected": -99.41236877441406,
|
|
"logps/rejected": -328.6900329589844,
|
|
"loss": 0.9131,
|
|
"margin_dpo/margin_mean": 117.93626403808594,
|
|
"margin_dpo/margin_std": 129.8984832763672,
|
|
"step": 227
|
|
},
|
|
{
|
|
"epoch": 0.33480176211453744,
|
|
"fcm_dpo/beta": 0.006433088798075914,
|
|
"fcm_dpo/delta": 0.07818908244371414,
|
|
"fcm_dpo/margin": 81.71975708007812,
|
|
"fcm_dpo/q_t": 0.38696879148483276,
|
|
"grad_norm": 28.652111053466797,
|
|
"learning_rate": 4.2218171073908463e-07,
|
|
"logits/chosen": -0.3662306070327759,
|
|
"logits/rejected": -0.3457154631614685,
|
|
"logps/chosen": -179.63031005859375,
|
|
"logps/ref_chosen": -65.89128875732422,
|
|
"logps/ref_rejected": -91.04875183105469,
|
|
"logps/rejected": -286.50750732421875,
|
|
"loss": 1.0584,
|
|
"margin_dpo/margin_mean": 81.71975708007812,
|
|
"margin_dpo/margin_std": 119.07015228271484,
|
|
"step": 228
|
|
},
|
|
{
|
|
"epoch": 0.33627019089574156,
|
|
"fcm_dpo/beta": 0.006459444761276245,
|
|
"fcm_dpo/delta": 0.005232607014477253,
|
|
"fcm_dpo/margin": 92.12015533447266,
|
|
"fcm_dpo/q_t": 0.3724709153175354,
|
|
"grad_norm": 33.15553665161133,
|
|
"learning_rate": 4.212490049118951e-07,
|
|
"logits/chosen": -0.44420498609542847,
|
|
"logits/rejected": -0.412958562374115,
|
|
"logps/chosen": -181.48049926757812,
|
|
"logps/ref_chosen": -70.70637512207031,
|
|
"logps/ref_rejected": -84.52741241455078,
|
|
"logps/rejected": -287.42169189453125,
|
|
"loss": 1.0091,
|
|
"margin_dpo/margin_mean": 92.12016296386719,
|
|
"margin_dpo/margin_std": 121.14804077148438,
|
|
"step": 229
|
|
},
|
|
{
|
|
"epoch": 0.3377386196769457,
|
|
"fcm_dpo/beta": 0.006237386725842953,
|
|
"fcm_dpo/delta": -0.20489397644996643,
|
|
"fcm_dpo/margin": 126.29830169677734,
|
|
"fcm_dpo/q_t": 0.33031266927719116,
|
|
"grad_norm": 31.86959457397461,
|
|
"learning_rate": 4.203117865141635e-07,
|
|
"logits/chosen": -0.32878124713897705,
|
|
"logits/rejected": -0.3298182487487793,
|
|
"logps/chosen": -130.9337158203125,
|
|
"logps/ref_chosen": -39.282005310058594,
|
|
"logps/ref_rejected": -85.62191009521484,
|
|
"logps/rejected": -303.5719299316406,
|
|
"loss": 0.8704,
|
|
"margin_dpo/margin_mean": 126.29830932617188,
|
|
"margin_dpo/margin_std": 123.00277709960938,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.3392070484581498,
|
|
"fcm_dpo/beta": 0.0062300898134708405,
|
|
"fcm_dpo/delta": 0.04776204377412796,
|
|
"fcm_dpo/margin": 89.06995391845703,
|
|
"fcm_dpo/q_t": 0.3806319832801819,
|
|
"grad_norm": 26.72955894470215,
|
|
"learning_rate": 4.1937008024246625e-07,
|
|
"logits/chosen": -0.36490845680236816,
|
|
"logits/rejected": -0.33270594477653503,
|
|
"logps/chosen": -161.59432983398438,
|
|
"logps/ref_chosen": -63.27644348144531,
|
|
"logps/ref_rejected": -74.1239013671875,
|
|
"logps/rejected": -261.5117492675781,
|
|
"loss": 1.0029,
|
|
"margin_dpo/margin_mean": 89.06995391845703,
|
|
"margin_dpo/margin_std": 107.24581909179688,
|
|
"step": 231
|
|
},
|
|
{
|
|
"epoch": 0.3406754772393539,
|
|
"fcm_dpo/beta": 0.006363061256706715,
|
|
"fcm_dpo/delta": 0.136710524559021,
|
|
"fcm_dpo/margin": 73.8527603149414,
|
|
"fcm_dpo/q_t": 0.40357333421707153,
|
|
"grad_norm": 31.16952896118164,
|
|
"learning_rate": 4.1842391091163933e-07,
|
|
"logits/chosen": -0.3764330744743347,
|
|
"logits/rejected": -0.3467271029949188,
|
|
"logps/chosen": -187.44082641601562,
|
|
"logps/ref_chosen": -70.74876403808594,
|
|
"logps/ref_rejected": -83.97706604003906,
|
|
"logps/rejected": -274.52191162109375,
|
|
"loss": 1.092,
|
|
"margin_dpo/margin_mean": 73.8527603149414,
|
|
"margin_dpo/margin_std": 118.7332763671875,
|
|
"step": 232
|
|
},
|
|
{
|
|
"epoch": 0.342143906020558,
|
|
"fcm_dpo/beta": 0.006316785700619221,
|
|
"fcm_dpo/delta": -0.11919762194156647,
|
|
"fcm_dpo/margin": 112.51909637451172,
|
|
"fcm_dpo/q_t": 0.3590048551559448,
|
|
"grad_norm": 28.269271850585938,
|
|
"learning_rate": 4.174733034541245e-07,
|
|
"logits/chosen": -0.3798902630805969,
|
|
"logits/rejected": -0.3772153854370117,
|
|
"logps/chosen": -157.00439453125,
|
|
"logps/ref_chosen": -54.8829345703125,
|
|
"logps/ref_rejected": -107.4800796508789,
|
|
"logps/rejected": -322.12060546875,
|
|
"loss": 0.9786,
|
|
"margin_dpo/margin_mean": 112.51909637451172,
|
|
"margin_dpo/margin_std": 150.15170288085938,
|
|
"step": 233
|
|
},
|
|
{
|
|
"epoch": 0.3436123348017621,
|
|
"fcm_dpo/beta": 0.006136967800557613,
|
|
"fcm_dpo/delta": -0.0946032702922821,
|
|
"fcm_dpo/margin": 111.787109375,
|
|
"fcm_dpo/q_t": 0.3509105145931244,
|
|
"grad_norm": 38.759918212890625,
|
|
"learning_rate": 4.165182829193126e-07,
|
|
"logits/chosen": -0.3641296625137329,
|
|
"logits/rejected": -0.39079636335372925,
|
|
"logps/chosen": -142.52503967285156,
|
|
"logps/ref_chosen": -44.094520568847656,
|
|
"logps/ref_rejected": -100.00663757324219,
|
|
"logps/rejected": -310.2242431640625,
|
|
"loss": 0.9207,
|
|
"margin_dpo/margin_mean": 111.787109375,
|
|
"margin_dpo/margin_std": 113.17666625976562,
|
|
"step": 234
|
|
},
|
|
{
|
|
"epoch": 0.34508076358296624,
|
|
"fcm_dpo/beta": 0.0063125560991466045,
|
|
"fcm_dpo/delta": 0.14529383182525635,
|
|
"fcm_dpo/margin": 72.99315643310547,
|
|
"fcm_dpo/q_t": 0.40060219168663025,
|
|
"grad_norm": 33.466941833496094,
|
|
"learning_rate": 4.1555887447288255e-07,
|
|
"logits/chosen": -0.4195903241634369,
|
|
"logits/rejected": -0.39497965574264526,
|
|
"logps/chosen": -181.33181762695312,
|
|
"logps/ref_chosen": -62.237911224365234,
|
|
"logps/ref_rejected": -90.39506530761719,
|
|
"logps/rejected": -282.48211669921875,
|
|
"loss": 1.0955,
|
|
"margin_dpo/margin_mean": 72.9931640625,
|
|
"margin_dpo/margin_std": 114.59500122070312,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.3465491923641703,
|
|
"fcm_dpo/beta": 0.006248786114156246,
|
|
"fcm_dpo/delta": -0.09669020771980286,
|
|
"fcm_dpo/margin": 110.42259216308594,
|
|
"fcm_dpo/q_t": 0.3470776677131653,
|
|
"grad_norm": 50.5585823059082,
|
|
"learning_rate": 4.1459510339613946e-07,
|
|
"logits/chosen": -0.36349114775657654,
|
|
"logits/rejected": -0.3634676933288574,
|
|
"logps/chosen": -136.66470336914062,
|
|
"logps/ref_chosen": -49.34136199951172,
|
|
"logps/ref_rejected": -103.51162719726562,
|
|
"logps/rejected": -301.257568359375,
|
|
"loss": 0.8949,
|
|
"margin_dpo/margin_mean": 110.42259216308594,
|
|
"margin_dpo/margin_std": 99.5032958984375,
|
|
"step": 236
|
|
},
|
|
{
|
|
"epoch": 0.34801762114537443,
|
|
"fcm_dpo/beta": 0.006244904361665249,
|
|
"fcm_dpo/delta": 0.023870917037129402,
|
|
"fcm_dpo/margin": 92.47233581542969,
|
|
"fcm_dpo/q_t": 0.37653428316116333,
|
|
"grad_norm": 30.452253341674805,
|
|
"learning_rate": 4.136269950853473e-07,
|
|
"logits/chosen": -0.35491400957107544,
|
|
"logits/rejected": -0.34882035851478577,
|
|
"logps/chosen": -170.31382751464844,
|
|
"logps/ref_chosen": -54.168121337890625,
|
|
"logps/ref_rejected": -94.78036499023438,
|
|
"logps/rejected": -303.3984069824219,
|
|
"loss": 1.0111,
|
|
"margin_dpo/margin_mean": 92.47232818603516,
|
|
"margin_dpo/margin_std": 120.4630126953125,
|
|
"step": 237
|
|
},
|
|
{
|
|
"epoch": 0.34948604992657856,
|
|
"fcm_dpo/beta": 0.006248504854738712,
|
|
"fcm_dpo/delta": 0.04543028026819229,
|
|
"fcm_dpo/margin": 89.06282043457031,
|
|
"fcm_dpo/q_t": 0.3825730085372925,
|
|
"grad_norm": 31.341205596923828,
|
|
"learning_rate": 4.126545750510605e-07,
|
|
"logits/chosen": -0.3699679374694824,
|
|
"logits/rejected": -0.3880308270454407,
|
|
"logps/chosen": -163.38356018066406,
|
|
"logps/ref_chosen": -53.973121643066406,
|
|
"logps/ref_rejected": -89.41795349121094,
|
|
"logps/rejected": -287.8912048339844,
|
|
"loss": 1.0143,
|
|
"margin_dpo/margin_mean": 89.06282043457031,
|
|
"margin_dpo/margin_std": 114.02665710449219,
|
|
"step": 238
|
|
},
|
|
{
|
|
"epoch": 0.3509544787077827,
|
|
"fcm_dpo/beta": 0.0062086135149002075,
|
|
"fcm_dpo/delta": -0.038354501128196716,
|
|
"fcm_dpo/margin": 102.07293701171875,
|
|
"fcm_dpo/q_t": 0.36239856481552124,
|
|
"grad_norm": 28.831066131591797,
|
|
"learning_rate": 4.116778689174514e-07,
|
|
"logits/chosen": -0.34534239768981934,
|
|
"logits/rejected": -0.32214316725730896,
|
|
"logps/chosen": -178.08087158203125,
|
|
"logps/ref_chosen": -58.09782409667969,
|
|
"logps/ref_rejected": -93.59294128417969,
|
|
"logps/rejected": -315.64892578125,
|
|
"loss": 0.9607,
|
|
"margin_dpo/margin_mean": 102.07293701171875,
|
|
"margin_dpo/margin_std": 110.16291809082031,
|
|
"step": 239
|
|
},
|
|
{
|
|
"epoch": 0.3524229074889868,
|
|
"fcm_dpo/beta": 0.006256358698010445,
|
|
"fcm_dpo/delta": 0.03319564089179039,
|
|
"fcm_dpo/margin": 90.84452056884766,
|
|
"fcm_dpo/q_t": 0.37809598445892334,
|
|
"grad_norm": 34.58857727050781,
|
|
"learning_rate": 4.106969024216348e-07,
|
|
"logits/chosen": -0.3797963261604309,
|
|
"logits/rejected": -0.3545226454734802,
|
|
"logps/chosen": -188.4025421142578,
|
|
"logps/ref_chosen": -60.6144905090332,
|
|
"logps/ref_rejected": -74.1185302734375,
|
|
"logps/rejected": -292.7510986328125,
|
|
"loss": 1.0351,
|
|
"margin_dpo/margin_mean": 90.84451293945312,
|
|
"margin_dpo/margin_std": 124.30812072753906,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.35389133627019087,
|
|
"fcm_dpo/beta": 0.006106827408075333,
|
|
"fcm_dpo/delta": -0.2318933755159378,
|
|
"fcm_dpo/margin": 133.21487426757812,
|
|
"fcm_dpo/q_t": 0.33365797996520996,
|
|
"grad_norm": 27.41342544555664,
|
|
"learning_rate": 4.097117014129903e-07,
|
|
"logits/chosen": -0.42394572496414185,
|
|
"logits/rejected": -0.39460113644599915,
|
|
"logps/chosen": -171.47601318359375,
|
|
"logps/ref_chosen": -66.091064453125,
|
|
"logps/ref_rejected": -88.06088256835938,
|
|
"logps/rejected": -326.66070556640625,
|
|
"loss": 0.8839,
|
|
"margin_dpo/margin_mean": 133.21487426757812,
|
|
"margin_dpo/margin_std": 142.80030822753906,
|
|
"step": 241
|
|
},
|
|
{
|
|
"epoch": 0.355359765051395,
|
|
"fcm_dpo/beta": 0.006066991947591305,
|
|
"fcm_dpo/delta": 0.01960517093539238,
|
|
"fcm_dpo/margin": 95.71859741210938,
|
|
"fcm_dpo/q_t": 0.3771246075630188,
|
|
"grad_norm": 29.23126983642578,
|
|
"learning_rate": 4.087222918524807e-07,
|
|
"logits/chosen": -0.3212859630584717,
|
|
"logits/rejected": -0.2920798659324646,
|
|
"logps/chosen": -186.41012573242188,
|
|
"logps/ref_chosen": -67.86392974853516,
|
|
"logps/ref_rejected": -83.36033630371094,
|
|
"logps/rejected": -297.6251220703125,
|
|
"loss": 1.0017,
|
|
"margin_dpo/margin_mean": 95.71859741210938,
|
|
"margin_dpo/margin_std": 121.07659912109375,
|
|
"step": 242
|
|
},
|
|
{
|
|
"epoch": 0.3568281938325991,
|
|
"fcm_dpo/beta": 0.005917678587138653,
|
|
"fcm_dpo/delta": -0.11988495290279388,
|
|
"fcm_dpo/margin": 120.15345764160156,
|
|
"fcm_dpo/q_t": 0.34703272581100464,
|
|
"grad_norm": 27.846588134765625,
|
|
"learning_rate": 4.07728699811968e-07,
|
|
"logits/chosen": -0.36647382378578186,
|
|
"logits/rejected": -0.3263128399848938,
|
|
"logps/chosen": -183.94918823242188,
|
|
"logps/ref_chosen": -63.0842399597168,
|
|
"logps/ref_rejected": -76.33563232421875,
|
|
"logps/rejected": -317.35406494140625,
|
|
"loss": 0.9108,
|
|
"margin_dpo/margin_mean": 120.1534652709961,
|
|
"margin_dpo/margin_std": 125.44760131835938,
|
|
"step": 243
|
|
},
|
|
{
|
|
"epoch": 0.35829662261380324,
|
|
"fcm_dpo/beta": 0.005800644867122173,
|
|
"fcm_dpo/delta": -0.08758918195962906,
|
|
"fcm_dpo/margin": 117.43812561035156,
|
|
"fcm_dpo/q_t": 0.3503507375717163,
|
|
"grad_norm": 31.091215133666992,
|
|
"learning_rate": 4.067309514735267e-07,
|
|
"logits/chosen": -0.42268094420433044,
|
|
"logits/rejected": -0.4104187488555908,
|
|
"logps/chosen": -163.81822204589844,
|
|
"logps/ref_chosen": -61.140689849853516,
|
|
"logps/ref_rejected": -94.89193725585938,
|
|
"logps/rejected": -315.00762939453125,
|
|
"loss": 0.9077,
|
|
"margin_dpo/margin_mean": 117.43812561035156,
|
|
"margin_dpo/margin_std": 110.86767578125,
|
|
"step": 244
|
|
},
|
|
{
|
|
"epoch": 0.35976505139500736,
|
|
"fcm_dpo/beta": 0.00573451304808259,
|
|
"fcm_dpo/delta": 0.01733044534921646,
|
|
"fcm_dpo/margin": 101.28717803955078,
|
|
"fcm_dpo/q_t": 0.3786167502403259,
|
|
"grad_norm": 33.05402755737305,
|
|
"learning_rate": 4.057290731287531e-07,
|
|
"logits/chosen": -0.40215417742729187,
|
|
"logits/rejected": -0.3637341856956482,
|
|
"logps/chosen": -191.681396484375,
|
|
"logps/ref_chosen": -67.26228332519531,
|
|
"logps/ref_rejected": -87.64010620117188,
|
|
"logps/rejected": -313.3464050292969,
|
|
"loss": 1.0265,
|
|
"margin_dpo/margin_mean": 101.28717041015625,
|
|
"margin_dpo/margin_std": 134.58218383789062,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.36123348017621143,
|
|
"fcm_dpo/beta": 0.005821553990244865,
|
|
"fcm_dpo/delta": 0.027677321806550026,
|
|
"fcm_dpo/margin": 98.57552337646484,
|
|
"fcm_dpo/q_t": 0.3820421099662781,
|
|
"grad_norm": 26.930212020874023,
|
|
"learning_rate": 4.047230911780736e-07,
|
|
"logits/chosen": -0.4357486963272095,
|
|
"logits/rejected": -0.394938200712204,
|
|
"logps/chosen": -191.74822998046875,
|
|
"logps/ref_chosen": -66.69696807861328,
|
|
"logps/ref_rejected": -84.34634399414062,
|
|
"logps/rejected": -307.97314453125,
|
|
"loss": 1.0254,
|
|
"margin_dpo/margin_mean": 98.57553100585938,
|
|
"margin_dpo/margin_std": 138.3719482421875,
|
|
"step": 246
|
|
},
|
|
{
|
|
"epoch": 0.36270190895741555,
|
|
"fcm_dpo/beta": 0.005621997639536858,
|
|
"fcm_dpo/delta": -0.23428848385810852,
|
|
"fcm_dpo/margin": 144.79278564453125,
|
|
"fcm_dpo/q_t": 0.33090120553970337,
|
|
"grad_norm": 26.57573890686035,
|
|
"learning_rate": 4.0371303213004814e-07,
|
|
"logits/chosen": -0.3413492441177368,
|
|
"logits/rejected": -0.33773866295814514,
|
|
"logps/chosen": -205.19589233398438,
|
|
"logps/ref_chosen": -56.6053466796875,
|
|
"logps/ref_rejected": -106.29326629638672,
|
|
"logps/rejected": -399.6766357421875,
|
|
"loss": 0.8815,
|
|
"margin_dpo/margin_mean": 144.79278564453125,
|
|
"margin_dpo/margin_std": 153.0501708984375,
|
|
"step": 247
|
|
},
|
|
{
|
|
"epoch": 0.3641703377386197,
|
|
"fcm_dpo/beta": 0.005500612780451775,
|
|
"fcm_dpo/delta": -0.0632907822728157,
|
|
"fcm_dpo/margin": 119.71734619140625,
|
|
"fcm_dpo/q_t": 0.35248565673828125,
|
|
"grad_norm": 40.54594039916992,
|
|
"learning_rate": 4.0269892260067197e-07,
|
|
"logits/chosen": -0.3447737991809845,
|
|
"logits/rejected": -0.358917772769928,
|
|
"logps/chosen": -171.75363159179688,
|
|
"logps/ref_chosen": -44.043216705322266,
|
|
"logps/ref_rejected": -91.85687255859375,
|
|
"logps/rejected": -339.2846374511719,
|
|
"loss": 0.908,
|
|
"margin_dpo/margin_mean": 119.71734619140625,
|
|
"margin_dpo/margin_std": 105.92500305175781,
|
|
"step": 248
|
|
},
|
|
{
|
|
"epoch": 0.3656387665198238,
|
|
"fcm_dpo/beta": 0.005672304891049862,
|
|
"fcm_dpo/delta": 0.20714300870895386,
|
|
"fcm_dpo/margin": 70.76122283935547,
|
|
"fcm_dpo/q_t": 0.4150072932243347,
|
|
"grad_norm": 52.06844711303711,
|
|
"learning_rate": 4.0168078931267426e-07,
|
|
"logits/chosen": -0.3665584921836853,
|
|
"logits/rejected": -0.3416873514652252,
|
|
"logps/chosen": -225.1993408203125,
|
|
"logps/ref_chosen": -62.442352294921875,
|
|
"logps/ref_rejected": -80.46806335449219,
|
|
"logps/rejected": -313.98626708984375,
|
|
"loss": 1.1559,
|
|
"margin_dpo/margin_mean": 70.76122283935547,
|
|
"margin_dpo/margin_std": 133.9126739501953,
|
|
"step": 249
|
|
},
|
|
{
|
|
"epoch": 0.3671071953010279,
|
|
"fcm_dpo/beta": 0.005674234591424465,
|
|
"fcm_dpo/delta": -0.10982278734445572,
|
|
"fcm_dpo/margin": 123.74378967285156,
|
|
"fcm_dpo/q_t": 0.34339314699172974,
|
|
"grad_norm": 34.09236526489258,
|
|
"learning_rate": 4.006586590948141e-07,
|
|
"logits/chosen": -0.38642430305480957,
|
|
"logits/rejected": -0.33151566982269287,
|
|
"logps/chosen": -202.39520263671875,
|
|
"logps/ref_chosen": -65.63668823242188,
|
|
"logps/ref_rejected": -73.87184143066406,
|
|
"logps/rejected": -334.3741455078125,
|
|
"loss": 0.8893,
|
|
"margin_dpo/margin_mean": 123.74378967285156,
|
|
"margin_dpo/margin_std": 108.41996765136719,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.368575624082232,
|
|
"fcm_dpo/beta": 0.005667783785611391,
|
|
"fcm_dpo/delta": 0.09314079582691193,
|
|
"fcm_dpo/margin": 90.27076721191406,
|
|
"fcm_dpo/q_t": 0.3912964463233948,
|
|
"grad_norm": 54.34296798706055,
|
|
"learning_rate": 3.9963255888117325e-07,
|
|
"logits/chosen": -0.33801060914993286,
|
|
"logits/rejected": -0.3055931329727173,
|
|
"logps/chosen": -211.67941284179688,
|
|
"logps/ref_chosen": -57.182716369628906,
|
|
"logps/ref_rejected": -77.66343688964844,
|
|
"logps/rejected": -322.430908203125,
|
|
"loss": 1.0705,
|
|
"margin_dpo/margin_mean": 90.27076721191406,
|
|
"margin_dpo/margin_std": 133.482421875,
|
|
"step": 251
|
|
},
|
|
{
|
|
"epoch": 0.3700440528634361,
|
|
"fcm_dpo/beta": 0.005632590502500534,
|
|
"fcm_dpo/delta": -0.06894432753324509,
|
|
"fcm_dpo/margin": 117.80094909667969,
|
|
"fcm_dpo/q_t": 0.34721171855926514,
|
|
"grad_norm": 34.09019088745117,
|
|
"learning_rate": 3.9860251571044666e-07,
|
|
"logits/chosen": -0.4143436849117279,
|
|
"logits/rejected": -0.3699311912059784,
|
|
"logps/chosen": -211.8092041015625,
|
|
"logps/ref_chosen": -71.68563842773438,
|
|
"logps/ref_rejected": -84.75799560546875,
|
|
"logps/rejected": -342.6824951171875,
|
|
"loss": 0.9072,
|
|
"margin_dpo/margin_mean": 117.80094909667969,
|
|
"margin_dpo/margin_std": 102.10981750488281,
|
|
"step": 252
|
|
},
|
|
{
|
|
"epoch": 0.37151248164464024,
|
|
"fcm_dpo/beta": 0.0056702555157244205,
|
|
"fcm_dpo/delta": -0.024740278720855713,
|
|
"fcm_dpo/margin": 109.68414306640625,
|
|
"fcm_dpo/q_t": 0.3643852472305298,
|
|
"grad_norm": 29.00065040588379,
|
|
"learning_rate": 3.9756855672522986e-07,
|
|
"logits/chosen": -0.40758126974105835,
|
|
"logits/rejected": -0.3990533649921417,
|
|
"logps/chosen": -192.28323364257812,
|
|
"logps/ref_chosen": -69.1339340209961,
|
|
"logps/ref_rejected": -98.70252990722656,
|
|
"logps/rejected": -331.5359802246094,
|
|
"loss": 0.9822,
|
|
"margin_dpo/margin_mean": 109.68414306640625,
|
|
"margin_dpo/margin_std": 129.01275634765625,
|
|
"step": 253
|
|
},
|
|
{
|
|
"epoch": 0.37298091042584436,
|
|
"fcm_dpo/beta": 0.005641551688313484,
|
|
"fcm_dpo/delta": 0.04004208371043205,
|
|
"fcm_dpo/margin": 99.6583251953125,
|
|
"fcm_dpo/q_t": 0.3856009840965271,
|
|
"grad_norm": 27.635013580322266,
|
|
"learning_rate": 3.965307091713037e-07,
|
|
"logits/chosen": -0.41606834530830383,
|
|
"logits/rejected": -0.397452175617218,
|
|
"logps/chosen": -183.28872680664062,
|
|
"logps/ref_chosen": -54.154998779296875,
|
|
"logps/ref_rejected": -90.30764770507812,
|
|
"logps/rejected": -319.0997009277344,
|
|
"loss": 1.0509,
|
|
"margin_dpo/margin_mean": 99.6583251953125,
|
|
"margin_dpo/margin_std": 151.13661193847656,
|
|
"step": 254
|
|
},
|
|
{
|
|
"epoch": 0.3744493392070485,
|
|
"fcm_dpo/beta": 0.005625682417303324,
|
|
"fcm_dpo/delta": 0.0007314831018447876,
|
|
"fcm_dpo/margin": 106.40800476074219,
|
|
"fcm_dpo/q_t": 0.37160855531692505,
|
|
"grad_norm": 29.136993408203125,
|
|
"learning_rate": 3.954890003969163e-07,
|
|
"logits/chosen": -0.3470228314399719,
|
|
"logits/rejected": -0.3381271958351135,
|
|
"logps/chosen": -193.8470458984375,
|
|
"logps/ref_chosen": -57.14167022705078,
|
|
"logps/ref_rejected": -90.2085952758789,
|
|
"logps/rejected": -333.32196044921875,
|
|
"loss": 1.0155,
|
|
"margin_dpo/margin_mean": 106.40800476074219,
|
|
"margin_dpo/margin_std": 140.4176483154297,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.37591776798825255,
|
|
"fcm_dpo/beta": 0.005624064244329929,
|
|
"fcm_dpo/delta": -0.03853630647063255,
|
|
"fcm_dpo/margin": 113.09561157226562,
|
|
"fcm_dpo/q_t": 0.3635770082473755,
|
|
"grad_norm": 43.678157806396484,
|
|
"learning_rate": 3.944434578520628e-07,
|
|
"logits/chosen": -0.3604806661605835,
|
|
"logits/rejected": -0.3640385866165161,
|
|
"logps/chosen": -175.4744110107422,
|
|
"logps/ref_chosen": -55.163490295410156,
|
|
"logps/ref_rejected": -92.56291961669922,
|
|
"logps/rejected": -325.969482421875,
|
|
"loss": 0.9694,
|
|
"margin_dpo/margin_mean": 113.09561157226562,
|
|
"margin_dpo/margin_std": 133.815185546875,
|
|
"step": 256
|
|
},
|
|
{
|
|
"epoch": 0.37738619676945667,
|
|
"fcm_dpo/beta": 0.005505690351128578,
|
|
"fcm_dpo/delta": -0.06898978352546692,
|
|
"fcm_dpo/margin": 120.23841857910156,
|
|
"fcm_dpo/q_t": 0.35991615056991577,
|
|
"grad_norm": 27.866411209106445,
|
|
"learning_rate": 3.933941090877615e-07,
|
|
"logits/chosen": -0.3533473610877991,
|
|
"logits/rejected": -0.33796215057373047,
|
|
"logps/chosen": -166.825439453125,
|
|
"logps/ref_chosen": -49.42369842529297,
|
|
"logps/ref_rejected": -79.53791809082031,
|
|
"logps/rejected": -317.1780700683594,
|
|
"loss": 0.9576,
|
|
"margin_dpo/margin_mean": 120.23841857910156,
|
|
"margin_dpo/margin_std": 135.40008544921875,
|
|
"step": 257
|
|
},
|
|
{
|
|
"epoch": 0.3788546255506608,
|
|
"fcm_dpo/beta": 0.005546920001506805,
|
|
"fcm_dpo/delta": -0.001775261014699936,
|
|
"fcm_dpo/margin": 108.45994567871094,
|
|
"fcm_dpo/q_t": 0.36892908811569214,
|
|
"grad_norm": 33.50674819946289,
|
|
"learning_rate": 3.923409817553284e-07,
|
|
"logits/chosen": -0.35097846388816833,
|
|
"logits/rejected": -0.34521350264549255,
|
|
"logps/chosen": -207.83615112304688,
|
|
"logps/ref_chosen": -59.384124755859375,
|
|
"logps/ref_rejected": -95.99010467529297,
|
|
"logps/rejected": -352.902099609375,
|
|
"loss": 1.0141,
|
|
"margin_dpo/margin_mean": 108.45994567871094,
|
|
"margin_dpo/margin_std": 145.33441162109375,
|
|
"step": 258
|
|
},
|
|
{
|
|
"epoch": 0.3803230543318649,
|
|
"fcm_dpo/beta": 0.005560112651437521,
|
|
"fcm_dpo/delta": 0.057319387793540955,
|
|
"fcm_dpo/margin": 98.13949584960938,
|
|
"fcm_dpo/q_t": 0.38358962535858154,
|
|
"grad_norm": 33.96443557739258,
|
|
"learning_rate": 3.9128410360564793e-07,
|
|
"logits/chosen": -0.41265004873275757,
|
|
"logits/rejected": -0.40723663568496704,
|
|
"logps/chosen": -191.66282653808594,
|
|
"logps/ref_chosen": -52.828346252441406,
|
|
"logps/ref_rejected": -89.191650390625,
|
|
"logps/rejected": -326.1656494140625,
|
|
"loss": 1.0362,
|
|
"margin_dpo/margin_mean": 98.13948822021484,
|
|
"margin_dpo/margin_std": 132.66073608398438,
|
|
"step": 259
|
|
},
|
|
{
|
|
"epoch": 0.38179148311306904,
|
|
"fcm_dpo/beta": 0.005574359092861414,
|
|
"fcm_dpo/delta": -0.05944538116455078,
|
|
"fcm_dpo/margin": 117.58509826660156,
|
|
"fcm_dpo/q_t": 0.35981056094169617,
|
|
"grad_norm": 32.101783752441406,
|
|
"learning_rate": 3.9022350248844246e-07,
|
|
"logits/chosen": -0.37826740741729736,
|
|
"logits/rejected": -0.3950265347957611,
|
|
"logps/chosen": -189.1026611328125,
|
|
"logps/ref_chosen": -47.41767501831055,
|
|
"logps/ref_rejected": -95.08978271484375,
|
|
"logps/rejected": -354.35986328125,
|
|
"loss": 0.9407,
|
|
"margin_dpo/margin_mean": 117.58509826660156,
|
|
"margin_dpo/margin_std": 128.20523071289062,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3832599118942731,
|
|
"fcm_dpo/beta": 0.005468597635626793,
|
|
"fcm_dpo/delta": -0.06998618692159653,
|
|
"fcm_dpo/margin": 121.63386535644531,
|
|
"fcm_dpo/q_t": 0.35736051201820374,
|
|
"grad_norm": 23.966012954711914,
|
|
"learning_rate": 3.891592063515376e-07,
|
|
"logits/chosen": -0.319491982460022,
|
|
"logits/rejected": -0.3146633505821228,
|
|
"logps/chosen": -190.02244567871094,
|
|
"logps/ref_chosen": -53.03137969970703,
|
|
"logps/ref_rejected": -88.51494598388672,
|
|
"logps/rejected": -347.139892578125,
|
|
"loss": 0.9529,
|
|
"margin_dpo/margin_mean": 121.63386535644531,
|
|
"margin_dpo/margin_std": 140.09190368652344,
|
|
"step": 261
|
|
},
|
|
{
|
|
"epoch": 0.38472834067547723,
|
|
"fcm_dpo/beta": 0.005526003893464804,
|
|
"fcm_dpo/delta": 0.0573844239115715,
|
|
"fcm_dpo/margin": 98.6690902709961,
|
|
"fcm_dpo/q_t": 0.37783992290496826,
|
|
"grad_norm": 33.454994201660156,
|
|
"learning_rate": 3.880912432401264e-07,
|
|
"logits/chosen": -0.2890551686286926,
|
|
"logits/rejected": -0.26014506816864014,
|
|
"logps/chosen": -221.21661376953125,
|
|
"logps/ref_chosen": -59.620140075683594,
|
|
"logps/ref_rejected": -86.41853332519531,
|
|
"logps/rejected": -346.68408203125,
|
|
"loss": 1.0033,
|
|
"margin_dpo/margin_mean": 98.6690902709961,
|
|
"margin_dpo/margin_std": 113.80564880371094,
|
|
"step": 262
|
|
},
|
|
{
|
|
"epoch": 0.38619676945668135,
|
|
"fcm_dpo/beta": 0.005366258788853884,
|
|
"fcm_dpo/delta": -0.16116134822368622,
|
|
"fcm_dpo/margin": 139.3936309814453,
|
|
"fcm_dpo/q_t": 0.3375219702720642,
|
|
"grad_norm": 30.789506912231445,
|
|
"learning_rate": 3.870196412960302e-07,
|
|
"logits/chosen": -0.3338819146156311,
|
|
"logits/rejected": -0.30775582790374756,
|
|
"logps/chosen": -204.53103637695312,
|
|
"logps/ref_chosen": -59.42094421386719,
|
|
"logps/ref_rejected": -96.85720825195312,
|
|
"logps/rejected": -381.3609619140625,
|
|
"loss": 0.8897,
|
|
"margin_dpo/margin_mean": 139.3936309814453,
|
|
"margin_dpo/margin_std": 136.70474243164062,
|
|
"step": 263
|
|
},
|
|
{
|
|
"epoch": 0.3876651982378855,
|
|
"fcm_dpo/beta": 0.0052995504811406136,
|
|
"fcm_dpo/delta": -0.014157909899950027,
|
|
"fcm_dpo/margin": 115.54821014404297,
|
|
"fcm_dpo/q_t": 0.3663901686668396,
|
|
"grad_norm": 33.63625717163086,
|
|
"learning_rate": 3.8594442875695665e-07,
|
|
"logits/chosen": -0.3557894825935364,
|
|
"logits/rejected": -0.3422415256500244,
|
|
"logps/chosen": -212.91517639160156,
|
|
"logps/ref_chosen": -62.722084045410156,
|
|
"logps/ref_rejected": -93.85620880126953,
|
|
"logps/rejected": -359.5975341796875,
|
|
"loss": 0.9682,
|
|
"margin_dpo/margin_mean": 115.5482177734375,
|
|
"margin_dpo/margin_std": 127.98780822753906,
|
|
"step": 264
|
|
},
|
|
{
|
|
"epoch": 0.3891336270190896,
|
|
"fcm_dpo/beta": 0.005331607535481453,
|
|
"fcm_dpo/delta": -0.0023941383697092533,
|
|
"fcm_dpo/margin": 112.95767211914062,
|
|
"fcm_dpo/q_t": 0.3753606081008911,
|
|
"grad_norm": 48.9280891418457,
|
|
"learning_rate": 3.848656339557562e-07,
|
|
"logits/chosen": -0.31224972009658813,
|
|
"logits/rejected": -0.2951231598854065,
|
|
"logps/chosen": -235.91409301757812,
|
|
"logps/ref_chosen": -61.971466064453125,
|
|
"logps/ref_rejected": -88.02059936523438,
|
|
"logps/rejected": -374.9208984375,
|
|
"loss": 1.0295,
|
|
"margin_dpo/margin_mean": 112.95767211914062,
|
|
"margin_dpo/margin_std": 163.15493774414062,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.39060205580029367,
|
|
"fcm_dpo/beta": 0.0053364490158855915,
|
|
"fcm_dpo/delta": 0.014164826832711697,
|
|
"fcm_dpo/margin": 109.93594360351562,
|
|
"fcm_dpo/q_t": 0.37322813272476196,
|
|
"grad_norm": 41.273826599121094,
|
|
"learning_rate": 3.8378328531967507e-07,
|
|
"logits/chosen": -0.343116819858551,
|
|
"logits/rejected": -0.2906397581100464,
|
|
"logps/chosen": -235.95025634765625,
|
|
"logps/ref_chosen": -67.09967041015625,
|
|
"logps/ref_rejected": -67.97122192382812,
|
|
"logps/rejected": -346.75775146484375,
|
|
"loss": 0.9942,
|
|
"margin_dpo/margin_mean": 109.93594360351562,
|
|
"margin_dpo/margin_std": 133.6815185546875,
|
|
"step": 266
|
|
},
|
|
{
|
|
"epoch": 0.3920704845814978,
|
|
"fcm_dpo/beta": 0.005292531102895737,
|
|
"fcm_dpo/delta": -0.037379078567028046,
|
|
"fcm_dpo/margin": 119.896484375,
|
|
"fcm_dpo/q_t": 0.36082565784454346,
|
|
"grad_norm": 37.10322189331055,
|
|
"learning_rate": 3.8269741136960646e-07,
|
|
"logits/chosen": -0.3480778932571411,
|
|
"logits/rejected": -0.30662471055984497,
|
|
"logps/chosen": -230.77191162109375,
|
|
"logps/ref_chosen": -68.97075653076172,
|
|
"logps/ref_rejected": -90.16844940185547,
|
|
"logps/rejected": -371.8660888671875,
|
|
"loss": 0.9604,
|
|
"margin_dpo/margin_mean": 119.896484375,
|
|
"margin_dpo/margin_std": 134.846435546875,
|
|
"step": 267
|
|
},
|
|
{
|
|
"epoch": 0.3935389133627019,
|
|
"fcm_dpo/beta": 0.0052706534042954445,
|
|
"fcm_dpo/delta": -0.005634669214487076,
|
|
"fcm_dpo/margin": 114.72981262207031,
|
|
"fcm_dpo/q_t": 0.37070363759994507,
|
|
"grad_norm": 36.76900100708008,
|
|
"learning_rate": 3.8160804071933894e-07,
|
|
"logits/chosen": -0.3364071846008301,
|
|
"logits/rejected": -0.34323662519454956,
|
|
"logps/chosen": -227.5570526123047,
|
|
"logps/ref_chosen": -55.90031051635742,
|
|
"logps/ref_rejected": -101.64763641357422,
|
|
"logps/rejected": -388.0341796875,
|
|
"loss": 0.9952,
|
|
"margin_dpo/margin_mean": 114.72982025146484,
|
|
"margin_dpo/margin_std": 145.24905395507812,
|
|
"step": 268
|
|
},
|
|
{
|
|
"epoch": 0.39500734214390604,
|
|
"fcm_dpo/beta": 0.005157118663191795,
|
|
"fcm_dpo/delta": -0.17022442817687988,
|
|
"fcm_dpo/margin": 146.7906494140625,
|
|
"fcm_dpo/q_t": 0.3414715826511383,
|
|
"grad_norm": 42.1302375793457,
|
|
"learning_rate": 3.8051520207480204e-07,
|
|
"logits/chosen": -0.3691613972187042,
|
|
"logits/rejected": -0.3452231287956238,
|
|
"logps/chosen": -251.450439453125,
|
|
"logps/ref_chosen": -70.03955841064453,
|
|
"logps/ref_rejected": -107.34937286376953,
|
|
"logps/rejected": -435.5509033203125,
|
|
"loss": 0.925,
|
|
"margin_dpo/margin_mean": 146.7906494140625,
|
|
"margin_dpo/margin_std": 168.05130004882812,
|
|
"step": 269
|
|
},
|
|
{
|
|
"epoch": 0.3964757709251101,
|
|
"fcm_dpo/beta": 0.005182851571589708,
|
|
"fcm_dpo/delta": 0.05792323499917984,
|
|
"fcm_dpo/margin": 105.11456298828125,
|
|
"fcm_dpo/q_t": 0.3828323185443878,
|
|
"grad_norm": 37.189781188964844,
|
|
"learning_rate": 3.794189242333106e-07,
|
|
"logits/chosen": -0.41337841749191284,
|
|
"logits/rejected": -0.40365487337112427,
|
|
"logps/chosen": -232.35296630859375,
|
|
"logps/ref_chosen": -69.53347778320312,
|
|
"logps/ref_rejected": -109.92864990234375,
|
|
"logps/rejected": -377.8627014160156,
|
|
"loss": 1.0389,
|
|
"margin_dpo/margin_mean": 105.11456298828125,
|
|
"margin_dpo/margin_std": 145.3082275390625,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.39794419970631423,
|
|
"fcm_dpo/beta": 0.0051218001171946526,
|
|
"fcm_dpo/delta": -0.057347435504198074,
|
|
"fcm_dpo/margin": 127.55348205566406,
|
|
"fcm_dpo/q_t": 0.35683873295783997,
|
|
"grad_norm": 39.590396881103516,
|
|
"learning_rate": 3.7831923608280514e-07,
|
|
"logits/chosen": -0.34910643100738525,
|
|
"logits/rejected": -0.3228174149990082,
|
|
"logps/chosen": -206.6493377685547,
|
|
"logps/ref_chosen": -56.76456832885742,
|
|
"logps/ref_rejected": -92.51383972167969,
|
|
"logps/rejected": -369.95208740234375,
|
|
"loss": 0.9256,
|
|
"margin_dpo/margin_mean": 127.55348205566406,
|
|
"margin_dpo/margin_std": 125.99525451660156,
|
|
"step": 271
|
|
},
|
|
{
|
|
"epoch": 0.39941262848751835,
|
|
"fcm_dpo/beta": 0.005027889274060726,
|
|
"fcm_dpo/delta": -0.1585131734609604,
|
|
"fcm_dpo/margin": 148.55596923828125,
|
|
"fcm_dpo/q_t": 0.33690333366394043,
|
|
"grad_norm": 36.002960205078125,
|
|
"learning_rate": 3.772161666010912e-07,
|
|
"logits/chosen": -0.331684410572052,
|
|
"logits/rejected": -0.3400617837905884,
|
|
"logps/chosen": -184.74874877929688,
|
|
"logps/ref_chosen": -49.497154235839844,
|
|
"logps/ref_rejected": -105.54279327392578,
|
|
"logps/rejected": -389.350341796875,
|
|
"loss": 0.8841,
|
|
"margin_dpo/margin_mean": 148.55596923828125,
|
|
"margin_dpo/margin_std": 140.95240783691406,
|
|
"step": 272
|
|
},
|
|
{
|
|
"epoch": 0.4008810572687225,
|
|
"fcm_dpo/beta": 0.004825687035918236,
|
|
"fcm_dpo/delta": -0.1409759819507599,
|
|
"fcm_dpo/margin": 151.05177307128906,
|
|
"fcm_dpo/q_t": 0.3416686952114105,
|
|
"grad_norm": 34.01618576049805,
|
|
"learning_rate": 3.761097448550755e-07,
|
|
"logits/chosen": -0.3630342483520508,
|
|
"logits/rejected": -0.3401733636856079,
|
|
"logps/chosen": -218.07992553710938,
|
|
"logps/ref_chosen": -62.97539520263672,
|
|
"logps/ref_rejected": -92.49858093261719,
|
|
"logps/rejected": -398.6549072265625,
|
|
"loss": 0.892,
|
|
"margin_dpo/margin_mean": 151.05178833007812,
|
|
"margin_dpo/margin_std": 145.12493896484375,
|
|
"step": 273
|
|
},
|
|
{
|
|
"epoch": 0.4023494860499266,
|
|
"fcm_dpo/beta": 0.004842686001211405,
|
|
"fcm_dpo/delta": 0.012656617909669876,
|
|
"fcm_dpo/margin": 121.4140625,
|
|
"fcm_dpo/q_t": 0.3711016774177551,
|
|
"grad_norm": 29.5135555267334,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": -0.2929585874080658,
|
|
"logits/rejected": -0.26695334911346436,
|
|
"logps/chosen": -246.96438598632812,
|
|
"logps/ref_chosen": -55.66770935058594,
|
|
"logps/ref_rejected": -77.33308410644531,
|
|
"logps/rejected": -390.0438232421875,
|
|
"loss": 0.9834,
|
|
"margin_dpo/margin_mean": 121.4140625,
|
|
"margin_dpo/margin_std": 138.95596313476562,
|
|
"step": 274
|
|
},
|
|
{
|
|
"epoch": 0.40381791483113066,
|
|
"fcm_dpo/beta": 0.0048539480194449425,
|
|
"fcm_dpo/delta": 0.006926249712705612,
|
|
"fcm_dpo/margin": 122.2330551147461,
|
|
"fcm_dpo/q_t": 0.3715580701828003,
|
|
"grad_norm": 31.085887908935547,
|
|
"learning_rate": 3.738869612786737e-07,
|
|
"logits/chosen": -0.3700248599052429,
|
|
"logits/rejected": -0.3734614849090576,
|
|
"logps/chosen": -204.39918518066406,
|
|
"logps/ref_chosen": -48.594703674316406,
|
|
"logps/ref_rejected": -93.30369567871094,
|
|
"logps/rejected": -371.34124755859375,
|
|
"loss": 0.995,
|
|
"margin_dpo/margin_mean": 122.2330551147461,
|
|
"margin_dpo/margin_std": 148.9960479736328,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.4052863436123348,
|
|
"fcm_dpo/beta": 0.004785975441336632,
|
|
"fcm_dpo/delta": -0.05713774263858795,
|
|
"fcm_dpo/margin": 136.44924926757812,
|
|
"fcm_dpo/q_t": 0.3614548146724701,
|
|
"grad_norm": 38.087745666503906,
|
|
"learning_rate": 3.7277065802070204e-07,
|
|
"logits/chosen": -0.3316674828529358,
|
|
"logits/rejected": -0.3020857274532318,
|
|
"logps/chosen": -219.54293823242188,
|
|
"logps/ref_chosen": -56.57740783691406,
|
|
"logps/ref_rejected": -70.36566925048828,
|
|
"logps/rejected": -369.78045654296875,
|
|
"loss": 0.9619,
|
|
"margin_dpo/margin_mean": 136.44924926757812,
|
|
"margin_dpo/margin_std": 161.114990234375,
|
|
"step": 276
|
|
},
|
|
{
|
|
"epoch": 0.4067547723935389,
|
|
"fcm_dpo/beta": 0.0047689881175756454,
|
|
"fcm_dpo/delta": -0.04522915184497833,
|
|
"fcm_dpo/margin": 134.67898559570312,
|
|
"fcm_dpo/q_t": 0.362157940864563,
|
|
"grad_norm": 36.099918365478516,
|
|
"learning_rate": 3.71651119641714e-07,
|
|
"logits/chosen": -0.3248156011104584,
|
|
"logits/rejected": -0.3068217635154724,
|
|
"logps/chosen": -242.65780639648438,
|
|
"logps/ref_chosen": -56.27156066894531,
|
|
"logps/ref_rejected": -92.88127136230469,
|
|
"logps/rejected": -413.946533203125,
|
|
"loss": 0.9632,
|
|
"margin_dpo/margin_mean": 134.67898559570312,
|
|
"margin_dpo/margin_std": 155.36151123046875,
|
|
"step": 277
|
|
},
|
|
{
|
|
"epoch": 0.40822320117474303,
|
|
"fcm_dpo/beta": 0.004655085504055023,
|
|
"fcm_dpo/delta": -0.10972905904054642,
|
|
"fcm_dpo/margin": 150.7223358154297,
|
|
"fcm_dpo/q_t": 0.3516245484352112,
|
|
"grad_norm": 29.135597229003906,
|
|
"learning_rate": 3.705283756425872e-07,
|
|
"logits/chosen": -0.3341606557369232,
|
|
"logits/rejected": -0.336972713470459,
|
|
"logps/chosen": -221.07388305664062,
|
|
"logps/ref_chosen": -52.94194030761719,
|
|
"logps/ref_rejected": -91.25357818603516,
|
|
"logps/rejected": -410.10784912109375,
|
|
"loss": 0.9254,
|
|
"margin_dpo/margin_mean": 150.72235107421875,
|
|
"margin_dpo/margin_std": 164.86074829101562,
|
|
"step": 278
|
|
},
|
|
{
|
|
"epoch": 0.40969162995594716,
|
|
"fcm_dpo/beta": 0.004513373598456383,
|
|
"fcm_dpo/delta": -0.13143345713615417,
|
|
"fcm_dpo/margin": 159.43524169921875,
|
|
"fcm_dpo/q_t": 0.35075873136520386,
|
|
"grad_norm": 37.86472702026367,
|
|
"learning_rate": 3.6940245560867e-07,
|
|
"logits/chosen": -0.2502022981643677,
|
|
"logits/rejected": -0.24658802151679993,
|
|
"logps/chosen": -250.07838439941406,
|
|
"logps/ref_chosen": -48.641319274902344,
|
|
"logps/ref_rejected": -87.8514404296875,
|
|
"logps/rejected": -448.7237548828125,
|
|
"loss": 0.9479,
|
|
"margin_dpo/margin_mean": 159.4352569580078,
|
|
"margin_dpo/margin_std": 191.3142852783203,
|
|
"step": 279
|
|
},
|
|
{
|
|
"epoch": 0.4111600587371512,
|
|
"fcm_dpo/beta": 0.00445135310292244,
|
|
"fcm_dpo/delta": -0.09706270694732666,
|
|
"fcm_dpo/margin": 155.06472778320312,
|
|
"fcm_dpo/q_t": 0.3463428020477295,
|
|
"grad_norm": 40.227508544921875,
|
|
"learning_rate": 3.6827338920900253e-07,
|
|
"logits/chosen": -0.28654709458351135,
|
|
"logits/rejected": -0.28727859258651733,
|
|
"logps/chosen": -259.53009033203125,
|
|
"logps/ref_chosen": -58.797122955322266,
|
|
"logps/ref_rejected": -98.61885070800781,
|
|
"logps/rejected": -454.41656494140625,
|
|
"loss": 0.9052,
|
|
"margin_dpo/margin_mean": 155.0647430419922,
|
|
"margin_dpo/margin_std": 146.40179443359375,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.41262848751835535,
|
|
"fcm_dpo/beta": 0.004433557391166687,
|
|
"fcm_dpo/delta": -0.03812451288104057,
|
|
"fcm_dpo/margin": 143.29185485839844,
|
|
"fcm_dpo/q_t": 0.3562266230583191,
|
|
"grad_norm": 30.542402267456055,
|
|
"learning_rate": 3.6714120619553435e-07,
|
|
"logits/chosen": -0.34395909309387207,
|
|
"logits/rejected": -0.3141595721244812,
|
|
"logps/chosen": -239.7939453125,
|
|
"logps/ref_chosen": -55.488521575927734,
|
|
"logps/ref_rejected": -80.88258361816406,
|
|
"logps/rejected": -408.4798583984375,
|
|
"loss": 0.9469,
|
|
"margin_dpo/margin_mean": 143.29185485839844,
|
|
"margin_dpo/margin_std": 147.6947021484375,
|
|
"step": 281
|
|
},
|
|
{
|
|
"epoch": 0.41409691629955947,
|
|
"fcm_dpo/beta": 0.004496179986745119,
|
|
"fcm_dpo/delta": 0.11047537624835968,
|
|
"fcm_dpo/margin": 109.85005187988281,
|
|
"fcm_dpo/q_t": 0.39475515484809875,
|
|
"grad_norm": 32.4976806640625,
|
|
"learning_rate": 3.660059364023408e-07,
|
|
"logits/chosen": -0.40445345640182495,
|
|
"logits/rejected": -0.37312695384025574,
|
|
"logps/chosen": -266.0508728027344,
|
|
"logps/ref_chosen": -73.07014465332031,
|
|
"logps/ref_rejected": -95.35098266601562,
|
|
"logps/rejected": -398.1817626953125,
|
|
"loss": 1.0544,
|
|
"margin_dpo/margin_mean": 109.85005187988281,
|
|
"margin_dpo/margin_std": 156.09814453125,
|
|
"step": 282
|
|
},
|
|
{
|
|
"epoch": 0.4155653450807636,
|
|
"fcm_dpo/beta": 0.004433175548911095,
|
|
"fcm_dpo/delta": -0.12632399797439575,
|
|
"fcm_dpo/margin": 161.82952880859375,
|
|
"fcm_dpo/q_t": 0.341400146484375,
|
|
"grad_norm": 44.17348098754883,
|
|
"learning_rate": 3.6486760974483685e-07,
|
|
"logits/chosen": -0.3220086991786957,
|
|
"logits/rejected": -0.3167068660259247,
|
|
"logps/chosen": -252.05734252929688,
|
|
"logps/ref_chosen": -61.89844512939453,
|
|
"logps/ref_rejected": -96.98655700683594,
|
|
"logps/rejected": -448.9749755859375,
|
|
"loss": 0.8904,
|
|
"margin_dpo/margin_mean": 161.8295135498047,
|
|
"margin_dpo/margin_std": 149.40249633789062,
|
|
"step": 283
|
|
},
|
|
{
|
|
"epoch": 0.4170337738619677,
|
|
"fcm_dpo/beta": 0.004331875126808882,
|
|
"fcm_dpo/delta": -0.0721510797739029,
|
|
"fcm_dpo/margin": 154.01272583007812,
|
|
"fcm_dpo/q_t": 0.35365206003189087,
|
|
"grad_norm": 28.796356201171875,
|
|
"learning_rate": 3.6372625621898863e-07,
|
|
"logits/chosen": -0.39249300956726074,
|
|
"logits/rejected": -0.37775149941444397,
|
|
"logps/chosen": -239.20748901367188,
|
|
"logps/ref_chosen": -58.4355354309082,
|
|
"logps/ref_rejected": -93.46926879882812,
|
|
"logps/rejected": -428.2539367675781,
|
|
"loss": 0.9158,
|
|
"margin_dpo/margin_mean": 154.01272583007812,
|
|
"margin_dpo/margin_std": 150.7726593017578,
|
|
"step": 284
|
|
},
|
|
{
|
|
"epoch": 0.4185022026431718,
|
|
"fcm_dpo/beta": 0.004361903294920921,
|
|
"fcm_dpo/delta": 0.014079738408327103,
|
|
"fcm_dpo/margin": 134.28880310058594,
|
|
"fcm_dpo/q_t": 0.3673364818096161,
|
|
"grad_norm": 34.644283294677734,
|
|
"learning_rate": 3.625819059005228e-07,
|
|
"logits/chosen": -0.3506457209587097,
|
|
"logits/rejected": -0.32794976234436035,
|
|
"logps/chosen": -276.463134765625,
|
|
"logps/ref_chosen": -66.23219299316406,
|
|
"logps/ref_rejected": -99.1268310546875,
|
|
"logps/rejected": -443.6466064453125,
|
|
"loss": 0.9694,
|
|
"margin_dpo/margin_mean": 134.28878784179688,
|
|
"margin_dpo/margin_std": 137.147216796875,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.4199706314243759,
|
|
"fcm_dpo/beta": 0.004304747097194195,
|
|
"fcm_dpo/delta": -0.060156404972076416,
|
|
"fcm_dpo/margin": 152.447265625,
|
|
"fcm_dpo/q_t": 0.3558097183704376,
|
|
"grad_norm": 35.37562561035156,
|
|
"learning_rate": 3.614345889441346e-07,
|
|
"logits/chosen": -0.3511343002319336,
|
|
"logits/rejected": -0.32436245679855347,
|
|
"logps/chosen": -274.97412109375,
|
|
"logps/ref_chosen": -72.95100402832031,
|
|
"logps/ref_rejected": -88.58845520019531,
|
|
"logps/rejected": -443.058837890625,
|
|
"loss": 0.9432,
|
|
"margin_dpo/margin_mean": 152.447265625,
|
|
"margin_dpo/margin_std": 162.77908325195312,
|
|
"step": 286
|
|
},
|
|
{
|
|
"epoch": 0.42143906020558003,
|
|
"fcm_dpo/beta": 0.0043206894770264626,
|
|
"fcm_dpo/delta": 0.0441654808819294,
|
|
"fcm_dpo/margin": 129.17588806152344,
|
|
"fcm_dpo/q_t": 0.3783441185951233,
|
|
"grad_norm": 31.608234405517578,
|
|
"learning_rate": 3.6028433558269275e-07,
|
|
"logits/chosen": -0.31515616178512573,
|
|
"logits/rejected": -0.28630921244621277,
|
|
"logps/chosen": -264.4310607910156,
|
|
"logps/ref_chosen": -61.54115295410156,
|
|
"logps/ref_rejected": -77.69607543945312,
|
|
"logps/rejected": -409.7618408203125,
|
|
"loss": 1.0024,
|
|
"margin_dpo/margin_mean": 129.17587280273438,
|
|
"margin_dpo/margin_std": 156.08370971679688,
|
|
"step": 287
|
|
},
|
|
{
|
|
"epoch": 0.42290748898678415,
|
|
"fcm_dpo/beta": 0.00423096539452672,
|
|
"fcm_dpo/delta": -0.06944943964481354,
|
|
"fcm_dpo/margin": 156.4290771484375,
|
|
"fcm_dpo/q_t": 0.3512691259384155,
|
|
"grad_norm": 27.641883850097656,
|
|
"learning_rate": 3.5913117612644327e-07,
|
|
"logits/chosen": -0.32602715492248535,
|
|
"logits/rejected": -0.3050415515899658,
|
|
"logps/chosen": -260.18255615234375,
|
|
"logps/ref_chosen": -56.661224365234375,
|
|
"logps/ref_rejected": -87.33570098876953,
|
|
"logps/rejected": -447.2861022949219,
|
|
"loss": 0.914,
|
|
"margin_dpo/margin_mean": 156.4290771484375,
|
|
"margin_dpo/margin_std": 137.47991943359375,
|
|
"step": 288
|
|
},
|
|
{
|
|
"epoch": 0.4243759177679883,
|
|
"fcm_dpo/beta": 0.0041911243461072445,
|
|
"fcm_dpo/delta": -0.14678727090358734,
|
|
"fcm_dpo/margin": 175.65328979492188,
|
|
"fcm_dpo/q_t": 0.34263110160827637,
|
|
"grad_norm": 36.062095642089844,
|
|
"learning_rate": 3.5797514096221024e-07,
|
|
"logits/chosen": -0.26659348607063293,
|
|
"logits/rejected": -0.25999683141708374,
|
|
"logps/chosen": -253.7037353515625,
|
|
"logps/ref_chosen": -45.23039245605469,
|
|
"logps/ref_rejected": -87.64266967773438,
|
|
"logps/rejected": -471.769287109375,
|
|
"loss": 0.8987,
|
|
"margin_dpo/margin_mean": 175.65328979492188,
|
|
"margin_dpo/margin_std": 176.63510131835938,
|
|
"step": 289
|
|
},
|
|
{
|
|
"epoch": 0.42584434654919234,
|
|
"fcm_dpo/beta": 0.004051923286169767,
|
|
"fcm_dpo/delta": -0.12010594457387924,
|
|
"fcm_dpo/margin": 175.46475219726562,
|
|
"fcm_dpo/q_t": 0.35006120800971985,
|
|
"grad_norm": 32.5576286315918,
|
|
"learning_rate": 3.568162605525952e-07,
|
|
"logits/chosen": -0.24551498889923096,
|
|
"logits/rejected": -0.262775182723999,
|
|
"logps/chosen": -283.1232604980469,
|
|
"logps/ref_chosen": -55.47149658203125,
|
|
"logps/ref_rejected": -116.70857238769531,
|
|
"logps/rejected": -519.8250732421875,
|
|
"loss": 0.9297,
|
|
"margin_dpo/margin_mean": 175.46475219726562,
|
|
"margin_dpo/margin_std": 197.48675537109375,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.42731277533039647,
|
|
"fcm_dpo/beta": 0.004028604365885258,
|
|
"fcm_dpo/delta": -0.06118813902139664,
|
|
"fcm_dpo/margin": 163.03326416015625,
|
|
"fcm_dpo/q_t": 0.35449501872062683,
|
|
"grad_norm": 26.545326232910156,
|
|
"learning_rate": 3.5565456543517485e-07,
|
|
"logits/chosen": -0.30797719955444336,
|
|
"logits/rejected": -0.2923903167247772,
|
|
"logps/chosen": -260.06689453125,
|
|
"logps/ref_chosen": -63.26036834716797,
|
|
"logps/ref_rejected": -89.29708862304688,
|
|
"logps/rejected": -449.1368713378906,
|
|
"loss": 0.9354,
|
|
"margin_dpo/margin_mean": 163.03326416015625,
|
|
"margin_dpo/margin_std": 164.8643798828125,
|
|
"step": 291
|
|
},
|
|
{
|
|
"epoch": 0.4287812041116006,
|
|
"fcm_dpo/beta": 0.003977569285780191,
|
|
"fcm_dpo/delta": -0.05237273871898651,
|
|
"fcm_dpo/margin": 163.1212615966797,
|
|
"fcm_dpo/q_t": 0.3588453233242035,
|
|
"grad_norm": 34.14498519897461,
|
|
"learning_rate": 3.5449008622169583e-07,
|
|
"logits/chosen": -0.262368768453598,
|
|
"logits/rejected": -0.23994597792625427,
|
|
"logps/chosen": -277.8984375,
|
|
"logps/ref_chosen": -53.91852951049805,
|
|
"logps/ref_rejected": -89.96138000488281,
|
|
"logps/rejected": -477.06256103515625,
|
|
"loss": 0.9758,
|
|
"margin_dpo/margin_mean": 163.12124633789062,
|
|
"margin_dpo/margin_std": 198.7785186767578,
|
|
"step": 292
|
|
},
|
|
{
|
|
"epoch": 0.4302496328928047,
|
|
"fcm_dpo/beta": 0.004008471965789795,
|
|
"fcm_dpo/delta": 0.07511234283447266,
|
|
"fcm_dpo/margin": 131.8068389892578,
|
|
"fcm_dpo/q_t": 0.38762548565864563,
|
|
"grad_norm": 37.924625396728516,
|
|
"learning_rate": 3.5332285359726846e-07,
|
|
"logits/chosen": -0.3276376724243164,
|
|
"logits/rejected": -0.3103697896003723,
|
|
"logps/chosen": -269.2470703125,
|
|
"logps/ref_chosen": -60.376033782958984,
|
|
"logps/ref_rejected": -77.85244750976562,
|
|
"logps/rejected": -418.53033447265625,
|
|
"loss": 1.0488,
|
|
"margin_dpo/margin_mean": 131.8068389892578,
|
|
"margin_dpo/margin_std": 185.1345672607422,
|
|
"step": 293
|
|
},
|
|
{
|
|
"epoch": 0.43171806167400884,
|
|
"fcm_dpo/beta": 0.0040005166083574295,
|
|
"fcm_dpo/delta": 0.008330628275871277,
|
|
"fcm_dpo/margin": 147.9698486328125,
|
|
"fcm_dpo/q_t": 0.37461739778518677,
|
|
"grad_norm": 28.923660278320312,
|
|
"learning_rate": 3.5215289831955786e-07,
|
|
"logits/chosen": -0.293660044670105,
|
|
"logits/rejected": -0.2946998178958893,
|
|
"logps/chosen": -237.821533203125,
|
|
"logps/ref_chosen": -48.0875358581543,
|
|
"logps/ref_rejected": -81.89698791503906,
|
|
"logps/rejected": -419.600830078125,
|
|
"loss": 0.9972,
|
|
"margin_dpo/margin_mean": 147.9698486328125,
|
|
"margin_dpo/margin_std": 184.6291961669922,
|
|
"step": 294
|
|
},
|
|
{
|
|
"epoch": 0.4331864904552129,
|
|
"fcm_dpo/beta": 0.0040347035974264145,
|
|
"fcm_dpo/delta": 0.02709888480603695,
|
|
"fcm_dpo/margin": 142.37374877929688,
|
|
"fcm_dpo/q_t": 0.3751312494277954,
|
|
"grad_norm": 33.199710845947266,
|
|
"learning_rate": 3.509802512179737e-07,
|
|
"logits/chosen": -0.3318241834640503,
|
|
"logits/rejected": -0.3298850655555725,
|
|
"logps/chosen": -268.1130065917969,
|
|
"logps/ref_chosen": -49.92467498779297,
|
|
"logps/ref_rejected": -87.45632934570312,
|
|
"logps/rejected": -448.01837158203125,
|
|
"loss": 1.0026,
|
|
"margin_dpo/margin_mean": 142.3737335205078,
|
|
"margin_dpo/margin_std": 173.57199096679688,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.434654919236417,
|
|
"fcm_dpo/beta": 0.0041085234843194485,
|
|
"fcm_dpo/delta": 0.1538042277097702,
|
|
"fcm_dpo/margin": 110.34952545166016,
|
|
"fcm_dpo/q_t": 0.40237492322921753,
|
|
"grad_norm": 50.468746185302734,
|
|
"learning_rate": 3.498049431928577e-07,
|
|
"logits/chosen": -0.38077211380004883,
|
|
"logits/rejected": -0.3572128415107727,
|
|
"logps/chosen": -313.87554931640625,
|
|
"logps/ref_chosen": -65.49124145507812,
|
|
"logps/ref_rejected": -93.08908081054688,
|
|
"logps/rejected": -451.82293701171875,
|
|
"loss": 1.1194,
|
|
"margin_dpo/margin_mean": 110.34951782226562,
|
|
"margin_dpo/margin_std": 187.090576171875,
|
|
"step": 296
|
|
},
|
|
{
|
|
"epoch": 0.43612334801762115,
|
|
"fcm_dpo/beta": 0.004196628928184509,
|
|
"fcm_dpo/delta": 0.063119076192379,
|
|
"fcm_dpo/margin": 128.75506591796875,
|
|
"fcm_dpo/q_t": 0.3794586956501007,
|
|
"grad_norm": 28.69700813293457,
|
|
"learning_rate": 3.486270052146694e-07,
|
|
"logits/chosen": -0.3626411557197571,
|
|
"logits/rejected": -0.3594970703125,
|
|
"logps/chosen": -282.9952697753906,
|
|
"logps/ref_chosen": -56.476951599121094,
|
|
"logps/ref_rejected": -95.1385498046875,
|
|
"logps/rejected": -450.41192626953125,
|
|
"loss": 1.0008,
|
|
"margin_dpo/margin_mean": 128.7550506591797,
|
|
"margin_dpo/margin_std": 144.84683227539062,
|
|
"step": 297
|
|
},
|
|
{
|
|
"epoch": 0.43759177679882527,
|
|
"fcm_dpo/beta": 0.004173451103270054,
|
|
"fcm_dpo/delta": -0.09957937896251678,
|
|
"fcm_dpo/margin": 165.98265075683594,
|
|
"fcm_dpo/q_t": 0.3572956323623657,
|
|
"grad_norm": 29.900131225585938,
|
|
"learning_rate": 3.474464683231698e-07,
|
|
"logits/chosen": -0.4068795442581177,
|
|
"logits/rejected": -0.42597341537475586,
|
|
"logps/chosen": -299.30560302734375,
|
|
"logps/ref_chosen": -67.32516479492188,
|
|
"logps/ref_rejected": -116.66217041015625,
|
|
"logps/rejected": -514.625244140625,
|
|
"loss": 0.954,
|
|
"margin_dpo/margin_mean": 165.98265075683594,
|
|
"margin_dpo/margin_std": 202.80752563476562,
|
|
"step": 298
|
|
},
|
|
{
|
|
"epoch": 0.4390602055800294,
|
|
"fcm_dpo/beta": 0.00413914117962122,
|
|
"fcm_dpo/delta": -0.04581880569458008,
|
|
"fcm_dpo/margin": 155.09854125976562,
|
|
"fcm_dpo/q_t": 0.3642904758453369,
|
|
"grad_norm": 38.76698684692383,
|
|
"learning_rate": 3.462633636266041e-07,
|
|
"logits/chosen": -0.3764153718948364,
|
|
"logits/rejected": -0.3700462579727173,
|
|
"logps/chosen": -246.14369201660156,
|
|
"logps/ref_chosen": -48.96209716796875,
|
|
"logps/ref_rejected": -84.32823944091797,
|
|
"logps/rejected": -436.6083984375,
|
|
"loss": 0.9702,
|
|
"margin_dpo/margin_mean": 155.0985565185547,
|
|
"margin_dpo/margin_std": 182.65501403808594,
|
|
"step": 299
|
|
},
|
|
{
|
|
"epoch": 0.44052863436123346,
|
|
"fcm_dpo/beta": 0.004044054076075554,
|
|
"fcm_dpo/delta": -0.1144830510020256,
|
|
"fcm_dpo/margin": 174.70237731933594,
|
|
"fcm_dpo/q_t": 0.3509829640388489,
|
|
"grad_norm": 42.70845413208008,
|
|
"learning_rate": 3.4507772230088147e-07,
|
|
"logits/chosen": -0.3672061562538147,
|
|
"logits/rejected": -0.36297306418418884,
|
|
"logps/chosen": -328.4141845703125,
|
|
"logps/ref_chosen": -59.07371139526367,
|
|
"logps/ref_rejected": -95.9664535522461,
|
|
"logps/rejected": -540.00927734375,
|
|
"loss": 0.9647,
|
|
"margin_dpo/margin_mean": 174.70237731933594,
|
|
"margin_dpo/margin_std": 214.1061248779297,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.4419970631424376,
|
|
"fcm_dpo/beta": 0.003927802667021751,
|
|
"fcm_dpo/delta": -0.099482960999012,
|
|
"fcm_dpo/margin": 176.13919067382812,
|
|
"fcm_dpo/q_t": 0.3530727028846741,
|
|
"grad_norm": 30.08980369567871,
|
|
"learning_rate": 3.4388957558875316e-07,
|
|
"logits/chosen": -0.40564534068107605,
|
|
"logits/rejected": -0.3970520496368408,
|
|
"logps/chosen": -284.28485107421875,
|
|
"logps/ref_chosen": -57.249366760253906,
|
|
"logps/ref_rejected": -92.35354614257812,
|
|
"logps/rejected": -495.5281982421875,
|
|
"loss": 0.9267,
|
|
"margin_dpo/margin_mean": 176.1392059326172,
|
|
"margin_dpo/margin_std": 191.83755493164062,
|
|
"step": 301
|
|
},
|
|
{
|
|
"epoch": 0.4434654919236417,
|
|
"fcm_dpo/beta": 0.0039380136877298355,
|
|
"fcm_dpo/delta": 0.05164004862308502,
|
|
"fcm_dpo/margin": 139.933349609375,
|
|
"fcm_dpo/q_t": 0.3805837631225586,
|
|
"grad_norm": 39.73493957519531,
|
|
"learning_rate": 3.426989547989902e-07,
|
|
"logits/chosen": -0.4200012683868408,
|
|
"logits/rejected": -0.4234361946582794,
|
|
"logps/chosen": -265.1363830566406,
|
|
"logps/ref_chosen": -51.197994232177734,
|
|
"logps/ref_rejected": -97.22636413574219,
|
|
"logps/rejected": -451.0981140136719,
|
|
"loss": 1.0152,
|
|
"margin_dpo/margin_mean": 139.933349609375,
|
|
"margin_dpo/margin_std": 175.3145751953125,
|
|
"step": 302
|
|
},
|
|
{
|
|
"epoch": 0.44493392070484583,
|
|
"fcm_dpo/beta": 0.003999094944447279,
|
|
"fcm_dpo/delta": 0.06958974152803421,
|
|
"fcm_dpo/margin": 133.57589721679688,
|
|
"fcm_dpo/q_t": 0.386226624250412,
|
|
"grad_norm": 40.41800308227539,
|
|
"learning_rate": 3.4150589130555773e-07,
|
|
"logits/chosen": -0.41884028911590576,
|
|
"logits/rejected": -0.39589619636535645,
|
|
"logps/chosen": -288.8157958984375,
|
|
"logps/ref_chosen": -66.71394348144531,
|
|
"logps/ref_rejected": -86.94542694091797,
|
|
"logps/rejected": -442.6231994628906,
|
|
"loss": 1.0405,
|
|
"margin_dpo/margin_mean": 133.57591247558594,
|
|
"margin_dpo/margin_std": 183.11703491210938,
|
|
"step": 303
|
|
},
|
|
{
|
|
"epoch": 0.44640234948604995,
|
|
"fcm_dpo/beta": 0.0040207491256296635,
|
|
"fcm_dpo/delta": -0.03515169024467468,
|
|
"fcm_dpo/margin": 157.3292694091797,
|
|
"fcm_dpo/q_t": 0.35517168045043945,
|
|
"grad_norm": 49.5794563293457,
|
|
"learning_rate": 3.403104165467883e-07,
|
|
"logits/chosen": -0.42296552658081055,
|
|
"logits/rejected": -0.4050954580307007,
|
|
"logps/chosen": -266.1893310546875,
|
|
"logps/ref_chosen": -71.95069885253906,
|
|
"logps/ref_rejected": -90.47203063964844,
|
|
"logps/rejected": -442.0399169921875,
|
|
"loss": 0.9141,
|
|
"margin_dpo/margin_mean": 157.32925415039062,
|
|
"margin_dpo/margin_std": 126.93194580078125,
|
|
"step": 304
|
|
},
|
|
{
|
|
"epoch": 0.447870778267254,
|
|
"fcm_dpo/beta": 0.003973768092691898,
|
|
"fcm_dpo/delta": 0.022460877895355225,
|
|
"fcm_dpo/margin": 145.38482666015625,
|
|
"fcm_dpo/q_t": 0.3763822913169861,
|
|
"grad_norm": 28.328157424926758,
|
|
"learning_rate": 3.391125620245535e-07,
|
|
"logits/chosen": -0.42863526940345764,
|
|
"logits/rejected": -0.40163588523864746,
|
|
"logps/chosen": -273.1666259765625,
|
|
"logps/ref_chosen": -66.79523468017578,
|
|
"logps/ref_rejected": -92.75459289550781,
|
|
"logps/rejected": -444.51080322265625,
|
|
"loss": 1.0001,
|
|
"margin_dpo/margin_mean": 145.38482666015625,
|
|
"margin_dpo/margin_std": 177.77105712890625,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.44933920704845814,
|
|
"fcm_dpo/beta": 0.004037954844534397,
|
|
"fcm_dpo/delta": 0.03192281723022461,
|
|
"fcm_dpo/margin": 141.09954833984375,
|
|
"fcm_dpo/q_t": 0.37558555603027344,
|
|
"grad_norm": 32.04235076904297,
|
|
"learning_rate": 3.3791235930343417e-07,
|
|
"logits/chosen": -0.3904664218425751,
|
|
"logits/rejected": -0.3591429889202118,
|
|
"logps/chosen": -269.64520263671875,
|
|
"logps/ref_chosen": -69.68389892578125,
|
|
"logps/ref_rejected": -85.15919494628906,
|
|
"logps/rejected": -426.2200622558594,
|
|
"loss": 0.9881,
|
|
"margin_dpo/margin_mean": 141.09954833984375,
|
|
"margin_dpo/margin_std": 163.01104736328125,
|
|
"step": 306
|
|
},
|
|
{
|
|
"epoch": 0.45080763582966227,
|
|
"fcm_dpo/beta": 0.004054022952914238,
|
|
"fcm_dpo/delta": 0.060959137976169586,
|
|
"fcm_dpo/margin": 133.7398681640625,
|
|
"fcm_dpo/q_t": 0.3805789649486542,
|
|
"grad_norm": 42.768367767333984,
|
|
"learning_rate": 3.367098400098881e-07,
|
|
"logits/chosen": -0.36754947900772095,
|
|
"logits/rejected": -0.3443043828010559,
|
|
"logps/chosen": -266.11358642578125,
|
|
"logps/ref_chosen": -70.16542053222656,
|
|
"logps/ref_rejected": -86.97230529785156,
|
|
"logps/rejected": -416.6603698730469,
|
|
"loss": 1.0175,
|
|
"margin_dpo/margin_mean": 133.7398681640625,
|
|
"margin_dpo/margin_std": 165.68634033203125,
|
|
"step": 307
|
|
},
|
|
{
|
|
"epoch": 0.4522760646108664,
|
|
"fcm_dpo/beta": 0.004061142913997173,
|
|
"fcm_dpo/delta": -0.046599872410297394,
|
|
"fcm_dpo/margin": 158.4833526611328,
|
|
"fcm_dpo/q_t": 0.3559141755104065,
|
|
"grad_norm": 34.26539993286133,
|
|
"learning_rate": 3.355050358314172e-07,
|
|
"logits/chosen": -0.3417690396308899,
|
|
"logits/rejected": -0.32422229647636414,
|
|
"logps/chosen": -250.93490600585938,
|
|
"logps/ref_chosen": -55.2449951171875,
|
|
"logps/ref_rejected": -79.37226104736328,
|
|
"logps/rejected": -433.5455017089844,
|
|
"loss": 0.9279,
|
|
"margin_dpo/margin_mean": 158.48333740234375,
|
|
"margin_dpo/margin_std": 150.41107177734375,
|
|
"step": 308
|
|
},
|
|
{
|
|
"epoch": 0.45374449339207046,
|
|
"fcm_dpo/beta": 0.004053506068885326,
|
|
"fcm_dpo/delta": -0.04356030374765396,
|
|
"fcm_dpo/margin": 157.86776733398438,
|
|
"fcm_dpo/q_t": 0.35849449038505554,
|
|
"grad_norm": 36.413551330566406,
|
|
"learning_rate": 3.3429797851573183e-07,
|
|
"logits/chosen": -0.3285176157951355,
|
|
"logits/rejected": -0.31304115056991577,
|
|
"logps/chosen": -240.2936248779297,
|
|
"logps/ref_chosen": -48.959083557128906,
|
|
"logps/ref_rejected": -82.34072875976562,
|
|
"logps/rejected": -431.54302978515625,
|
|
"loss": 0.9503,
|
|
"margin_dpo/margin_mean": 157.86776733398438,
|
|
"margin_dpo/margin_std": 165.03907775878906,
|
|
"step": 309
|
|
},
|
|
{
|
|
"epoch": 0.4552129221732746,
|
|
"fcm_dpo/beta": 0.004054386168718338,
|
|
"fcm_dpo/delta": 0.03481416404247284,
|
|
"fcm_dpo/margin": 139.72027587890625,
|
|
"fcm_dpo/q_t": 0.3711395859718323,
|
|
"grad_norm": 30.295249938964844,
|
|
"learning_rate": 3.3308869986991487e-07,
|
|
"logits/chosen": -0.35680243372917175,
|
|
"logits/rejected": -0.33559316396713257,
|
|
"logps/chosen": -279.739501953125,
|
|
"logps/ref_chosen": -62.74177932739258,
|
|
"logps/ref_rejected": -79.9302978515625,
|
|
"logps/rejected": -436.6482849121094,
|
|
"loss": 0.9662,
|
|
"margin_dpo/margin_mean": 139.72027587890625,
|
|
"margin_dpo/margin_std": 132.34963989257812,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.4566813509544787,
|
|
"fcm_dpo/beta": 0.003984658978879452,
|
|
"fcm_dpo/delta": -0.08475086092948914,
|
|
"fcm_dpo/margin": 170.3639373779297,
|
|
"fcm_dpo/q_t": 0.3561347723007202,
|
|
"grad_norm": 27.640430450439453,
|
|
"learning_rate": 3.3187723175958346e-07,
|
|
"logits/chosen": -0.33530759811401367,
|
|
"logits/rejected": -0.30850982666015625,
|
|
"logps/chosen": -278.0128479003906,
|
|
"logps/ref_chosen": -53.02798080444336,
|
|
"logps/ref_rejected": -77.43820190429688,
|
|
"logps/rejected": -472.7869873046875,
|
|
"loss": 0.9432,
|
|
"margin_dpo/margin_mean": 170.3639373779297,
|
|
"margin_dpo/margin_std": 193.53854370117188,
|
|
"step": 311
|
|
},
|
|
{
|
|
"epoch": 0.4581497797356828,
|
|
"fcm_dpo/beta": 0.003978079650551081,
|
|
"fcm_dpo/delta": -0.023921042680740356,
|
|
"fcm_dpo/margin": 156.37979125976562,
|
|
"fcm_dpo/q_t": 0.36489611864089966,
|
|
"grad_norm": 29.57942008972168,
|
|
"learning_rate": 3.306636061080487e-07,
|
|
"logits/chosen": -0.27948933839797974,
|
|
"logits/rejected": -0.261229932308197,
|
|
"logps/chosen": -254.8425750732422,
|
|
"logps/ref_chosen": -49.39221954345703,
|
|
"logps/ref_rejected": -75.79280853271484,
|
|
"logps/rejected": -437.62298583984375,
|
|
"loss": 0.9616,
|
|
"margin_dpo/margin_mean": 156.37979125976562,
|
|
"margin_dpo/margin_std": 174.12229919433594,
|
|
"step": 312
|
|
},
|
|
{
|
|
"epoch": 0.45961820851688695,
|
|
"fcm_dpo/beta": 0.0040033310651779175,
|
|
"fcm_dpo/delta": 0.01650937646627426,
|
|
"fcm_dpo/margin": 145.47708129882812,
|
|
"fcm_dpo/q_t": 0.3724866807460785,
|
|
"grad_norm": 33.476741790771484,
|
|
"learning_rate": 3.2944785489547537e-07,
|
|
"logits/chosen": -0.3220197558403015,
|
|
"logits/rejected": -0.3162776529788971,
|
|
"logps/chosen": -249.13385009765625,
|
|
"logps/ref_chosen": -50.152740478515625,
|
|
"logps/ref_rejected": -86.40620422363281,
|
|
"logps/rejected": -430.8643798828125,
|
|
"loss": 1.0053,
|
|
"margin_dpo/margin_mean": 145.47706604003906,
|
|
"margin_dpo/margin_std": 175.97396850585938,
|
|
"step": 313
|
|
},
|
|
{
|
|
"epoch": 0.461086637298091,
|
|
"fcm_dpo/beta": 0.0039779325015842915,
|
|
"fcm_dpo/delta": 0.053729020059108734,
|
|
"fcm_dpo/margin": 137.9976806640625,
|
|
"fcm_dpo/q_t": 0.38239920139312744,
|
|
"grad_norm": 28.024948120117188,
|
|
"learning_rate": 3.2823001015803857e-07,
|
|
"logits/chosen": -0.37483978271484375,
|
|
"logits/rejected": -0.37171846628189087,
|
|
"logps/chosen": -271.1210632324219,
|
|
"logps/ref_chosen": -57.237579345703125,
|
|
"logps/ref_rejected": -97.5965347290039,
|
|
"logps/rejected": -449.4776611328125,
|
|
"loss": 1.0322,
|
|
"margin_dpo/margin_mean": 137.99769592285156,
|
|
"margin_dpo/margin_std": 185.14695739746094,
|
|
"step": 314
|
|
},
|
|
{
|
|
"epoch": 0.46255506607929514,
|
|
"fcm_dpo/beta": 0.004043739754706621,
|
|
"fcm_dpo/delta": 0.06402629613876343,
|
|
"fcm_dpo/margin": 133.40902709960938,
|
|
"fcm_dpo/q_t": 0.3819577395915985,
|
|
"grad_norm": 34.34666442871094,
|
|
"learning_rate": 3.270101039870797e-07,
|
|
"logits/chosen": -0.25970029830932617,
|
|
"logits/rejected": -0.2599654793739319,
|
|
"logps/chosen": -258.28570556640625,
|
|
"logps/ref_chosen": -49.06958770751953,
|
|
"logps/ref_rejected": -85.68087768554688,
|
|
"logps/rejected": -428.30596923828125,
|
|
"loss": 1.0081,
|
|
"margin_dpo/margin_mean": 133.40902709960938,
|
|
"margin_dpo/margin_std": 158.57159423828125,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.46402349486049926,
|
|
"fcm_dpo/beta": 0.0039628720842301846,
|
|
"fcm_dpo/delta": -0.11757272481918335,
|
|
"fcm_dpo/margin": 178.52488708496094,
|
|
"fcm_dpo/q_t": 0.34840020537376404,
|
|
"grad_norm": 29.264596939086914,
|
|
"learning_rate": 3.2578816852826086e-07,
|
|
"logits/chosen": -0.30190056562423706,
|
|
"logits/rejected": -0.3046305775642395,
|
|
"logps/chosen": -262.7852783203125,
|
|
"logps/ref_chosen": -54.26074981689453,
|
|
"logps/ref_rejected": -101.2814712524414,
|
|
"logps/rejected": -488.33087158203125,
|
|
"loss": 0.9125,
|
|
"margin_dpo/margin_mean": 178.52488708496094,
|
|
"margin_dpo/margin_std": 183.82940673828125,
|
|
"step": 316
|
|
},
|
|
{
|
|
"epoch": 0.4654919236417034,
|
|
"fcm_dpo/beta": 0.0038559194654226303,
|
|
"fcm_dpo/delta": -0.21507573127746582,
|
|
"fcm_dpo/margin": 207.04443359375,
|
|
"fcm_dpo/q_t": 0.32492557168006897,
|
|
"grad_norm": 30.406768798828125,
|
|
"learning_rate": 3.2456423598071783e-07,
|
|
"logits/chosen": -0.3299999237060547,
|
|
"logits/rejected": -0.3157616853713989,
|
|
"logps/chosen": -272.45648193359375,
|
|
"logps/ref_chosen": -56.094207763671875,
|
|
"logps/ref_rejected": -100.69905090332031,
|
|
"logps/rejected": -524.105712890625,
|
|
"loss": 0.8408,
|
|
"margin_dpo/margin_mean": 207.04443359375,
|
|
"margin_dpo/margin_std": 174.73843383789062,
|
|
"step": 317
|
|
},
|
|
{
|
|
"epoch": 0.4669603524229075,
|
|
"fcm_dpo/beta": 0.0038027470000088215,
|
|
"fcm_dpo/delta": -0.02990717813372612,
|
|
"fcm_dpo/margin": 165.0573272705078,
|
|
"fcm_dpo/q_t": 0.3614851236343384,
|
|
"grad_norm": 39.84989929199219,
|
|
"learning_rate": 3.233383385962115e-07,
|
|
"logits/chosen": -0.39633870124816895,
|
|
"logits/rejected": -0.3627668023109436,
|
|
"logps/chosen": -288.56982421875,
|
|
"logps/ref_chosen": -64.64569854736328,
|
|
"logps/ref_rejected": -82.76425170898438,
|
|
"logps/rejected": -471.7456970214844,
|
|
"loss": 0.9456,
|
|
"margin_dpo/margin_mean": 165.0573272705078,
|
|
"margin_dpo/margin_std": 168.24526977539062,
|
|
"step": 318
|
|
},
|
|
{
|
|
"epoch": 0.4684287812041116,
|
|
"fcm_dpo/beta": 0.0037455919664353132,
|
|
"fcm_dpo/delta": -0.06087436154484749,
|
|
"fcm_dpo/margin": 175.37954711914062,
|
|
"fcm_dpo/q_t": 0.35514286160469055,
|
|
"grad_norm": 32.62771987915039,
|
|
"learning_rate": 3.2211050867827805e-07,
|
|
"logits/chosen": -0.32384127378463745,
|
|
"logits/rejected": -0.33828234672546387,
|
|
"logps/chosen": -260.467529296875,
|
|
"logps/ref_chosen": -49.383758544921875,
|
|
"logps/ref_rejected": -113.90650939941406,
|
|
"logps/rejected": -500.3698425292969,
|
|
"loss": 0.9384,
|
|
"margin_dpo/margin_mean": 175.37954711914062,
|
|
"margin_dpo/margin_std": 184.76806640625,
|
|
"step": 319
|
|
},
|
|
{
|
|
"epoch": 0.4698972099853157,
|
|
"fcm_dpo/beta": 0.0036399452947080135,
|
|
"fcm_dpo/delta": -0.14344662427902222,
|
|
"fcm_dpo/margin": 201.1544189453125,
|
|
"fcm_dpo/q_t": 0.3414286971092224,
|
|
"grad_norm": 35.252044677734375,
|
|
"learning_rate": 3.208807785813777e-07,
|
|
"logits/chosen": -0.34068095684051514,
|
|
"logits/rejected": -0.34001022577285767,
|
|
"logps/chosen": -272.8780212402344,
|
|
"logps/ref_chosen": -59.50489044189453,
|
|
"logps/ref_rejected": -97.66717529296875,
|
|
"logps/rejected": -512.1947021484375,
|
|
"loss": 0.8862,
|
|
"margin_dpo/margin_mean": 201.1544189453125,
|
|
"margin_dpo/margin_std": 194.9528045654297,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.4713656387665198,
|
|
"fcm_dpo/beta": 0.0035794866271317005,
|
|
"fcm_dpo/delta": -0.025970421731472015,
|
|
"fcm_dpo/margin": 173.90859985351562,
|
|
"fcm_dpo/q_t": 0.3651740550994873,
|
|
"grad_norm": 39.09996795654297,
|
|
"learning_rate": 3.1964918071004217e-07,
|
|
"logits/chosen": -0.3051578104496002,
|
|
"logits/rejected": -0.29157111048698425,
|
|
"logps/chosen": -323.6648254394531,
|
|
"logps/ref_chosen": -61.548683166503906,
|
|
"logps/ref_rejected": -91.64103698730469,
|
|
"logps/rejected": -527.665771484375,
|
|
"loss": 0.9826,
|
|
"margin_dpo/margin_mean": 173.90859985351562,
|
|
"margin_dpo/margin_std": 204.27359008789062,
|
|
"step": 321
|
|
},
|
|
{
|
|
"epoch": 0.47283406754772395,
|
|
"fcm_dpo/beta": 0.003550457302480936,
|
|
"fcm_dpo/delta": -0.07277655601501465,
|
|
"fcm_dpo/margin": 187.88018798828125,
|
|
"fcm_dpo/q_t": 0.3536107838153839,
|
|
"grad_norm": 34.10807800292969,
|
|
"learning_rate": 3.184157475180207e-07,
|
|
"logits/chosen": -0.3065683841705322,
|
|
"logits/rejected": -0.300385057926178,
|
|
"logps/chosen": -284.23797607421875,
|
|
"logps/ref_chosen": -57.29003143310547,
|
|
"logps/ref_rejected": -95.74992370605469,
|
|
"logps/rejected": -510.5780334472656,
|
|
"loss": 0.9286,
|
|
"margin_dpo/margin_mean": 187.88018798828125,
|
|
"margin_dpo/margin_std": 189.9258270263672,
|
|
"step": 322
|
|
},
|
|
{
|
|
"epoch": 0.47430249632892807,
|
|
"fcm_dpo/beta": 0.003559872740879655,
|
|
"fcm_dpo/delta": 0.03238154202699661,
|
|
"fcm_dpo/margin": 159.92843627929688,
|
|
"fcm_dpo/q_t": 0.374622106552124,
|
|
"grad_norm": 46.58732604980469,
|
|
"learning_rate": 3.171805115074251e-07,
|
|
"logits/chosen": -0.3481108546257019,
|
|
"logits/rejected": -0.343014121055603,
|
|
"logps/chosen": -301.1619873046875,
|
|
"logps/ref_chosen": -51.23395919799805,
|
|
"logps/ref_rejected": -75.06192016601562,
|
|
"logps/rejected": -484.91839599609375,
|
|
"loss": 1.0045,
|
|
"margin_dpo/margin_mean": 159.92843627929688,
|
|
"margin_dpo/margin_std": 189.8189697265625,
|
|
"step": 323
|
|
},
|
|
{
|
|
"epoch": 0.47577092511013214,
|
|
"fcm_dpo/beta": 0.003606216050684452,
|
|
"fcm_dpo/delta": -0.016858771443367004,
|
|
"fcm_dpo/margin": 170.2562713623047,
|
|
"fcm_dpo/q_t": 0.3706227242946625,
|
|
"grad_norm": 61.91301345825195,
|
|
"learning_rate": 3.1594350522787295e-07,
|
|
"logits/chosen": -0.3303983509540558,
|
|
"logits/rejected": -0.3054184913635254,
|
|
"logps/chosen": -341.5509033203125,
|
|
"logps/ref_chosen": -65.13516998291016,
|
|
"logps/ref_rejected": -86.47750854492188,
|
|
"logps/rejected": -533.1495361328125,
|
|
"loss": 1.0022,
|
|
"margin_dpo/margin_mean": 170.2562713623047,
|
|
"margin_dpo/margin_std": 211.9681396484375,
|
|
"step": 324
|
|
},
|
|
{
|
|
"epoch": 0.47723935389133626,
|
|
"fcm_dpo/beta": 0.003626979887485504,
|
|
"fcm_dpo/delta": 0.10734298825263977,
|
|
"fcm_dpo/margin": 137.27645874023438,
|
|
"fcm_dpo/q_t": 0.38791322708129883,
|
|
"grad_norm": 33.29164123535156,
|
|
"learning_rate": 3.147047612756302e-07,
|
|
"logits/chosen": -0.4132434129714966,
|
|
"logits/rejected": -0.3767145872116089,
|
|
"logps/chosen": -273.2174377441406,
|
|
"logps/ref_chosen": -56.215599060058594,
|
|
"logps/ref_rejected": -70.08592987060547,
|
|
"logps/rejected": -424.3642272949219,
|
|
"loss": 1.0305,
|
|
"margin_dpo/margin_mean": 137.27645874023438,
|
|
"margin_dpo/margin_std": 158.93292236328125,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.4787077826725404,
|
|
"fcm_dpo/beta": 0.0036813640035688877,
|
|
"fcm_dpo/delta": 0.08160503208637238,
|
|
"fcm_dpo/margin": 141.9955596923828,
|
|
"fcm_dpo/q_t": 0.3813520669937134,
|
|
"grad_norm": 49.46158981323242,
|
|
"learning_rate": 3.134643122927519e-07,
|
|
"logits/chosen": -0.42192065715789795,
|
|
"logits/rejected": -0.38546887040138245,
|
|
"logps/chosen": -309.13665771484375,
|
|
"logps/ref_chosen": -72.72496032714844,
|
|
"logps/ref_rejected": -79.8467788696289,
|
|
"logps/rejected": -458.25408935546875,
|
|
"loss": 0.9932,
|
|
"margin_dpo/margin_mean": 141.9955596923828,
|
|
"margin_dpo/margin_std": 141.18475341796875,
|
|
"step": 326
|
|
},
|
|
{
|
|
"epoch": 0.4801762114537445,
|
|
"fcm_dpo/beta": 0.0036152596585452557,
|
|
"fcm_dpo/delta": -0.12888285517692566,
|
|
"fcm_dpo/margin": 198.74432373046875,
|
|
"fcm_dpo/q_t": 0.34069618582725525,
|
|
"grad_norm": 56.69047546386719,
|
|
"learning_rate": 3.1222219096622264e-07,
|
|
"logits/chosen": -0.3953353762626648,
|
|
"logits/rejected": -0.3728201389312744,
|
|
"logps/chosen": -278.8756103515625,
|
|
"logps/ref_chosen": -69.13441467285156,
|
|
"logps/ref_rejected": -111.93377685546875,
|
|
"logps/rejected": -520.4193115234375,
|
|
"loss": 0.8922,
|
|
"margin_dpo/margin_mean": 198.74432373046875,
|
|
"margin_dpo/margin_std": 184.82196044921875,
|
|
"step": 327
|
|
},
|
|
{
|
|
"epoch": 0.48164464023494863,
|
|
"fcm_dpo/beta": 0.0036046169698238373,
|
|
"fcm_dpo/delta": -0.01882966235280037,
|
|
"fcm_dpo/margin": 171.35028076171875,
|
|
"fcm_dpo/q_t": 0.3656262159347534,
|
|
"grad_norm": 41.368961334228516,
|
|
"learning_rate": 3.1097843002709427e-07,
|
|
"logits/chosen": -0.3795957565307617,
|
|
"logits/rejected": -0.3735014796257019,
|
|
"logps/chosen": -278.96380615234375,
|
|
"logps/ref_chosen": -59.68719482421875,
|
|
"logps/ref_rejected": -90.85499572753906,
|
|
"logps/rejected": -481.48187255859375,
|
|
"loss": 0.963,
|
|
"margin_dpo/margin_mean": 171.35028076171875,
|
|
"margin_dpo/margin_std": 190.85531616210938,
|
|
"step": 328
|
|
},
|
|
{
|
|
"epoch": 0.4831130690161527,
|
|
"fcm_dpo/beta": 0.0035419906489551067,
|
|
"fcm_dpo/delta": -0.06768238544464111,
|
|
"fcm_dpo/margin": 187.00729370117188,
|
|
"fcm_dpo/q_t": 0.3545988202095032,
|
|
"grad_norm": 28.033584594726562,
|
|
"learning_rate": 3.0973306224962437e-07,
|
|
"logits/chosen": -0.35659074783325195,
|
|
"logits/rejected": -0.33611446619033813,
|
|
"logps/chosen": -300.49114990234375,
|
|
"logps/ref_chosen": -65.2461929321289,
|
|
"logps/ref_rejected": -100.69770812988281,
|
|
"logps/rejected": -522.949951171875,
|
|
"loss": 0.9397,
|
|
"margin_dpo/margin_mean": 187.00729370117188,
|
|
"margin_dpo/margin_std": 195.09939575195312,
|
|
"step": 329
|
|
},
|
|
{
|
|
"epoch": 0.4845814977973568,
|
|
"fcm_dpo/beta": 0.0035197772085666656,
|
|
"fcm_dpo/delta": -0.04879715293645859,
|
|
"fcm_dpo/margin": 183.4234619140625,
|
|
"fcm_dpo/q_t": 0.3568430542945862,
|
|
"grad_norm": 40.50788116455078,
|
|
"learning_rate": 3.084861204504122e-07,
|
|
"logits/chosen": -0.3393748104572296,
|
|
"logits/rejected": -0.34215378761291504,
|
|
"logps/chosen": -263.3137512207031,
|
|
"logps/ref_chosen": -46.998348236083984,
|
|
"logps/ref_rejected": -86.87684631347656,
|
|
"logps/rejected": -486.61572265625,
|
|
"loss": 0.9342,
|
|
"margin_dpo/margin_mean": 183.4234619140625,
|
|
"margin_dpo/margin_std": 184.44473266601562,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.48604992657856094,
|
|
"fcm_dpo/beta": 0.003468625945970416,
|
|
"fcm_dpo/delta": -0.08099709451198578,
|
|
"fcm_dpo/margin": 194.73370361328125,
|
|
"fcm_dpo/q_t": 0.34509968757629395,
|
|
"grad_norm": 34.197383880615234,
|
|
"learning_rate": 3.072376374875335e-07,
|
|
"logits/chosen": -0.33651411533355713,
|
|
"logits/rejected": -0.32489413022994995,
|
|
"logps/chosen": -278.44921875,
|
|
"logps/ref_chosen": -50.52424621582031,
|
|
"logps/ref_rejected": -89.01544189453125,
|
|
"logps/rejected": -511.67413330078125,
|
|
"loss": 0.8733,
|
|
"margin_dpo/margin_mean": 194.73370361328125,
|
|
"margin_dpo/margin_std": 140.58489990234375,
|
|
"step": 331
|
|
},
|
|
{
|
|
"epoch": 0.48751835535976507,
|
|
"fcm_dpo/beta": 0.003518771380186081,
|
|
"fcm_dpo/delta": 0.11069121956825256,
|
|
"fcm_dpo/margin": 140.563232421875,
|
|
"fcm_dpo/q_t": 0.38987618684768677,
|
|
"grad_norm": 32.90526580810547,
|
|
"learning_rate": 3.059876462596758e-07,
|
|
"logits/chosen": -0.3346155285835266,
|
|
"logits/rejected": -0.3171945810317993,
|
|
"logps/chosen": -291.15472412109375,
|
|
"logps/ref_chosen": -49.18028259277344,
|
|
"logps/ref_rejected": -76.48515319824219,
|
|
"logps/rejected": -459.0228271484375,
|
|
"loss": 1.029,
|
|
"margin_dpo/margin_mean": 140.563232421875,
|
|
"margin_dpo/margin_std": 166.2570037841797,
|
|
"step": 332
|
|
},
|
|
{
|
|
"epoch": 0.4889867841409692,
|
|
"fcm_dpo/beta": 0.0034761871211230755,
|
|
"fcm_dpo/delta": -0.055231112986803055,
|
|
"fcm_dpo/margin": 186.94793701171875,
|
|
"fcm_dpo/q_t": 0.3612428307533264,
|
|
"grad_norm": 31.33414649963379,
|
|
"learning_rate": 3.0473617970527015e-07,
|
|
"logits/chosen": -0.37555959820747375,
|
|
"logits/rejected": -0.36939913034439087,
|
|
"logps/chosen": -315.4451904296875,
|
|
"logps/ref_chosen": -63.75574493408203,
|
|
"logps/ref_rejected": -95.04411315917969,
|
|
"logps/rejected": -533.6814575195312,
|
|
"loss": 0.9718,
|
|
"margin_dpo/margin_mean": 186.94793701171875,
|
|
"margin_dpo/margin_std": 219.6304931640625,
|
|
"step": 333
|
|
},
|
|
{
|
|
"epoch": 0.49045521292217326,
|
|
"fcm_dpo/beta": 0.003508550813421607,
|
|
"fcm_dpo/delta": 0.022331170737743378,
|
|
"fcm_dpo/margin": 165.01031494140625,
|
|
"fcm_dpo/q_t": 0.3745434284210205,
|
|
"grad_norm": 45.59312438964844,
|
|
"learning_rate": 3.034832708016243e-07,
|
|
"logits/chosen": -0.37708473205566406,
|
|
"logits/rejected": -0.37125474214553833,
|
|
"logps/chosen": -313.27618408203125,
|
|
"logps/ref_chosen": -66.97975158691406,
|
|
"logps/ref_rejected": -95.31692504882812,
|
|
"logps/rejected": -506.6236877441406,
|
|
"loss": 1.0114,
|
|
"margin_dpo/margin_mean": 165.01031494140625,
|
|
"margin_dpo/margin_std": 209.47122192382812,
|
|
"step": 334
|
|
},
|
|
{
|
|
"epoch": 0.4919236417033774,
|
|
"fcm_dpo/beta": 0.003607717342674732,
|
|
"fcm_dpo/delta": 0.14469006657600403,
|
|
"fcm_dpo/margin": 127.66055297851562,
|
|
"fcm_dpo/q_t": 0.4004567563533783,
|
|
"grad_norm": 63.05012512207031,
|
|
"learning_rate": 3.022289525640531e-07,
|
|
"logits/chosen": -0.38800978660583496,
|
|
"logits/rejected": -0.36463358998298645,
|
|
"logps/chosen": -334.33935546875,
|
|
"logps/ref_chosen": -62.54248046875,
|
|
"logps/ref_rejected": -87.61770629882812,
|
|
"logps/rejected": -487.07513427734375,
|
|
"loss": 1.0921,
|
|
"margin_dpo/margin_mean": 127.66055297851562,
|
|
"margin_dpo/margin_std": 194.76068115234375,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.4933920704845815,
|
|
"fcm_dpo/beta": 0.0035587395541369915,
|
|
"fcm_dpo/delta": -0.1215284913778305,
|
|
"fcm_dpo/margin": 200.35911560058594,
|
|
"fcm_dpo/q_t": 0.34965717792510986,
|
|
"grad_norm": 34.31383514404297,
|
|
"learning_rate": 3.009732580450086e-07,
|
|
"logits/chosen": -0.3676879405975342,
|
|
"logits/rejected": -0.36403754353523254,
|
|
"logps/chosen": -313.09222412109375,
|
|
"logps/ref_chosen": -54.53115463256836,
|
|
"logps/ref_rejected": -104.40424346923828,
|
|
"logps/rejected": -563.3244018554688,
|
|
"loss": 0.9395,
|
|
"margin_dpo/margin_mean": 200.35910034179688,
|
|
"margin_dpo/margin_std": 232.01760864257812,
|
|
"step": 336
|
|
},
|
|
{
|
|
"epoch": 0.4948604992657856,
|
|
"fcm_dpo/beta": 0.003459386760368943,
|
|
"fcm_dpo/delta": -0.11083994805812836,
|
|
"fcm_dpo/margin": 203.0913848876953,
|
|
"fcm_dpo/q_t": 0.34715068340301514,
|
|
"grad_norm": 32.61246109008789,
|
|
"learning_rate": 2.9971622033320914e-07,
|
|
"logits/chosen": -0.3618990480899811,
|
|
"logits/rejected": -0.3507819175720215,
|
|
"logps/chosen": -311.53759765625,
|
|
"logps/ref_chosen": -65.12869262695312,
|
|
"logps/ref_rejected": -101.72701263427734,
|
|
"logps/rejected": -551.2273559570312,
|
|
"loss": 0.9156,
|
|
"margin_dpo/margin_mean": 203.09136962890625,
|
|
"margin_dpo/margin_std": 209.97573852539062,
|
|
"step": 337
|
|
},
|
|
{
|
|
"epoch": 0.49632892804698975,
|
|
"fcm_dpo/beta": 0.0034060748293995857,
|
|
"fcm_dpo/delta": -0.08523920178413391,
|
|
"fcm_dpo/margin": 199.4775848388672,
|
|
"fcm_dpo/q_t": 0.34894174337387085,
|
|
"grad_norm": 29.552736282348633,
|
|
"learning_rate": 2.984578725527675e-07,
|
|
"logits/chosen": -0.4156501591205597,
|
|
"logits/rejected": -0.40091991424560547,
|
|
"logps/chosen": -276.4804382324219,
|
|
"logps/ref_chosen": -58.422706604003906,
|
|
"logps/ref_rejected": -89.06854248046875,
|
|
"logps/rejected": -506.6038513183594,
|
|
"loss": 0.9049,
|
|
"margin_dpo/margin_mean": 199.4775848388672,
|
|
"margin_dpo/margin_std": 184.65594482421875,
|
|
"step": 338
|
|
},
|
|
{
|
|
"epoch": 0.4977973568281938,
|
|
"fcm_dpo/beta": 0.0033988947980105877,
|
|
"fcm_dpo/delta": -0.04927036911249161,
|
|
"fcm_dpo/margin": 189.7136993408203,
|
|
"fcm_dpo/q_t": 0.35729217529296875,
|
|
"grad_norm": 32.228424072265625,
|
|
"learning_rate": 2.9719824786231796e-07,
|
|
"logits/chosen": -0.40810322761535645,
|
|
"logits/rejected": -0.3912397623062134,
|
|
"logps/chosen": -287.6553649902344,
|
|
"logps/ref_chosen": -59.99531555175781,
|
|
"logps/ref_rejected": -103.9109115600586,
|
|
"logps/rejected": -521.28466796875,
|
|
"loss": 0.9323,
|
|
"margin_dpo/margin_mean": 189.7136993408203,
|
|
"margin_dpo/margin_std": 180.0070343017578,
|
|
"step": 339
|
|
},
|
|
{
|
|
"epoch": 0.49926578560939794,
|
|
"fcm_dpo/beta": 0.0033571631647646427,
|
|
"fcm_dpo/delta": 0.030759550631046295,
|
|
"fcm_dpo/margin": 169.98486328125,
|
|
"fcm_dpo/q_t": 0.3748738765716553,
|
|
"grad_norm": 34.929901123046875,
|
|
"learning_rate": 2.959373794541426e-07,
|
|
"logits/chosen": -0.3299640417098999,
|
|
"logits/rejected": -0.3012125492095947,
|
|
"logps/chosen": -285.945556640625,
|
|
"logps/ref_chosen": -52.83022689819336,
|
|
"logps/ref_rejected": -73.10723114013672,
|
|
"logps/rejected": -476.2073974609375,
|
|
"loss": 0.9978,
|
|
"margin_dpo/margin_mean": 169.98486328125,
|
|
"margin_dpo/margin_std": 201.46371459960938,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.5007342143906021,
|
|
"fcm_dpo/beta": 0.0033384612761437893,
|
|
"fcm_dpo/delta": -0.0509408563375473,
|
|
"fcm_dpo/margin": 193.81198120117188,
|
|
"fcm_dpo/q_t": 0.3578472137451172,
|
|
"grad_norm": 29.954818725585938,
|
|
"learning_rate": 2.946753005532965e-07,
|
|
"logits/chosen": -0.358737051486969,
|
|
"logits/rejected": -0.3582429587841034,
|
|
"logps/chosen": -274.4048767089844,
|
|
"logps/ref_chosen": -47.899803161621094,
|
|
"logps/ref_rejected": -101.80987548828125,
|
|
"logps/rejected": -522.126953125,
|
|
"loss": 0.9392,
|
|
"margin_dpo/margin_mean": 193.81198120117188,
|
|
"margin_dpo/margin_std": 198.76821899414062,
|
|
"step": 341
|
|
},
|
|
{
|
|
"epoch": 0.5022026431718062,
|
|
"fcm_dpo/beta": 0.003372443839907646,
|
|
"fcm_dpo/delta": 0.02195318415760994,
|
|
"fcm_dpo/margin": 171.60679626464844,
|
|
"fcm_dpo/q_t": 0.37457433342933655,
|
|
"grad_norm": 34.212318420410156,
|
|
"learning_rate": 2.934120444167326e-07,
|
|
"logits/chosen": -0.38501685857772827,
|
|
"logits/rejected": -0.35319679975509644,
|
|
"logps/chosen": -302.15545654296875,
|
|
"logps/ref_chosen": -71.99664306640625,
|
|
"logps/ref_rejected": -92.58959197998047,
|
|
"logps/rejected": -494.355224609375,
|
|
"loss": 0.9967,
|
|
"margin_dpo/margin_mean": 171.60679626464844,
|
|
"margin_dpo/margin_std": 208.8232421875,
|
|
"step": 342
|
|
},
|
|
{
|
|
"epoch": 0.5036710719530103,
|
|
"fcm_dpo/beta": 0.0033103451132774353,
|
|
"fcm_dpo/delta": -0.0885283350944519,
|
|
"fcm_dpo/margin": 206.1126708984375,
|
|
"fcm_dpo/q_t": 0.3450053334236145,
|
|
"grad_norm": 34.08480453491211,
|
|
"learning_rate": 2.9214764433242476e-07,
|
|
"logits/chosen": -0.3982047438621521,
|
|
"logits/rejected": -0.3942341208457947,
|
|
"logps/chosen": -267.8955383300781,
|
|
"logps/ref_chosen": -54.405616760253906,
|
|
"logps/ref_rejected": -111.04142761230469,
|
|
"logps/rejected": -530.64404296875,
|
|
"loss": 0.8828,
|
|
"margin_dpo/margin_mean": 206.1126708984375,
|
|
"margin_dpo/margin_std": 162.10682678222656,
|
|
"step": 343
|
|
},
|
|
{
|
|
"epoch": 0.5051395007342144,
|
|
"fcm_dpo/beta": 0.0033293329179286957,
|
|
"fcm_dpo/delta": -0.014567896723747253,
|
|
"fcm_dpo/margin": 183.69131469726562,
|
|
"fcm_dpo/q_t": 0.3665677309036255,
|
|
"grad_norm": 27.15936851501465,
|
|
"learning_rate": 2.9088213361849126e-07,
|
|
"logits/chosen": -0.38978269696235657,
|
|
"logits/rejected": -0.3856002688407898,
|
|
"logps/chosen": -273.5009460449219,
|
|
"logps/ref_chosen": -53.96466827392578,
|
|
"logps/ref_rejected": -90.62336730957031,
|
|
"logps/rejected": -493.8509521484375,
|
|
"loss": 0.9563,
|
|
"margin_dpo/margin_mean": 183.69131469726562,
|
|
"margin_dpo/margin_std": 185.71316528320312,
|
|
"step": 344
|
|
},
|
|
{
|
|
"epoch": 0.5066079295154186,
|
|
"fcm_dpo/beta": 0.0032876995392143726,
|
|
"fcm_dpo/delta": -0.037146538496017456,
|
|
"fcm_dpo/margin": 193.00808715820312,
|
|
"fcm_dpo/q_t": 0.36083394289016724,
|
|
"grad_norm": 27.978347778320312,
|
|
"learning_rate": 2.896155456223163e-07,
|
|
"logits/chosen": -0.39028286933898926,
|
|
"logits/rejected": -0.3786432147026062,
|
|
"logps/chosen": -334.9024963378906,
|
|
"logps/ref_chosen": -61.685699462890625,
|
|
"logps/ref_rejected": -99.49041748046875,
|
|
"logps/rejected": -565.7152709960938,
|
|
"loss": 0.952,
|
|
"margin_dpo/margin_mean": 193.00807189941406,
|
|
"margin_dpo/margin_std": 207.6051025390625,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.5080763582966226,
|
|
"fcm_dpo/beta": 0.003262849524617195,
|
|
"fcm_dpo/delta": -0.004547563381493092,
|
|
"fcm_dpo/margin": 185.1959686279297,
|
|
"fcm_dpo/q_t": 0.3652818500995636,
|
|
"grad_norm": 34.94870376586914,
|
|
"learning_rate": 2.883479137196714e-07,
|
|
"logits/chosen": -0.3476218581199646,
|
|
"logits/rejected": -0.3306209444999695,
|
|
"logps/chosen": -337.10003662109375,
|
|
"logps/ref_chosen": -55.256263732910156,
|
|
"logps/ref_rejected": -77.41532135009766,
|
|
"logps/rejected": -544.455078125,
|
|
"loss": 0.9697,
|
|
"margin_dpo/margin_mean": 185.1959686279297,
|
|
"margin_dpo/margin_std": 203.53085327148438,
|
|
"step": 346
|
|
},
|
|
{
|
|
"epoch": 0.5095447870778267,
|
|
"fcm_dpo/beta": 0.0032533821649849415,
|
|
"fcm_dpo/delta": -0.015425489284098148,
|
|
"fcm_dpo/margin": 188.865478515625,
|
|
"fcm_dpo/q_t": 0.3658721446990967,
|
|
"grad_norm": 38.90462112426758,
|
|
"learning_rate": 2.8707927131383614e-07,
|
|
"logits/chosen": -0.3579314947128296,
|
|
"logits/rejected": -0.3465426564216614,
|
|
"logps/chosen": -343.6216125488281,
|
|
"logps/ref_chosen": -57.56623840332031,
|
|
"logps/ref_rejected": -92.35509490966797,
|
|
"logps/rejected": -567.2759399414062,
|
|
"loss": 0.9762,
|
|
"margin_dpo/margin_mean": 188.865478515625,
|
|
"margin_dpo/margin_std": 218.76934814453125,
|
|
"step": 347
|
|
},
|
|
{
|
|
"epoch": 0.5110132158590308,
|
|
"fcm_dpo/beta": 0.0032581479754298925,
|
|
"fcm_dpo/delta": 0.0629473477602005,
|
|
"fcm_dpo/margin": 165.63992309570312,
|
|
"fcm_dpo/q_t": 0.381051242351532,
|
|
"grad_norm": 30.090953826904297,
|
|
"learning_rate": 2.858096518347179e-07,
|
|
"logits/chosen": -0.4058574438095093,
|
|
"logits/rejected": -0.4066629707813263,
|
|
"logps/chosen": -306.64599609375,
|
|
"logps/ref_chosen": -56.31770324707031,
|
|
"logps/ref_rejected": -89.13836669921875,
|
|
"logps/rejected": -505.1065673828125,
|
|
"loss": 1.0098,
|
|
"margin_dpo/margin_mean": 165.63992309570312,
|
|
"margin_dpo/margin_std": 191.34750366210938,
|
|
"step": 348
|
|
},
|
|
{
|
|
"epoch": 0.5124816446402349,
|
|
"fcm_dpo/beta": 0.003329505678266287,
|
|
"fcm_dpo/delta": 0.02792828157544136,
|
|
"fcm_dpo/margin": 172.01544189453125,
|
|
"fcm_dpo/q_t": 0.3782421946525574,
|
|
"grad_norm": 26.204023361206055,
|
|
"learning_rate": 2.845390887379706e-07,
|
|
"logits/chosen": -0.3858356475830078,
|
|
"logits/rejected": -0.3851817548274994,
|
|
"logps/chosen": -302.1793212890625,
|
|
"logps/ref_chosen": -58.025516510009766,
|
|
"logps/ref_rejected": -97.50515747070312,
|
|
"logps/rejected": -513.6744384765625,
|
|
"loss": 1.0217,
|
|
"margin_dpo/margin_mean": 172.01544189453125,
|
|
"margin_dpo/margin_std": 228.81277465820312,
|
|
"step": 349
|
|
},
|
|
{
|
|
"epoch": 0.5139500734214391,
|
|
"fcm_dpo/beta": 0.003294072113931179,
|
|
"fcm_dpo/delta": -0.006476100534200668,
|
|
"fcm_dpo/margin": 183.84515380859375,
|
|
"fcm_dpo/q_t": 0.36599087715148926,
|
|
"grad_norm": 39.012115478515625,
|
|
"learning_rate": 2.8326761550411346e-07,
|
|
"logits/chosen": -0.4358096122741699,
|
|
"logits/rejected": -0.43361157178878784,
|
|
"logps/chosen": -314.4185485839844,
|
|
"logps/ref_chosen": -64.33049011230469,
|
|
"logps/ref_rejected": -89.87164306640625,
|
|
"logps/rejected": -523.8048706054688,
|
|
"loss": 0.9903,
|
|
"margin_dpo/margin_mean": 183.84518432617188,
|
|
"margin_dpo/margin_std": 221.12252807617188,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.5154185022026432,
|
|
"fcm_dpo/beta": 0.00327136623673141,
|
|
"fcm_dpo/delta": -0.07544407248497009,
|
|
"fcm_dpo/margin": 204.92010498046875,
|
|
"fcm_dpo/q_t": 0.355930358171463,
|
|
"grad_norm": 33.651607513427734,
|
|
"learning_rate": 2.819952656376487e-07,
|
|
"logits/chosen": -0.40549007058143616,
|
|
"logits/rejected": -0.3955671489238739,
|
|
"logps/chosen": -296.75994873046875,
|
|
"logps/ref_chosen": -60.6721305847168,
|
|
"logps/ref_rejected": -101.5654296875,
|
|
"logps/rejected": -542.5733642578125,
|
|
"loss": 0.9437,
|
|
"margin_dpo/margin_mean": 204.92010498046875,
|
|
"margin_dpo/margin_std": 227.87014770507812,
|
|
"step": 351
|
|
},
|
|
{
|
|
"epoch": 0.5168869309838473,
|
|
"fcm_dpo/beta": 0.003319011302664876,
|
|
"fcm_dpo/delta": 0.15692806243896484,
|
|
"fcm_dpo/margin": 135.69808959960938,
|
|
"fcm_dpo/q_t": 0.40290194749832153,
|
|
"grad_norm": 41.35541915893555,
|
|
"learning_rate": 2.8072207266617854e-07,
|
|
"logits/chosen": -0.39126765727996826,
|
|
"logits/rejected": -0.35691601037979126,
|
|
"logps/chosen": -348.217041015625,
|
|
"logps/ref_chosen": -70.9434585571289,
|
|
"logps/ref_rejected": -76.6419677734375,
|
|
"logps/rejected": -489.6136169433594,
|
|
"loss": 1.0962,
|
|
"margin_dpo/margin_mean": 135.69808959960938,
|
|
"margin_dpo/margin_std": 207.16409301757812,
|
|
"step": 352
|
|
},
|
|
{
|
|
"epoch": 0.5183553597650514,
|
|
"fcm_dpo/beta": 0.003364278469234705,
|
|
"fcm_dpo/delta": 0.043343670666217804,
|
|
"fcm_dpo/margin": 166.03475952148438,
|
|
"fcm_dpo/q_t": 0.3799568712711334,
|
|
"grad_norm": 41.904701232910156,
|
|
"learning_rate": 2.794480701395219e-07,
|
|
"logits/chosen": -0.43321138620376587,
|
|
"logits/rejected": -0.41663485765457153,
|
|
"logps/chosen": -319.3279113769531,
|
|
"logps/ref_chosen": -58.39533996582031,
|
|
"logps/ref_rejected": -80.33553314208984,
|
|
"logps/rejected": -507.3028259277344,
|
|
"loss": 1.0207,
|
|
"margin_dpo/margin_mean": 166.03475952148438,
|
|
"margin_dpo/margin_std": 212.48361206054688,
|
|
"step": 353
|
|
},
|
|
{
|
|
"epoch": 0.5198237885462555,
|
|
"fcm_dpo/beta": 0.0033673213329166174,
|
|
"fcm_dpo/delta": -0.036777447909116745,
|
|
"fcm_dpo/margin": 188.36825561523438,
|
|
"fcm_dpo/q_t": 0.3579559922218323,
|
|
"grad_norm": 39.02664566040039,
|
|
"learning_rate": 2.781732916288303e-07,
|
|
"logits/chosen": -0.4032512307167053,
|
|
"logits/rejected": -0.383810818195343,
|
|
"logps/chosen": -275.3306579589844,
|
|
"logps/ref_chosen": -59.80299377441406,
|
|
"logps/ref_rejected": -88.75750732421875,
|
|
"logps/rejected": -492.6534423828125,
|
|
"loss": 0.9294,
|
|
"margin_dpo/margin_mean": 188.36827087402344,
|
|
"margin_dpo/margin_std": 177.9414520263672,
|
|
"step": 354
|
|
},
|
|
{
|
|
"epoch": 0.5212922173274597,
|
|
"fcm_dpo/beta": 0.003320076037198305,
|
|
"fcm_dpo/delta": -0.06938491016626358,
|
|
"fcm_dpo/margin": 200.06072998046875,
|
|
"fcm_dpo/q_t": 0.3513967990875244,
|
|
"grad_norm": 39.97414779663086,
|
|
"learning_rate": 2.7689777072570284e-07,
|
|
"logits/chosen": -0.4452924132347107,
|
|
"logits/rejected": -0.42499247193336487,
|
|
"logps/chosen": -272.80706787109375,
|
|
"logps/ref_chosen": -54.12849807739258,
|
|
"logps/ref_rejected": -82.40606689453125,
|
|
"logps/rejected": -501.1453552246094,
|
|
"loss": 0.9087,
|
|
"margin_dpo/margin_mean": 200.06072998046875,
|
|
"margin_dpo/margin_std": 179.74571228027344,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.5227606461086637,
|
|
"fcm_dpo/beta": 0.0034437617287039757,
|
|
"fcm_dpo/delta": 0.1814279407262802,
|
|
"fcm_dpo/margin": 122.82986450195312,
|
|
"fcm_dpo/q_t": 0.41210052371025085,
|
|
"grad_norm": 50.92268753051758,
|
|
"learning_rate": 2.7562154104130176e-07,
|
|
"logits/chosen": -0.4147285223007202,
|
|
"logits/rejected": -0.39140522480010986,
|
|
"logps/chosen": -330.02154541015625,
|
|
"logps/ref_chosen": -64.6738052368164,
|
|
"logps/ref_rejected": -75.89926147460938,
|
|
"logps/rejected": -464.07684326171875,
|
|
"loss": 1.1595,
|
|
"margin_dpo/margin_mean": 122.82986450195312,
|
|
"margin_dpo/margin_std": 233.60147094726562,
|
|
"step": 356
|
|
},
|
|
{
|
|
"epoch": 0.5242290748898678,
|
|
"fcm_dpo/beta": 0.0034146038815379143,
|
|
"fcm_dpo/delta": -0.019891854375600815,
|
|
"fcm_dpo/margin": 180.96995544433594,
|
|
"fcm_dpo/q_t": 0.3629704713821411,
|
|
"grad_norm": 36.10367202758789,
|
|
"learning_rate": 2.7434463620546594e-07,
|
|
"logits/chosen": -0.4508202075958252,
|
|
"logits/rejected": -0.43361175060272217,
|
|
"logps/chosen": -294.203857421875,
|
|
"logps/ref_chosen": -52.725799560546875,
|
|
"logps/ref_rejected": -86.84115600585938,
|
|
"logps/rejected": -509.2891845703125,
|
|
"loss": 0.9528,
|
|
"margin_dpo/margin_mean": 180.96995544433594,
|
|
"margin_dpo/margin_std": 186.50103759765625,
|
|
"step": 357
|
|
},
|
|
{
|
|
"epoch": 0.5256975036710719,
|
|
"fcm_dpo/beta": 0.003465921152383089,
|
|
"fcm_dpo/delta": 0.06651946902275085,
|
|
"fcm_dpo/margin": 154.9437255859375,
|
|
"fcm_dpo/q_t": 0.3825710117816925,
|
|
"grad_norm": 31.915424346923828,
|
|
"learning_rate": 2.730670898658255e-07,
|
|
"logits/chosen": -0.4602763056755066,
|
|
"logits/rejected": -0.44098126888275146,
|
|
"logps/chosen": -284.6385192871094,
|
|
"logps/ref_chosen": -63.20543670654297,
|
|
"logps/ref_rejected": -88.373291015625,
|
|
"logps/rejected": -464.7501220703125,
|
|
"loss": 1.0108,
|
|
"margin_dpo/margin_mean": 154.9437255859375,
|
|
"margin_dpo/margin_std": 186.69895935058594,
|
|
"step": 358
|
|
},
|
|
{
|
|
"epoch": 0.527165932452276,
|
|
"fcm_dpo/beta": 0.003439284861087799,
|
|
"fcm_dpo/delta": -0.05646362528204918,
|
|
"fcm_dpo/margin": 189.73446655273438,
|
|
"fcm_dpo/q_t": 0.36093398928642273,
|
|
"grad_norm": 38.223670959472656,
|
|
"learning_rate": 2.717889356869146e-07,
|
|
"logits/chosen": -0.39621424674987793,
|
|
"logits/rejected": -0.3747130036354065,
|
|
"logps/chosen": -310.32684326171875,
|
|
"logps/ref_chosen": -56.370216369628906,
|
|
"logps/ref_rejected": -82.17375183105469,
|
|
"logps/rejected": -525.8648681640625,
|
|
"loss": 0.9534,
|
|
"margin_dpo/margin_mean": 189.73448181152344,
|
|
"margin_dpo/margin_std": 211.576416015625,
|
|
"step": 359
|
|
},
|
|
{
|
|
"epoch": 0.5286343612334802,
|
|
"fcm_dpo/beta": 0.0034922873601317406,
|
|
"fcm_dpo/delta": 0.09579764306545258,
|
|
"fcm_dpo/margin": 145.714599609375,
|
|
"fcm_dpo/q_t": 0.3853462338447571,
|
|
"grad_norm": 46.432003021240234,
|
|
"learning_rate": 2.7051020734928443e-07,
|
|
"logits/chosen": -0.37489083409309387,
|
|
"logits/rejected": -0.3580286502838135,
|
|
"logps/chosen": -298.4937744140625,
|
|
"logps/ref_chosen": -51.460384368896484,
|
|
"logps/ref_rejected": -69.83892059326172,
|
|
"logps/rejected": -462.5869445800781,
|
|
"loss": 1.0106,
|
|
"margin_dpo/margin_mean": 145.714599609375,
|
|
"margin_dpo/margin_std": 157.42027282714844,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.5301027900146843,
|
|
"fcm_dpo/beta": 0.0035505923442542553,
|
|
"fcm_dpo/delta": 0.06202498450875282,
|
|
"fcm_dpo/margin": 152.30416870117188,
|
|
"fcm_dpo/q_t": 0.3825526237487793,
|
|
"grad_norm": 37.47382736206055,
|
|
"learning_rate": 2.6923093854861593e-07,
|
|
"logits/chosen": -0.3717784285545349,
|
|
"logits/rejected": -0.36584901809692383,
|
|
"logps/chosen": -310.5164794921875,
|
|
"logps/ref_chosen": -53.86951446533203,
|
|
"logps/ref_rejected": -90.7692642211914,
|
|
"logps/rejected": -499.7203674316406,
|
|
"loss": 1.0306,
|
|
"margin_dpo/margin_mean": 152.30416870117188,
|
|
"margin_dpo/margin_std": 200.70108032226562,
|
|
"step": 361
|
|
},
|
|
{
|
|
"epoch": 0.5315712187958884,
|
|
"fcm_dpo/beta": 0.0034302864223718643,
|
|
"fcm_dpo/delta": -0.19361282885074615,
|
|
"fcm_dpo/margin": 226.57215881347656,
|
|
"fcm_dpo/q_t": 0.3321119248867035,
|
|
"grad_norm": 34.013301849365234,
|
|
"learning_rate": 2.679511629948319e-07,
|
|
"logits/chosen": -0.4100894331932068,
|
|
"logits/rejected": -0.4160076677799225,
|
|
"logps/chosen": -279.9183654785156,
|
|
"logps/ref_chosen": -58.639060974121094,
|
|
"logps/ref_rejected": -105.58195495605469,
|
|
"logps/rejected": -553.4334106445312,
|
|
"loss": 0.8572,
|
|
"margin_dpo/margin_mean": 226.57215881347656,
|
|
"margin_dpo/margin_std": 208.89010620117188,
|
|
"step": 362
|
|
},
|
|
{
|
|
"epoch": 0.5330396475770925,
|
|
"fcm_dpo/beta": 0.003321536350995302,
|
|
"fcm_dpo/delta": -0.20300859212875366,
|
|
"fcm_dpo/margin": 237.05946350097656,
|
|
"fcm_dpo/q_t": 0.32788795232772827,
|
|
"grad_norm": 25.153287887573242,
|
|
"learning_rate": 2.6667091441120816e-07,
|
|
"logits/chosen": -0.3687742054462433,
|
|
"logits/rejected": -0.3484545350074768,
|
|
"logps/chosen": -246.83908081054688,
|
|
"logps/ref_chosen": -44.558380126953125,
|
|
"logps/ref_rejected": -74.69496154785156,
|
|
"logps/rejected": -514.03515625,
|
|
"loss": 0.8467,
|
|
"margin_dpo/margin_mean": 237.0594482421875,
|
|
"margin_dpo/margin_std": 207.8511962890625,
|
|
"step": 363
|
|
},
|
|
{
|
|
"epoch": 0.5345080763582967,
|
|
"fcm_dpo/beta": 0.003306190250441432,
|
|
"fcm_dpo/delta": 0.02047722041606903,
|
|
"fcm_dpo/margin": 175.40988159179688,
|
|
"fcm_dpo/q_t": 0.37260884046554565,
|
|
"grad_norm": 35.79054641723633,
|
|
"learning_rate": 2.6539022653348575e-07,
|
|
"logits/chosen": -0.40862590074539185,
|
|
"logits/rejected": -0.41578084230422974,
|
|
"logps/chosen": -270.4530944824219,
|
|
"logps/ref_chosen": -48.894622802734375,
|
|
"logps/ref_rejected": -91.395751953125,
|
|
"logps/rejected": -488.36407470703125,
|
|
"loss": 0.9919,
|
|
"margin_dpo/margin_mean": 175.40988159179688,
|
|
"margin_dpo/margin_std": 205.0936737060547,
|
|
"step": 364
|
|
},
|
|
{
|
|
"epoch": 0.5359765051395007,
|
|
"fcm_dpo/beta": 0.0033004698343575,
|
|
"fcm_dpo/delta": 0.016225282102823257,
|
|
"fcm_dpo/margin": 177.1582794189453,
|
|
"fcm_dpo/q_t": 0.3734648823738098,
|
|
"grad_norm": 36.35362243652344,
|
|
"learning_rate": 2.641091331089811e-07,
|
|
"logits/chosen": -0.38673701882362366,
|
|
"logits/rejected": -0.3943500518798828,
|
|
"logps/chosen": -276.78192138671875,
|
|
"logps/ref_chosen": -51.49274444580078,
|
|
"logps/ref_rejected": -92.70166778564453,
|
|
"logps/rejected": -495.1491394042969,
|
|
"loss": 0.9778,
|
|
"margin_dpo/margin_mean": 177.1582794189453,
|
|
"margin_dpo/margin_std": 200.8518524169922,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.5374449339207048,
|
|
"fcm_dpo/beta": 0.0032841768115758896,
|
|
"fcm_dpo/delta": 0.0016945749521255493,
|
|
"fcm_dpo/margin": 182.0388946533203,
|
|
"fcm_dpo/q_t": 0.3690670430660248,
|
|
"grad_norm": 28.885610580444336,
|
|
"learning_rate": 2.6282766789569736e-07,
|
|
"logits/chosen": -0.36415767669677734,
|
|
"logits/rejected": -0.37583303451538086,
|
|
"logps/chosen": -248.95938110351562,
|
|
"logps/ref_chosen": -44.7205696105957,
|
|
"logps/ref_rejected": -83.31040954589844,
|
|
"logps/rejected": -469.588134765625,
|
|
"loss": 0.9792,
|
|
"margin_dpo/margin_mean": 182.0388946533203,
|
|
"margin_dpo/margin_std": 206.19607543945312,
|
|
"step": 366
|
|
},
|
|
{
|
|
"epoch": 0.5389133627019089,
|
|
"fcm_dpo/beta": 0.0033485644962638617,
|
|
"fcm_dpo/delta": 0.08677735924720764,
|
|
"fcm_dpo/margin": 154.576416015625,
|
|
"fcm_dpo/q_t": 0.3839992582798004,
|
|
"grad_norm": 24.783180236816406,
|
|
"learning_rate": 2.615458646614349e-07,
|
|
"logits/chosen": -0.39714837074279785,
|
|
"logits/rejected": -0.37421467900276184,
|
|
"logps/chosen": -266.13519287109375,
|
|
"logps/ref_chosen": -58.405418395996094,
|
|
"logps/ref_rejected": -76.75132751464844,
|
|
"logps/rejected": -439.0574951171875,
|
|
"loss": 1.016,
|
|
"margin_dpo/margin_mean": 154.576416015625,
|
|
"margin_dpo/margin_std": 178.5806884765625,
|
|
"step": 367
|
|
},
|
|
{
|
|
"epoch": 0.540381791483113,
|
|
"fcm_dpo/beta": 0.003304690821096301,
|
|
"fcm_dpo/delta": -0.12389971315860748,
|
|
"fcm_dpo/margin": 216.4133758544922,
|
|
"fcm_dpo/q_t": 0.33748477697372437,
|
|
"grad_norm": 53.478485107421875,
|
|
"learning_rate": 2.6026375718290083e-07,
|
|
"logits/chosen": -0.4093177318572998,
|
|
"logits/rejected": -0.4137042760848999,
|
|
"logps/chosen": -243.46835327148438,
|
|
"logps/ref_chosen": -44.452518463134766,
|
|
"logps/ref_rejected": -98.55526733398438,
|
|
"logps/rejected": -513.9844970703125,
|
|
"loss": 0.8591,
|
|
"margin_dpo/margin_mean": 216.4133758544922,
|
|
"margin_dpo/margin_std": 163.95318603515625,
|
|
"step": 368
|
|
},
|
|
{
|
|
"epoch": 0.5418502202643172,
|
|
"fcm_dpo/beta": 0.0033651897683739662,
|
|
"fcm_dpo/delta": 0.14462685585021973,
|
|
"fcm_dpo/margin": 136.97042846679688,
|
|
"fcm_dpo/q_t": 0.39847463369369507,
|
|
"grad_norm": 42.55653762817383,
|
|
"learning_rate": 2.589813792448196e-07,
|
|
"logits/chosen": -0.3915749788284302,
|
|
"logits/rejected": -0.3701251447200775,
|
|
"logps/chosen": -312.446533203125,
|
|
"logps/ref_chosen": -71.38150024414062,
|
|
"logps/ref_rejected": -91.29582214355469,
|
|
"logps/rejected": -469.33123779296875,
|
|
"loss": 1.083,
|
|
"margin_dpo/margin_mean": 136.9704132080078,
|
|
"margin_dpo/margin_std": 200.96856689453125,
|
|
"step": 369
|
|
},
|
|
{
|
|
"epoch": 0.5433186490455213,
|
|
"fcm_dpo/beta": 0.003478851169347763,
|
|
"fcm_dpo/delta": 0.18450786173343658,
|
|
"fcm_dpo/margin": 121.46247863769531,
|
|
"fcm_dpo/q_t": 0.40680354833602905,
|
|
"grad_norm": 36.18484878540039,
|
|
"learning_rate": 2.5769876463904263e-07,
|
|
"logits/chosen": -0.4049254357814789,
|
|
"logits/rejected": -0.39879292249679565,
|
|
"logps/chosen": -330.7537841796875,
|
|
"logps/ref_chosen": -71.60749816894531,
|
|
"logps/ref_rejected": -97.25978088378906,
|
|
"logps/rejected": -477.86859130859375,
|
|
"loss": 1.1047,
|
|
"margin_dpo/margin_mean": 121.46247863769531,
|
|
"margin_dpo/margin_std": 185.816162109375,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.5447870778267254,
|
|
"fcm_dpo/beta": 0.003517968812957406,
|
|
"fcm_dpo/delta": 0.012149915099143982,
|
|
"fcm_dpo/margin": 167.24020385742188,
|
|
"fcm_dpo/q_t": 0.37212252616882324,
|
|
"grad_norm": 32.21125030517578,
|
|
"learning_rate": 2.5641594716365744e-07,
|
|
"logits/chosen": -0.41469210386276245,
|
|
"logits/rejected": -0.39996105432510376,
|
|
"logps/chosen": -320.701171875,
|
|
"logps/ref_chosen": -69.41448974609375,
|
|
"logps/ref_rejected": -99.17217254638672,
|
|
"logps/rejected": -517.6990966796875,
|
|
"loss": 1.0022,
|
|
"margin_dpo/margin_mean": 167.24020385742188,
|
|
"margin_dpo/margin_std": 208.5806121826172,
|
|
"step": 371
|
|
},
|
|
{
|
|
"epoch": 0.5462555066079295,
|
|
"fcm_dpo/beta": 0.00341993011534214,
|
|
"fcm_dpo/delta": -0.14982974529266357,
|
|
"fcm_dpo/margin": 215.7982635498047,
|
|
"fcm_dpo/q_t": 0.3453604578971863,
|
|
"grad_norm": 25.910696029663086,
|
|
"learning_rate": 2.551329606220976e-07,
|
|
"logits/chosen": -0.4080229103565216,
|
|
"logits/rejected": -0.38077276945114136,
|
|
"logps/chosen": -302.6246643066406,
|
|
"logps/ref_chosen": -61.8179931640625,
|
|
"logps/ref_rejected": -78.53948974609375,
|
|
"logps/rejected": -535.1444091796875,
|
|
"loss": 0.9163,
|
|
"margin_dpo/margin_mean": 215.79824829101562,
|
|
"margin_dpo/margin_std": 238.89315795898438,
|
|
"step": 372
|
|
},
|
|
{
|
|
"epoch": 0.5477239353891337,
|
|
"fcm_dpo/beta": 0.0033825889695435762,
|
|
"fcm_dpo/delta": -0.09976015239953995,
|
|
"fcm_dpo/margin": 204.6912841796875,
|
|
"fcm_dpo/q_t": 0.3472912907600403,
|
|
"grad_norm": 46.24806213378906,
|
|
"learning_rate": 2.538498388222517e-07,
|
|
"logits/chosen": -0.3898068368434906,
|
|
"logits/rejected": -0.36087289452552795,
|
|
"logps/chosen": -328.5521240234375,
|
|
"logps/ref_chosen": -64.21713256835938,
|
|
"logps/ref_rejected": -85.95960998535156,
|
|
"logps/rejected": -554.9859619140625,
|
|
"loss": 0.9146,
|
|
"margin_dpo/margin_mean": 204.6912841796875,
|
|
"margin_dpo/margin_std": 197.73797607421875,
|
|
"step": 373
|
|
},
|
|
{
|
|
"epoch": 0.5491923641703378,
|
|
"fcm_dpo/beta": 0.0033088005147874355,
|
|
"fcm_dpo/delta": -0.009856566786766052,
|
|
"fcm_dpo/margin": 183.782958984375,
|
|
"fcm_dpo/q_t": 0.3736165761947632,
|
|
"grad_norm": 32.12522888183594,
|
|
"learning_rate": 2.525666155755725e-07,
|
|
"logits/chosen": -0.46630632877349854,
|
|
"logits/rejected": -0.44315189123153687,
|
|
"logps/chosen": -309.981201171875,
|
|
"logps/ref_chosen": -70.65018463134766,
|
|
"logps/ref_rejected": -93.64016723632812,
|
|
"logps/rejected": -516.754150390625,
|
|
"loss": 1.0042,
|
|
"margin_dpo/margin_mean": 183.782958984375,
|
|
"margin_dpo/margin_std": 240.5712890625,
|
|
"step": 374
|
|
},
|
|
{
|
|
"epoch": 0.5506607929515418,
|
|
"fcm_dpo/beta": 0.003304019570350647,
|
|
"fcm_dpo/delta": 0.007909119129180908,
|
|
"fcm_dpo/margin": 178.82217407226562,
|
|
"fcm_dpo/q_t": 0.3718445301055908,
|
|
"grad_norm": 32.83172607421875,
|
|
"learning_rate": 2.512833246961859e-07,
|
|
"logits/chosen": -0.4039704203605652,
|
|
"logits/rejected": -0.40034106373786926,
|
|
"logps/chosen": -325.0075378417969,
|
|
"logps/ref_chosen": -60.080223083496094,
|
|
"logps/ref_rejected": -88.93830871582031,
|
|
"logps/rejected": -532.6878051757812,
|
|
"loss": 1.0158,
|
|
"margin_dpo/margin_mean": 178.8221893310547,
|
|
"margin_dpo/margin_std": 229.82867431640625,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.5521292217327459,
|
|
"fcm_dpo/beta": 0.003269658889621496,
|
|
"fcm_dpo/delta": -0.13697855174541473,
|
|
"fcm_dpo/margin": 222.34677124023438,
|
|
"fcm_dpo/q_t": 0.3445650041103363,
|
|
"grad_norm": 41.24235153198242,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": -0.42362266778945923,
|
|
"logits/rejected": -0.41283369064331055,
|
|
"logps/chosen": -312.84063720703125,
|
|
"logps/ref_chosen": -62.660308837890625,
|
|
"logps/ref_rejected": -105.52660369873047,
|
|
"logps/rejected": -578.0537109375,
|
|
"loss": 0.9182,
|
|
"margin_dpo/margin_mean": 222.34677124023438,
|
|
"margin_dpo/margin_std": 240.65061950683594,
|
|
"step": 376
|
|
},
|
|
{
|
|
"epoch": 0.55359765051395,
|
|
"fcm_dpo/beta": 0.0032267069909721613,
|
|
"fcm_dpo/delta": -0.05978459492325783,
|
|
"fcm_dpo/margin": 203.26560974121094,
|
|
"fcm_dpo/q_t": 0.3565753102302551,
|
|
"grad_norm": 30.29025650024414,
|
|
"learning_rate": 2.487166753038141e-07,
|
|
"logits/chosen": -0.37672150135040283,
|
|
"logits/rejected": -0.3738042712211609,
|
|
"logps/chosen": -322.83746337890625,
|
|
"logps/ref_chosen": -54.478736877441406,
|
|
"logps/ref_rejected": -98.70335388183594,
|
|
"logps/rejected": -570.3276977539062,
|
|
"loss": 0.9434,
|
|
"margin_dpo/margin_mean": 203.265625,
|
|
"margin_dpo/margin_std": 216.9762420654297,
|
|
"step": 377
|
|
},
|
|
{
|
|
"epoch": 0.5550660792951542,
|
|
"fcm_dpo/beta": 0.0031417347490787506,
|
|
"fcm_dpo/delta": -0.11842157691717148,
|
|
"fcm_dpo/margin": 225.80239868164062,
|
|
"fcm_dpo/q_t": 0.3420916795730591,
|
|
"grad_norm": 36.97312545776367,
|
|
"learning_rate": 2.4743338442442754e-07,
|
|
"logits/chosen": -0.37459754943847656,
|
|
"logits/rejected": -0.3812897205352783,
|
|
"logps/chosen": -291.92535400390625,
|
|
"logps/ref_chosen": -45.02053451538086,
|
|
"logps/ref_rejected": -88.0469741821289,
|
|
"logps/rejected": -560.754150390625,
|
|
"loss": 0.8968,
|
|
"margin_dpo/margin_mean": 225.80239868164062,
|
|
"margin_dpo/margin_std": 211.446533203125,
|
|
"step": 378
|
|
},
|
|
{
|
|
"epoch": 0.5565345080763583,
|
|
"fcm_dpo/beta": 0.0030931192450225353,
|
|
"fcm_dpo/delta": -0.05514880642294884,
|
|
"fcm_dpo/margin": 210.42945861816406,
|
|
"fcm_dpo/q_t": 0.35938382148742676,
|
|
"grad_norm": 45.417083740234375,
|
|
"learning_rate": 2.461501611777483e-07,
|
|
"logits/chosen": -0.34937772154808044,
|
|
"logits/rejected": -0.36817657947540283,
|
|
"logps/chosen": -330.56414794921875,
|
|
"logps/ref_chosen": -53.182098388671875,
|
|
"logps/ref_rejected": -114.3001708984375,
|
|
"logps/rejected": -602.1116333007812,
|
|
"loss": 0.9546,
|
|
"margin_dpo/margin_mean": 210.429443359375,
|
|
"margin_dpo/margin_std": 237.06201171875,
|
|
"step": 379
|
|
},
|
|
{
|
|
"epoch": 0.5580029368575624,
|
|
"fcm_dpo/beta": 0.0030490856152027845,
|
|
"fcm_dpo/delta": -0.1288592517375946,
|
|
"fcm_dpo/margin": 236.06048583984375,
|
|
"fcm_dpo/q_t": 0.34213754534721375,
|
|
"grad_norm": 34.14671325683594,
|
|
"learning_rate": 2.4486703937790243e-07,
|
|
"logits/chosen": -0.38159555196762085,
|
|
"logits/rejected": -0.4034666419029236,
|
|
"logps/chosen": -325.86065673828125,
|
|
"logps/ref_chosen": -51.3530387878418,
|
|
"logps/ref_rejected": -104.19169616699219,
|
|
"logps/rejected": -614.759765625,
|
|
"loss": 0.9184,
|
|
"margin_dpo/margin_mean": 236.06048583984375,
|
|
"margin_dpo/margin_std": 250.18896484375,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5594713656387665,
|
|
"fcm_dpo/beta": 0.0030402536503970623,
|
|
"fcm_dpo/delta": 0.06344518810510635,
|
|
"fcm_dpo/margin": 177.61572265625,
|
|
"fcm_dpo/q_t": 0.38519516587257385,
|
|
"grad_norm": 35.91572189331055,
|
|
"learning_rate": 2.435840528363426e-07,
|
|
"logits/chosen": -0.4085123538970947,
|
|
"logits/rejected": -0.386846125125885,
|
|
"logps/chosen": -332.9157409667969,
|
|
"logps/ref_chosen": -57.80306625366211,
|
|
"logps/ref_rejected": -79.21940612792969,
|
|
"logps/rejected": -531.9478149414062,
|
|
"loss": 1.0584,
|
|
"margin_dpo/margin_mean": 177.61572265625,
|
|
"margin_dpo/margin_std": 262.201904296875,
|
|
"step": 381
|
|
},
|
|
{
|
|
"epoch": 0.5609397944199707,
|
|
"fcm_dpo/beta": 0.0030455852393060923,
|
|
"fcm_dpo/delta": -0.024747373536229134,
|
|
"fcm_dpo/margin": 204.61878967285156,
|
|
"fcm_dpo/q_t": 0.3605649173259735,
|
|
"grad_norm": 28.64333724975586,
|
|
"learning_rate": 2.4230123536095745e-07,
|
|
"logits/chosen": -0.43623343110084534,
|
|
"logits/rejected": -0.43505585193634033,
|
|
"logps/chosen": -338.3182067871094,
|
|
"logps/ref_chosen": -66.02030181884766,
|
|
"logps/ref_rejected": -110.71016693115234,
|
|
"logps/rejected": -587.6268310546875,
|
|
"loss": 0.9376,
|
|
"margin_dpo/margin_mean": 204.6187744140625,
|
|
"margin_dpo/margin_std": 196.5201416015625,
|
|
"step": 382
|
|
},
|
|
{
|
|
"epoch": 0.5624082232011748,
|
|
"fcm_dpo/beta": 0.003025210928171873,
|
|
"fcm_dpo/delta": -0.03023645095527172,
|
|
"fcm_dpo/margin": 207.67013549804688,
|
|
"fcm_dpo/q_t": 0.36507272720336914,
|
|
"grad_norm": 36.634037017822266,
|
|
"learning_rate": 2.4101862075518037e-07,
|
|
"logits/chosen": -0.38792383670806885,
|
|
"logits/rejected": -0.39373254776000977,
|
|
"logps/chosen": -328.1868896484375,
|
|
"logps/ref_chosen": -50.39148712158203,
|
|
"logps/ref_rejected": -93.71589660644531,
|
|
"logps/rejected": -579.181396484375,
|
|
"loss": 0.9913,
|
|
"margin_dpo/margin_mean": 207.67013549804688,
|
|
"margin_dpo/margin_std": 263.74371337890625,
|
|
"step": 383
|
|
},
|
|
{
|
|
"epoch": 0.5638766519823789,
|
|
"fcm_dpo/beta": 0.003078867681324482,
|
|
"fcm_dpo/delta": 0.08402273058891296,
|
|
"fcm_dpo/margin": 168.70809936523438,
|
|
"fcm_dpo/q_t": 0.38155868649482727,
|
|
"grad_norm": 27.592374801635742,
|
|
"learning_rate": 2.397362428170992e-07,
|
|
"logits/chosen": -0.425597608089447,
|
|
"logits/rejected": -0.4135643243789673,
|
|
"logps/chosen": -330.5281677246094,
|
|
"logps/ref_chosen": -52.046104431152344,
|
|
"logps/ref_rejected": -85.76089477539062,
|
|
"logps/rejected": -532.9510498046875,
|
|
"loss": 0.9975,
|
|
"margin_dpo/margin_mean": 168.70809936523438,
|
|
"margin_dpo/margin_std": 170.51783752441406,
|
|
"step": 384
|
|
},
|
|
{
|
|
"epoch": 0.5653450807635829,
|
|
"fcm_dpo/beta": 0.0030736164189875126,
|
|
"fcm_dpo/delta": -0.00494825653731823,
|
|
"fcm_dpo/margin": 196.7218475341797,
|
|
"fcm_dpo/q_t": 0.3634437322616577,
|
|
"grad_norm": 29.260663986206055,
|
|
"learning_rate": 2.3845413533856514e-07,
|
|
"logits/chosen": -0.4318080544471741,
|
|
"logits/rejected": -0.39295560121536255,
|
|
"logps/chosen": -317.27215576171875,
|
|
"logps/ref_chosen": -65.55215454101562,
|
|
"logps/ref_rejected": -77.82792663574219,
|
|
"logps/rejected": -526.269775390625,
|
|
"loss": 0.9452,
|
|
"margin_dpo/margin_mean": 196.7218475341797,
|
|
"margin_dpo/margin_std": 186.86444091796875,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.566813509544787,
|
|
"fcm_dpo/beta": 0.003056345507502556,
|
|
"fcm_dpo/delta": -0.04953234642744064,
|
|
"fcm_dpo/margin": 211.47879028320312,
|
|
"fcm_dpo/q_t": 0.3594558835029602,
|
|
"grad_norm": 32.3004264831543,
|
|
"learning_rate": 2.3717233210430254e-07,
|
|
"logits/chosen": -0.4355229139328003,
|
|
"logits/rejected": -0.4245448112487793,
|
|
"logps/chosen": -326.9559326171875,
|
|
"logps/ref_chosen": -58.22185516357422,
|
|
"logps/ref_rejected": -92.32742309570312,
|
|
"logps/rejected": -572.5403442382812,
|
|
"loss": 0.9513,
|
|
"margin_dpo/margin_mean": 211.4788055419922,
|
|
"margin_dpo/margin_std": 234.755859375,
|
|
"step": 386
|
|
},
|
|
{
|
|
"epoch": 0.5682819383259912,
|
|
"fcm_dpo/beta": 0.003059773938730359,
|
|
"fcm_dpo/delta": 0.06919734925031662,
|
|
"fcm_dpo/margin": 174.63983154296875,
|
|
"fcm_dpo/q_t": 0.37939631938934326,
|
|
"grad_norm": 37.251277923583984,
|
|
"learning_rate": 2.3589086689101889e-07,
|
|
"logits/chosen": -0.5007586479187012,
|
|
"logits/rejected": -0.4742280840873718,
|
|
"logps/chosen": -350.56463623046875,
|
|
"logps/ref_chosen": -66.41944885253906,
|
|
"logps/ref_rejected": -92.16915893554688,
|
|
"logps/rejected": -550.9541625976562,
|
|
"loss": 0.9959,
|
|
"margin_dpo/margin_mean": 174.63983154296875,
|
|
"margin_dpo/margin_std": 184.31121826171875,
|
|
"step": 387
|
|
},
|
|
{
|
|
"epoch": 0.5697503671071953,
|
|
"fcm_dpo/beta": 0.0030251401476562023,
|
|
"fcm_dpo/delta": -0.11198361963033676,
|
|
"fcm_dpo/margin": 232.57754516601562,
|
|
"fcm_dpo/q_t": 0.34893184900283813,
|
|
"grad_norm": 34.62775421142578,
|
|
"learning_rate": 2.3460977346651428e-07,
|
|
"logits/chosen": -0.37753403186798096,
|
|
"logits/rejected": -0.384876012802124,
|
|
"logps/chosen": -325.88348388671875,
|
|
"logps/ref_chosen": -50.129459381103516,
|
|
"logps/ref_rejected": -104.43305969238281,
|
|
"logps/rejected": -612.7646484375,
|
|
"loss": 0.9198,
|
|
"margin_dpo/margin_mean": 232.57754516601562,
|
|
"margin_dpo/margin_std": 249.87030029296875,
|
|
"step": 388
|
|
},
|
|
{
|
|
"epoch": 0.5712187958883994,
|
|
"fcm_dpo/beta": 0.0030121058225631714,
|
|
"fcm_dpo/delta": -0.0019872207194566727,
|
|
"fcm_dpo/margin": 199.80288696289062,
|
|
"fcm_dpo/q_t": 0.36984607577323914,
|
|
"grad_norm": 36.19541549682617,
|
|
"learning_rate": 2.3332908558879177e-07,
|
|
"logits/chosen": -0.45936498045921326,
|
|
"logits/rejected": -0.44065529108047485,
|
|
"logps/chosen": -367.11614990234375,
|
|
"logps/ref_chosen": -57.906593322753906,
|
|
"logps/ref_rejected": -77.91454315185547,
|
|
"logps/rejected": -586.927001953125,
|
|
"loss": 0.9889,
|
|
"margin_dpo/margin_mean": 199.80288696289062,
|
|
"margin_dpo/margin_std": 242.24429321289062,
|
|
"step": 389
|
|
},
|
|
{
|
|
"epoch": 0.5726872246696035,
|
|
"fcm_dpo/beta": 0.0030045639723539352,
|
|
"fcm_dpo/delta": 0.008383665233850479,
|
|
"fcm_dpo/margin": 196.9191131591797,
|
|
"fcm_dpo/q_t": 0.37548089027404785,
|
|
"grad_norm": 35.78683853149414,
|
|
"learning_rate": 2.320488370051681e-07,
|
|
"logits/chosen": -0.4140510559082031,
|
|
"logits/rejected": -0.40045636892318726,
|
|
"logps/chosen": -360.5129699707031,
|
|
"logps/ref_chosen": -49.22591781616211,
|
|
"logps/ref_rejected": -85.5281982421875,
|
|
"logps/rejected": -593.734375,
|
|
"loss": 1.0231,
|
|
"margin_dpo/margin_mean": 196.9191131591797,
|
|
"margin_dpo/margin_std": 272.89666748046875,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.5741556534508077,
|
|
"fcm_dpo/beta": 0.0031170903239399195,
|
|
"fcm_dpo/delta": 0.19101738929748535,
|
|
"fcm_dpo/margin": 133.54165649414062,
|
|
"fcm_dpo/q_t": 0.41037672758102417,
|
|
"grad_norm": 45.97058868408203,
|
|
"learning_rate": 2.3076906145138405e-07,
|
|
"logits/chosen": -0.41732552647590637,
|
|
"logits/rejected": -0.40580475330352783,
|
|
"logps/chosen": -364.70263671875,
|
|
"logps/ref_chosen": -64.32965087890625,
|
|
"logps/ref_rejected": -86.73820495605469,
|
|
"logps/rejected": -520.65283203125,
|
|
"loss": 1.1084,
|
|
"margin_dpo/margin_mean": 133.54165649414062,
|
|
"margin_dpo/margin_std": 206.95831298828125,
|
|
"step": 391
|
|
},
|
|
{
|
|
"epoch": 0.5756240822320118,
|
|
"fcm_dpo/beta": 0.003100549802184105,
|
|
"fcm_dpo/delta": -0.10810496658086777,
|
|
"fcm_dpo/margin": 225.9717559814453,
|
|
"fcm_dpo/q_t": 0.3456001877784729,
|
|
"grad_norm": 33.09029769897461,
|
|
"learning_rate": 2.294897926507156e-07,
|
|
"logits/chosen": -0.4238108992576599,
|
|
"logits/rejected": -0.41437631845474243,
|
|
"logps/chosen": -324.51971435546875,
|
|
"logps/ref_chosen": -53.50397872924805,
|
|
"logps/ref_rejected": -102.34584045410156,
|
|
"logps/rejected": -599.3333129882812,
|
|
"loss": 0.8946,
|
|
"margin_dpo/margin_mean": 225.9717559814453,
|
|
"margin_dpo/margin_std": 207.6659393310547,
|
|
"step": 392
|
|
},
|
|
{
|
|
"epoch": 0.5770925110132159,
|
|
"fcm_dpo/beta": 0.003088606521487236,
|
|
"fcm_dpo/delta": 0.04707473888993263,
|
|
"fcm_dpo/margin": 179.875244140625,
|
|
"fcm_dpo/q_t": 0.3865922689437866,
|
|
"grad_norm": 37.300777435302734,
|
|
"learning_rate": 2.2821106431308543e-07,
|
|
"logits/chosen": -0.4085625410079956,
|
|
"logits/rejected": -0.40135449171066284,
|
|
"logps/chosen": -313.4761962890625,
|
|
"logps/ref_chosen": -46.473915100097656,
|
|
"logps/ref_rejected": -71.96885681152344,
|
|
"logps/rejected": -518.846435546875,
|
|
"loss": 1.0513,
|
|
"margin_dpo/margin_mean": 179.87525939941406,
|
|
"margin_dpo/margin_std": 273.9698791503906,
|
|
"step": 393
|
|
},
|
|
{
|
|
"epoch": 0.57856093979442,
|
|
"fcm_dpo/beta": 0.003088343655690551,
|
|
"fcm_dpo/delta": -0.005594061687588692,
|
|
"fcm_dpo/margin": 195.913330078125,
|
|
"fcm_dpo/q_t": 0.371346116065979,
|
|
"grad_norm": 37.6540641784668,
|
|
"learning_rate": 2.2693291013417452e-07,
|
|
"logits/chosen": -0.4252457022666931,
|
|
"logits/rejected": -0.4192967116832733,
|
|
"logps/chosen": -353.8863525390625,
|
|
"logps/ref_chosen": -52.91154861450195,
|
|
"logps/ref_rejected": -90.8226318359375,
|
|
"logps/rejected": -587.7107543945312,
|
|
"loss": 0.9933,
|
|
"margin_dpo/margin_mean": 195.913330078125,
|
|
"margin_dpo/margin_std": 244.0496826171875,
|
|
"step": 394
|
|
},
|
|
{
|
|
"epoch": 0.580029368575624,
|
|
"fcm_dpo/beta": 0.0030713113956153393,
|
|
"fcm_dpo/delta": -0.03419704735279083,
|
|
"fcm_dpo/margin": 205.683837890625,
|
|
"fcm_dpo/q_t": 0.36777931451797485,
|
|
"grad_norm": 43.52488708496094,
|
|
"learning_rate": 2.2565536379453404e-07,
|
|
"logits/chosen": -0.43619823455810547,
|
|
"logits/rejected": -0.4260198175907135,
|
|
"logps/chosen": -357.0216369628906,
|
|
"logps/ref_chosen": -62.546112060546875,
|
|
"logps/ref_rejected": -83.78262329101562,
|
|
"logps/rejected": -583.9420166015625,
|
|
"loss": 0.9959,
|
|
"margin_dpo/margin_mean": 205.683837890625,
|
|
"margin_dpo/margin_std": 268.2707214355469,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.5814977973568282,
|
|
"fcm_dpo/beta": 0.0030712145380675793,
|
|
"fcm_dpo/delta": 0.00153273344039917,
|
|
"fcm_dpo/margin": 194.87359619140625,
|
|
"fcm_dpo/q_t": 0.36835265159606934,
|
|
"grad_norm": 34.06664276123047,
|
|
"learning_rate": 2.2437845895869825e-07,
|
|
"logits/chosen": -0.4365978240966797,
|
|
"logits/rejected": -0.40913087129592896,
|
|
"logps/chosen": -365.53350830078125,
|
|
"logps/ref_chosen": -68.99594116210938,
|
|
"logps/ref_rejected": -88.64665985107422,
|
|
"logps/rejected": -580.0578002929688,
|
|
"loss": 0.9636,
|
|
"margin_dpo/margin_mean": 194.87359619140625,
|
|
"margin_dpo/margin_std": 210.96783447265625,
|
|
"step": 396
|
|
},
|
|
{
|
|
"epoch": 0.5829662261380323,
|
|
"fcm_dpo/beta": 0.0029885326512157917,
|
|
"fcm_dpo/delta": -0.15819747745990753,
|
|
"fcm_dpo/margin": 249.2646942138672,
|
|
"fcm_dpo/q_t": 0.3350715637207031,
|
|
"grad_norm": 35.845821380615234,
|
|
"learning_rate": 2.2310222927429716e-07,
|
|
"logits/chosen": -0.40235501527786255,
|
|
"logits/rejected": -0.3899458944797516,
|
|
"logps/chosen": -331.36700439453125,
|
|
"logps/ref_chosen": -61.27716827392578,
|
|
"logps/ref_rejected": -103.11612701416016,
|
|
"logps/rejected": -622.4706420898438,
|
|
"loss": 0.8687,
|
|
"margin_dpo/margin_mean": 249.26467895507812,
|
|
"margin_dpo/margin_std": 219.42138671875,
|
|
"step": 397
|
|
},
|
|
{
|
|
"epoch": 0.5844346549192364,
|
|
"fcm_dpo/beta": 0.0029659317806363106,
|
|
"fcm_dpo/delta": -0.029825370758771896,
|
|
"fcm_dpo/margin": 211.71212768554688,
|
|
"fcm_dpo/q_t": 0.3643757104873657,
|
|
"grad_norm": 33.558929443359375,
|
|
"learning_rate": 2.2182670837116972e-07,
|
|
"logits/chosen": -0.46954572200775146,
|
|
"logits/rejected": -0.458831787109375,
|
|
"logps/chosen": -372.77703857421875,
|
|
"logps/ref_chosen": -68.15155029296875,
|
|
"logps/ref_rejected": -108.52360534667969,
|
|
"logps/rejected": -624.8612060546875,
|
|
"loss": 0.9628,
|
|
"margin_dpo/margin_mean": 211.71212768554688,
|
|
"margin_dpo/margin_std": 241.5102081298828,
|
|
"step": 398
|
|
},
|
|
{
|
|
"epoch": 0.5859030837004405,
|
|
"fcm_dpo/beta": 0.0029565119184553623,
|
|
"fcm_dpo/delta": 0.0046921707689762115,
|
|
"fcm_dpo/margin": 201.39321899414062,
|
|
"fcm_dpo/q_t": 0.3706039786338806,
|
|
"grad_norm": 38.02647399902344,
|
|
"learning_rate": 2.2055192986047804e-07,
|
|
"logits/chosen": -0.4315683841705322,
|
|
"logits/rejected": -0.38041120767593384,
|
|
"logps/chosen": -330.4461975097656,
|
|
"logps/ref_chosen": -60.889801025390625,
|
|
"logps/ref_rejected": -77.965576171875,
|
|
"logps/rejected": -548.9151611328125,
|
|
"loss": 0.9914,
|
|
"margin_dpo/margin_mean": 201.39321899414062,
|
|
"margin_dpo/margin_std": 242.23544311523438,
|
|
"step": 399
|
|
},
|
|
{
|
|
"epoch": 0.5873715124816447,
|
|
"fcm_dpo/beta": 0.002852116245776415,
|
|
"fcm_dpo/delta": -0.23298129439353943,
|
|
"fcm_dpo/margin": 284.9051818847656,
|
|
"fcm_dpo/q_t": 0.31991565227508545,
|
|
"grad_norm": 30.26348876953125,
|
|
"learning_rate": 2.192779273338215e-07,
|
|
"logits/chosen": -0.40956538915634155,
|
|
"logits/rejected": -0.3953632712364197,
|
|
"logps/chosen": -309.8514404296875,
|
|
"logps/ref_chosen": -63.64359664916992,
|
|
"logps/ref_rejected": -105.252685546875,
|
|
"logps/rejected": -636.36572265625,
|
|
"loss": 0.8283,
|
|
"margin_dpo/margin_mean": 284.9051818847656,
|
|
"margin_dpo/margin_std": 236.2659912109375,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5873715124816447,
|
|
"eval_fcm_dpo/beta": 0.0028311661444604397,
|
|
"eval_logits/chosen": -0.44635966420173645,
|
|
"eval_logits/rejected": -0.42632707953453064,
|
|
"eval_logps/chosen": -410.6361999511719,
|
|
"eval_logps/ref_chosen": -79.05104064941406,
|
|
"eval_logps/ref_rejected": -86.79793548583984,
|
|
"eval_logps/rejected": -548.2955932617188,
|
|
"eval_loss": 0.579971969127655,
|
|
"eval_margin_dpo/margin_mean": 129.91244506835938,
|
|
"eval_margin_dpo/margin_std": 237.20855712890625,
|
|
"eval_runtime": 39.3402,
|
|
"eval_samples_per_second": 59.456,
|
|
"eval_steps_per_second": 1.881,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5888399412628488,
|
|
"fcm_dpo/beta": 0.0028878701850771904,
|
|
"fcm_dpo/delta": 0.15109677612781525,
|
|
"fcm_dpo/margin": 157.91604614257812,
|
|
"fcm_dpo/q_t": 0.40476858615875244,
|
|
"grad_norm": 35.52607727050781,
|
|
"learning_rate": 2.1800473436235136e-07,
|
|
"logits/chosen": -0.39845049381256104,
|
|
"logits/rejected": -0.3905218243598938,
|
|
"logps/chosen": -345.0669250488281,
|
|
"logps/ref_chosen": -57.16303253173828,
|
|
"logps/ref_rejected": -83.79249572753906,
|
|
"logps/rejected": -529.6124267578125,
|
|
"loss": 1.1349,
|
|
"margin_dpo/margin_mean": 157.91604614257812,
|
|
"margin_dpo/margin_std": 286.06787109375,
|
|
"step": 401
|
|
},
|
|
{
|
|
"epoch": 0.5903083700440529,
|
|
"fcm_dpo/beta": 0.002814686391502619,
|
|
"fcm_dpo/delta": -0.2406725436449051,
|
|
"fcm_dpo/margin": 291.61102294921875,
|
|
"fcm_dpo/q_t": 0.3189072012901306,
|
|
"grad_norm": 26.38422203063965,
|
|
"learning_rate": 2.1673238449588665e-07,
|
|
"logits/chosen": -0.423443078994751,
|
|
"logits/rejected": -0.3997713625431061,
|
|
"logps/chosen": -261.9085998535156,
|
|
"logps/ref_chosen": -50.74037170410156,
|
|
"logps/ref_rejected": -81.0460433959961,
|
|
"logps/rejected": -583.8253173828125,
|
|
"loss": 0.814,
|
|
"margin_dpo/margin_mean": 291.61102294921875,
|
|
"margin_dpo/margin_std": 232.8087158203125,
|
|
"step": 402
|
|
},
|
|
{
|
|
"epoch": 0.591776798825257,
|
|
"fcm_dpo/beta": 0.002770167775452137,
|
|
"fcm_dpo/delta": -0.014987404458224773,
|
|
"fcm_dpo/margin": 221.64427185058594,
|
|
"fcm_dpo/q_t": 0.36400288343429565,
|
|
"grad_norm": 28.025609970092773,
|
|
"learning_rate": 2.154609112620295e-07,
|
|
"logits/chosen": -0.3887700140476227,
|
|
"logits/rejected": -0.3797057271003723,
|
|
"logps/chosen": -287.8169860839844,
|
|
"logps/ref_chosen": -47.14731216430664,
|
|
"logps/ref_rejected": -77.2666015625,
|
|
"logps/rejected": -539.58056640625,
|
|
"loss": 0.9562,
|
|
"margin_dpo/margin_mean": 221.64425659179688,
|
|
"margin_dpo/margin_std": 230.0044708251953,
|
|
"step": 403
|
|
},
|
|
{
|
|
"epoch": 0.593245227606461,
|
|
"fcm_dpo/beta": 0.0027550768572837114,
|
|
"fcm_dpo/delta": -0.033548858016729355,
|
|
"fcm_dpo/margin": 229.15975952148438,
|
|
"fcm_dpo/q_t": 0.36247456073760986,
|
|
"grad_norm": 29.232345581054688,
|
|
"learning_rate": 2.1419034816528218e-07,
|
|
"logits/chosen": -0.35594630241394043,
|
|
"logits/rejected": -0.3386983871459961,
|
|
"logps/chosen": -307.4197998046875,
|
|
"logps/ref_chosen": -47.875274658203125,
|
|
"logps/ref_rejected": -77.15499877929688,
|
|
"logps/rejected": -565.8592529296875,
|
|
"loss": 0.9693,
|
|
"margin_dpo/margin_mean": 229.15975952148438,
|
|
"margin_dpo/margin_std": 263.5882568359375,
|
|
"step": 404
|
|
},
|
|
{
|
|
"epoch": 0.5947136563876652,
|
|
"fcm_dpo/beta": 0.0028160212095826864,
|
|
"fcm_dpo/delta": 0.0803837925195694,
|
|
"fcm_dpo/margin": 184.99761962890625,
|
|
"fcm_dpo/q_t": 0.38957273960113525,
|
|
"grad_norm": 46.26492691040039,
|
|
"learning_rate": 2.129207286861638e-07,
|
|
"logits/chosen": -0.3900597095489502,
|
|
"logits/rejected": -0.37792617082595825,
|
|
"logps/chosen": -363.26409912109375,
|
|
"logps/ref_chosen": -65.16290283203125,
|
|
"logps/ref_rejected": -87.18678283691406,
|
|
"logps/rejected": -570.2855834960938,
|
|
"loss": 1.0657,
|
|
"margin_dpo/margin_mean": 184.99761962890625,
|
|
"margin_dpo/margin_std": 266.91748046875,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.5961820851688693,
|
|
"fcm_dpo/beta": 0.002798373345285654,
|
|
"fcm_dpo/delta": -0.047089073807001114,
|
|
"fcm_dpo/margin": 229.93914794921875,
|
|
"fcm_dpo/q_t": 0.3619244396686554,
|
|
"grad_norm": 38.67859649658203,
|
|
"learning_rate": 2.1165208628032861e-07,
|
|
"logits/chosen": -0.4114735722541809,
|
|
"logits/rejected": -0.4083394408226013,
|
|
"logps/chosen": -326.46710205078125,
|
|
"logps/ref_chosen": -49.740814208984375,
|
|
"logps/ref_rejected": -92.07862854003906,
|
|
"logps/rejected": -598.744140625,
|
|
"loss": 0.9584,
|
|
"margin_dpo/margin_mean": 229.93914794921875,
|
|
"margin_dpo/margin_std": 259.556640625,
|
|
"step": 406
|
|
},
|
|
{
|
|
"epoch": 0.5976505139500734,
|
|
"fcm_dpo/beta": 0.002859012223780155,
|
|
"fcm_dpo/delta": 0.14943772554397583,
|
|
"fcm_dpo/margin": 159.05093383789062,
|
|
"fcm_dpo/q_t": 0.3990488648414612,
|
|
"grad_norm": 51.8092041015625,
|
|
"learning_rate": 2.1038445437768375e-07,
|
|
"logits/chosen": -0.41653358936309814,
|
|
"logits/rejected": -0.38666877150535583,
|
|
"logps/chosen": -360.2216796875,
|
|
"logps/ref_chosen": -56.33069610595703,
|
|
"logps/ref_rejected": -77.51209259033203,
|
|
"logps/rejected": -540.4539794921875,
|
|
"loss": 1.1054,
|
|
"margin_dpo/margin_mean": 159.05093383789062,
|
|
"margin_dpo/margin_std": 249.28366088867188,
|
|
"step": 407
|
|
},
|
|
{
|
|
"epoch": 0.5991189427312775,
|
|
"fcm_dpo/beta": 0.0029165763407945633,
|
|
"fcm_dpo/delta": 0.1471884399652481,
|
|
"fcm_dpo/margin": 157.62078857421875,
|
|
"fcm_dpo/q_t": 0.39709725975990295,
|
|
"grad_norm": 37.17081832885742,
|
|
"learning_rate": 2.0911786638150872e-07,
|
|
"logits/chosen": -0.43049564957618713,
|
|
"logits/rejected": -0.4026661217212677,
|
|
"logps/chosen": -369.4244384765625,
|
|
"logps/ref_chosen": -69.789306640625,
|
|
"logps/ref_rejected": -90.09693908691406,
|
|
"logps/rejected": -547.3529052734375,
|
|
"loss": 1.0566,
|
|
"margin_dpo/margin_mean": 157.62078857421875,
|
|
"margin_dpo/margin_std": 198.9236602783203,
|
|
"step": 408
|
|
},
|
|
{
|
|
"epoch": 0.6005873715124816,
|
|
"fcm_dpo/beta": 0.0029886472038924694,
|
|
"fcm_dpo/delta": 0.08729657530784607,
|
|
"fcm_dpo/margin": 172.93191528320312,
|
|
"fcm_dpo/q_t": 0.38554543256759644,
|
|
"grad_norm": 46.94636154174805,
|
|
"learning_rate": 2.0785235566757517e-07,
|
|
"logits/chosen": -0.3939806818962097,
|
|
"logits/rejected": -0.3780689239501953,
|
|
"logps/chosen": -353.55657958984375,
|
|
"logps/ref_chosen": -67.31744384765625,
|
|
"logps/ref_rejected": -84.904296875,
|
|
"logps/rejected": -544.0753173828125,
|
|
"loss": 1.0233,
|
|
"margin_dpo/margin_mean": 172.93191528320312,
|
|
"margin_dpo/margin_std": 209.38848876953125,
|
|
"step": 409
|
|
},
|
|
{
|
|
"epoch": 0.6020558002936858,
|
|
"fcm_dpo/beta": 0.0030058538541197777,
|
|
"fcm_dpo/delta": 0.03576916456222534,
|
|
"fcm_dpo/margin": 188.38836669921875,
|
|
"fcm_dpo/q_t": 0.3729361891746521,
|
|
"grad_norm": 36.37736129760742,
|
|
"learning_rate": 2.065879555832674e-07,
|
|
"logits/chosen": -0.3966106176376343,
|
|
"logits/rejected": -0.39567676186561584,
|
|
"logps/chosen": -317.6330261230469,
|
|
"logps/ref_chosen": -51.465354919433594,
|
|
"logps/ref_rejected": -83.198974609375,
|
|
"logps/rejected": -537.7550048828125,
|
|
"loss": 0.9902,
|
|
"margin_dpo/margin_mean": 188.38836669921875,
|
|
"margin_dpo/margin_std": 211.78468322753906,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.6035242290748899,
|
|
"fcm_dpo/beta": 0.0030007443856447935,
|
|
"fcm_dpo/delta": 0.03217388689517975,
|
|
"fcm_dpo/margin": 189.33251953125,
|
|
"fcm_dpo/q_t": 0.37786054611206055,
|
|
"grad_norm": 50.11013412475586,
|
|
"learning_rate": 2.0532469944670343e-07,
|
|
"logits/chosen": -0.3935418725013733,
|
|
"logits/rejected": -0.3961487412452698,
|
|
"logps/chosen": -325.54656982421875,
|
|
"logps/ref_chosen": -52.30727005004883,
|
|
"logps/ref_rejected": -80.69495391845703,
|
|
"logps/rejected": -543.2667846679688,
|
|
"loss": 1.0067,
|
|
"margin_dpo/margin_mean": 189.33250427246094,
|
|
"margin_dpo/margin_std": 229.78692626953125,
|
|
"step": 411
|
|
},
|
|
{
|
|
"epoch": 0.604992657856094,
|
|
"fcm_dpo/beta": 0.0030541394371539354,
|
|
"fcm_dpo/delta": 0.038746751844882965,
|
|
"fcm_dpo/margin": 184.47250366210938,
|
|
"fcm_dpo/q_t": 0.37744957208633423,
|
|
"grad_norm": 60.06779861450195,
|
|
"learning_rate": 2.0406262054585738e-07,
|
|
"logits/chosen": -0.4395294785499573,
|
|
"logits/rejected": -0.45951640605926514,
|
|
"logps/chosen": -333.88372802734375,
|
|
"logps/ref_chosen": -53.144126892089844,
|
|
"logps/ref_rejected": -100.0608139038086,
|
|
"logps/rejected": -565.27294921875,
|
|
"loss": 1.0095,
|
|
"margin_dpo/margin_mean": 184.47250366210938,
|
|
"margin_dpo/margin_std": 230.4259490966797,
|
|
"step": 412
|
|
},
|
|
{
|
|
"epoch": 0.6064610866372981,
|
|
"fcm_dpo/beta": 0.003098301589488983,
|
|
"fcm_dpo/delta": 0.049318261444568634,
|
|
"fcm_dpo/margin": 178.3287353515625,
|
|
"fcm_dpo/q_t": 0.37706446647644043,
|
|
"grad_norm": 51.303245544433594,
|
|
"learning_rate": 2.0280175213768205e-07,
|
|
"logits/chosen": -0.4190981388092041,
|
|
"logits/rejected": -0.414195716381073,
|
|
"logps/chosen": -356.578369140625,
|
|
"logps/ref_chosen": -61.58196258544922,
|
|
"logps/ref_rejected": -99.47340393066406,
|
|
"logps/rejected": -572.7985229492188,
|
|
"loss": 1.0196,
|
|
"margin_dpo/margin_mean": 178.3287353515625,
|
|
"margin_dpo/margin_std": 226.3348388671875,
|
|
"step": 413
|
|
},
|
|
{
|
|
"epoch": 0.6079295154185022,
|
|
"fcm_dpo/beta": 0.0030625397339463234,
|
|
"fcm_dpo/delta": -0.10935955494642258,
|
|
"fcm_dpo/margin": 229.00564575195312,
|
|
"fcm_dpo/q_t": 0.3444908857345581,
|
|
"grad_norm": 45.593589782714844,
|
|
"learning_rate": 2.0154212744723247e-07,
|
|
"logits/chosen": -0.377156138420105,
|
|
"logits/rejected": -0.3661056160926819,
|
|
"logps/chosen": -312.2870178222656,
|
|
"logps/ref_chosen": -46.63148498535156,
|
|
"logps/ref_rejected": -87.64653015136719,
|
|
"logps/rejected": -582.3077392578125,
|
|
"loss": 0.9087,
|
|
"margin_dpo/margin_mean": 229.00564575195312,
|
|
"margin_dpo/margin_std": 222.5440216064453,
|
|
"step": 414
|
|
},
|
|
{
|
|
"epoch": 0.6093979441997063,
|
|
"fcm_dpo/beta": 0.0030559361912310123,
|
|
"fcm_dpo/delta": 0.09887358546257019,
|
|
"fcm_dpo/margin": 165.65187072753906,
|
|
"fcm_dpo/q_t": 0.3890509009361267,
|
|
"grad_norm": 40.08522033691406,
|
|
"learning_rate": 2.002837796667909e-07,
|
|
"logits/chosen": -0.43923091888427734,
|
|
"logits/rejected": -0.4275917410850525,
|
|
"logps/chosen": -386.06353759765625,
|
|
"logps/ref_chosen": -78.6182861328125,
|
|
"logps/ref_rejected": -100.47752380371094,
|
|
"logps/rejected": -573.5745849609375,
|
|
"loss": 1.0445,
|
|
"margin_dpo/margin_mean": 165.65187072753906,
|
|
"margin_dpo/margin_std": 219.1352081298828,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.6108663729809104,
|
|
"fcm_dpo/beta": 0.003010387998074293,
|
|
"fcm_dpo/delta": -0.19164395332336426,
|
|
"fcm_dpo/margin": 258.158447265625,
|
|
"fcm_dpo/q_t": 0.3281788229942322,
|
|
"grad_norm": 34.98270797729492,
|
|
"learning_rate": 1.990267419549914e-07,
|
|
"logits/chosen": -0.40908828377723694,
|
|
"logits/rejected": -0.39938801527023315,
|
|
"logps/chosen": -331.32080078125,
|
|
"logps/ref_chosen": -58.27912521362305,
|
|
"logps/ref_rejected": -90.56871795654297,
|
|
"logps/rejected": -621.7688598632812,
|
|
"loss": 0.8446,
|
|
"margin_dpo/margin_mean": 258.158447265625,
|
|
"margin_dpo/margin_std": 212.94345092773438,
|
|
"step": 416
|
|
},
|
|
{
|
|
"epoch": 0.6123348017621145,
|
|
"fcm_dpo/beta": 0.002966091735288501,
|
|
"fcm_dpo/delta": -0.016102882102131844,
|
|
"fcm_dpo/margin": 207.3574981689453,
|
|
"fcm_dpo/q_t": 0.360975980758667,
|
|
"grad_norm": 30.207260131835938,
|
|
"learning_rate": 1.9777104743594686e-07,
|
|
"logits/chosen": -0.3575609624385834,
|
|
"logits/rejected": -0.3195306360721588,
|
|
"logps/chosen": -326.48291015625,
|
|
"logps/ref_chosen": -50.1987190246582,
|
|
"logps/ref_rejected": -68.15184020996094,
|
|
"logps/rejected": -551.7935791015625,
|
|
"loss": 0.935,
|
|
"margin_dpo/margin_mean": 207.35751342773438,
|
|
"margin_dpo/margin_std": 190.68011474609375,
|
|
"step": 417
|
|
},
|
|
{
|
|
"epoch": 0.6138032305433186,
|
|
"fcm_dpo/beta": 0.002962034661322832,
|
|
"fcm_dpo/delta": -0.04637201130390167,
|
|
"fcm_dpo/margin": 216.85317993164062,
|
|
"fcm_dpo/q_t": 0.3637494742870331,
|
|
"grad_norm": 25.070138931274414,
|
|
"learning_rate": 1.965167291983757e-07,
|
|
"logits/chosen": -0.46340426802635193,
|
|
"logits/rejected": -0.4286314845085144,
|
|
"logps/chosen": -367.762451171875,
|
|
"logps/ref_chosen": -81.97846984863281,
|
|
"logps/ref_rejected": -104.69148254394531,
|
|
"logps/rejected": -607.32861328125,
|
|
"loss": 0.9707,
|
|
"margin_dpo/margin_mean": 216.85317993164062,
|
|
"margin_dpo/margin_std": 255.80337524414062,
|
|
"step": 418
|
|
},
|
|
{
|
|
"epoch": 0.6152716593245228,
|
|
"fcm_dpo/beta": 0.0028951792046427727,
|
|
"fcm_dpo/delta": -0.08015348017215729,
|
|
"fcm_dpo/margin": 233.0614013671875,
|
|
"fcm_dpo/q_t": 0.3508976697921753,
|
|
"grad_norm": 24.830020904541016,
|
|
"learning_rate": 1.9526382029472988e-07,
|
|
"logits/chosen": -0.39471104741096497,
|
|
"logits/rejected": -0.3786032199859619,
|
|
"logps/chosen": -315.84515380859375,
|
|
"logps/ref_chosen": -52.948646545410156,
|
|
"logps/ref_rejected": -91.58309936523438,
|
|
"logps/rejected": -587.541015625,
|
|
"loss": 0.9221,
|
|
"margin_dpo/margin_mean": 233.0614013671875,
|
|
"margin_dpo/margin_std": 232.522705078125,
|
|
"step": 419
|
|
},
|
|
{
|
|
"epoch": 0.6167400881057269,
|
|
"fcm_dpo/beta": 0.0029447104316204786,
|
|
"fcm_dpo/delta": 0.1310030221939087,
|
|
"fcm_dpo/margin": 161.28619384765625,
|
|
"fcm_dpo/q_t": 0.40055570006370544,
|
|
"grad_norm": 52.39372253417969,
|
|
"learning_rate": 1.9401235374032425e-07,
|
|
"logits/chosen": -0.44154638051986694,
|
|
"logits/rejected": -0.3922814130783081,
|
|
"logps/chosen": -384.56658935546875,
|
|
"logps/ref_chosen": -77.7699203491211,
|
|
"logps/ref_rejected": -69.31985473632812,
|
|
"logps/rejected": -537.4027099609375,
|
|
"loss": 1.0989,
|
|
"margin_dpo/margin_mean": 161.28619384765625,
|
|
"margin_dpo/margin_std": 265.3115234375,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.618208516886931,
|
|
"fcm_dpo/beta": 0.0030234779696911573,
|
|
"fcm_dpo/delta": 0.11057442426681519,
|
|
"fcm_dpo/margin": 163.34596252441406,
|
|
"fcm_dpo/q_t": 0.3888086974620819,
|
|
"grad_norm": 33.57830810546875,
|
|
"learning_rate": 1.9276236251246653e-07,
|
|
"logits/chosen": -0.45666855573654175,
|
|
"logits/rejected": -0.43738096952438354,
|
|
"logps/chosen": -311.1793212890625,
|
|
"logps/ref_chosen": -53.765865325927734,
|
|
"logps/ref_rejected": -89.28144836425781,
|
|
"logps/rejected": -510.0408630371094,
|
|
"loss": 1.0469,
|
|
"margin_dpo/margin_mean": 163.34596252441406,
|
|
"margin_dpo/margin_std": 208.17727661132812,
|
|
"step": 421
|
|
},
|
|
{
|
|
"epoch": 0.6196769456681351,
|
|
"fcm_dpo/beta": 0.0030494979582726955,
|
|
"fcm_dpo/delta": 0.04655279591679573,
|
|
"fcm_dpo/margin": 182.31634521484375,
|
|
"fcm_dpo/q_t": 0.3765103816986084,
|
|
"grad_norm": 35.29560852050781,
|
|
"learning_rate": 1.9151387954958792e-07,
|
|
"logits/chosen": -0.4439750909805298,
|
|
"logits/rejected": -0.42819076776504517,
|
|
"logps/chosen": -358.15643310546875,
|
|
"logps/ref_chosen": -68.6337661743164,
|
|
"logps/ref_rejected": -87.86351013183594,
|
|
"logps/rejected": -559.7025146484375,
|
|
"loss": 1.0112,
|
|
"margin_dpo/margin_mean": 182.31634521484375,
|
|
"margin_dpo/margin_std": 225.18934631347656,
|
|
"step": 422
|
|
},
|
|
{
|
|
"epoch": 0.6211453744493393,
|
|
"fcm_dpo/beta": 0.003037120681256056,
|
|
"fcm_dpo/delta": -0.029378529638051987,
|
|
"fcm_dpo/margin": 206.6036376953125,
|
|
"fcm_dpo/q_t": 0.3606680631637573,
|
|
"grad_norm": 30.540699005126953,
|
|
"learning_rate": 1.902669377503756e-07,
|
|
"logits/chosen": -0.42354506254196167,
|
|
"logits/rejected": -0.42130446434020996,
|
|
"logps/chosen": -310.9537353515625,
|
|
"logps/ref_chosen": -54.99030303955078,
|
|
"logps/ref_rejected": -86.30654907226562,
|
|
"logps/rejected": -548.8735961914062,
|
|
"loss": 0.9463,
|
|
"margin_dpo/margin_mean": 206.6036376953125,
|
|
"margin_dpo/margin_std": 212.5541229248047,
|
|
"step": 423
|
|
},
|
|
{
|
|
"epoch": 0.6226138032305433,
|
|
"fcm_dpo/beta": 0.0030368948355317116,
|
|
"fcm_dpo/delta": 0.032666608691215515,
|
|
"fcm_dpo/margin": 187.3394775390625,
|
|
"fcm_dpo/q_t": 0.37737834453582764,
|
|
"grad_norm": 32.117645263671875,
|
|
"learning_rate": 1.890215699729057e-07,
|
|
"logits/chosen": -0.4487920105457306,
|
|
"logits/rejected": -0.41544824838638306,
|
|
"logps/chosen": -282.232177734375,
|
|
"logps/ref_chosen": -56.01192092895508,
|
|
"logps/ref_rejected": -66.47896575927734,
|
|
"logps/rejected": -480.0386962890625,
|
|
"loss": 1.0066,
|
|
"margin_dpo/margin_mean": 187.3394775390625,
|
|
"margin_dpo/margin_std": 233.7342529296875,
|
|
"step": 424
|
|
},
|
|
{
|
|
"epoch": 0.6240822320117474,
|
|
"fcm_dpo/beta": 0.0031079701147973537,
|
|
"fcm_dpo/delta": 0.09193390607833862,
|
|
"fcm_dpo/margin": 164.8294677734375,
|
|
"fcm_dpo/q_t": 0.3847385048866272,
|
|
"grad_norm": 31.431589126586914,
|
|
"learning_rate": 1.8777780903377732e-07,
|
|
"logits/chosen": -0.38863712549209595,
|
|
"logits/rejected": -0.3869234323501587,
|
|
"logps/chosen": -284.51678466796875,
|
|
"logps/ref_chosen": -46.86899948120117,
|
|
"logps/ref_rejected": -95.92545318603516,
|
|
"logps/rejected": -498.4027099609375,
|
|
"loss": 1.0322,
|
|
"margin_dpo/margin_mean": 164.8294677734375,
|
|
"margin_dpo/margin_std": 204.4449920654297,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.6255506607929515,
|
|
"fcm_dpo/beta": 0.0031212703324854374,
|
|
"fcm_dpo/delta": -0.033512186259031296,
|
|
"fcm_dpo/margin": 201.99557495117188,
|
|
"fcm_dpo/q_t": 0.3606271743774414,
|
|
"grad_norm": 38.98627853393555,
|
|
"learning_rate": 1.8653568770724803e-07,
|
|
"logits/chosen": -0.4529041051864624,
|
|
"logits/rejected": -0.4022233486175537,
|
|
"logps/chosen": -299.26812744140625,
|
|
"logps/ref_chosen": -76.58354187011719,
|
|
"logps/ref_rejected": -81.26658630371094,
|
|
"logps/rejected": -505.94671630859375,
|
|
"loss": 0.9505,
|
|
"margin_dpo/margin_mean": 201.99557495117188,
|
|
"margin_dpo/margin_std": 206.97454833984375,
|
|
"step": 426
|
|
},
|
|
{
|
|
"epoch": 0.6270190895741556,
|
|
"fcm_dpo/beta": 0.003162715584039688,
|
|
"fcm_dpo/delta": 0.15372638404369354,
|
|
"fcm_dpo/margin": 143.31716918945312,
|
|
"fcm_dpo/q_t": 0.3991009294986725,
|
|
"grad_norm": 34.75946044921875,
|
|
"learning_rate": 1.8529523872436977e-07,
|
|
"logits/chosen": -0.45945197343826294,
|
|
"logits/rejected": -0.4228222370147705,
|
|
"logps/chosen": -276.22698974609375,
|
|
"logps/ref_chosen": -64.8538818359375,
|
|
"logps/ref_rejected": -78.5660171508789,
|
|
"logps/rejected": -433.2562561035156,
|
|
"loss": 1.0598,
|
|
"margin_dpo/margin_mean": 143.31715393066406,
|
|
"margin_dpo/margin_std": 183.68850708007812,
|
|
"step": 427
|
|
},
|
|
{
|
|
"epoch": 0.6284875183553598,
|
|
"fcm_dpo/beta": 0.003189402399584651,
|
|
"fcm_dpo/delta": 0.009054990485310555,
|
|
"fcm_dpo/margin": 185.44857788085938,
|
|
"fcm_dpo/q_t": 0.37188664078712463,
|
|
"grad_norm": 29.212984085083008,
|
|
"learning_rate": 1.8405649477212697e-07,
|
|
"logits/chosen": -0.4812716245651245,
|
|
"logits/rejected": -0.47259581089019775,
|
|
"logps/chosen": -344.0758972167969,
|
|
"logps/ref_chosen": -62.63666534423828,
|
|
"logps/ref_rejected": -103.28181457519531,
|
|
"logps/rejected": -570.169677734375,
|
|
"loss": 1.0084,
|
|
"margin_dpo/margin_mean": 185.44857788085938,
|
|
"margin_dpo/margin_std": 239.06033325195312,
|
|
"step": 428
|
|
},
|
|
{
|
|
"epoch": 0.6299559471365639,
|
|
"fcm_dpo/beta": 0.0032644562888890505,
|
|
"fcm_dpo/delta": 0.08702676743268967,
|
|
"fcm_dpo/margin": 157.81967163085938,
|
|
"fcm_dpo/q_t": 0.3883768320083618,
|
|
"grad_norm": 37.52164077758789,
|
|
"learning_rate": 1.828194884925749e-07,
|
|
"logits/chosen": -0.46438342332839966,
|
|
"logits/rejected": -0.4186452627182007,
|
|
"logps/chosen": -361.667236328125,
|
|
"logps/ref_chosen": -81.23401641845703,
|
|
"logps/ref_rejected": -91.79493713378906,
|
|
"logps/rejected": -530.0478515625,
|
|
"loss": 1.0732,
|
|
"margin_dpo/margin_mean": 157.81967163085938,
|
|
"margin_dpo/margin_std": 228.81576538085938,
|
|
"step": 429
|
|
},
|
|
{
|
|
"epoch": 0.631424375917768,
|
|
"fcm_dpo/beta": 0.0032723741605877876,
|
|
"fcm_dpo/delta": 0.02481374703347683,
|
|
"fcm_dpo/margin": 176.12640380859375,
|
|
"fcm_dpo/q_t": 0.37325915694236755,
|
|
"grad_norm": 30.306285858154297,
|
|
"learning_rate": 1.8158425248197928e-07,
|
|
"logits/chosen": -0.4715721011161804,
|
|
"logits/rejected": -0.46700191497802734,
|
|
"logps/chosen": -303.973388671875,
|
|
"logps/ref_chosen": -60.920326232910156,
|
|
"logps/ref_rejected": -104.42280578613281,
|
|
"logps/rejected": -523.602294921875,
|
|
"loss": 0.9759,
|
|
"margin_dpo/margin_mean": 176.12640380859375,
|
|
"margin_dpo/margin_std": 191.27938842773438,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.6328928046989721,
|
|
"fcm_dpo/beta": 0.0031659076921641827,
|
|
"fcm_dpo/delta": -0.19049368798732758,
|
|
"fcm_dpo/margin": 244.8009033203125,
|
|
"fcm_dpo/q_t": 0.3305261731147766,
|
|
"grad_norm": 26.351268768310547,
|
|
"learning_rate": 1.8035081928995788e-07,
|
|
"logits/chosen": -0.3872508406639099,
|
|
"logits/rejected": -0.3752667307853699,
|
|
"logps/chosen": -293.5878601074219,
|
|
"logps/ref_chosen": -57.34874725341797,
|
|
"logps/ref_rejected": -92.84022521972656,
|
|
"logps/rejected": -573.8802490234375,
|
|
"loss": 0.8614,
|
|
"margin_dpo/margin_mean": 244.8009033203125,
|
|
"margin_dpo/margin_std": 219.5264892578125,
|
|
"step": 431
|
|
},
|
|
{
|
|
"epoch": 0.6343612334801763,
|
|
"fcm_dpo/beta": 0.0031368029303848743,
|
|
"fcm_dpo/delta": -0.083712637424469,
|
|
"fcm_dpo/margin": 215.79534912109375,
|
|
"fcm_dpo/q_t": 0.35200947523117065,
|
|
"grad_norm": 28.590843200683594,
|
|
"learning_rate": 1.791192214186223e-07,
|
|
"logits/chosen": -0.4346217215061188,
|
|
"logits/rejected": -0.4086345434188843,
|
|
"logps/chosen": -320.74261474609375,
|
|
"logps/ref_chosen": -71.07479095458984,
|
|
"logps/ref_rejected": -98.57952880859375,
|
|
"logps/rejected": -564.042724609375,
|
|
"loss": 0.9165,
|
|
"margin_dpo/margin_mean": 215.79534912109375,
|
|
"margin_dpo/margin_std": 203.4432373046875,
|
|
"step": 432
|
|
},
|
|
{
|
|
"epoch": 0.6358296622613803,
|
|
"fcm_dpo/beta": 0.0031476817093789577,
|
|
"fcm_dpo/delta": 0.1059950441122055,
|
|
"fcm_dpo/margin": 158.52908325195312,
|
|
"fcm_dpo/q_t": 0.3886904716491699,
|
|
"grad_norm": 33.54300308227539,
|
|
"learning_rate": 1.7788949132172193e-07,
|
|
"logits/chosen": -0.3967306911945343,
|
|
"logits/rejected": -0.37517887353897095,
|
|
"logps/chosen": -373.89337158203125,
|
|
"logps/ref_chosen": -58.273193359375,
|
|
"logps/ref_rejected": -95.95089721679688,
|
|
"logps/rejected": -570.1001586914062,
|
|
"loss": 1.0645,
|
|
"margin_dpo/margin_mean": 158.52908325195312,
|
|
"margin_dpo/margin_std": 224.95269775390625,
|
|
"step": 433
|
|
},
|
|
{
|
|
"epoch": 0.6372980910425844,
|
|
"fcm_dpo/beta": 0.0031643174588680267,
|
|
"fcm_dpo/delta": 0.033846087753772736,
|
|
"fcm_dpo/margin": 179.49761962890625,
|
|
"fcm_dpo/q_t": 0.38102903962135315,
|
|
"grad_norm": 28.94676399230957,
|
|
"learning_rate": 1.7666166140378853e-07,
|
|
"logits/chosen": -0.42005491256713867,
|
|
"logits/rejected": -0.40054017305374146,
|
|
"logps/chosen": -327.9237060546875,
|
|
"logps/ref_chosen": -61.97370147705078,
|
|
"logps/ref_rejected": -78.49861145019531,
|
|
"logps/rejected": -523.9462280273438,
|
|
"loss": 1.0064,
|
|
"margin_dpo/margin_mean": 179.49761962890625,
|
|
"margin_dpo/margin_std": 235.10498046875,
|
|
"step": 434
|
|
},
|
|
{
|
|
"epoch": 0.6387665198237885,
|
|
"fcm_dpo/beta": 0.0031498861499130726,
|
|
"fcm_dpo/delta": -0.05088559538125992,
|
|
"fcm_dpo/margin": 205.5423583984375,
|
|
"fcm_dpo/q_t": 0.3588791489601135,
|
|
"grad_norm": 28.799325942993164,
|
|
"learning_rate": 1.7543576401928218e-07,
|
|
"logits/chosen": -0.4303112328052521,
|
|
"logits/rejected": -0.40882986783981323,
|
|
"logps/chosen": -297.64056396484375,
|
|
"logps/ref_chosen": -51.502052307128906,
|
|
"logps/ref_rejected": -87.56689453125,
|
|
"logps/rejected": -539.247802734375,
|
|
"loss": 0.9575,
|
|
"margin_dpo/margin_mean": 205.54234313964844,
|
|
"margin_dpo/margin_std": 227.2718048095703,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.6402349486049926,
|
|
"fcm_dpo/beta": 0.003141575027257204,
|
|
"fcm_dpo/delta": -0.0027038920670747757,
|
|
"fcm_dpo/margin": 191.78115844726562,
|
|
"fcm_dpo/q_t": 0.36727362871170044,
|
|
"grad_norm": 53.2458610534668,
|
|
"learning_rate": 1.742118314717391e-07,
|
|
"logits/chosen": -0.4502115249633789,
|
|
"logits/rejected": -0.4046837091445923,
|
|
"logps/chosen": -322.1340026855469,
|
|
"logps/ref_chosen": -71.40371704101562,
|
|
"logps/ref_rejected": -82.72775268554688,
|
|
"logps/rejected": -525.2391967773438,
|
|
"loss": 0.9633,
|
|
"margin_dpo/margin_mean": 191.78115844726562,
|
|
"margin_dpo/margin_std": 206.23292541503906,
|
|
"step": 436
|
|
},
|
|
{
|
|
"epoch": 0.6417033773861968,
|
|
"fcm_dpo/beta": 0.0031595812179148197,
|
|
"fcm_dpo/delta": 0.026493970304727554,
|
|
"fcm_dpo/margin": 181.99075317382812,
|
|
"fcm_dpo/q_t": 0.372455894947052,
|
|
"grad_norm": 30.233713150024414,
|
|
"learning_rate": 1.7298989601292036e-07,
|
|
"logits/chosen": -0.43642017245292664,
|
|
"logits/rejected": -0.4024639129638672,
|
|
"logps/chosen": -321.0672912597656,
|
|
"logps/ref_chosen": -64.7442626953125,
|
|
"logps/ref_rejected": -82.04356384277344,
|
|
"logps/rejected": -520.3573608398438,
|
|
"loss": 0.9863,
|
|
"margin_dpo/margin_mean": 181.99075317382812,
|
|
"margin_dpo/margin_std": 205.34609985351562,
|
|
"step": 437
|
|
},
|
|
{
|
|
"epoch": 0.6431718061674009,
|
|
"fcm_dpo/beta": 0.00310856057330966,
|
|
"fcm_dpo/delta": -0.08261559903621674,
|
|
"fcm_dpo/margin": 217.52000427246094,
|
|
"fcm_dpo/q_t": 0.34879451990127563,
|
|
"grad_norm": 41.74407196044922,
|
|
"learning_rate": 1.7176998984196144e-07,
|
|
"logits/chosen": -0.3906601667404175,
|
|
"logits/rejected": -0.35248345136642456,
|
|
"logps/chosen": -311.93084716796875,
|
|
"logps/ref_chosen": -59.0186653137207,
|
|
"logps/ref_rejected": -83.07682800292969,
|
|
"logps/rejected": -553.5089721679688,
|
|
"loss": 0.9029,
|
|
"margin_dpo/margin_mean": 217.51998901367188,
|
|
"margin_dpo/margin_std": 194.8534393310547,
|
|
"step": 438
|
|
},
|
|
{
|
|
"epoch": 0.644640234948605,
|
|
"fcm_dpo/beta": 0.0031026997603476048,
|
|
"fcm_dpo/delta": 0.05992882698774338,
|
|
"fcm_dpo/margin": 174.4813232421875,
|
|
"fcm_dpo/q_t": 0.38289695978164673,
|
|
"grad_norm": 30.436159133911133,
|
|
"learning_rate": 1.7055214510452458e-07,
|
|
"logits/chosen": -0.38404345512390137,
|
|
"logits/rejected": -0.37690842151641846,
|
|
"logps/chosen": -328.8792724609375,
|
|
"logps/ref_chosen": -53.78407669067383,
|
|
"logps/ref_rejected": -83.98545837402344,
|
|
"logps/rejected": -533.56201171875,
|
|
"loss": 1.0235,
|
|
"margin_dpo/margin_mean": 174.4813232421875,
|
|
"margin_dpo/margin_std": 211.0643768310547,
|
|
"step": 439
|
|
},
|
|
{
|
|
"epoch": 0.6461086637298091,
|
|
"fcm_dpo/beta": 0.0031752916984260082,
|
|
"fcm_dpo/delta": 0.037204399704933167,
|
|
"fcm_dpo/margin": 177.81304931640625,
|
|
"fcm_dpo/q_t": 0.3790428638458252,
|
|
"grad_norm": 36.150543212890625,
|
|
"learning_rate": 1.6933639389195134e-07,
|
|
"logits/chosen": -0.45105117559432983,
|
|
"logits/rejected": -0.4272584915161133,
|
|
"logps/chosen": -372.9900207519531,
|
|
"logps/ref_chosen": -78.56671905517578,
|
|
"logps/ref_rejected": -96.49775695800781,
|
|
"logps/rejected": -568.734130859375,
|
|
"loss": 1.0121,
|
|
"margin_dpo/margin_mean": 177.81304931640625,
|
|
"margin_dpo/margin_std": 225.9781494140625,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.6475770925110133,
|
|
"fcm_dpo/beta": 0.0031558992341160774,
|
|
"fcm_dpo/delta": -0.05471445247530937,
|
|
"fcm_dpo/margin": 206.31964111328125,
|
|
"fcm_dpo/q_t": 0.36530476808547974,
|
|
"grad_norm": 41.47607421875,
|
|
"learning_rate": 1.681227682404166e-07,
|
|
"logits/chosen": -0.42267531156539917,
|
|
"logits/rejected": -0.4021593928337097,
|
|
"logps/chosen": -371.01458740234375,
|
|
"logps/ref_chosen": -60.824440002441406,
|
|
"logps/ref_rejected": -96.47080993652344,
|
|
"logps/rejected": -612.9805908203125,
|
|
"loss": 0.9977,
|
|
"margin_dpo/margin_mean": 206.31964111328125,
|
|
"margin_dpo/margin_std": 277.037841796875,
|
|
"step": 441
|
|
},
|
|
{
|
|
"epoch": 0.6490455212922174,
|
|
"fcm_dpo/beta": 0.0031145289540290833,
|
|
"fcm_dpo/delta": -0.10305148363113403,
|
|
"fcm_dpo/margin": 223.28561401367188,
|
|
"fcm_dpo/q_t": 0.3511419892311096,
|
|
"grad_norm": 28.58584213256836,
|
|
"learning_rate": 1.669113001300851e-07,
|
|
"logits/chosen": -0.4091102182865143,
|
|
"logits/rejected": -0.3904363512992859,
|
|
"logps/chosen": -326.81689453125,
|
|
"logps/ref_chosen": -47.01121520996094,
|
|
"logps/ref_rejected": -76.53926086425781,
|
|
"logps/rejected": -579.6304931640625,
|
|
"loss": 0.9206,
|
|
"margin_dpo/margin_mean": 223.28561401367188,
|
|
"margin_dpo/margin_std": 229.4536590576172,
|
|
"step": 442
|
|
},
|
|
{
|
|
"epoch": 0.6505139500734214,
|
|
"fcm_dpo/beta": 0.003117314074188471,
|
|
"fcm_dpo/delta": 0.14508318901062012,
|
|
"fcm_dpo/margin": 148.06362915039062,
|
|
"fcm_dpo/q_t": 0.40172627568244934,
|
|
"grad_norm": 37.50111389160156,
|
|
"learning_rate": 1.6570202148426815e-07,
|
|
"logits/chosen": -0.408527135848999,
|
|
"logits/rejected": -0.38471612334251404,
|
|
"logps/chosen": -397.83453369140625,
|
|
"logps/ref_chosen": -71.27301788330078,
|
|
"logps/ref_rejected": -86.679931640625,
|
|
"logps/rejected": -561.3050537109375,
|
|
"loss": 1.1228,
|
|
"margin_dpo/margin_mean": 148.06362915039062,
|
|
"margin_dpo/margin_std": 257.1429138183594,
|
|
"step": 443
|
|
},
|
|
{
|
|
"epoch": 0.6519823788546255,
|
|
"fcm_dpo/beta": 0.00309735513292253,
|
|
"fcm_dpo/delta": -0.12234368920326233,
|
|
"fcm_dpo/margin": 230.2260284423828,
|
|
"fcm_dpo/q_t": 0.34877198934555054,
|
|
"grad_norm": 35.36920928955078,
|
|
"learning_rate": 1.6449496416858282e-07,
|
|
"logits/chosen": -0.36250776052474976,
|
|
"logits/rejected": -0.3549155592918396,
|
|
"logps/chosen": -372.5677185058594,
|
|
"logps/ref_chosen": -57.213706970214844,
|
|
"logps/ref_rejected": -97.25489807128906,
|
|
"logps/rejected": -642.8349609375,
|
|
"loss": 0.93,
|
|
"margin_dpo/margin_mean": 230.22604370117188,
|
|
"margin_dpo/margin_std": 260.1424865722656,
|
|
"step": 444
|
|
},
|
|
{
|
|
"epoch": 0.6534508076358296,
|
|
"fcm_dpo/beta": 0.003053246531635523,
|
|
"fcm_dpo/delta": -0.07174232602119446,
|
|
"fcm_dpo/margin": 218.43179321289062,
|
|
"fcm_dpo/q_t": 0.3583186864852905,
|
|
"grad_norm": 34.992069244384766,
|
|
"learning_rate": 1.6329015999011182e-07,
|
|
"logits/chosen": -0.39381998777389526,
|
|
"logits/rejected": -0.37214112281799316,
|
|
"logps/chosen": -351.34881591796875,
|
|
"logps/ref_chosen": -67.29979705810547,
|
|
"logps/ref_rejected": -92.68267059326172,
|
|
"logps/rejected": -595.1634521484375,
|
|
"loss": 0.9516,
|
|
"margin_dpo/margin_mean": 218.43177795410156,
|
|
"margin_dpo/margin_std": 246.44183349609375,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.6549192364170338,
|
|
"fcm_dpo/beta": 0.0030171829275786877,
|
|
"fcm_dpo/delta": -0.10393056273460388,
|
|
"fcm_dpo/margin": 230.83352661132812,
|
|
"fcm_dpo/q_t": 0.34617435932159424,
|
|
"grad_norm": 26.35415267944336,
|
|
"learning_rate": 1.6208764069656578e-07,
|
|
"logits/chosen": -0.43713754415512085,
|
|
"logits/rejected": -0.43820661306381226,
|
|
"logps/chosen": -321.90740966796875,
|
|
"logps/ref_chosen": -59.098487854003906,
|
|
"logps/ref_rejected": -101.26419067382812,
|
|
"logps/rejected": -594.9066162109375,
|
|
"loss": 0.8993,
|
|
"margin_dpo/margin_mean": 230.83352661132812,
|
|
"margin_dpo/margin_std": 212.1956024169922,
|
|
"step": 446
|
|
},
|
|
{
|
|
"epoch": 0.6563876651982379,
|
|
"fcm_dpo/beta": 0.0029287966899573803,
|
|
"fcm_dpo/delta": -0.08693499863147736,
|
|
"fcm_dpo/margin": 232.25527954101562,
|
|
"fcm_dpo/q_t": 0.35467979311943054,
|
|
"grad_norm": 32.44998550415039,
|
|
"learning_rate": 1.608874379754465e-07,
|
|
"logits/chosen": -0.421570360660553,
|
|
"logits/rejected": -0.42356860637664795,
|
|
"logps/chosen": -343.432861328125,
|
|
"logps/ref_chosen": -56.07533264160156,
|
|
"logps/ref_rejected": -98.69475555419922,
|
|
"logps/rejected": -618.3075561523438,
|
|
"loss": 0.9272,
|
|
"margin_dpo/margin_mean": 232.2552947998047,
|
|
"margin_dpo/margin_std": 253.9686737060547,
|
|
"step": 447
|
|
},
|
|
{
|
|
"epoch": 0.657856093979442,
|
|
"fcm_dpo/beta": 0.002891901880502701,
|
|
"fcm_dpo/delta": -0.11446341872215271,
|
|
"fcm_dpo/margin": 244.29922485351562,
|
|
"fcm_dpo/q_t": 0.34647947549819946,
|
|
"grad_norm": 37.00398254394531,
|
|
"learning_rate": 1.5968958345321177e-07,
|
|
"logits/chosen": -0.3976057767868042,
|
|
"logits/rejected": -0.39330196380615234,
|
|
"logps/chosen": -364.0402526855469,
|
|
"logps/ref_chosen": -60.00384521484375,
|
|
"logps/ref_rejected": -102.26465606689453,
|
|
"logps/rejected": -650.6002807617188,
|
|
"loss": 0.9,
|
|
"margin_dpo/margin_mean": 244.29922485351562,
|
|
"margin_dpo/margin_std": 234.91090393066406,
|
|
"step": 448
|
|
},
|
|
{
|
|
"epoch": 0.6593245227606461,
|
|
"fcm_dpo/beta": 0.002847596537321806,
|
|
"fcm_dpo/delta": -0.026260854676365852,
|
|
"fcm_dpo/margin": 219.3227081298828,
|
|
"fcm_dpo/q_t": 0.3709370791912079,
|
|
"grad_norm": 35.01579284667969,
|
|
"learning_rate": 1.584941086944423e-07,
|
|
"logits/chosen": -0.40996721386909485,
|
|
"logits/rejected": -0.3869406580924988,
|
|
"logps/chosen": -385.76275634765625,
|
|
"logps/ref_chosen": -67.52661895751953,
|
|
"logps/ref_rejected": -88.59690856933594,
|
|
"logps/rejected": -626.15576171875,
|
|
"loss": 1.0099,
|
|
"margin_dpo/margin_mean": 219.3227081298828,
|
|
"margin_dpo/margin_std": 304.9261474609375,
|
|
"step": 449
|
|
},
|
|
{
|
|
"epoch": 0.6607929515418502,
|
|
"fcm_dpo/beta": 0.0028021461330354214,
|
|
"fcm_dpo/delta": -0.12278854101896286,
|
|
"fcm_dpo/margin": 254.86483764648438,
|
|
"fcm_dpo/q_t": 0.33995717763900757,
|
|
"grad_norm": 35.38984680175781,
|
|
"learning_rate": 1.573010452010098e-07,
|
|
"logits/chosen": -0.4261053204536438,
|
|
"logits/rejected": -0.42196720838546753,
|
|
"logps/chosen": -306.01397705078125,
|
|
"logps/ref_chosen": -57.10811996459961,
|
|
"logps/ref_rejected": -102.75494384765625,
|
|
"logps/rejected": -606.525634765625,
|
|
"loss": 0.8839,
|
|
"margin_dpo/margin_mean": 254.86483764648438,
|
|
"margin_dpo/margin_std": 228.66526794433594,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.6622613803230544,
|
|
"fcm_dpo/beta": 0.002811973448842764,
|
|
"fcm_dpo/delta": 0.04851696640253067,
|
|
"fcm_dpo/margin": 196.84996032714844,
|
|
"fcm_dpo/q_t": 0.3752724528312683,
|
|
"grad_norm": 42.07421112060547,
|
|
"learning_rate": 1.5611042441124687e-07,
|
|
"logits/chosen": -0.415117084980011,
|
|
"logits/rejected": -0.38188982009887695,
|
|
"logps/chosen": -367.1541748046875,
|
|
"logps/ref_chosen": -58.46883010864258,
|
|
"logps/ref_rejected": -72.92941284179688,
|
|
"logps/rejected": -578.4647216796875,
|
|
"loss": 1.0271,
|
|
"margin_dpo/margin_mean": 196.8499755859375,
|
|
"margin_dpo/margin_std": 257.9520263671875,
|
|
"step": 451
|
|
},
|
|
{
|
|
"epoch": 0.6637298091042585,
|
|
"fcm_dpo/beta": 0.002792142331600189,
|
|
"fcm_dpo/delta": -0.020108355209231377,
|
|
"fcm_dpo/margin": 221.60552978515625,
|
|
"fcm_dpo/q_t": 0.36079761385917664,
|
|
"grad_norm": 25.38531494140625,
|
|
"learning_rate": 1.549222776991186e-07,
|
|
"logits/chosen": -0.3913271129131317,
|
|
"logits/rejected": -0.3999473750591278,
|
|
"logps/chosen": -272.1903076171875,
|
|
"logps/ref_chosen": -50.39055252075195,
|
|
"logps/ref_rejected": -97.77142333984375,
|
|
"logps/rejected": -541.1766967773438,
|
|
"loss": 0.9344,
|
|
"margin_dpo/margin_mean": 221.60552978515625,
|
|
"margin_dpo/margin_std": 206.4978790283203,
|
|
"step": 452
|
|
},
|
|
{
|
|
"epoch": 0.6651982378854625,
|
|
"fcm_dpo/beta": 0.0028024991042912006,
|
|
"fcm_dpo/delta": 0.04751697555184364,
|
|
"fcm_dpo/margin": 198.021484375,
|
|
"fcm_dpo/q_t": 0.37886279821395874,
|
|
"grad_norm": 26.131412506103516,
|
|
"learning_rate": 1.5373663637339584e-07,
|
|
"logits/chosen": -0.4181209206581116,
|
|
"logits/rejected": -0.38588768243789673,
|
|
"logps/chosen": -332.87738037109375,
|
|
"logps/ref_chosen": -57.71485137939453,
|
|
"logps/ref_rejected": -82.20741271972656,
|
|
"logps/rejected": -555.3914184570312,
|
|
"loss": 0.9948,
|
|
"margin_dpo/margin_mean": 198.021484375,
|
|
"margin_dpo/margin_std": 228.34547424316406,
|
|
"step": 453
|
|
},
|
|
{
|
|
"epoch": 0.6666666666666666,
|
|
"fcm_dpo/beta": 0.0027867662720382214,
|
|
"fcm_dpo/delta": -0.059162285178899765,
|
|
"fcm_dpo/margin": 234.98402404785156,
|
|
"fcm_dpo/q_t": 0.35987627506256104,
|
|
"grad_norm": 29.379961013793945,
|
|
"learning_rate": 1.5255353167683017e-07,
|
|
"logits/chosen": -0.40170955657958984,
|
|
"logits/rejected": -0.3783451318740845,
|
|
"logps/chosen": -366.53466796875,
|
|
"logps/ref_chosen": -60.945648193359375,
|
|
"logps/ref_rejected": -84.95079040527344,
|
|
"logps/rejected": -625.5238037109375,
|
|
"loss": 0.9475,
|
|
"margin_dpo/margin_mean": 234.98403930664062,
|
|
"margin_dpo/margin_std": 266.738037109375,
|
|
"step": 454
|
|
},
|
|
{
|
|
"epoch": 0.6681350954478708,
|
|
"fcm_dpo/beta": 0.002752738306298852,
|
|
"fcm_dpo/delta": -0.13085488975048065,
|
|
"fcm_dpo/margin": 262.0626220703125,
|
|
"fcm_dpo/q_t": 0.34647172689437866,
|
|
"grad_norm": 29.516231536865234,
|
|
"learning_rate": 1.5137299478533064e-07,
|
|
"logits/chosen": -0.4226793646812439,
|
|
"logits/rejected": -0.4276910424232483,
|
|
"logps/chosen": -327.73028564453125,
|
|
"logps/ref_chosen": -44.88671112060547,
|
|
"logps/ref_rejected": -115.30147552490234,
|
|
"logps/rejected": -660.2076416015625,
|
|
"loss": 0.9117,
|
|
"margin_dpo/margin_mean": 262.0626220703125,
|
|
"margin_dpo/margin_std": 271.49981689453125,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.6696035242290749,
|
|
"fcm_dpo/beta": 0.0026842241641134024,
|
|
"fcm_dpo/delta": -0.08822280168533325,
|
|
"fcm_dpo/margin": 254.17987060546875,
|
|
"fcm_dpo/q_t": 0.3486155569553375,
|
|
"grad_norm": 30.855884552001953,
|
|
"learning_rate": 1.5019505680714232e-07,
|
|
"logits/chosen": -0.42566657066345215,
|
|
"logits/rejected": -0.4341641068458557,
|
|
"logps/chosen": -339.21624755859375,
|
|
"logps/ref_chosen": -57.036781311035156,
|
|
"logps/ref_rejected": -105.21784210205078,
|
|
"logps/rejected": -641.5771484375,
|
|
"loss": 0.8963,
|
|
"margin_dpo/margin_mean": 254.17987060546875,
|
|
"margin_dpo/margin_std": 227.32037353515625,
|
|
"step": 456
|
|
},
|
|
{
|
|
"epoch": 0.671071953010279,
|
|
"fcm_dpo/beta": 0.002638116478919983,
|
|
"fcm_dpo/delta": -0.051346272230148315,
|
|
"fcm_dpo/margin": 245.4560546875,
|
|
"fcm_dpo/q_t": 0.3536643385887146,
|
|
"grad_norm": 43.32398223876953,
|
|
"learning_rate": 1.4901974878202627e-07,
|
|
"logits/chosen": -0.4410599172115326,
|
|
"logits/rejected": -0.4189218282699585,
|
|
"logps/chosen": -335.3031311035156,
|
|
"logps/ref_chosen": -54.24253845214844,
|
|
"logps/ref_rejected": -85.10956573486328,
|
|
"logps/rejected": -611.626220703125,
|
|
"loss": 0.9209,
|
|
"margin_dpo/margin_mean": 245.4560546875,
|
|
"margin_dpo/margin_std": 225.08139038085938,
|
|
"step": 457
|
|
},
|
|
{
|
|
"epoch": 0.6725403817914831,
|
|
"fcm_dpo/beta": 0.0026078918017446995,
|
|
"fcm_dpo/delta": -0.07165485620498657,
|
|
"fcm_dpo/margin": 255.62677001953125,
|
|
"fcm_dpo/q_t": 0.35423409938812256,
|
|
"grad_norm": 27.470394134521484,
|
|
"learning_rate": 1.4784710168044212e-07,
|
|
"logits/chosen": -0.42076390981674194,
|
|
"logits/rejected": -0.40564727783203125,
|
|
"logps/chosen": -329.96343994140625,
|
|
"logps/ref_chosen": -55.40888214111328,
|
|
"logps/ref_rejected": -97.68325805664062,
|
|
"logps/rejected": -627.8646240234375,
|
|
"loss": 0.9208,
|
|
"margin_dpo/margin_mean": 255.6267852783203,
|
|
"margin_dpo/margin_std": 259.33697509765625,
|
|
"step": 458
|
|
},
|
|
{
|
|
"epoch": 0.6740088105726872,
|
|
"fcm_dpo/beta": 0.0025895205326378345,
|
|
"fcm_dpo/delta": -0.016186371445655823,
|
|
"fcm_dpo/margin": 237.4794158935547,
|
|
"fcm_dpo/q_t": 0.36387407779693604,
|
|
"grad_norm": 28.839862823486328,
|
|
"learning_rate": 1.466771464027316e-07,
|
|
"logits/chosen": -0.4365195035934448,
|
|
"logits/rejected": -0.43345052003860474,
|
|
"logps/chosen": -352.8772277832031,
|
|
"logps/ref_chosen": -46.55748748779297,
|
|
"logps/ref_rejected": -86.16854095458984,
|
|
"logps/rejected": -629.9677124023438,
|
|
"loss": 0.9595,
|
|
"margin_dpo/margin_mean": 237.4794158935547,
|
|
"margin_dpo/margin_std": 249.88954162597656,
|
|
"step": 459
|
|
},
|
|
{
|
|
"epoch": 0.6754772393538914,
|
|
"fcm_dpo/beta": 0.0025813106913119555,
|
|
"fcm_dpo/delta": -0.04352150857448578,
|
|
"fcm_dpo/margin": 248.228271484375,
|
|
"fcm_dpo/q_t": 0.35783928632736206,
|
|
"grad_norm": 29.873188018798828,
|
|
"learning_rate": 1.4550991377830423e-07,
|
|
"logits/chosen": -0.45997023582458496,
|
|
"logits/rejected": -0.47314104437828064,
|
|
"logps/chosen": -400.25897216796875,
|
|
"logps/ref_chosen": -51.63489532470703,
|
|
"logps/ref_rejected": -104.11935424804688,
|
|
"logps/rejected": -700.9716796875,
|
|
"loss": 0.9376,
|
|
"margin_dpo/margin_mean": 248.22825622558594,
|
|
"margin_dpo/margin_std": 249.94369506835938,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6769456681350955,
|
|
"fcm_dpo/beta": 0.002606459427624941,
|
|
"fcm_dpo/delta": 0.06772121787071228,
|
|
"fcm_dpo/margin": 205.47274780273438,
|
|
"fcm_dpo/q_t": 0.3833613097667694,
|
|
"grad_norm": 33.35320281982422,
|
|
"learning_rate": 1.4434543456482518e-07,
|
|
"logits/chosen": -0.49127471446990967,
|
|
"logits/rejected": -0.4890371561050415,
|
|
"logps/chosen": -408.4715576171875,
|
|
"logps/ref_chosen": -55.18195724487305,
|
|
"logps/ref_rejected": -86.47689819335938,
|
|
"logps/rejected": -645.2392578125,
|
|
"loss": 1.0183,
|
|
"margin_dpo/margin_mean": 205.47274780273438,
|
|
"margin_dpo/margin_std": 255.91586303710938,
|
|
"step": 461
|
|
},
|
|
{
|
|
"epoch": 0.6784140969162996,
|
|
"fcm_dpo/beta": 0.002664759289473295,
|
|
"fcm_dpo/delta": 0.15602310001850128,
|
|
"fcm_dpo/margin": 169.3271026611328,
|
|
"fcm_dpo/q_t": 0.40088993310928345,
|
|
"grad_norm": 56.87949752807617,
|
|
"learning_rate": 1.4318373944740484e-07,
|
|
"logits/chosen": -0.5266599655151367,
|
|
"logits/rejected": -0.5125139951705933,
|
|
"logps/chosen": -429.41357421875,
|
|
"logps/ref_chosen": -69.92803192138672,
|
|
"logps/ref_rejected": -78.84111022949219,
|
|
"logps/rejected": -607.6537475585938,
|
|
"loss": 1.0795,
|
|
"margin_dpo/margin_mean": 169.3271026611328,
|
|
"margin_dpo/margin_std": 241.06423950195312,
|
|
"step": 462
|
|
},
|
|
{
|
|
"epoch": 0.6798825256975036,
|
|
"fcm_dpo/beta": 0.0027028373442590237,
|
|
"fcm_dpo/delta": -0.00691063329577446,
|
|
"fcm_dpo/margin": 224.18450927734375,
|
|
"fcm_dpo/q_t": 0.37208259105682373,
|
|
"grad_norm": 59.45559310913086,
|
|
"learning_rate": 1.4202485903778976e-07,
|
|
"logits/chosen": -0.48024556040763855,
|
|
"logits/rejected": -0.4725441634654999,
|
|
"logps/chosen": -393.7677001953125,
|
|
"logps/ref_chosen": -55.27437210083008,
|
|
"logps/ref_rejected": -89.02497863769531,
|
|
"logps/rejected": -651.7028198242188,
|
|
"loss": 1.0023,
|
|
"margin_dpo/margin_mean": 224.18450927734375,
|
|
"margin_dpo/margin_std": 288.2208557128906,
|
|
"step": 463
|
|
},
|
|
{
|
|
"epoch": 0.6813509544787077,
|
|
"fcm_dpo/beta": 0.002575058490037918,
|
|
"fcm_dpo/delta": -0.24060693383216858,
|
|
"fcm_dpo/margin": 317.8370361328125,
|
|
"fcm_dpo/q_t": 0.32363659143447876,
|
|
"grad_norm": 35.653228759765625,
|
|
"learning_rate": 1.4086882387355658e-07,
|
|
"logits/chosen": -0.4769009053707123,
|
|
"logits/rejected": -0.5046235918998718,
|
|
"logps/chosen": -376.1098327636719,
|
|
"logps/ref_chosen": -50.91230010986328,
|
|
"logps/ref_rejected": -102.4893798828125,
|
|
"logps/rejected": -745.52392578125,
|
|
"loss": 0.8444,
|
|
"margin_dpo/margin_mean": 317.8370361328125,
|
|
"margin_dpo/margin_std": 292.25225830078125,
|
|
"step": 464
|
|
},
|
|
{
|
|
"epoch": 0.6828193832599119,
|
|
"fcm_dpo/beta": 0.0025294728111475706,
|
|
"fcm_dpo/delta": -0.11184176802635193,
|
|
"fcm_dpo/margin": 278.3428039550781,
|
|
"fcm_dpo/q_t": 0.34381166100502014,
|
|
"grad_norm": 32.800621032714844,
|
|
"learning_rate": 1.3971566441730714e-07,
|
|
"logits/chosen": -0.4514947533607483,
|
|
"logits/rejected": -0.4547080993652344,
|
|
"logps/chosen": -367.2335205078125,
|
|
"logps/ref_chosen": -60.116851806640625,
|
|
"logps/ref_rejected": -113.94602966308594,
|
|
"logps/rejected": -699.405517578125,
|
|
"loss": 0.8945,
|
|
"margin_dpo/margin_mean": 278.3428039550781,
|
|
"margin_dpo/margin_std": 257.3295593261719,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.684287812041116,
|
|
"fcm_dpo/beta": 0.0025112878065556288,
|
|
"fcm_dpo/delta": 0.04471685737371445,
|
|
"fcm_dpo/margin": 222.06112670898438,
|
|
"fcm_dpo/q_t": 0.37593716382980347,
|
|
"grad_norm": 33.81809616088867,
|
|
"learning_rate": 1.3856541105586545e-07,
|
|
"logits/chosen": -0.46073633432388306,
|
|
"logits/rejected": -0.45000267028808594,
|
|
"logps/chosen": -391.29534912109375,
|
|
"logps/ref_chosen": -52.920921325683594,
|
|
"logps/ref_rejected": -90.3154296875,
|
|
"logps/rejected": -650.7509765625,
|
|
"loss": 1.0039,
|
|
"margin_dpo/margin_mean": 222.06109619140625,
|
|
"margin_dpo/margin_std": 262.95599365234375,
|
|
"step": 466
|
|
},
|
|
{
|
|
"epoch": 0.6857562408223201,
|
|
"fcm_dpo/beta": 0.00250377063639462,
|
|
"fcm_dpo/delta": 0.003041490912437439,
|
|
"fcm_dpo/margin": 237.8001251220703,
|
|
"fcm_dpo/q_t": 0.37523841857910156,
|
|
"grad_norm": 46.20204544067383,
|
|
"learning_rate": 1.3741809409947729e-07,
|
|
"logits/chosen": -0.5411770343780518,
|
|
"logits/rejected": -0.5251420736312866,
|
|
"logps/chosen": -482.17987060546875,
|
|
"logps/ref_chosen": -78.7158203125,
|
|
"logps/ref_rejected": -102.86019897460938,
|
|
"logps/rejected": -744.1243896484375,
|
|
"loss": 1.0294,
|
|
"margin_dpo/margin_mean": 237.80014038085938,
|
|
"margin_dpo/margin_std": 325.20404052734375,
|
|
"step": 467
|
|
},
|
|
{
|
|
"epoch": 0.6872246696035242,
|
|
"fcm_dpo/beta": 0.002495876979082823,
|
|
"fcm_dpo/delta": -0.055481918156147,
|
|
"fcm_dpo/margin": 260.9195251464844,
|
|
"fcm_dpo/q_t": 0.35720133781433105,
|
|
"grad_norm": 33.269073486328125,
|
|
"learning_rate": 1.362737437810114e-07,
|
|
"logits/chosen": -0.5177669525146484,
|
|
"logits/rejected": -0.49751150608062744,
|
|
"logps/chosen": -376.0140075683594,
|
|
"logps/ref_chosen": -69.93536376953125,
|
|
"logps/ref_rejected": -101.02880859375,
|
|
"logps/rejected": -668.0269775390625,
|
|
"loss": 0.9373,
|
|
"margin_dpo/margin_mean": 260.9195556640625,
|
|
"margin_dpo/margin_std": 270.485595703125,
|
|
"step": 468
|
|
},
|
|
{
|
|
"epoch": 0.6886930983847284,
|
|
"fcm_dpo/beta": 0.002456499030813575,
|
|
"fcm_dpo/delta": -0.07530368864536285,
|
|
"fcm_dpo/margin": 272.3406982421875,
|
|
"fcm_dpo/q_t": 0.35292261838912964,
|
|
"grad_norm": 29.80097770690918,
|
|
"learning_rate": 1.351323902551631e-07,
|
|
"logits/chosen": -0.5581041574478149,
|
|
"logits/rejected": -0.5368698835372925,
|
|
"logps/chosen": -412.0981140136719,
|
|
"logps/ref_chosen": -68.12469482421875,
|
|
"logps/ref_rejected": -104.78640747070312,
|
|
"logps/rejected": -721.1005249023438,
|
|
"loss": 0.9203,
|
|
"margin_dpo/margin_mean": 272.3406982421875,
|
|
"margin_dpo/margin_std": 266.16925048828125,
|
|
"step": 469
|
|
},
|
|
{
|
|
"epoch": 0.6901615271659325,
|
|
"fcm_dpo/beta": 0.0024513239040970802,
|
|
"fcm_dpo/delta": -0.03624919056892395,
|
|
"fcm_dpo/margin": 258.62164306640625,
|
|
"fcm_dpo/q_t": 0.35686296224594116,
|
|
"grad_norm": 25.734210968017578,
|
|
"learning_rate": 1.339940635976592e-07,
|
|
"logits/chosen": -0.5032815933227539,
|
|
"logits/rejected": -0.49432438611984253,
|
|
"logps/chosen": -312.69384765625,
|
|
"logps/ref_chosen": -43.791927337646484,
|
|
"logps/ref_rejected": -82.70285034179688,
|
|
"logps/rejected": -610.2263793945312,
|
|
"loss": 0.9388,
|
|
"margin_dpo/margin_mean": 258.62164306640625,
|
|
"margin_dpo/margin_std": 256.03778076171875,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.6916299559471366,
|
|
"fcm_dpo/beta": 0.0024561528116464615,
|
|
"fcm_dpo/delta": 0.06438558548688889,
|
|
"fcm_dpo/margin": 219.4058074951172,
|
|
"fcm_dpo/q_t": 0.38015252351760864,
|
|
"grad_norm": 27.724517822265625,
|
|
"learning_rate": 1.3285879380446563e-07,
|
|
"logits/chosen": -0.5248172879219055,
|
|
"logits/rejected": -0.5118055939674377,
|
|
"logps/chosen": -382.17425537109375,
|
|
"logps/ref_chosen": -63.33952331542969,
|
|
"logps/ref_rejected": -83.61048126220703,
|
|
"logps/rejected": -621.85107421875,
|
|
"loss": 1.001,
|
|
"margin_dpo/margin_mean": 219.40579223632812,
|
|
"margin_dpo/margin_std": 245.6639404296875,
|
|
"step": 471
|
|
},
|
|
{
|
|
"epoch": 0.6930983847283406,
|
|
"fcm_dpo/beta": 0.0024520205333828926,
|
|
"fcm_dpo/delta": -0.06850513815879822,
|
|
"fcm_dpo/margin": 270.7828063964844,
|
|
"fcm_dpo/q_t": 0.3614341616630554,
|
|
"grad_norm": 26.621925354003906,
|
|
"learning_rate": 1.317266107909975e-07,
|
|
"logits/chosen": -0.5987716317176819,
|
|
"logits/rejected": -0.5613675117492676,
|
|
"logps/chosen": -406.96942138671875,
|
|
"logps/ref_chosen": -83.66610717773438,
|
|
"logps/ref_rejected": -117.20919799804688,
|
|
"logps/rejected": -711.2952880859375,
|
|
"loss": 0.9546,
|
|
"margin_dpo/margin_mean": 270.78277587890625,
|
|
"margin_dpo/margin_std": 321.0069885253906,
|
|
"step": 472
|
|
},
|
|
{
|
|
"epoch": 0.6945668135095447,
|
|
"fcm_dpo/beta": 0.00249496940523386,
|
|
"fcm_dpo/delta": 0.14937041699886322,
|
|
"fcm_dpo/margin": 183.41294860839844,
|
|
"fcm_dpo/q_t": 0.40312713384628296,
|
|
"grad_norm": 36.99402618408203,
|
|
"learning_rate": 1.3059754439133002e-07,
|
|
"logits/chosen": -0.5006664395332336,
|
|
"logits/rejected": -0.4722933769226074,
|
|
"logps/chosen": -418.47216796875,
|
|
"logps/ref_chosen": -63.49696731567383,
|
|
"logps/ref_rejected": -81.14657592773438,
|
|
"logps/rejected": -619.5347900390625,
|
|
"loss": 1.1136,
|
|
"margin_dpo/margin_mean": 183.4129638671875,
|
|
"margin_dpo/margin_std": 317.38916015625,
|
|
"step": 473
|
|
},
|
|
{
|
|
"epoch": 0.6960352422907489,
|
|
"fcm_dpo/beta": 0.002567752031609416,
|
|
"fcm_dpo/delta": 0.09946488589048386,
|
|
"fcm_dpo/margin": 196.4961700439453,
|
|
"fcm_dpo/q_t": 0.38934361934661865,
|
|
"grad_norm": 33.70315933227539,
|
|
"learning_rate": 1.2947162435741277e-07,
|
|
"logits/chosen": -0.5230387449264526,
|
|
"logits/rejected": -0.5138710737228394,
|
|
"logps/chosen": -361.4368591308594,
|
|
"logps/ref_chosen": -52.6119384765625,
|
|
"logps/ref_rejected": -90.08041381835938,
|
|
"logps/rejected": -595.4014892578125,
|
|
"loss": 1.0559,
|
|
"margin_dpo/margin_mean": 196.49615478515625,
|
|
"margin_dpo/margin_std": 272.44683837890625,
|
|
"step": 474
|
|
},
|
|
{
|
|
"epoch": 0.697503671071953,
|
|
"fcm_dpo/beta": 0.002530063269659877,
|
|
"fcm_dpo/delta": -0.09672711789608002,
|
|
"fcm_dpo/margin": 272.7248840332031,
|
|
"fcm_dpo/q_t": 0.3485596776008606,
|
|
"grad_norm": 23.911319732666016,
|
|
"learning_rate": 1.2834888035828596e-07,
|
|
"logits/chosen": -0.48518672585487366,
|
|
"logits/rejected": -0.49074888229370117,
|
|
"logps/chosen": -290.80499267578125,
|
|
"logps/ref_chosen": -42.49519348144531,
|
|
"logps/ref_rejected": -90.06294250488281,
|
|
"logps/rejected": -611.09765625,
|
|
"loss": 0.9061,
|
|
"margin_dpo/margin_mean": 272.7248840332031,
|
|
"margin_dpo/margin_std": 265.2176208496094,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.6989720998531571,
|
|
"fcm_dpo/beta": 0.002525830641388893,
|
|
"fcm_dpo/delta": 0.016785459592938423,
|
|
"fcm_dpo/margin": 231.28424072265625,
|
|
"fcm_dpo/q_t": 0.3694440722465515,
|
|
"grad_norm": 32.7138786315918,
|
|
"learning_rate": 1.2722934197929802e-07,
|
|
"logits/chosen": -0.5376179814338684,
|
|
"logits/rejected": -0.5307664275169373,
|
|
"logps/chosen": -305.538818359375,
|
|
"logps/ref_chosen": -42.94938278198242,
|
|
"logps/ref_rejected": -73.71023559570312,
|
|
"logps/rejected": -567.5839233398438,
|
|
"loss": 0.9606,
|
|
"margin_dpo/margin_mean": 231.28424072265625,
|
|
"margin_dpo/margin_std": 231.99273681640625,
|
|
"step": 476
|
|
},
|
|
{
|
|
"epoch": 0.7004405286343612,
|
|
"fcm_dpo/beta": 0.0025543358642607927,
|
|
"fcm_dpo/delta": 0.06146734952926636,
|
|
"fcm_dpo/margin": 212.0760498046875,
|
|
"fcm_dpo/q_t": 0.3796027898788452,
|
|
"grad_norm": 30.43485450744629,
|
|
"learning_rate": 1.2611303872132631e-07,
|
|
"logits/chosen": -0.5261760354042053,
|
|
"logits/rejected": -0.4716786742210388,
|
|
"logps/chosen": -365.0345458984375,
|
|
"logps/ref_chosen": -70.77261352539062,
|
|
"logps/ref_rejected": -76.13737487792969,
|
|
"logps/rejected": -582.475341796875,
|
|
"loss": 1.02,
|
|
"margin_dpo/margin_mean": 212.0760498046875,
|
|
"margin_dpo/margin_std": 266.8735656738281,
|
|
"step": 477
|
|
},
|
|
{
|
|
"epoch": 0.7019089574155654,
|
|
"fcm_dpo/beta": 0.0025457185693085194,
|
|
"fcm_dpo/delta": -0.05643880367279053,
|
|
"fcm_dpo/margin": 256.39349365234375,
|
|
"fcm_dpo/q_t": 0.3551124632358551,
|
|
"grad_norm": 30.269575119018555,
|
|
"learning_rate": 1.2500000000000005e-07,
|
|
"logits/chosen": -0.4699116349220276,
|
|
"logits/rejected": -0.4670166075229645,
|
|
"logps/chosen": -282.0042724609375,
|
|
"logps/ref_chosen": -41.440513610839844,
|
|
"logps/ref_rejected": -85.36196899414062,
|
|
"logps/rejected": -582.3192138671875,
|
|
"loss": 0.9133,
|
|
"margin_dpo/margin_mean": 256.39349365234375,
|
|
"margin_dpo/margin_std": 231.91416931152344,
|
|
"step": 478
|
|
},
|
|
{
|
|
"epoch": 0.7033773861967695,
|
|
"fcm_dpo/beta": 0.002581070177257061,
|
|
"fcm_dpo/delta": 0.0580781027674675,
|
|
"fcm_dpo/margin": 209.93270874023438,
|
|
"fcm_dpo/q_t": 0.3800834119319916,
|
|
"grad_norm": 34.0640754699707,
|
|
"learning_rate": 1.2389025514492456e-07,
|
|
"logits/chosen": -0.49057966470718384,
|
|
"logits/rejected": -0.49215784668922424,
|
|
"logps/chosen": -384.4983215332031,
|
|
"logps/ref_chosen": -53.907920837402344,
|
|
"logps/ref_rejected": -95.1163330078125,
|
|
"logps/rejected": -635.639404296875,
|
|
"loss": 1.0201,
|
|
"margin_dpo/margin_mean": 209.93270874023438,
|
|
"margin_dpo/margin_std": 248.3424530029297,
|
|
"step": 479
|
|
},
|
|
{
|
|
"epoch": 0.7048458149779736,
|
|
"fcm_dpo/beta": 0.002632645657286048,
|
|
"fcm_dpo/delta": 0.1320512592792511,
|
|
"fcm_dpo/margin": 179.1344757080078,
|
|
"fcm_dpo/q_t": 0.3967488408088684,
|
|
"grad_norm": 46.27069091796875,
|
|
"learning_rate": 1.227838333989088e-07,
|
|
"logits/chosen": -0.4810282588005066,
|
|
"logits/rejected": -0.4517630636692047,
|
|
"logps/chosen": -410.599853515625,
|
|
"logps/ref_chosen": -58.682701110839844,
|
|
"logps/ref_rejected": -82.93248748779297,
|
|
"logps/rejected": -613.984130859375,
|
|
"loss": 1.083,
|
|
"margin_dpo/margin_mean": 179.1344757080078,
|
|
"margin_dpo/margin_std": 258.1438293457031,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.7063142437591777,
|
|
"fcm_dpo/beta": 0.002601025393232703,
|
|
"fcm_dpo/delta": -0.051007017493247986,
|
|
"fcm_dpo/margin": 248.92288208007812,
|
|
"fcm_dpo/q_t": 0.3576533794403076,
|
|
"grad_norm": 35.464439392089844,
|
|
"learning_rate": 1.2168076391719489e-07,
|
|
"logits/chosen": -0.5199482440948486,
|
|
"logits/rejected": -0.5128802061080933,
|
|
"logps/chosen": -346.0648498535156,
|
|
"logps/ref_chosen": -54.964271545410156,
|
|
"logps/ref_rejected": -92.42044067382812,
|
|
"logps/rejected": -632.44384765625,
|
|
"loss": 0.94,
|
|
"margin_dpo/margin_mean": 248.92288208007812,
|
|
"margin_dpo/margin_std": 260.282958984375,
|
|
"step": 481
|
|
},
|
|
{
|
|
"epoch": 0.7077826725403817,
|
|
"fcm_dpo/beta": 0.002659988822415471,
|
|
"fcm_dpo/delta": 0.1587131917476654,
|
|
"fcm_dpo/margin": 168.63143920898438,
|
|
"fcm_dpo/q_t": 0.4008188843727112,
|
|
"grad_norm": 38.83755111694336,
|
|
"learning_rate": 1.2058107576668938e-07,
|
|
"logits/chosen": -0.47180038690567017,
|
|
"logits/rejected": -0.45411762595176697,
|
|
"logps/chosen": -398.88873291015625,
|
|
"logps/ref_chosen": -67.553466796875,
|
|
"logps/ref_rejected": -87.58953857421875,
|
|
"logps/rejected": -587.5562133789062,
|
|
"loss": 1.0902,
|
|
"margin_dpo/margin_mean": 168.63143920898438,
|
|
"margin_dpo/margin_std": 254.7269287109375,
|
|
"step": 482
|
|
},
|
|
{
|
|
"epoch": 0.7092511013215859,
|
|
"fcm_dpo/beta": 0.002619517268612981,
|
|
"fcm_dpo/delta": -0.1474650502204895,
|
|
"fcm_dpo/margin": 281.0007019042969,
|
|
"fcm_dpo/q_t": 0.34216371178627014,
|
|
"grad_norm": 40.65428161621094,
|
|
"learning_rate": 1.194847979251979e-07,
|
|
"logits/chosen": -0.5000494718551636,
|
|
"logits/rejected": -0.48498857021331787,
|
|
"logps/chosen": -357.7904052734375,
|
|
"logps/ref_chosen": -63.32981872558594,
|
|
"logps/ref_rejected": -95.78697204589844,
|
|
"logps/rejected": -671.248291015625,
|
|
"loss": 0.8912,
|
|
"margin_dpo/margin_mean": 281.0007019042969,
|
|
"margin_dpo/margin_std": 279.4383544921875,
|
|
"step": 483
|
|
},
|
|
{
|
|
"epoch": 0.71071953010279,
|
|
"fcm_dpo/beta": 0.0026109693571925163,
|
|
"fcm_dpo/delta": -0.03564995154738426,
|
|
"fcm_dpo/margin": 242.32601928710938,
|
|
"fcm_dpo/q_t": 0.3604073226451874,
|
|
"grad_norm": 27.125112533569336,
|
|
"learning_rate": 1.1839195928066101e-07,
|
|
"logits/chosen": -0.48026251792907715,
|
|
"logits/rejected": -0.46726343035697937,
|
|
"logps/chosen": -332.4265441894531,
|
|
"logps/ref_chosen": -59.13812255859375,
|
|
"logps/ref_rejected": -84.37144470214844,
|
|
"logps/rejected": -599.9859008789062,
|
|
"loss": 0.9377,
|
|
"margin_dpo/margin_mean": 242.32603454589844,
|
|
"margin_dpo/margin_std": 237.0671844482422,
|
|
"step": 484
|
|
},
|
|
{
|
|
"epoch": 0.7121879588839941,
|
|
"fcm_dpo/beta": 0.0025863349437713623,
|
|
"fcm_dpo/delta": -0.01730707660317421,
|
|
"fcm_dpo/margin": 238.25384521484375,
|
|
"fcm_dpo/q_t": 0.36559098958969116,
|
|
"grad_norm": 36.989036560058594,
|
|
"learning_rate": 1.1730258863039347e-07,
|
|
"logits/chosen": -0.468616247177124,
|
|
"logits/rejected": -0.466255784034729,
|
|
"logps/chosen": -326.2464904785156,
|
|
"logps/ref_chosen": -58.849571228027344,
|
|
"logps/ref_rejected": -103.36408233642578,
|
|
"logps/rejected": -609.0148315429688,
|
|
"loss": 0.967,
|
|
"margin_dpo/margin_mean": 238.25384521484375,
|
|
"margin_dpo/margin_std": 268.4663391113281,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.7136563876651982,
|
|
"fcm_dpo/beta": 0.0025734733790159225,
|
|
"fcm_dpo/delta": -0.033280279487371445,
|
|
"fcm_dpo/margin": 245.24191284179688,
|
|
"fcm_dpo/q_t": 0.36465585231781006,
|
|
"grad_norm": 40.85445022583008,
|
|
"learning_rate": 1.1621671468032493e-07,
|
|
"logits/chosen": -0.50041663646698,
|
|
"logits/rejected": -0.48742035031318665,
|
|
"logps/chosen": -367.93310546875,
|
|
"logps/ref_chosen": -55.25966262817383,
|
|
"logps/ref_rejected": -92.13936614990234,
|
|
"logps/rejected": -650.0546875,
|
|
"loss": 0.9882,
|
|
"margin_dpo/margin_mean": 245.24192810058594,
|
|
"margin_dpo/margin_std": 306.00677490234375,
|
|
"step": 486
|
|
},
|
|
{
|
|
"epoch": 0.7151248164464024,
|
|
"fcm_dpo/beta": 0.002564340131357312,
|
|
"fcm_dpo/delta": 0.024611998349428177,
|
|
"fcm_dpo/margin": 224.8999786376953,
|
|
"fcm_dpo/q_t": 0.3688669800758362,
|
|
"grad_norm": 35.55867004394531,
|
|
"learning_rate": 1.1513436604424378e-07,
|
|
"logits/chosen": -0.48910582065582275,
|
|
"logits/rejected": -0.4775072932243347,
|
|
"logps/chosen": -351.23388671875,
|
|
"logps/ref_chosen": -53.06330871582031,
|
|
"logps/ref_rejected": -92.41883087158203,
|
|
"logps/rejected": -615.4893798828125,
|
|
"loss": 0.9727,
|
|
"margin_dpo/margin_mean": 224.8999786376953,
|
|
"margin_dpo/margin_std": 231.9244842529297,
|
|
"step": 487
|
|
},
|
|
{
|
|
"epoch": 0.7165932452276065,
|
|
"fcm_dpo/beta": 0.0025914846919476986,
|
|
"fcm_dpo/delta": 0.017736222594976425,
|
|
"fcm_dpo/margin": 224.94769287109375,
|
|
"fcm_dpo/q_t": 0.3664684593677521,
|
|
"grad_norm": 39.95408248901367,
|
|
"learning_rate": 1.1405557124304335e-07,
|
|
"logits/chosen": -0.4504004716873169,
|
|
"logits/rejected": -0.43368983268737793,
|
|
"logps/chosen": -295.21221923828125,
|
|
"logps/ref_chosen": -52.22815704345703,
|
|
"logps/ref_rejected": -84.00656127929688,
|
|
"logps/rejected": -551.9383544921875,
|
|
"loss": 0.9446,
|
|
"margin_dpo/margin_mean": 224.94769287109375,
|
|
"margin_dpo/margin_std": 193.76791381835938,
|
|
"step": 488
|
|
},
|
|
{
|
|
"epoch": 0.7180616740088106,
|
|
"fcm_dpo/beta": 0.00260374927893281,
|
|
"fcm_dpo/delta": 0.053067468106746674,
|
|
"fcm_dpo/margin": 211.18069458007812,
|
|
"fcm_dpo/q_t": 0.378859281539917,
|
|
"grad_norm": 33.997283935546875,
|
|
"learning_rate": 1.1298035870396985e-07,
|
|
"logits/chosen": -0.49653559923171997,
|
|
"logits/rejected": -0.47625601291656494,
|
|
"logps/chosen": -304.00372314453125,
|
|
"logps/ref_chosen": -55.989627838134766,
|
|
"logps/ref_rejected": -79.39812469482422,
|
|
"logps/rejected": -538.5928955078125,
|
|
"loss": 0.9855,
|
|
"margin_dpo/margin_mean": 211.18069458007812,
|
|
"margin_dpo/margin_std": 227.5222930908203,
|
|
"step": 489
|
|
},
|
|
{
|
|
"epoch": 0.7195301027900147,
|
|
"fcm_dpo/beta": 0.002616376616060734,
|
|
"fcm_dpo/delta": -0.01173383742570877,
|
|
"fcm_dpo/margin": 233.45465087890625,
|
|
"fcm_dpo/q_t": 0.36978480219841003,
|
|
"grad_norm": 51.60263442993164,
|
|
"learning_rate": 1.1190875675987355e-07,
|
|
"logits/chosen": -0.5432174801826477,
|
|
"logits/rejected": -0.5576708316802979,
|
|
"logps/chosen": -326.5618896484375,
|
|
"logps/ref_chosen": -52.36639404296875,
|
|
"logps/ref_rejected": -110.4090576171875,
|
|
"logps/rejected": -618.0592041015625,
|
|
"loss": 0.995,
|
|
"margin_dpo/margin_mean": 233.45465087890625,
|
|
"margin_dpo/margin_std": 293.9273376464844,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.7209985315712188,
|
|
"fcm_dpo/beta": 0.002662881277501583,
|
|
"fcm_dpo/delta": 0.1412055492401123,
|
|
"fcm_dpo/margin": 174.71432495117188,
|
|
"fcm_dpo/q_t": 0.39850908517837524,
|
|
"grad_norm": 46.16297912597656,
|
|
"learning_rate": 1.1084079364846241e-07,
|
|
"logits/chosen": -0.4562872052192688,
|
|
"logits/rejected": -0.4420876204967499,
|
|
"logps/chosen": -348.5367431640625,
|
|
"logps/ref_chosen": -60.11626434326172,
|
|
"logps/ref_rejected": -73.27278900146484,
|
|
"logps/rejected": -536.4075927734375,
|
|
"loss": 1.072,
|
|
"margin_dpo/margin_mean": 174.71432495117188,
|
|
"margin_dpo/margin_std": 248.17523193359375,
|
|
"step": 491
|
|
},
|
|
{
|
|
"epoch": 0.7224669603524229,
|
|
"fcm_dpo/beta": 0.002773004351183772,
|
|
"fcm_dpo/delta": 0.22383955121040344,
|
|
"fcm_dpo/margin": 138.69735717773438,
|
|
"fcm_dpo/q_t": 0.4172734022140503,
|
|
"grad_norm": 42.61263656616211,
|
|
"learning_rate": 1.097764975115576e-07,
|
|
"logits/chosen": -0.44189101457595825,
|
|
"logits/rejected": -0.4200980067253113,
|
|
"logps/chosen": -361.65570068359375,
|
|
"logps/ref_chosen": -53.994178771972656,
|
|
"logps/ref_rejected": -72.65962219238281,
|
|
"logps/rejected": -519.0184936523438,
|
|
"loss": 1.1625,
|
|
"margin_dpo/margin_mean": 138.69735717773438,
|
|
"margin_dpo/margin_std": 266.28662109375,
|
|
"step": 492
|
|
},
|
|
{
|
|
"epoch": 0.723935389133627,
|
|
"fcm_dpo/beta": 0.0028385447803884745,
|
|
"fcm_dpo/delta": 0.1169082373380661,
|
|
"fcm_dpo/margin": 172.2346954345703,
|
|
"fcm_dpo/q_t": 0.39189258217811584,
|
|
"grad_norm": 43.42572021484375,
|
|
"learning_rate": 1.0871589639435203e-07,
|
|
"logits/chosen": -0.5289523601531982,
|
|
"logits/rejected": -0.5010632276535034,
|
|
"logps/chosen": -380.6925048828125,
|
|
"logps/ref_chosen": -75.49723815917969,
|
|
"logps/ref_rejected": -87.32301330566406,
|
|
"logps/rejected": -564.7529296875,
|
|
"loss": 1.0419,
|
|
"margin_dpo/margin_mean": 172.2346954345703,
|
|
"margin_dpo/margin_std": 214.199951171875,
|
|
"step": 493
|
|
},
|
|
{
|
|
"epoch": 0.7254038179148311,
|
|
"fcm_dpo/beta": 0.00280939694494009,
|
|
"fcm_dpo/delta": -0.12965027987957,
|
|
"fcm_dpo/margin": 256.29547119140625,
|
|
"fcm_dpo/q_t": 0.3384782075881958,
|
|
"grad_norm": 34.547096252441406,
|
|
"learning_rate": 1.0765901824467166e-07,
|
|
"logits/chosen": -0.4465922713279724,
|
|
"logits/rejected": -0.45816731452941895,
|
|
"logps/chosen": -296.73284912109375,
|
|
"logps/ref_chosen": -41.35926818847656,
|
|
"logps/ref_rejected": -86.09136962890625,
|
|
"logps/rejected": -597.7604370117188,
|
|
"loss": 0.8787,
|
|
"margin_dpo/margin_mean": 256.2955017089844,
|
|
"margin_dpo/margin_std": 221.27516174316406,
|
|
"step": 494
|
|
},
|
|
{
|
|
"epoch": 0.7268722466960352,
|
|
"fcm_dpo/beta": 0.002777573186904192,
|
|
"fcm_dpo/delta": -0.08846628665924072,
|
|
"fcm_dpo/margin": 245.58688354492188,
|
|
"fcm_dpo/q_t": 0.35333937406539917,
|
|
"grad_norm": 34.23904037475586,
|
|
"learning_rate": 1.0660589091223854e-07,
|
|
"logits/chosen": -0.493415504693985,
|
|
"logits/rejected": -0.4882367253303528,
|
|
"logps/chosen": -356.96539306640625,
|
|
"logps/ref_chosen": -63.53507995605469,
|
|
"logps/ref_rejected": -91.42443084716797,
|
|
"logps/rejected": -630.441650390625,
|
|
"loss": 0.929,
|
|
"margin_dpo/margin_mean": 245.58688354492188,
|
|
"margin_dpo/margin_std": 258.7152099609375,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.7283406754772394,
|
|
"fcm_dpo/beta": 0.002794483909383416,
|
|
"fcm_dpo/delta": 0.13067297637462616,
|
|
"fcm_dpo/margin": 170.2359619140625,
|
|
"fcm_dpo/q_t": 0.39404138922691345,
|
|
"grad_norm": 46.63887405395508,
|
|
"learning_rate": 1.0555654214793722e-07,
|
|
"logits/chosen": -0.4721938669681549,
|
|
"logits/rejected": -0.4511212706565857,
|
|
"logps/chosen": -414.6033630371094,
|
|
"logps/ref_chosen": -72.5919189453125,
|
|
"logps/ref_rejected": -84.32933807373047,
|
|
"logps/rejected": -596.5767822265625,
|
|
"loss": 1.0508,
|
|
"margin_dpo/margin_mean": 170.2359619140625,
|
|
"margin_dpo/margin_std": 214.48468017578125,
|
|
"step": 496
|
|
},
|
|
{
|
|
"epoch": 0.7298091042584435,
|
|
"fcm_dpo/beta": 0.0028514941222965717,
|
|
"fcm_dpo/delta": 0.13596858084201813,
|
|
"fcm_dpo/margin": 164.8284912109375,
|
|
"fcm_dpo/q_t": 0.3949023485183716,
|
|
"grad_norm": 35.32782745361328,
|
|
"learning_rate": 1.0451099960308374e-07,
|
|
"logits/chosen": -0.4963914752006531,
|
|
"logits/rejected": -0.479754239320755,
|
|
"logps/chosen": -384.9673156738281,
|
|
"logps/ref_chosen": -58.59397506713867,
|
|
"logps/ref_rejected": -76.28836822509766,
|
|
"logps/rejected": -567.490234375,
|
|
"loss": 1.0561,
|
|
"margin_dpo/margin_mean": 164.8284912109375,
|
|
"margin_dpo/margin_std": 201.14633178710938,
|
|
"step": 497
|
|
},
|
|
{
|
|
"epoch": 0.7312775330396476,
|
|
"fcm_dpo/beta": 0.0028984027449041605,
|
|
"fcm_dpo/delta": 0.004962385166436434,
|
|
"fcm_dpo/margin": 205.40037536621094,
|
|
"fcm_dpo/q_t": 0.37139469385147095,
|
|
"grad_norm": 38.93318557739258,
|
|
"learning_rate": 1.0346929082869641e-07,
|
|
"logits/chosen": -0.4890047311782837,
|
|
"logits/rejected": -0.4707816243171692,
|
|
"logps/chosen": -396.84356689453125,
|
|
"logps/ref_chosen": -71.20565795898438,
|
|
"logps/ref_rejected": -83.95803833007812,
|
|
"logps/rejected": -614.996337890625,
|
|
"loss": 1.0052,
|
|
"margin_dpo/margin_mean": 205.40037536621094,
|
|
"margin_dpo/margin_std": 263.6328125,
|
|
"step": 498
|
|
},
|
|
{
|
|
"epoch": 0.7327459618208517,
|
|
"fcm_dpo/beta": 0.0028373058885335922,
|
|
"fcm_dpo/delta": -0.13358387351036072,
|
|
"fcm_dpo/margin": 255.02593994140625,
|
|
"fcm_dpo/q_t": 0.3414595425128937,
|
|
"grad_norm": 47.647823333740234,
|
|
"learning_rate": 1.0243144327477013e-07,
|
|
"logits/chosen": -0.4490722417831421,
|
|
"logits/rejected": -0.45660898089408875,
|
|
"logps/chosen": -363.71142578125,
|
|
"logps/ref_chosen": -51.25519561767578,
|
|
"logps/ref_rejected": -101.07870483398438,
|
|
"logps/rejected": -668.5609130859375,
|
|
"loss": 0.9152,
|
|
"margin_dpo/margin_mean": 255.02593994140625,
|
|
"margin_dpo/margin_std": 267.2959899902344,
|
|
"step": 499
|
|
},
|
|
{
|
|
"epoch": 0.7342143906020558,
|
|
"fcm_dpo/beta": 0.0028089380357414484,
|
|
"fcm_dpo/delta": -0.024006161838769913,
|
|
"fcm_dpo/margin": 221.58847045898438,
|
|
"fcm_dpo/q_t": 0.3665502965450287,
|
|
"grad_norm": 52.78643798828125,
|
|
"learning_rate": 1.0139748428955333e-07,
|
|
"logits/chosen": -0.4591681957244873,
|
|
"logits/rejected": -0.4673272371292114,
|
|
"logps/chosen": -429.0184326171875,
|
|
"logps/ref_chosen": -57.027442932128906,
|
|
"logps/ref_rejected": -93.93421173095703,
|
|
"logps/rejected": -687.513671875,
|
|
"loss": 0.9865,
|
|
"margin_dpo/margin_mean": 221.58847045898438,
|
|
"margin_dpo/margin_std": 274.5337829589844,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.73568281938326,
|
|
"fcm_dpo/beta": 0.002789230551570654,
|
|
"fcm_dpo/delta": -0.06634011119604111,
|
|
"fcm_dpo/margin": 237.33590698242188,
|
|
"fcm_dpo/q_t": 0.3573167026042938,
|
|
"grad_norm": 34.69628143310547,
|
|
"learning_rate": 1.0036744111882672e-07,
|
|
"logits/chosen": -0.45637276768684387,
|
|
"logits/rejected": -0.4298150837421417,
|
|
"logps/chosen": -364.97088623046875,
|
|
"logps/ref_chosen": -54.359527587890625,
|
|
"logps/ref_rejected": -80.15670013427734,
|
|
"logps/rejected": -628.10400390625,
|
|
"loss": 0.9556,
|
|
"margin_dpo/margin_mean": 237.33590698242188,
|
|
"margin_dpo/margin_std": 273.72589111328125,
|
|
"step": 501
|
|
},
|
|
{
|
|
"epoch": 0.737151248164464,
|
|
"fcm_dpo/beta": 0.002761396113783121,
|
|
"fcm_dpo/delta": -0.06494729220867157,
|
|
"fcm_dpo/margin": 239.1594696044922,
|
|
"fcm_dpo/q_t": 0.3486095666885376,
|
|
"grad_norm": 28.551965713500977,
|
|
"learning_rate": 9.934134090518592e-08,
|
|
"logits/chosen": -0.4536617398262024,
|
|
"logits/rejected": -0.4335412085056305,
|
|
"logps/chosen": -356.140380859375,
|
|
"logps/ref_chosen": -67.60050964355469,
|
|
"logps/ref_rejected": -82.94876098632812,
|
|
"logps/rejected": -610.6480712890625,
|
|
"loss": 0.8913,
|
|
"margin_dpo/margin_mean": 239.1594696044922,
|
|
"margin_dpo/margin_std": 181.16015625,
|
|
"step": 502
|
|
},
|
|
{
|
|
"epoch": 0.7386196769456681,
|
|
"fcm_dpo/beta": 0.002728442195802927,
|
|
"fcm_dpo/delta": 0.006640031933784485,
|
|
"fcm_dpo/margin": 217.5033721923828,
|
|
"fcm_dpo/q_t": 0.36779502034187317,
|
|
"grad_norm": 33.08012008666992,
|
|
"learning_rate": 9.831921068732571e-08,
|
|
"logits/chosen": -0.43443652987480164,
|
|
"logits/rejected": -0.4157460927963257,
|
|
"logps/chosen": -348.41259765625,
|
|
"logps/ref_chosen": -55.078407287597656,
|
|
"logps/ref_rejected": -82.50544738769531,
|
|
"logps/rejected": -593.343017578125,
|
|
"loss": 0.9555,
|
|
"margin_dpo/margin_mean": 217.50338745117188,
|
|
"margin_dpo/margin_std": 215.96286010742188,
|
|
"step": 503
|
|
},
|
|
{
|
|
"epoch": 0.7400881057268722,
|
|
"fcm_dpo/beta": 0.0026825086679309607,
|
|
"fcm_dpo/delta": -0.13238787651062012,
|
|
"fcm_dpo/margin": 269.34735107421875,
|
|
"fcm_dpo/q_t": 0.34110093116760254,
|
|
"grad_norm": 35.52935791015625,
|
|
"learning_rate": 9.730107739932805e-08,
|
|
"logits/chosen": -0.45639944076538086,
|
|
"logits/rejected": -0.4605451822280884,
|
|
"logps/chosen": -375.00860595703125,
|
|
"logps/ref_chosen": -59.96575164794922,
|
|
"logps/ref_rejected": -103.76212310791016,
|
|
"logps/rejected": -688.15234375,
|
|
"loss": 0.8908,
|
|
"margin_dpo/margin_mean": 269.34735107421875,
|
|
"margin_dpo/margin_std": 249.264892578125,
|
|
"step": 504
|
|
},
|
|
{
|
|
"epoch": 0.7415565345080763,
|
|
"fcm_dpo/beta": 0.0027532954700291157,
|
|
"fcm_dpo/delta": 0.1878781020641327,
|
|
"fcm_dpo/margin": 152.27743530273438,
|
|
"fcm_dpo/q_t": 0.4108385741710663,
|
|
"grad_norm": 54.627689361572266,
|
|
"learning_rate": 9.628696786995188e-08,
|
|
"logits/chosen": -0.5202739834785461,
|
|
"logits/rejected": -0.49124228954315186,
|
|
"logps/chosen": -429.90838623046875,
|
|
"logps/ref_chosen": -76.1549072265625,
|
|
"logps/ref_rejected": -88.58537292480469,
|
|
"logps/rejected": -594.6163330078125,
|
|
"loss": 1.1332,
|
|
"margin_dpo/margin_mean": 152.27743530273438,
|
|
"margin_dpo/margin_std": 267.0574951171875,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.7430249632892805,
|
|
"fcm_dpo/beta": 0.002741964068263769,
|
|
"fcm_dpo/delta": -0.04416107013821602,
|
|
"fcm_dpo/margin": 233.71510314941406,
|
|
"fcm_dpo/q_t": 0.358504056930542,
|
|
"grad_norm": 38.65158462524414,
|
|
"learning_rate": 9.527690882192635e-08,
|
|
"logits/chosen": -0.47684890031814575,
|
|
"logits/rejected": -0.4706017076969147,
|
|
"logps/chosen": -328.3465576171875,
|
|
"logps/ref_chosen": -48.96050262451172,
|
|
"logps/ref_rejected": -78.41505432128906,
|
|
"logps/rejected": -591.5161743164062,
|
|
"loss": 0.9438,
|
|
"margin_dpo/margin_mean": 233.71510314941406,
|
|
"margin_dpo/margin_std": 243.26914978027344,
|
|
"step": 506
|
|
},
|
|
{
|
|
"epoch": 0.7444933920704846,
|
|
"fcm_dpo/beta": 0.002767894882708788,
|
|
"fcm_dpo/delta": 0.04769434034824371,
|
|
"fcm_dpo/margin": 200.48585510253906,
|
|
"fcm_dpo/q_t": 0.38139885663986206,
|
|
"grad_norm": 31.428619384765625,
|
|
"learning_rate": 9.427092687124691e-08,
|
|
"logits/chosen": -0.4897286891937256,
|
|
"logits/rejected": -0.4771530032157898,
|
|
"logps/chosen": -389.1927185058594,
|
|
"logps/ref_chosen": -66.80149841308594,
|
|
"logps/ref_rejected": -95.37289428710938,
|
|
"logps/rejected": -618.25,
|
|
"loss": 1.0113,
|
|
"margin_dpo/margin_mean": 200.48583984375,
|
|
"margin_dpo/margin_std": 256.5914001464844,
|
|
"step": 507
|
|
},
|
|
{
|
|
"epoch": 0.7459618208516887,
|
|
"fcm_dpo/beta": 0.002809921745210886,
|
|
"fcm_dpo/delta": 0.07570519298315048,
|
|
"fcm_dpo/margin": 187.93719482421875,
|
|
"fcm_dpo/q_t": 0.38831907510757446,
|
|
"grad_norm": 37.678199768066406,
|
|
"learning_rate": 9.326904852647344e-08,
|
|
"logits/chosen": -0.4942491054534912,
|
|
"logits/rejected": -0.48562514781951904,
|
|
"logps/chosen": -381.1417236328125,
|
|
"logps/ref_chosen": -71.303466796875,
|
|
"logps/ref_rejected": -95.6275405883789,
|
|
"logps/rejected": -593.4030151367188,
|
|
"loss": 1.0468,
|
|
"margin_dpo/margin_mean": 187.93719482421875,
|
|
"margin_dpo/margin_std": 266.2950744628906,
|
|
"step": 508
|
|
},
|
|
{
|
|
"epoch": 0.7474302496328928,
|
|
"fcm_dpo/beta": 0.0028523635119199753,
|
|
"fcm_dpo/delta": 0.05893818289041519,
|
|
"fcm_dpo/margin": 190.60777282714844,
|
|
"fcm_dpo/q_t": 0.38169676065444946,
|
|
"grad_norm": 39.39438247680664,
|
|
"learning_rate": 9.227130018803195e-08,
|
|
"logits/chosen": -0.5167714357376099,
|
|
"logits/rejected": -0.5027569532394409,
|
|
"logps/chosen": -326.60260009765625,
|
|
"logps/ref_chosen": -63.81895065307617,
|
|
"logps/ref_rejected": -83.25643920898438,
|
|
"logps/rejected": -536.6478881835938,
|
|
"loss": 1.0086,
|
|
"margin_dpo/margin_mean": 190.60775756835938,
|
|
"margin_dpo/margin_std": 227.0200653076172,
|
|
"step": 509
|
|
},
|
|
{
|
|
"epoch": 0.748898678414097,
|
|
"fcm_dpo/beta": 0.002829122357070446,
|
|
"fcm_dpo/delta": -0.050209492444992065,
|
|
"fcm_dpo/margin": 228.68527221679688,
|
|
"fcm_dpo/q_t": 0.35418280959129333,
|
|
"grad_norm": 33.37362289428711,
|
|
"learning_rate": 9.127770814751932e-08,
|
|
"logits/chosen": -0.47170090675354004,
|
|
"logits/rejected": -0.4742484390735626,
|
|
"logps/chosen": -364.83221435546875,
|
|
"logps/ref_chosen": -51.878448486328125,
|
|
"logps/ref_rejected": -102.7651596069336,
|
|
"logps/rejected": -644.4041748046875,
|
|
"loss": 0.9209,
|
|
"margin_dpo/margin_mean": 228.68527221679688,
|
|
"margin_dpo/margin_std": 210.78187561035156,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.750367107195301,
|
|
"fcm_dpo/beta": 0.0028122500516474247,
|
|
"fcm_dpo/delta": -0.04460298642516136,
|
|
"fcm_dpo/margin": 228.15643310546875,
|
|
"fcm_dpo/q_t": 0.3623583912849426,
|
|
"grad_norm": 28.944910049438477,
|
|
"learning_rate": 9.028829858700973e-08,
|
|
"logits/chosen": -0.4922507405281067,
|
|
"logits/rejected": -0.4849390387535095,
|
|
"logps/chosen": -339.91571044921875,
|
|
"logps/ref_chosen": -60.23811721801758,
|
|
"logps/ref_rejected": -92.85676574707031,
|
|
"logps/rejected": -600.6907958984375,
|
|
"loss": 0.9745,
|
|
"margin_dpo/margin_mean": 228.15643310546875,
|
|
"margin_dpo/margin_std": 276.27374267578125,
|
|
"step": 511
|
|
},
|
|
{
|
|
"epoch": 0.7518355359765051,
|
|
"fcm_dpo/beta": 0.002760728355497122,
|
|
"fcm_dpo/delta": -0.08452171087265015,
|
|
"fcm_dpo/margin": 245.87234497070312,
|
|
"fcm_dpo/q_t": 0.3484812378883362,
|
|
"grad_norm": 27.64395523071289,
|
|
"learning_rate": 8.930309757836516e-08,
|
|
"logits/chosen": -0.4628169536590576,
|
|
"logits/rejected": -0.44934043288230896,
|
|
"logps/chosen": -307.468994140625,
|
|
"logps/ref_chosen": -54.905494689941406,
|
|
"logps/ref_rejected": -81.87586975097656,
|
|
"logps/rejected": -580.3117065429688,
|
|
"loss": 0.8986,
|
|
"margin_dpo/margin_mean": 245.87234497070312,
|
|
"margin_dpo/margin_std": 219.15728759765625,
|
|
"step": 512
|
|
},
|
|
{
|
|
"epoch": 0.7533039647577092,
|
|
"fcm_dpo/beta": 0.002783479169011116,
|
|
"fcm_dpo/delta": 0.1268652230501175,
|
|
"fcm_dpo/margin": 172.15696716308594,
|
|
"fcm_dpo/q_t": 0.39400118589401245,
|
|
"grad_norm": 46.16716766357422,
|
|
"learning_rate": 8.832213108254863e-08,
|
|
"logits/chosen": -0.4615207314491272,
|
|
"logits/rejected": -0.42703330516815186,
|
|
"logps/chosen": -352.5402526855469,
|
|
"logps/ref_chosen": -64.91644287109375,
|
|
"logps/ref_rejected": -76.06245422363281,
|
|
"logps/rejected": -535.84326171875,
|
|
"loss": 1.0662,
|
|
"margin_dpo/margin_mean": 172.15696716308594,
|
|
"margin_dpo/margin_std": 235.25601196289062,
|
|
"step": 513
|
|
},
|
|
{
|
|
"epoch": 0.7547723935389133,
|
|
"fcm_dpo/beta": 0.0028532061260193586,
|
|
"fcm_dpo/delta": 0.07234474271535873,
|
|
"fcm_dpo/margin": 186.24154663085938,
|
|
"fcm_dpo/q_t": 0.38589781522750854,
|
|
"grad_norm": 40.779319763183594,
|
|
"learning_rate": 8.734542494893954e-08,
|
|
"logits/chosen": -0.47675463557243347,
|
|
"logits/rejected": -0.44973114132881165,
|
|
"logps/chosen": -357.24749755859375,
|
|
"logps/ref_chosen": -74.22957611083984,
|
|
"logps/ref_rejected": -78.945556640625,
|
|
"logps/rejected": -548.2050170898438,
|
|
"loss": 1.0352,
|
|
"margin_dpo/margin_mean": 186.24154663085938,
|
|
"margin_dpo/margin_std": 250.20767211914062,
|
|
"step": 514
|
|
},
|
|
{
|
|
"epoch": 0.7562408223201175,
|
|
"fcm_dpo/beta": 0.0028960055205971003,
|
|
"fcm_dpo/delta": 0.09402909129858017,
|
|
"fcm_dpo/margin": 176.40408325195312,
|
|
"fcm_dpo/q_t": 0.3851974606513977,
|
|
"grad_norm": 31.811119079589844,
|
|
"learning_rate": 8.637300491465272e-08,
|
|
"logits/chosen": -0.47848212718963623,
|
|
"logits/rejected": -0.47844403982162476,
|
|
"logps/chosen": -294.98284912109375,
|
|
"logps/ref_chosen": -50.40156555175781,
|
|
"logps/ref_rejected": -87.09774780273438,
|
|
"logps/rejected": -508.0831604003906,
|
|
"loss": 1.0331,
|
|
"margin_dpo/margin_mean": 176.40408325195312,
|
|
"margin_dpo/margin_std": 221.04446411132812,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.7577092511013216,
|
|
"fcm_dpo/beta": 0.00288777775131166,
|
|
"fcm_dpo/delta": -0.08516449481248856,
|
|
"fcm_dpo/margin": 235.27685546875,
|
|
"fcm_dpo/q_t": 0.3478805422782898,
|
|
"grad_norm": 32.073699951171875,
|
|
"learning_rate": 8.540489660386064e-08,
|
|
"logits/chosen": -0.50160813331604,
|
|
"logits/rejected": -0.5122185945510864,
|
|
"logps/chosen": -312.9991455078125,
|
|
"logps/ref_chosen": -64.64956665039062,
|
|
"logps/ref_rejected": -111.72237396240234,
|
|
"logps/rejected": -595.3488159179688,
|
|
"loss": 0.8995,
|
|
"margin_dpo/margin_mean": 235.27685546875,
|
|
"margin_dpo/margin_std": 209.3588104248047,
|
|
"step": 516
|
|
},
|
|
{
|
|
"epoch": 0.7591776798825257,
|
|
"fcm_dpo/beta": 0.002824044320732355,
|
|
"fcm_dpo/delta": -0.06978069990873337,
|
|
"fcm_dpo/margin": 235.31272888183594,
|
|
"fcm_dpo/q_t": 0.35607290267944336,
|
|
"grad_norm": 34.7307014465332,
|
|
"learning_rate": 8.444112552711752e-08,
|
|
"logits/chosen": -0.498099148273468,
|
|
"logits/rejected": -0.47780922055244446,
|
|
"logps/chosen": -338.8596496582031,
|
|
"logps/ref_chosen": -60.913551330566406,
|
|
"logps/ref_rejected": -89.08308410644531,
|
|
"logps/rejected": -602.3419189453125,
|
|
"loss": 0.9334,
|
|
"margin_dpo/margin_mean": 235.31271362304688,
|
|
"margin_dpo/margin_std": 250.22042846679688,
|
|
"step": 517
|
|
},
|
|
{
|
|
"epoch": 0.7606461086637298,
|
|
"fcm_dpo/beta": 0.002825916511937976,
|
|
"fcm_dpo/delta": 0.01667827181518078,
|
|
"fcm_dpo/margin": 206.73744201660156,
|
|
"fcm_dpo/q_t": 0.37291136384010315,
|
|
"grad_norm": 30.245616912841797,
|
|
"learning_rate": 8.348171708068747e-08,
|
|
"logits/chosen": -0.48336124420166016,
|
|
"logits/rejected": -0.4803961515426636,
|
|
"logps/chosen": -331.5633850097656,
|
|
"logps/ref_chosen": -57.45589065551758,
|
|
"logps/ref_rejected": -85.31269836425781,
|
|
"logps/rejected": -566.1575927734375,
|
|
"loss": 1.0028,
|
|
"margin_dpo/margin_mean": 206.73744201660156,
|
|
"margin_dpo/margin_std": 258.0599365234375,
|
|
"step": 518
|
|
},
|
|
{
|
|
"epoch": 0.762114537444934,
|
|
"fcm_dpo/beta": 0.0028772333171218634,
|
|
"fcm_dpo/delta": 0.08668302744626999,
|
|
"fcm_dpo/margin": 179.88906860351562,
|
|
"fcm_dpo/q_t": 0.3874738812446594,
|
|
"grad_norm": 30.495445251464844,
|
|
"learning_rate": 8.25266965458755e-08,
|
|
"logits/chosen": -0.49129921197891235,
|
|
"logits/rejected": -0.470879465341568,
|
|
"logps/chosen": -341.64581298828125,
|
|
"logps/ref_chosen": -74.06331634521484,
|
|
"logps/ref_rejected": -104.44416809082031,
|
|
"logps/rejected": -551.9157104492188,
|
|
"loss": 1.035,
|
|
"margin_dpo/margin_mean": 179.88906860351562,
|
|
"margin_dpo/margin_std": 234.95318603515625,
|
|
"step": 519
|
|
},
|
|
{
|
|
"epoch": 0.7635829662261381,
|
|
"fcm_dpo/beta": 0.002877956721931696,
|
|
"fcm_dpo/delta": 0.02885432541370392,
|
|
"fcm_dpo/margin": 198.79312133789062,
|
|
"fcm_dpo/q_t": 0.3750660717487335,
|
|
"grad_norm": 30.533952713012695,
|
|
"learning_rate": 8.15760890883607e-08,
|
|
"logits/chosen": -0.4489268660545349,
|
|
"logits/rejected": -0.4413859248161316,
|
|
"logps/chosen": -339.8787841796875,
|
|
"logps/ref_chosen": -70.2998275756836,
|
|
"logps/ref_rejected": -99.98133850097656,
|
|
"logps/rejected": -568.3533935546875,
|
|
"loss": 0.9855,
|
|
"margin_dpo/margin_mean": 198.79312133789062,
|
|
"margin_dpo/margin_std": 215.6772918701172,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.7650513950073421,
|
|
"fcm_dpo/beta": 0.0029383916407823563,
|
|
"fcm_dpo/delta": 0.026043057441711426,
|
|
"fcm_dpo/margin": 195.25540161132812,
|
|
"fcm_dpo/q_t": 0.3743501305580139,
|
|
"grad_norm": 37.878440856933594,
|
|
"learning_rate": 8.062991975753378e-08,
|
|
"logits/chosen": -0.46159619092941284,
|
|
"logits/rejected": -0.44562065601348877,
|
|
"logps/chosen": -301.5888366699219,
|
|
"logps/ref_chosen": -58.14292526245117,
|
|
"logps/ref_rejected": -83.28060913085938,
|
|
"logps/rejected": -521.98193359375,
|
|
"loss": 0.9898,
|
|
"margin_dpo/margin_mean": 195.2554168701172,
|
|
"margin_dpo/margin_std": 218.83047485351562,
|
|
"step": 521
|
|
},
|
|
{
|
|
"epoch": 0.7665198237885462,
|
|
"fcm_dpo/beta": 0.002963225357234478,
|
|
"fcm_dpo/delta": 0.09250913560390472,
|
|
"fcm_dpo/margin": 172.7684326171875,
|
|
"fcm_dpo/q_t": 0.3848268687725067,
|
|
"grad_norm": 41.32555389404297,
|
|
"learning_rate": 7.968821348583643e-08,
|
|
"logits/chosen": -0.45410579442977905,
|
|
"logits/rejected": -0.44396814703941345,
|
|
"logps/chosen": -321.1343994140625,
|
|
"logps/ref_chosen": -46.54766845703125,
|
|
"logps/ref_rejected": -66.01388549804688,
|
|
"logps/rejected": -513.3690185546875,
|
|
"loss": 1.0304,
|
|
"margin_dpo/margin_mean": 172.7684326171875,
|
|
"margin_dpo/margin_std": 213.2279052734375,
|
|
"step": 522
|
|
},
|
|
{
|
|
"epoch": 0.7679882525697503,
|
|
"fcm_dpo/beta": 0.002987142652273178,
|
|
"fcm_dpo/delta": 0.034964967519044876,
|
|
"fcm_dpo/margin": 189.8165283203125,
|
|
"fcm_dpo/q_t": 0.3737178146839142,
|
|
"grad_norm": 38.12350082397461,
|
|
"learning_rate": 7.875099508810484e-08,
|
|
"logits/chosen": -0.5127274394035339,
|
|
"logits/rejected": -0.48612135648727417,
|
|
"logps/chosen": -336.416015625,
|
|
"logps/ref_chosen": -61.76960372924805,
|
|
"logps/ref_rejected": -83.76141357421875,
|
|
"logps/rejected": -548.2243041992188,
|
|
"loss": 1.0043,
|
|
"margin_dpo/margin_mean": 189.8165283203125,
|
|
"margin_dpo/margin_std": 229.8765869140625,
|
|
"step": 523
|
|
},
|
|
{
|
|
"epoch": 0.7694566813509545,
|
|
"fcm_dpo/beta": 0.002963773440569639,
|
|
"fcm_dpo/delta": -0.03581948205828667,
|
|
"fcm_dpo/margin": 213.60824584960938,
|
|
"fcm_dpo/q_t": 0.3576499819755554,
|
|
"grad_norm": 43.33149719238281,
|
|
"learning_rate": 7.781828926091535e-08,
|
|
"logits/chosen": -0.5263937711715698,
|
|
"logits/rejected": -0.49376797676086426,
|
|
"logps/chosen": -350.075439453125,
|
|
"logps/ref_chosen": -78.0720443725586,
|
|
"logps/ref_rejected": -81.30198669433594,
|
|
"logps/rejected": -566.9136352539062,
|
|
"loss": 0.9439,
|
|
"margin_dpo/margin_mean": 213.60824584960938,
|
|
"margin_dpo/margin_std": 214.0634765625,
|
|
"step": 524
|
|
},
|
|
{
|
|
"epoch": 0.7709251101321586,
|
|
"fcm_dpo/beta": 0.0028939242474734783,
|
|
"fcm_dpo/delta": -0.140090212225914,
|
|
"fcm_dpo/margin": 251.78009033203125,
|
|
"fcm_dpo/q_t": 0.3388723134994507,
|
|
"grad_norm": 34.86455154418945,
|
|
"learning_rate": 7.689012058193384e-08,
|
|
"logits/chosen": -0.4875791668891907,
|
|
"logits/rejected": -0.4962403476238251,
|
|
"logps/chosen": -318.8263244628906,
|
|
"logps/ref_chosen": -50.827857971191406,
|
|
"logps/ref_rejected": -100.05294036865234,
|
|
"logps/rejected": -619.83154296875,
|
|
"loss": 0.8763,
|
|
"margin_dpo/margin_mean": 251.7801055908203,
|
|
"margin_dpo/margin_std": 222.5386505126953,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.7723935389133627,
|
|
"fcm_dpo/beta": 0.0028644688427448273,
|
|
"fcm_dpo/delta": -0.102485790848732,
|
|
"fcm_dpo/margin": 242.66949462890625,
|
|
"fcm_dpo/q_t": 0.34745296835899353,
|
|
"grad_norm": 30.96253776550293,
|
|
"learning_rate": 7.596651350926836e-08,
|
|
"logits/chosen": -0.5117580890655518,
|
|
"logits/rejected": -0.48653119802474976,
|
|
"logps/chosen": -344.20849609375,
|
|
"logps/ref_chosen": -63.167236328125,
|
|
"logps/ref_rejected": -86.30934143066406,
|
|
"logps/rejected": -610.0201416015625,
|
|
"loss": 0.9091,
|
|
"margin_dpo/margin_mean": 242.6695098876953,
|
|
"margin_dpo/margin_std": 234.477294921875,
|
|
"step": 526
|
|
},
|
|
{
|
|
"epoch": 0.7738619676945668,
|
|
"fcm_dpo/beta": 0.002855823142454028,
|
|
"fcm_dpo/delta": 0.0754547268152237,
|
|
"fcm_dpo/margin": 185.0948486328125,
|
|
"fcm_dpo/q_t": 0.38190531730651855,
|
|
"grad_norm": 37.24220275878906,
|
|
"learning_rate": 7.504749238082414e-08,
|
|
"logits/chosen": -0.551037073135376,
|
|
"logits/rejected": -0.5188230276107788,
|
|
"logps/chosen": -381.26397705078125,
|
|
"logps/ref_chosen": -71.12867736816406,
|
|
"logps/ref_rejected": -78.3425521850586,
|
|
"logps/rejected": -573.57275390625,
|
|
"loss": 1.004,
|
|
"margin_dpo/margin_mean": 185.0948486328125,
|
|
"margin_dpo/margin_std": 205.111572265625,
|
|
"step": 527
|
|
},
|
|
{
|
|
"epoch": 0.775330396475771,
|
|
"fcm_dpo/beta": 0.0028600182849913836,
|
|
"fcm_dpo/delta": -0.007810359820723534,
|
|
"fcm_dpo/margin": 212.27090454101562,
|
|
"fcm_dpo/q_t": 0.37373942136764526,
|
|
"grad_norm": 35.34698486328125,
|
|
"learning_rate": 7.413308141366254e-08,
|
|
"logits/chosen": -0.5303002595901489,
|
|
"logits/rejected": -0.509202241897583,
|
|
"logps/chosen": -384.4117736816406,
|
|
"logps/ref_chosen": -68.0894546508789,
|
|
"logps/ref_rejected": -93.91006469726562,
|
|
"logps/rejected": -622.5032958984375,
|
|
"loss": 1.0079,
|
|
"margin_dpo/margin_mean": 212.27090454101562,
|
|
"margin_dpo/margin_std": 285.313720703125,
|
|
"step": 528
|
|
},
|
|
{
|
|
"epoch": 0.7767988252569751,
|
|
"fcm_dpo/beta": 0.002924061380326748,
|
|
"fcm_dpo/delta": 0.13557088375091553,
|
|
"fcm_dpo/margin": 161.04995727539062,
|
|
"fcm_dpo/q_t": 0.3966936767101288,
|
|
"grad_norm": 46.321571350097656,
|
|
"learning_rate": 7.322330470336313e-08,
|
|
"logits/chosen": -0.49917709827423096,
|
|
"logits/rejected": -0.49793556332588196,
|
|
"logps/chosen": -410.2076416015625,
|
|
"logps/ref_chosen": -55.57495880126953,
|
|
"logps/ref_rejected": -89.20909118652344,
|
|
"logps/rejected": -604.8917236328125,
|
|
"loss": 1.0838,
|
|
"margin_dpo/margin_mean": 161.04995727539062,
|
|
"margin_dpo/margin_std": 237.48895263671875,
|
|
"step": 529
|
|
},
|
|
{
|
|
"epoch": 0.7782672540381792,
|
|
"fcm_dpo/beta": 0.0028766831383109093,
|
|
"fcm_dpo/delta": -0.14135703444480896,
|
|
"fcm_dpo/margin": 253.9014892578125,
|
|
"fcm_dpo/q_t": 0.3434082269668579,
|
|
"grad_norm": 33.014739990234375,
|
|
"learning_rate": 7.231818622338822e-08,
|
|
"logits/chosen": -0.47442853450775146,
|
|
"logits/rejected": -0.45949307084083557,
|
|
"logps/chosen": -333.4599609375,
|
|
"logps/ref_chosen": -47.601417541503906,
|
|
"logps/ref_rejected": -87.2845230102539,
|
|
"logps/rejected": -627.0445556640625,
|
|
"loss": 0.9293,
|
|
"margin_dpo/margin_mean": 253.90150451660156,
|
|
"margin_dpo/margin_std": 285.0568542480469,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.7797356828193832,
|
|
"fcm_dpo/beta": 0.0028520459309220314,
|
|
"fcm_dpo/delta": -0.02389775589108467,
|
|
"fcm_dpo/margin": 218.2183837890625,
|
|
"fcm_dpo/q_t": 0.36491817235946655,
|
|
"grad_norm": 39.8703498840332,
|
|
"learning_rate": 7.141774982445147e-08,
|
|
"logits/chosen": -0.5686095952987671,
|
|
"logits/rejected": -0.5440300107002258,
|
|
"logps/chosen": -381.2401123046875,
|
|
"logps/ref_chosen": -55.246063232421875,
|
|
"logps/ref_rejected": -70.60598754882812,
|
|
"logps/rejected": -614.8184204101562,
|
|
"loss": 0.9667,
|
|
"margin_dpo/margin_mean": 218.2183837890625,
|
|
"margin_dpo/margin_std": 248.57521057128906,
|
|
"step": 531
|
|
},
|
|
{
|
|
"epoch": 0.7812041116005873,
|
|
"fcm_dpo/beta": 0.0027955423574894667,
|
|
"fcm_dpo/delta": -0.043103598058223724,
|
|
"fcm_dpo/margin": 227.61962890625,
|
|
"fcm_dpo/q_t": 0.36520931124687195,
|
|
"grad_norm": 60.421783447265625,
|
|
"learning_rate": 7.052201923388953e-08,
|
|
"logits/chosen": -0.5076245069503784,
|
|
"logits/rejected": -0.478635311126709,
|
|
"logps/chosen": -414.247314453125,
|
|
"logps/ref_chosen": -70.28601837158203,
|
|
"logps/ref_rejected": -86.5913314819336,
|
|
"logps/rejected": -658.1722412109375,
|
|
"loss": 0.998,
|
|
"margin_dpo/margin_mean": 227.61961364746094,
|
|
"margin_dpo/margin_std": 282.49871826171875,
|
|
"step": 532
|
|
},
|
|
{
|
|
"epoch": 0.7826725403817915,
|
|
"fcm_dpo/beta": 0.0028768163174390793,
|
|
"fcm_dpo/delta": 0.06540602445602417,
|
|
"fcm_dpo/margin": 186.47171020507812,
|
|
"fcm_dpo/q_t": 0.38702893257141113,
|
|
"grad_norm": 41.66936111450195,
|
|
"learning_rate": 6.963101805503646e-08,
|
|
"logits/chosen": -0.4856870174407959,
|
|
"logits/rejected": -0.45800524950027466,
|
|
"logps/chosen": -395.514892578125,
|
|
"logps/ref_chosen": -64.8551025390625,
|
|
"logps/ref_rejected": -76.58805847167969,
|
|
"logps/rejected": -593.7196044921875,
|
|
"loss": 1.0558,
|
|
"margin_dpo/margin_mean": 186.47171020507812,
|
|
"margin_dpo/margin_std": 268.0407409667969,
|
|
"step": 533
|
|
},
|
|
{
|
|
"epoch": 0.7841409691629956,
|
|
"fcm_dpo/beta": 0.0028785793110728264,
|
|
"fcm_dpo/delta": 0.0326891764998436,
|
|
"fcm_dpo/margin": 197.72434997558594,
|
|
"fcm_dpo/q_t": 0.37530991435050964,
|
|
"grad_norm": 38.83979034423828,
|
|
"learning_rate": 6.874476976660184e-08,
|
|
"logits/chosen": -0.510763943195343,
|
|
"logits/rejected": -0.49589037895202637,
|
|
"logps/chosen": -392.6068115234375,
|
|
"logps/ref_chosen": -60.119388580322266,
|
|
"logps/ref_rejected": -78.54347229003906,
|
|
"logps/rejected": -608.7552490234375,
|
|
"loss": 1.0056,
|
|
"margin_dpo/margin_mean": 197.724365234375,
|
|
"margin_dpo/margin_std": 245.38973999023438,
|
|
"step": 534
|
|
},
|
|
{
|
|
"epoch": 0.7856093979441997,
|
|
"fcm_dpo/beta": 0.0028537982143461704,
|
|
"fcm_dpo/delta": -0.11794188618659973,
|
|
"fcm_dpo/margin": 248.56651306152344,
|
|
"fcm_dpo/q_t": 0.34491124749183655,
|
|
"grad_norm": 37.186885833740234,
|
|
"learning_rate": 6.786329772205246e-08,
|
|
"logits/chosen": -0.5178133845329285,
|
|
"logits/rejected": -0.49928510189056396,
|
|
"logps/chosen": -351.3582763671875,
|
|
"logps/ref_chosen": -54.330238342285156,
|
|
"logps/ref_rejected": -96.30763244628906,
|
|
"logps/rejected": -641.9022216796875,
|
|
"loss": 0.899,
|
|
"margin_dpo/margin_mean": 248.56649780273438,
|
|
"margin_dpo/margin_std": 232.72039794921875,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.7870778267254038,
|
|
"fcm_dpo/beta": 0.002728927880525589,
|
|
"fcm_dpo/delta": -0.21693715453147888,
|
|
"fcm_dpo/margin": 293.05352783203125,
|
|
"fcm_dpo/q_t": 0.33220839500427246,
|
|
"grad_norm": 31.354650497436523,
|
|
"learning_rate": 6.698662514899638e-08,
|
|
"logits/chosen": -0.5063174962997437,
|
|
"logits/rejected": -0.5071820616722107,
|
|
"logps/chosen": -320.23358154296875,
|
|
"logps/ref_chosen": -47.08053207397461,
|
|
"logps/ref_rejected": -89.09783935546875,
|
|
"logps/rejected": -655.304443359375,
|
|
"loss": 0.8628,
|
|
"margin_dpo/margin_mean": 293.05352783203125,
|
|
"margin_dpo/margin_std": 300.212890625,
|
|
"step": 536
|
|
},
|
|
{
|
|
"epoch": 0.788546255506608,
|
|
"fcm_dpo/beta": 0.0027053924277424812,
|
|
"fcm_dpo/delta": -0.014062870293855667,
|
|
"fcm_dpo/margin": 226.46954345703125,
|
|
"fcm_dpo/q_t": 0.3674009442329407,
|
|
"grad_norm": 38.02225875854492,
|
|
"learning_rate": 6.611477514857114e-08,
|
|
"logits/chosen": -0.47698020935058594,
|
|
"logits/rejected": -0.4474596381187439,
|
|
"logps/chosen": -329.1002197265625,
|
|
"logps/ref_chosen": -57.747467041015625,
|
|
"logps/ref_rejected": -70.43838500976562,
|
|
"logps/rejected": -568.2606811523438,
|
|
"loss": 0.9875,
|
|
"margin_dpo/margin_mean": 226.46954345703125,
|
|
"margin_dpo/margin_std": 272.7801513671875,
|
|
"step": 537
|
|
},
|
|
{
|
|
"epoch": 0.7900146842878121,
|
|
"fcm_dpo/beta": 0.0026679965667426586,
|
|
"fcm_dpo/delta": -0.04222983494400978,
|
|
"fcm_dpo/margin": 239.66842651367188,
|
|
"fcm_dpo/q_t": 0.35988813638687134,
|
|
"grad_norm": 38.96491241455078,
|
|
"learning_rate": 6.524777069483525e-08,
|
|
"logits/chosen": -0.5299701690673828,
|
|
"logits/rejected": -0.5038875341415405,
|
|
"logps/chosen": -405.4573669433594,
|
|
"logps/ref_chosen": -66.41594696044922,
|
|
"logps/ref_rejected": -84.22808837890625,
|
|
"logps/rejected": -662.9379272460938,
|
|
"loss": 0.9442,
|
|
"margin_dpo/margin_mean": 239.66842651367188,
|
|
"margin_dpo/margin_std": 251.71209716796875,
|
|
"step": 538
|
|
},
|
|
{
|
|
"epoch": 0.7914831130690162,
|
|
"fcm_dpo/beta": 0.0026902840472757816,
|
|
"fcm_dpo/delta": 0.07448217272758484,
|
|
"fcm_dpo/margin": 196.82864379882812,
|
|
"fcm_dpo/q_t": 0.3829989731311798,
|
|
"grad_norm": 29.287826538085938,
|
|
"learning_rate": 6.438563463416221e-08,
|
|
"logits/chosen": -0.5244389176368713,
|
|
"logits/rejected": -0.4966394901275635,
|
|
"logps/chosen": -368.57196044921875,
|
|
"logps/ref_chosen": -58.492855072021484,
|
|
"logps/ref_rejected": -91.85395050048828,
|
|
"logps/rejected": -598.76171875,
|
|
"loss": 1.0163,
|
|
"margin_dpo/margin_mean": 196.82864379882812,
|
|
"margin_dpo/margin_std": 235.43844604492188,
|
|
"step": 539
|
|
},
|
|
{
|
|
"epoch": 0.7929515418502202,
|
|
"fcm_dpo/beta": 0.002647720742970705,
|
|
"fcm_dpo/delta": -0.18606501817703247,
|
|
"fcm_dpo/margin": 291.5929260253906,
|
|
"fcm_dpo/q_t": 0.33625712990760803,
|
|
"grad_norm": 37.2298698425293,
|
|
"learning_rate": 6.352838968463919e-08,
|
|
"logits/chosen": -0.5106328129768372,
|
|
"logits/rejected": -0.5080406665802002,
|
|
"logps/chosen": -361.15386962890625,
|
|
"logps/ref_chosen": -63.482513427734375,
|
|
"logps/ref_rejected": -116.42999267578125,
|
|
"logps/rejected": -705.6942749023438,
|
|
"loss": 0.8864,
|
|
"margin_dpo/margin_mean": 291.5928955078125,
|
|
"margin_dpo/margin_std": 289.0450744628906,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7944199706314243,
|
|
"fcm_dpo/beta": 0.0026333509013056755,
|
|
"fcm_dpo/delta": 0.12476900219917297,
|
|
"fcm_dpo/margin": 182.57830810546875,
|
|
"fcm_dpo/q_t": 0.3981369137763977,
|
|
"grad_norm": 36.667198181152344,
|
|
"learning_rate": 6.267605843546767e-08,
|
|
"logits/chosen": -0.5467498302459717,
|
|
"logits/rejected": -0.5339305400848389,
|
|
"logps/chosen": -430.84771728515625,
|
|
"logps/ref_chosen": -78.28036499023438,
|
|
"logps/ref_rejected": -103.273681640625,
|
|
"logps/rejected": -638.4193725585938,
|
|
"loss": 1.0807,
|
|
"margin_dpo/margin_mean": 182.57830810546875,
|
|
"margin_dpo/margin_std": 271.44097900390625,
|
|
"step": 541
|
|
},
|
|
{
|
|
"epoch": 0.7958883994126285,
|
|
"fcm_dpo/beta": 0.002633909694850445,
|
|
"fcm_dpo/delta": -0.05343947932124138,
|
|
"fcm_dpo/margin": 246.30075073242188,
|
|
"fcm_dpo/q_t": 0.3598957359790802,
|
|
"grad_norm": 37.33884811401367,
|
|
"learning_rate": 6.182866334636888e-08,
|
|
"logits/chosen": -0.5174318552017212,
|
|
"logits/rejected": -0.5168588161468506,
|
|
"logps/chosen": -388.119384765625,
|
|
"logps/ref_chosen": -57.48497009277344,
|
|
"logps/ref_rejected": -96.47506713867188,
|
|
"logps/rejected": -673.4102783203125,
|
|
"loss": 0.9588,
|
|
"margin_dpo/margin_mean": 246.30075073242188,
|
|
"margin_dpo/margin_std": 279.5341796875,
|
|
"step": 542
|
|
},
|
|
{
|
|
"epoch": 0.7973568281938326,
|
|
"fcm_dpo/beta": 0.0026966403238475323,
|
|
"fcm_dpo/delta": 0.13497576117515564,
|
|
"fcm_dpo/margin": 174.83229064941406,
|
|
"fcm_dpo/q_t": 0.40193819999694824,
|
|
"grad_norm": 33.94786834716797,
|
|
"learning_rate": 6.098622674699147e-08,
|
|
"logits/chosen": -0.519938588142395,
|
|
"logits/rejected": -0.5251273512840271,
|
|
"logps/chosen": -390.4471435546875,
|
|
"logps/ref_chosen": -60.61750793457031,
|
|
"logps/ref_rejected": -105.59896850585938,
|
|
"logps/rejected": -610.2608642578125,
|
|
"loss": 1.099,
|
|
"margin_dpo/margin_mean": 174.83230590820312,
|
|
"margin_dpo/margin_std": 286.04852294921875,
|
|
"step": 543
|
|
},
|
|
{
|
|
"epoch": 0.7988252569750367,
|
|
"fcm_dpo/beta": 0.002676740288734436,
|
|
"fcm_dpo/delta": -0.09121545404195786,
|
|
"fcm_dpo/margin": 255.8384246826172,
|
|
"fcm_dpo/q_t": 0.3496483564376831,
|
|
"grad_norm": 39.29571533203125,
|
|
"learning_rate": 6.01487708363232e-08,
|
|
"logits/chosen": -0.4991987347602844,
|
|
"logits/rejected": -0.4953753650188446,
|
|
"logps/chosen": -407.407470703125,
|
|
"logps/ref_chosen": -59.642303466796875,
|
|
"logps/ref_rejected": -100.95469665527344,
|
|
"logps/rejected": -704.5582275390625,
|
|
"loss": 0.9054,
|
|
"margin_dpo/margin_mean": 255.8384246826172,
|
|
"margin_dpo/margin_std": 249.53448486328125,
|
|
"step": 544
|
|
},
|
|
{
|
|
"epoch": 0.8002936857562408,
|
|
"fcm_dpo/beta": 0.0026431018486618996,
|
|
"fcm_dpo/delta": -0.0723901242017746,
|
|
"fcm_dpo/margin": 252.58734130859375,
|
|
"fcm_dpo/q_t": 0.3531061112880707,
|
|
"grad_norm": 27.791589736938477,
|
|
"learning_rate": 5.9316317682106294e-08,
|
|
"logits/chosen": -0.49018025398254395,
|
|
"logits/rejected": -0.4865438938140869,
|
|
"logps/chosen": -402.95513916015625,
|
|
"logps/ref_chosen": -67.64859771728516,
|
|
"logps/ref_rejected": -95.90800476074219,
|
|
"logps/rejected": -683.8018798828125,
|
|
"loss": 0.9299,
|
|
"margin_dpo/margin_mean": 252.58734130859375,
|
|
"margin_dpo/margin_std": 258.5457763671875,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.801762114537445,
|
|
"fcm_dpo/beta": 0.0026444008108228445,
|
|
"fcm_dpo/delta": 0.05075571686029434,
|
|
"fcm_dpo/margin": 208.76841735839844,
|
|
"fcm_dpo/q_t": 0.37396320700645447,
|
|
"grad_norm": 34.60557174682617,
|
|
"learning_rate": 5.848888922025552e-08,
|
|
"logits/chosen": -0.48540985584259033,
|
|
"logits/rejected": -0.47076770663261414,
|
|
"logps/chosen": -358.11322021484375,
|
|
"logps/ref_chosen": -50.744232177734375,
|
|
"logps/ref_rejected": -81.86622619628906,
|
|
"logps/rejected": -598.003662109375,
|
|
"loss": 0.9881,
|
|
"margin_dpo/margin_mean": 208.7684326171875,
|
|
"margin_dpo/margin_std": 221.3150634765625,
|
|
"step": 546
|
|
},
|
|
{
|
|
"epoch": 0.8032305433186491,
|
|
"fcm_dpo/beta": 0.002690389985218644,
|
|
"fcm_dpo/delta": 0.04553506523370743,
|
|
"fcm_dpo/margin": 206.67840576171875,
|
|
"fcm_dpo/q_t": 0.37883201241493225,
|
|
"grad_norm": 47.69929122924805,
|
|
"learning_rate": 5.7666507254280265e-08,
|
|
"logits/chosen": -0.5229824185371399,
|
|
"logits/rejected": -0.5123274326324463,
|
|
"logps/chosen": -416.7408447265625,
|
|
"logps/ref_chosen": -73.6877212524414,
|
|
"logps/ref_rejected": -90.76136779785156,
|
|
"logps/rejected": -640.492919921875,
|
|
"loss": 1.0268,
|
|
"margin_dpo/margin_mean": 206.67840576171875,
|
|
"margin_dpo/margin_std": 272.1181945800781,
|
|
"step": 547
|
|
},
|
|
{
|
|
"epoch": 0.8046989720998532,
|
|
"fcm_dpo/beta": 0.0026915257330983877,
|
|
"fcm_dpo/delta": 0.021523747593164444,
|
|
"fcm_dpo/margin": 215.3766632080078,
|
|
"fcm_dpo/q_t": 0.37684351205825806,
|
|
"grad_norm": 32.53667449951172,
|
|
"learning_rate": 5.684919345471029e-08,
|
|
"logits/chosen": -0.5300377011299133,
|
|
"logits/rejected": -0.512967586517334,
|
|
"logps/chosen": -406.23773193359375,
|
|
"logps/ref_chosen": -65.24634552001953,
|
|
"logps/ref_rejected": -94.11807250976562,
|
|
"logps/rejected": -650.486083984375,
|
|
"loss": 1.0016,
|
|
"margin_dpo/margin_mean": 215.3766632080078,
|
|
"margin_dpo/margin_std": 280.05401611328125,
|
|
"step": 548
|
|
},
|
|
{
|
|
"epoch": 0.8061674008810573,
|
|
"fcm_dpo/beta": 0.00273230392485857,
|
|
"fcm_dpo/delta": 0.08910378813743591,
|
|
"fcm_dpo/margin": 188.5994873046875,
|
|
"fcm_dpo/q_t": 0.38830626010894775,
|
|
"grad_norm": 39.737220764160156,
|
|
"learning_rate": 5.603696935852426e-08,
|
|
"logits/chosen": -0.4929801821708679,
|
|
"logits/rejected": -0.4741705656051636,
|
|
"logps/chosen": -383.95849609375,
|
|
"logps/ref_chosen": -49.21235656738281,
|
|
"logps/ref_rejected": -73.91031646728516,
|
|
"logps/rejected": -597.2559814453125,
|
|
"loss": 1.0374,
|
|
"margin_dpo/margin_mean": 188.5994873046875,
|
|
"margin_dpo/margin_std": 250.06410217285156,
|
|
"step": 549
|
|
},
|
|
{
|
|
"epoch": 0.8076358296622613,
|
|
"fcm_dpo/beta": 0.0027629181277006865,
|
|
"fcm_dpo/delta": 0.0815601795911789,
|
|
"fcm_dpo/margin": 189.18296813964844,
|
|
"fcm_dpo/q_t": 0.38146719336509705,
|
|
"grad_norm": 38.10478973388672,
|
|
"learning_rate": 5.5229856368582376e-08,
|
|
"logits/chosen": -0.4668349623680115,
|
|
"logits/rejected": -0.4747786521911621,
|
|
"logps/chosen": -374.70623779296875,
|
|
"logps/ref_chosen": -56.80695343017578,
|
|
"logps/ref_rejected": -95.12580871582031,
|
|
"logps/rejected": -602.2080688476562,
|
|
"loss": 1.0057,
|
|
"margin_dpo/margin_mean": 189.18296813964844,
|
|
"margin_dpo/margin_std": 203.63247680664062,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.8091042584434655,
|
|
"fcm_dpo/beta": 0.0027016932144761086,
|
|
"fcm_dpo/delta": -0.18426547944545746,
|
|
"fcm_dpo/margin": 284.74884033203125,
|
|
"fcm_dpo/q_t": 0.3298325836658478,
|
|
"grad_norm": 41.22262191772461,
|
|
"learning_rate": 5.4427875753062734e-08,
|
|
"logits/chosen": -0.4827745258808136,
|
|
"logits/rejected": -0.48983511328697205,
|
|
"logps/chosen": -374.94500732421875,
|
|
"logps/ref_chosen": -59.10633087158203,
|
|
"logps/ref_rejected": -111.67280578613281,
|
|
"logps/rejected": -712.2603149414062,
|
|
"loss": 0.8561,
|
|
"margin_dpo/margin_mean": 284.74884033203125,
|
|
"margin_dpo/margin_std": 247.297607421875,
|
|
"step": 551
|
|
},
|
|
{
|
|
"epoch": 0.8105726872246696,
|
|
"fcm_dpo/beta": 0.002569342264905572,
|
|
"fcm_dpo/delta": -0.22732627391815186,
|
|
"fcm_dpo/margin": 313.23541259765625,
|
|
"fcm_dpo/q_t": 0.32994934916496277,
|
|
"grad_norm": 25.86652946472168,
|
|
"learning_rate": 5.363104864490034e-08,
|
|
"logits/chosen": -0.4956032335758209,
|
|
"logits/rejected": -0.4835580885410309,
|
|
"logps/chosen": -360.0677490234375,
|
|
"logps/ref_chosen": -62.35459899902344,
|
|
"logps/ref_rejected": -104.56210327148438,
|
|
"logps/rejected": -715.5106201171875,
|
|
"loss": 0.8604,
|
|
"margin_dpo/margin_mean": 313.23541259765625,
|
|
"margin_dpo/margin_std": 302.6081237792969,
|
|
"step": 552
|
|
},
|
|
{
|
|
"epoch": 0.8120411160058737,
|
|
"fcm_dpo/beta": 0.0025847742799669504,
|
|
"fcm_dpo/delta": 0.07955043762922287,
|
|
"fcm_dpo/margin": 202.94935607910156,
|
|
"fcm_dpo/q_t": 0.38608789443969727,
|
|
"grad_norm": 36.25725555419922,
|
|
"learning_rate": 5.2839396041230415e-08,
|
|
"logits/chosen": -0.4990794062614441,
|
|
"logits/rejected": -0.48621460795402527,
|
|
"logps/chosen": -384.55615234375,
|
|
"logps/ref_chosen": -68.25881958007812,
|
|
"logps/ref_rejected": -98.0971450805664,
|
|
"logps/rejected": -617.3438110351562,
|
|
"loss": 1.0248,
|
|
"margin_dpo/margin_mean": 202.94935607910156,
|
|
"margin_dpo/margin_std": 253.7549285888672,
|
|
"step": 553
|
|
},
|
|
{
|
|
"epoch": 0.8135095447870778,
|
|
"fcm_dpo/beta": 0.0025886285584419966,
|
|
"fcm_dpo/delta": -0.09040947258472443,
|
|
"fcm_dpo/margin": 264.222900390625,
|
|
"fcm_dpo/q_t": 0.35030698776245117,
|
|
"grad_norm": 31.871034622192383,
|
|
"learning_rate": 5.205293880283551e-08,
|
|
"logits/chosen": -0.5079976916313171,
|
|
"logits/rejected": -0.4783214330673218,
|
|
"logps/chosen": -389.82989501953125,
|
|
"logps/ref_chosen": -67.94767761230469,
|
|
"logps/ref_rejected": -89.78272247314453,
|
|
"logps/rejected": -675.8878173828125,
|
|
"loss": 0.9239,
|
|
"margin_dpo/margin_mean": 264.222900390625,
|
|
"margin_dpo/margin_std": 268.23663330078125,
|
|
"step": 554
|
|
},
|
|
{
|
|
"epoch": 0.8149779735682819,
|
|
"fcm_dpo/beta": 0.0025238655507564545,
|
|
"fcm_dpo/delta": -0.07741110026836395,
|
|
"fcm_dpo/margin": 266.2066650390625,
|
|
"fcm_dpo/q_t": 0.35779866576194763,
|
|
"grad_norm": 37.653892517089844,
|
|
"learning_rate": 5.127169765359515e-08,
|
|
"logits/chosen": -0.5288249254226685,
|
|
"logits/rejected": -0.5379273891448975,
|
|
"logps/chosen": -376.24407958984375,
|
|
"logps/ref_chosen": -53.33049011230469,
|
|
"logps/ref_rejected": -108.47937774658203,
|
|
"logps/rejected": -697.599609375,
|
|
"loss": 0.9471,
|
|
"margin_dpo/margin_mean": 266.2066650390625,
|
|
"margin_dpo/margin_std": 305.3093566894531,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.8164464023494861,
|
|
"fcm_dpo/beta": 0.002562987618148327,
|
|
"fcm_dpo/delta": 0.11464083939790726,
|
|
"fcm_dpo/margin": 191.5848388671875,
|
|
"fcm_dpo/q_t": 0.394670307636261,
|
|
"grad_norm": 48.968658447265625,
|
|
"learning_rate": 5.049569317994012e-08,
|
|
"logits/chosen": -0.45059263706207275,
|
|
"logits/rejected": -0.43395426869392395,
|
|
"logps/chosen": -389.5137939453125,
|
|
"logps/ref_chosen": -58.64447021484375,
|
|
"logps/ref_rejected": -101.34040832519531,
|
|
"logps/rejected": -623.7945556640625,
|
|
"loss": 1.0517,
|
|
"margin_dpo/margin_mean": 191.5848388671875,
|
|
"margin_dpo/margin_std": 260.8348388671875,
|
|
"step": 556
|
|
},
|
|
{
|
|
"epoch": 0.8179148311306902,
|
|
"fcm_dpo/beta": 0.002571512944996357,
|
|
"fcm_dpo/delta": 0.0015640445053577423,
|
|
"fcm_dpo/margin": 232.67929077148438,
|
|
"fcm_dpo/q_t": 0.37047192454338074,
|
|
"grad_norm": 40.12662887573242,
|
|
"learning_rate": 4.9724945830310144e-08,
|
|
"logits/chosen": -0.49551674723625183,
|
|
"logits/rejected": -0.49951496720314026,
|
|
"logps/chosen": -406.4974365234375,
|
|
"logps/ref_chosen": -67.84066009521484,
|
|
"logps/ref_rejected": -109.93965911865234,
|
|
"logps/rejected": -681.2757568359375,
|
|
"loss": 0.9819,
|
|
"margin_dpo/margin_mean": 232.6792755126953,
|
|
"margin_dpo/margin_std": 272.00067138671875,
|
|
"step": 557
|
|
},
|
|
{
|
|
"epoch": 0.8193832599118943,
|
|
"fcm_dpo/beta": 0.0024929968640208244,
|
|
"fcm_dpo/delta": -0.20179688930511475,
|
|
"fcm_dpo/margin": 314.78643798828125,
|
|
"fcm_dpo/q_t": 0.3285095691680908,
|
|
"grad_norm": 42.07298278808594,
|
|
"learning_rate": 4.8959475914614554e-08,
|
|
"logits/chosen": -0.5093335509300232,
|
|
"logits/rejected": -0.4982109069824219,
|
|
"logps/chosen": -371.7452392578125,
|
|
"logps/ref_chosen": -62.36824035644531,
|
|
"logps/ref_rejected": -102.16102600097656,
|
|
"logps/rejected": -726.324462890625,
|
|
"loss": 0.866,
|
|
"margin_dpo/margin_mean": 314.78643798828125,
|
|
"margin_dpo/margin_std": 296.3161926269531,
|
|
"step": 558
|
|
},
|
|
{
|
|
"epoch": 0.8208516886930984,
|
|
"fcm_dpo/beta": 0.0024765529669821262,
|
|
"fcm_dpo/delta": 0.002801853232085705,
|
|
"fcm_dpo/margin": 241.2025146484375,
|
|
"fcm_dpo/q_t": 0.3673396706581116,
|
|
"grad_norm": 28.609155654907227,
|
|
"learning_rate": 4.8199303603697614e-08,
|
|
"logits/chosen": -0.5868000984191895,
|
|
"logits/rejected": -0.5612949728965759,
|
|
"logps/chosen": -414.205322265625,
|
|
"logps/ref_chosen": -60.752323150634766,
|
|
"logps/ref_rejected": -93.44229125976562,
|
|
"logps/rejected": -688.0977783203125,
|
|
"loss": 0.9684,
|
|
"margin_dpo/margin_mean": 241.2025146484375,
|
|
"margin_dpo/margin_std": 260.7628173828125,
|
|
"step": 559
|
|
},
|
|
{
|
|
"epoch": 0.8223201174743024,
|
|
"fcm_dpo/beta": 0.0025273123756051064,
|
|
"fcm_dpo/delta": 0.15692362189292908,
|
|
"fcm_dpo/margin": 178.22430419921875,
|
|
"fcm_dpo/q_t": 0.3967198431491852,
|
|
"grad_norm": 32.44375991821289,
|
|
"learning_rate": 4.7444448928806615e-08,
|
|
"logits/chosen": -0.5102473497390747,
|
|
"logits/rejected": -0.4792303442955017,
|
|
"logps/chosen": -362.985595703125,
|
|
"logps/ref_chosen": -58.10382080078125,
|
|
"logps/ref_rejected": -79.99122619628906,
|
|
"logps/rejected": -563.0972900390625,
|
|
"loss": 1.0412,
|
|
"margin_dpo/margin_mean": 178.22430419921875,
|
|
"margin_dpo/margin_std": 187.90447998046875,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.8237885462555066,
|
|
"fcm_dpo/beta": 0.0026212262455374002,
|
|
"fcm_dpo/delta": 0.1312703937292099,
|
|
"fcm_dpo/margin": 180.79673767089844,
|
|
"fcm_dpo/q_t": 0.3930835723876953,
|
|
"grad_norm": 34.945152282714844,
|
|
"learning_rate": 4.669493178106432e-08,
|
|
"logits/chosen": -0.46148979663848877,
|
|
"logits/rejected": -0.4639427661895752,
|
|
"logps/chosen": -401.359619140625,
|
|
"logps/ref_chosen": -50.912879943847656,
|
|
"logps/ref_rejected": -99.06856536865234,
|
|
"logps/rejected": -630.3120727539062,
|
|
"loss": 1.0486,
|
|
"margin_dpo/margin_mean": 180.79672241210938,
|
|
"margin_dpo/margin_std": 226.11021423339844,
|
|
"step": 561
|
|
},
|
|
{
|
|
"epoch": 0.8252569750367107,
|
|
"fcm_dpo/beta": 0.0025770908687263727,
|
|
"fcm_dpo/delta": -0.08462567627429962,
|
|
"fcm_dpo/margin": 262.9320068359375,
|
|
"fcm_dpo/q_t": 0.35398316383361816,
|
|
"grad_norm": 43.5445442199707,
|
|
"learning_rate": 4.5950771910944596e-08,
|
|
"logits/chosen": -0.5065794587135315,
|
|
"logits/rejected": -0.4925631284713745,
|
|
"logps/chosen": -369.410400390625,
|
|
"logps/ref_chosen": -59.46440124511719,
|
|
"logps/ref_rejected": -96.54266357421875,
|
|
"logps/rejected": -669.420654296875,
|
|
"loss": 0.9218,
|
|
"margin_dpo/margin_mean": 262.9320068359375,
|
|
"margin_dpo/margin_std": 273.57208251953125,
|
|
"step": 562
|
|
},
|
|
{
|
|
"epoch": 0.8267254038179148,
|
|
"fcm_dpo/beta": 0.002649601548910141,
|
|
"fcm_dpo/delta": 0.11059555411338806,
|
|
"fcm_dpo/margin": 185.759521484375,
|
|
"fcm_dpo/q_t": 0.38974693417549133,
|
|
"grad_norm": 38.421958923339844,
|
|
"learning_rate": 4.521198892775202e-08,
|
|
"logits/chosen": -0.4199938178062439,
|
|
"logits/rejected": -0.41303732991218567,
|
|
"logps/chosen": -402.4117736816406,
|
|
"logps/ref_chosen": -60.60819625854492,
|
|
"logps/ref_rejected": -94.56770324707031,
|
|
"logps/rejected": -622.1307983398438,
|
|
"loss": 1.0664,
|
|
"margin_dpo/margin_mean": 185.759521484375,
|
|
"margin_dpo/margin_std": 253.02633666992188,
|
|
"step": 563
|
|
},
|
|
{
|
|
"epoch": 0.8281938325991189,
|
|
"fcm_dpo/beta": 0.002637513680383563,
|
|
"fcm_dpo/delta": -0.013331831432878971,
|
|
"fcm_dpo/margin": 232.2291717529297,
|
|
"fcm_dpo/q_t": 0.35960835218429565,
|
|
"grad_norm": 34.64234161376953,
|
|
"learning_rate": 4.447860229910544e-08,
|
|
"logits/chosen": -0.5244062542915344,
|
|
"logits/rejected": -0.4892103672027588,
|
|
"logps/chosen": -395.12603759765625,
|
|
"logps/ref_chosen": -74.26837921142578,
|
|
"logps/ref_rejected": -93.23818969726562,
|
|
"logps/rejected": -646.3250732421875,
|
|
"loss": 0.9258,
|
|
"margin_dpo/margin_mean": 232.22918701171875,
|
|
"margin_dpo/margin_std": 195.64129638671875,
|
|
"step": 564
|
|
},
|
|
{
|
|
"epoch": 0.8296622613803231,
|
|
"fcm_dpo/beta": 0.0026156511157751083,
|
|
"fcm_dpo/delta": -0.02939961664378643,
|
|
"fcm_dpo/margin": 239.86215209960938,
|
|
"fcm_dpo/q_t": 0.37034720182418823,
|
|
"grad_norm": 41.29433059692383,
|
|
"learning_rate": 4.375063135042445e-08,
|
|
"logits/chosen": -0.5047638416290283,
|
|
"logits/rejected": -0.48947322368621826,
|
|
"logps/chosen": -412.850830078125,
|
|
"logps/ref_chosen": -69.0199203491211,
|
|
"logps/ref_rejected": -85.7789306640625,
|
|
"logps/rejected": -669.4719848632812,
|
|
"loss": 0.9974,
|
|
"margin_dpo/margin_mean": 239.86215209960938,
|
|
"margin_dpo/margin_std": 316.59368896484375,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.8311306901615272,
|
|
"fcm_dpo/beta": 0.002625175751745701,
|
|
"fcm_dpo/delta": -0.04155733436346054,
|
|
"fcm_dpo/margin": 242.91168212890625,
|
|
"fcm_dpo/q_t": 0.36505499482154846,
|
|
"grad_norm": 34.86541748046875,
|
|
"learning_rate": 4.3028095264420525e-08,
|
|
"logits/chosen": -0.46465927362442017,
|
|
"logits/rejected": -0.46675699949264526,
|
|
"logps/chosen": -391.67828369140625,
|
|
"logps/ref_chosen": -66.5453109741211,
|
|
"logps/ref_rejected": -103.86932373046875,
|
|
"logps/rejected": -671.9140014648438,
|
|
"loss": 0.9961,
|
|
"margin_dpo/margin_mean": 242.91168212890625,
|
|
"margin_dpo/margin_std": 304.49200439453125,
|
|
"step": 566
|
|
},
|
|
{
|
|
"epoch": 0.8325991189427313,
|
|
"fcm_dpo/beta": 0.0026026610285043716,
|
|
"fcm_dpo/delta": 0.01975536160171032,
|
|
"fcm_dpo/margin": 223.38568115234375,
|
|
"fcm_dpo/q_t": 0.3706703186035156,
|
|
"grad_norm": 39.21678924560547,
|
|
"learning_rate": 4.231101308059165e-08,
|
|
"logits/chosen": -0.4942345917224884,
|
|
"logits/rejected": -0.4810565710067749,
|
|
"logps/chosen": -371.0935974121094,
|
|
"logps/ref_chosen": -52.85829544067383,
|
|
"logps/ref_rejected": -85.37095642089844,
|
|
"logps/rejected": -626.991943359375,
|
|
"loss": 0.9786,
|
|
"margin_dpo/margin_mean": 223.38568115234375,
|
|
"margin_dpo/margin_std": 245.55088806152344,
|
|
"step": 567
|
|
},
|
|
{
|
|
"epoch": 0.8340675477239354,
|
|
"fcm_dpo/beta": 0.00256351288408041,
|
|
"fcm_dpo/delta": -0.08483800292015076,
|
|
"fcm_dpo/margin": 264.6658630371094,
|
|
"fcm_dpo/q_t": 0.34867846965789795,
|
|
"grad_norm": 28.661598205566406,
|
|
"learning_rate": 4.1599403694720145e-08,
|
|
"logits/chosen": -0.44072189927101135,
|
|
"logits/rejected": -0.44052451848983765,
|
|
"logps/chosen": -348.7388916015625,
|
|
"logps/ref_chosen": -45.1923828125,
|
|
"logps/ref_rejected": -89.09236907958984,
|
|
"logps/rejected": -657.3046875,
|
|
"loss": 0.9103,
|
|
"margin_dpo/margin_mean": 264.66583251953125,
|
|
"margin_dpo/margin_std": 249.04701232910156,
|
|
"step": 568
|
|
},
|
|
{
|
|
"epoch": 0.8355359765051396,
|
|
"fcm_dpo/beta": 0.0026110014878213406,
|
|
"fcm_dpo/delta": 0.09269944578409195,
|
|
"fcm_dpo/margin": 195.74871826171875,
|
|
"fcm_dpo/q_t": 0.3880962133407593,
|
|
"grad_norm": 41.08034896850586,
|
|
"learning_rate": 4.089328585837512e-08,
|
|
"logits/chosen": -0.4750403165817261,
|
|
"logits/rejected": -0.46098893880844116,
|
|
"logps/chosen": -413.0689697265625,
|
|
"logps/ref_chosen": -63.72056198120117,
|
|
"logps/ref_rejected": -79.10325622558594,
|
|
"logps/rejected": -624.2003173828125,
|
|
"loss": 1.0448,
|
|
"margin_dpo/margin_mean": 195.74871826171875,
|
|
"margin_dpo/margin_std": 259.3326110839844,
|
|
"step": 569
|
|
},
|
|
{
|
|
"epoch": 0.8370044052863436,
|
|
"fcm_dpo/beta": 0.002593663055449724,
|
|
"fcm_dpo/delta": -0.01997148059308529,
|
|
"fcm_dpo/margin": 238.44491577148438,
|
|
"fcm_dpo/q_t": 0.3628031015396118,
|
|
"grad_norm": 33.436527252197266,
|
|
"learning_rate": 4.019267817841834e-08,
|
|
"logits/chosen": -0.5121582746505737,
|
|
"logits/rejected": -0.4927957057952881,
|
|
"logps/chosen": -382.0532531738281,
|
|
"logps/ref_chosen": -61.61454391479492,
|
|
"logps/ref_rejected": -82.14186096191406,
|
|
"logps/rejected": -641.0254516601562,
|
|
"loss": 0.9467,
|
|
"margin_dpo/margin_mean": 238.44491577148438,
|
|
"margin_dpo/margin_std": 236.92184448242188,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.8384728340675477,
|
|
"fcm_dpo/beta": 0.002562489826232195,
|
|
"fcm_dpo/delta": -0.06662685424089432,
|
|
"fcm_dpo/margin": 258.15692138671875,
|
|
"fcm_dpo/q_t": 0.3554573655128479,
|
|
"grad_norm": 45.022804260253906,
|
|
"learning_rate": 3.9497599116513705e-08,
|
|
"logits/chosen": -0.4584212005138397,
|
|
"logits/rejected": -0.4587005376815796,
|
|
"logps/chosen": -373.08074951171875,
|
|
"logps/ref_chosen": -53.05406188964844,
|
|
"logps/ref_rejected": -91.33682250976562,
|
|
"logps/rejected": -669.5204467773438,
|
|
"loss": 0.9544,
|
|
"margin_dpo/margin_mean": 258.15692138671875,
|
|
"margin_dpo/margin_std": 291.7584228515625,
|
|
"step": 571
|
|
},
|
|
{
|
|
"epoch": 0.8399412628487518,
|
|
"fcm_dpo/beta": 0.0025576986372470856,
|
|
"fcm_dpo/delta": -0.0027488693594932556,
|
|
"fcm_dpo/margin": 235.54122924804688,
|
|
"fcm_dpo/q_t": 0.3760080337524414,
|
|
"grad_norm": 36.91169357299805,
|
|
"learning_rate": 3.880806698864086e-08,
|
|
"logits/chosen": -0.4423763155937195,
|
|
"logits/rejected": -0.43832147121429443,
|
|
"logps/chosen": -395.4063720703125,
|
|
"logps/ref_chosen": -48.45928955078125,
|
|
"logps/ref_rejected": -83.55703735351562,
|
|
"logps/rejected": -666.0453491210938,
|
|
"loss": 1.0158,
|
|
"margin_dpo/margin_mean": 235.54122924804688,
|
|
"margin_dpo/margin_std": 325.2923889160156,
|
|
"step": 572
|
|
},
|
|
{
|
|
"epoch": 0.8414096916299559,
|
|
"fcm_dpo/beta": 0.002582044806331396,
|
|
"fcm_dpo/delta": 0.0030916333198547363,
|
|
"fcm_dpo/margin": 230.921142578125,
|
|
"fcm_dpo/q_t": 0.36895716190338135,
|
|
"grad_norm": 34.18012237548828,
|
|
"learning_rate": 3.812409996461275e-08,
|
|
"logits/chosen": -0.4861369729042053,
|
|
"logits/rejected": -0.4677738547325134,
|
|
"logps/chosen": -373.53863525390625,
|
|
"logps/ref_chosen": -51.62262725830078,
|
|
"logps/ref_rejected": -85.32499694824219,
|
|
"logps/rejected": -638.162109375,
|
|
"loss": 0.9723,
|
|
"margin_dpo/margin_mean": 230.921142578125,
|
|
"margin_dpo/margin_std": 252.47198486328125,
|
|
"step": 573
|
|
},
|
|
{
|
|
"epoch": 0.8428781204111601,
|
|
"fcm_dpo/beta": 0.0025438209995627403,
|
|
"fcm_dpo/delta": -0.07464863359928131,
|
|
"fcm_dpo/margin": 263.26629638671875,
|
|
"fcm_dpo/q_t": 0.3528655171394348,
|
|
"grad_norm": 30.160154342651367,
|
|
"learning_rate": 3.74457160675965e-08,
|
|
"logits/chosen": -0.5087519288063049,
|
|
"logits/rejected": -0.5088317394256592,
|
|
"logps/chosen": -369.17791748046875,
|
|
"logps/ref_chosen": -51.04446029663086,
|
|
"logps/ref_rejected": -92.80640411376953,
|
|
"logps/rejected": -674.2061157226562,
|
|
"loss": 0.9268,
|
|
"margin_dpo/margin_mean": 263.26629638671875,
|
|
"margin_dpo/margin_std": 267.4271240234375,
|
|
"step": 574
|
|
},
|
|
{
|
|
"epoch": 0.8443465491923642,
|
|
"fcm_dpo/beta": 0.0025178524665534496,
|
|
"fcm_dpo/delta": 0.048557281494140625,
|
|
"fcm_dpo/margin": 219.31741333007812,
|
|
"fcm_dpo/q_t": 0.3802528381347656,
|
|
"grad_norm": 39.644771575927734,
|
|
"learning_rate": 3.677293317363864e-08,
|
|
"logits/chosen": -0.4346257448196411,
|
|
"logits/rejected": -0.4225711226463318,
|
|
"logps/chosen": -427.237060546875,
|
|
"logps/ref_chosen": -71.7901382446289,
|
|
"logps/ref_rejected": -95.38619995117188,
|
|
"logps/rejected": -670.1505737304688,
|
|
"loss": 1.0547,
|
|
"margin_dpo/margin_mean": 219.31741333007812,
|
|
"margin_dpo/margin_std": 301.30224609375,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.8458149779735683,
|
|
"fcm_dpo/beta": 0.0025599338114261627,
|
|
"fcm_dpo/delta": 0.06943172216415405,
|
|
"fcm_dpo/margin": 208.48135375976562,
|
|
"fcm_dpo/q_t": 0.38438957929611206,
|
|
"grad_norm": 35.327213287353516,
|
|
"learning_rate": 3.6105769011194224e-08,
|
|
"logits/chosen": -0.4814299941062927,
|
|
"logits/rejected": -0.4900047481060028,
|
|
"logps/chosen": -380.19384765625,
|
|
"logps/ref_chosen": -54.262962341308594,
|
|
"logps/ref_rejected": -100.75428009033203,
|
|
"logps/rejected": -635.16650390625,
|
|
"loss": 1.0487,
|
|
"margin_dpo/margin_mean": 208.48135375976562,
|
|
"margin_dpo/margin_std": 286.2848205566406,
|
|
"step": 576
|
|
},
|
|
{
|
|
"epoch": 0.8472834067547724,
|
|
"fcm_dpo/beta": 0.0025888322852551937,
|
|
"fcm_dpo/delta": -0.0014976090751588345,
|
|
"fcm_dpo/margin": 232.30636596679688,
|
|
"fcm_dpo/q_t": 0.36417728662490845,
|
|
"grad_norm": 31.62501335144043,
|
|
"learning_rate": 3.5444241160659304e-08,
|
|
"logits/chosen": -0.4902263581752777,
|
|
"logits/rejected": -0.4602411389350891,
|
|
"logps/chosen": -351.259033203125,
|
|
"logps/ref_chosen": -61.909706115722656,
|
|
"logps/ref_rejected": -84.07069396972656,
|
|
"logps/rejected": -605.7263793945312,
|
|
"loss": 0.9617,
|
|
"margin_dpo/margin_mean": 232.30638122558594,
|
|
"margin_dpo/margin_std": 235.41848754882812,
|
|
"step": 577
|
|
},
|
|
{
|
|
"epoch": 0.8487518355359766,
|
|
"fcm_dpo/beta": 0.0025621510576456785,
|
|
"fcm_dpo/delta": -0.015007011592388153,
|
|
"fcm_dpo/margin": 239.20761108398438,
|
|
"fcm_dpo/q_t": 0.3611851930618286,
|
|
"grad_norm": 30.414794921875,
|
|
"learning_rate": 3.478836705390808e-08,
|
|
"logits/chosen": -0.3913380205631256,
|
|
"logits/rejected": -0.3857693076133728,
|
|
"logps/chosen": -340.91937255859375,
|
|
"logps/ref_chosen": -49.26368713378906,
|
|
"logps/ref_rejected": -83.4362564086914,
|
|
"logps/rejected": -614.299560546875,
|
|
"loss": 0.9374,
|
|
"margin_dpo/margin_mean": 239.20761108398438,
|
|
"margin_dpo/margin_std": 213.71923828125,
|
|
"step": 578
|
|
},
|
|
{
|
|
"epoch": 0.8502202643171806,
|
|
"fcm_dpo/beta": 0.0026131728664040565,
|
|
"fcm_dpo/delta": 0.09434732794761658,
|
|
"fcm_dpo/margin": 195.37240600585938,
|
|
"fcm_dpo/q_t": 0.3860264718532562,
|
|
"grad_norm": 37.3227424621582,
|
|
"learning_rate": 3.41381639738331e-08,
|
|
"logits/chosen": -0.47116217017173767,
|
|
"logits/rejected": -0.4621252417564392,
|
|
"logps/chosen": -367.91162109375,
|
|
"logps/ref_chosen": -58.88581848144531,
|
|
"logps/ref_rejected": -94.78762817382812,
|
|
"logps/rejected": -599.185791015625,
|
|
"loss": 1.0322,
|
|
"margin_dpo/margin_mean": 195.37240600585938,
|
|
"margin_dpo/margin_std": 244.83892822265625,
|
|
"step": 579
|
|
},
|
|
{
|
|
"epoch": 0.8516886930983847,
|
|
"fcm_dpo/beta": 0.0025751434732228518,
|
|
"fcm_dpo/delta": -0.1056961864233017,
|
|
"fcm_dpo/margin": 270.815673828125,
|
|
"fcm_dpo/q_t": 0.34934502840042114,
|
|
"grad_norm": 27.840450286865234,
|
|
"learning_rate": 3.349364905389032e-08,
|
|
"logits/chosen": -0.4681975841522217,
|
|
"logits/rejected": -0.4637313485145569,
|
|
"logps/chosen": -287.3629150390625,
|
|
"logps/ref_chosen": -48.70683670043945,
|
|
"logps/ref_rejected": -81.7583999633789,
|
|
"logps/rejected": -591.2301635742188,
|
|
"loss": 0.9179,
|
|
"margin_dpo/margin_mean": 270.815673828125,
|
|
"margin_dpo/margin_std": 285.8489685058594,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.8531571218795888,
|
|
"fcm_dpo/beta": 0.002611548872664571,
|
|
"fcm_dpo/delta": 0.10047388076782227,
|
|
"fcm_dpo/margin": 193.2278289794922,
|
|
"fcm_dpo/q_t": 0.3879375159740448,
|
|
"grad_norm": 31.592443466186523,
|
|
"learning_rate": 3.285483927764726e-08,
|
|
"logits/chosen": -0.5031832456588745,
|
|
"logits/rejected": -0.4919954836368561,
|
|
"logps/chosen": -390.05816650390625,
|
|
"logps/ref_chosen": -62.22235107421875,
|
|
"logps/ref_rejected": -91.73568725585938,
|
|
"logps/rejected": -612.79931640625,
|
|
"loss": 1.0351,
|
|
"margin_dpo/margin_mean": 193.22781372070312,
|
|
"margin_dpo/margin_std": 241.17733764648438,
|
|
"step": 581
|
|
},
|
|
{
|
|
"epoch": 0.8546255506607929,
|
|
"fcm_dpo/beta": 0.002609074581414461,
|
|
"fcm_dpo/delta": -0.10906706750392914,
|
|
"fcm_dpo/margin": 268.527587890625,
|
|
"fcm_dpo/q_t": 0.34740155935287476,
|
|
"grad_norm": 30.773542404174805,
|
|
"learning_rate": 3.222175147833556e-08,
|
|
"logits/chosen": -0.4649215638637543,
|
|
"logits/rejected": -0.4725267291069031,
|
|
"logps/chosen": -334.1369323730469,
|
|
"logps/ref_chosen": -58.228660583496094,
|
|
"logps/ref_rejected": -110.06959533691406,
|
|
"logps/rejected": -654.50537109375,
|
|
"loss": 0.9329,
|
|
"margin_dpo/margin_mean": 268.527587890625,
|
|
"margin_dpo/margin_std": 281.925048828125,
|
|
"step": 582
|
|
},
|
|
{
|
|
"epoch": 0.856093979441997,
|
|
"fcm_dpo/beta": 0.0025950425770133734,
|
|
"fcm_dpo/delta": 0.12001574784517288,
|
|
"fcm_dpo/margin": 187.0833282470703,
|
|
"fcm_dpo/q_t": 0.39638835191726685,
|
|
"grad_norm": 38.84540557861328,
|
|
"learning_rate": 3.159440233840763e-08,
|
|
"logits/chosen": -0.4593685269355774,
|
|
"logits/rejected": -0.44617611169815063,
|
|
"logps/chosen": -376.9676513671875,
|
|
"logps/ref_chosen": -56.86286163330078,
|
|
"logps/ref_rejected": -88.4039306640625,
|
|
"logps/rejected": -595.5921020507812,
|
|
"loss": 1.0772,
|
|
"margin_dpo/margin_mean": 187.0833282470703,
|
|
"margin_dpo/margin_std": 274.46728515625,
|
|
"step": 583
|
|
},
|
|
{
|
|
"epoch": 0.8575624082232012,
|
|
"fcm_dpo/beta": 0.0025689757894724607,
|
|
"fcm_dpo/delta": -0.1522265374660492,
|
|
"fcm_dpo/margin": 288.31964111328125,
|
|
"fcm_dpo/q_t": 0.335731565952301,
|
|
"grad_norm": 27.584877014160156,
|
|
"learning_rate": 3.0972808389096635e-08,
|
|
"logits/chosen": -0.48634427785873413,
|
|
"logits/rejected": -0.47195422649383545,
|
|
"logps/chosen": -321.5301818847656,
|
|
"logps/ref_chosen": -56.90068054199219,
|
|
"logps/ref_rejected": -97.63606262207031,
|
|
"logps/rejected": -650.585205078125,
|
|
"loss": 0.8585,
|
|
"margin_dpo/margin_mean": 288.3196105957031,
|
|
"margin_dpo/margin_std": 241.53466796875,
|
|
"step": 584
|
|
},
|
|
{
|
|
"epoch": 0.8590308370044053,
|
|
"fcm_dpo/beta": 0.0025610639713704586,
|
|
"fcm_dpo/delta": 0.02514045313000679,
|
|
"fcm_dpo/margin": 225.02957153320312,
|
|
"fcm_dpo/q_t": 0.37413886189460754,
|
|
"grad_norm": 35.07202911376953,
|
|
"learning_rate": 3.035698600998121e-08,
|
|
"logits/chosen": -0.49202650785446167,
|
|
"logits/rejected": -0.4775129556655884,
|
|
"logps/chosen": -386.05023193359375,
|
|
"logps/ref_chosen": -60.973968505859375,
|
|
"logps/ref_rejected": -84.16952514648438,
|
|
"logps/rejected": -634.275390625,
|
|
"loss": 0.9943,
|
|
"margin_dpo/margin_mean": 225.02957153320312,
|
|
"margin_dpo/margin_std": 258.579345703125,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.8604992657856094,
|
|
"fcm_dpo/beta": 0.0026022857055068016,
|
|
"fcm_dpo/delta": 0.09514912962913513,
|
|
"fcm_dpo/margin": 195.86416625976562,
|
|
"fcm_dpo/q_t": 0.38909363746643066,
|
|
"grad_norm": 41.226749420166016,
|
|
"learning_rate": 2.974695142855388e-08,
|
|
"logits/chosen": -0.44712215662002563,
|
|
"logits/rejected": -0.4477439820766449,
|
|
"logps/chosen": -386.0147705078125,
|
|
"logps/ref_chosen": -56.85559844970703,
|
|
"logps/ref_rejected": -91.80261993408203,
|
|
"logps/rejected": -616.825927734375,
|
|
"loss": 1.0511,
|
|
"margin_dpo/margin_mean": 195.86416625976562,
|
|
"margin_dpo/margin_std": 270.68035888671875,
|
|
"step": 586
|
|
},
|
|
{
|
|
"epoch": 0.8619676945668135,
|
|
"fcm_dpo/beta": 0.0026168739423155785,
|
|
"fcm_dpo/delta": 0.0030817219521850348,
|
|
"fcm_dpo/margin": 228.1727294921875,
|
|
"fcm_dpo/q_t": 0.368298202753067,
|
|
"grad_norm": 30.468931198120117,
|
|
"learning_rate": 2.9142720719793122e-08,
|
|
"logits/chosen": -0.48182883858680725,
|
|
"logits/rejected": -0.48729079961776733,
|
|
"logps/chosen": -289.46923828125,
|
|
"logps/ref_chosen": -44.69159698486328,
|
|
"logps/ref_rejected": -82.62385559082031,
|
|
"logps/rejected": -555.57421875,
|
|
"loss": 0.9793,
|
|
"margin_dpo/margin_mean": 228.17274475097656,
|
|
"margin_dpo/margin_std": 261.90203857421875,
|
|
"step": 587
|
|
},
|
|
{
|
|
"epoch": 0.8634361233480177,
|
|
"fcm_dpo/beta": 0.0026173333171755075,
|
|
"fcm_dpo/delta": 0.0325515940785408,
|
|
"fcm_dpo/margin": 217.41366577148438,
|
|
"fcm_dpo/q_t": 0.3760989308357239,
|
|
"grad_norm": 35.79490280151367,
|
|
"learning_rate": 2.8544309805740018e-08,
|
|
"logits/chosen": -0.5010935664176941,
|
|
"logits/rejected": -0.5093272924423218,
|
|
"logps/chosen": -365.68310546875,
|
|
"logps/ref_chosen": -50.29494857788086,
|
|
"logps/ref_rejected": -107.36988067626953,
|
|
"logps/rejected": -640.1717529296875,
|
|
"loss": 0.9967,
|
|
"margin_dpo/margin_mean": 217.41366577148438,
|
|
"margin_dpo/margin_std": 257.3772277832031,
|
|
"step": 588
|
|
},
|
|
{
|
|
"epoch": 0.8649045521292217,
|
|
"fcm_dpo/beta": 0.002576986327767372,
|
|
"fcm_dpo/delta": -0.15281669795513153,
|
|
"fcm_dpo/margin": 287.77593994140625,
|
|
"fcm_dpo/q_t": 0.34083396196365356,
|
|
"grad_norm": 29.585044860839844,
|
|
"learning_rate": 2.7951734455078786e-08,
|
|
"logits/chosen": -0.48489910364151,
|
|
"logits/rejected": -0.47208333015441895,
|
|
"logps/chosen": -363.57073974609375,
|
|
"logps/ref_chosen": -59.929908752441406,
|
|
"logps/ref_rejected": -111.65534973144531,
|
|
"logps/rejected": -703.0721435546875,
|
|
"loss": 0.8936,
|
|
"margin_dpo/margin_mean": 287.77593994140625,
|
|
"margin_dpo/margin_std": 287.7421875,
|
|
"step": 589
|
|
},
|
|
{
|
|
"epoch": 0.8663729809104258,
|
|
"fcm_dpo/beta": 0.0025251221377402544,
|
|
"fcm_dpo/delta": -0.0640040785074234,
|
|
"fcm_dpo/margin": 261.2105712890625,
|
|
"fcm_dpo/q_t": 0.3543946146965027,
|
|
"grad_norm": 30.528892517089844,
|
|
"learning_rate": 2.736501028272095e-08,
|
|
"logits/chosen": -0.5115870237350464,
|
|
"logits/rejected": -0.5127223134040833,
|
|
"logps/chosen": -343.80242919921875,
|
|
"logps/ref_chosen": -55.80979537963867,
|
|
"logps/ref_rejected": -106.06282043457031,
|
|
"logps/rejected": -655.2659912109375,
|
|
"loss": 0.9296,
|
|
"margin_dpo/margin_mean": 261.2105712890625,
|
|
"margin_dpo/margin_std": 264.1900939941406,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.8678414096916299,
|
|
"fcm_dpo/beta": 0.002512441948056221,
|
|
"fcm_dpo/delta": -0.01745918020606041,
|
|
"fcm_dpo/margin": 245.31549072265625,
|
|
"fcm_dpo/q_t": 0.3611946403980255,
|
|
"grad_norm": 34.154884338378906,
|
|
"learning_rate": 2.678415274939408e-08,
|
|
"logits/chosen": -0.4548659920692444,
|
|
"logits/rejected": -0.4189767837524414,
|
|
"logps/chosen": -348.8348693847656,
|
|
"logps/ref_chosen": -56.24061965942383,
|
|
"logps/ref_rejected": -83.78629302978516,
|
|
"logps/rejected": -621.696044921875,
|
|
"loss": 0.9438,
|
|
"margin_dpo/margin_mean": 245.3155059814453,
|
|
"margin_dpo/margin_std": 240.31198120117188,
|
|
"step": 591
|
|
},
|
|
{
|
|
"epoch": 0.869309838472834,
|
|
"fcm_dpo/beta": 0.0025183933321386576,
|
|
"fcm_dpo/delta": 0.03698636591434479,
|
|
"fcm_dpo/margin": 224.35015869140625,
|
|
"fcm_dpo/q_t": 0.37596046924591064,
|
|
"grad_norm": 36.619808197021484,
|
|
"learning_rate": 2.6209177161234442e-08,
|
|
"logits/chosen": -0.41440728306770325,
|
|
"logits/rejected": -0.39510399103164673,
|
|
"logps/chosen": -373.01190185546875,
|
|
"logps/ref_chosen": -47.94025421142578,
|
|
"logps/ref_rejected": -75.73287963867188,
|
|
"logps/rejected": -625.1546630859375,
|
|
"loss": 1.0503,
|
|
"margin_dpo/margin_mean": 224.35015869140625,
|
|
"margin_dpo/margin_std": 322.49993896484375,
|
|
"step": 592
|
|
},
|
|
{
|
|
"epoch": 0.8707782672540382,
|
|
"fcm_dpo/beta": 0.002595474012196064,
|
|
"fcm_dpo/delta": 0.17214420437812805,
|
|
"fcm_dpo/margin": 167.77935791015625,
|
|
"fcm_dpo/q_t": 0.4035273790359497,
|
|
"grad_norm": 44.139076232910156,
|
|
"learning_rate": 2.564009866938349e-08,
|
|
"logits/chosen": -0.4011484682559967,
|
|
"logits/rejected": -0.3781154155731201,
|
|
"logps/chosen": -341.73126220703125,
|
|
"logps/ref_chosen": -48.690757751464844,
|
|
"logps/ref_rejected": -60.90800094604492,
|
|
"logps/rejected": -521.7278442382812,
|
|
"loss": 1.0966,
|
|
"margin_dpo/margin_mean": 167.7793426513672,
|
|
"margin_dpo/margin_std": 248.49221801757812,
|
|
"step": 593
|
|
},
|
|
{
|
|
"epoch": 0.8722466960352423,
|
|
"fcm_dpo/beta": 0.00264896173030138,
|
|
"fcm_dpo/delta": -0.018487006425857544,
|
|
"fcm_dpo/margin": 231.80612182617188,
|
|
"fcm_dpo/q_t": 0.36759766936302185,
|
|
"grad_norm": 32.71717071533203,
|
|
"learning_rate": 2.5076932269588708e-08,
|
|
"logits/chosen": -0.5203255414962769,
|
|
"logits/rejected": -0.49811041355133057,
|
|
"logps/chosen": -332.0072021484375,
|
|
"logps/ref_chosen": -54.93488693237305,
|
|
"logps/ref_rejected": -86.09967803955078,
|
|
"logps/rejected": -594.9780883789062,
|
|
"loss": 0.9936,
|
|
"margin_dpo/margin_mean": 231.80612182617188,
|
|
"margin_dpo/margin_std": 272.37933349609375,
|
|
"step": 594
|
|
},
|
|
{
|
|
"epoch": 0.8737151248164464,
|
|
"fcm_dpo/beta": 0.002612186362966895,
|
|
"fcm_dpo/delta": 0.022652659565210342,
|
|
"fcm_dpo/margin": 221.48776245117188,
|
|
"fcm_dpo/q_t": 0.37508952617645264,
|
|
"grad_norm": 33.132080078125,
|
|
"learning_rate": 2.451969280180849e-08,
|
|
"logits/chosen": -0.4913725256919861,
|
|
"logits/rejected": -0.4721420407295227,
|
|
"logps/chosen": -338.8521728515625,
|
|
"logps/ref_chosen": -49.4204216003418,
|
|
"logps/ref_rejected": -80.62731170654297,
|
|
"logps/rejected": -591.546875,
|
|
"loss": 0.9833,
|
|
"margin_dpo/margin_mean": 221.48776245117188,
|
|
"margin_dpo/margin_std": 257.7921142578125,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.8751835535976505,
|
|
"fcm_dpo/beta": 0.002664618194103241,
|
|
"fcm_dpo/delta": 0.08821612596511841,
|
|
"fcm_dpo/margin": 193.6279296875,
|
|
"fcm_dpo/q_t": 0.38701266050338745,
|
|
"grad_norm": 41.59341049194336,
|
|
"learning_rate": 2.396839494982103e-08,
|
|
"logits/chosen": -0.5178302526473999,
|
|
"logits/rejected": -0.48967409133911133,
|
|
"logps/chosen": -358.09478759765625,
|
|
"logps/ref_chosen": -59.791683197021484,
|
|
"logps/ref_rejected": -80.09111785888672,
|
|
"logps/rejected": -572.0221557617188,
|
|
"loss": 1.0508,
|
|
"margin_dpo/margin_mean": 193.6279296875,
|
|
"margin_dpo/margin_std": 269.0942687988281,
|
|
"step": 596
|
|
},
|
|
{
|
|
"epoch": 0.8766519823788547,
|
|
"fcm_dpo/beta": 0.0026114017236977816,
|
|
"fcm_dpo/delta": -0.08313372731208801,
|
|
"fcm_dpo/margin": 258.4861755371094,
|
|
"fcm_dpo/q_t": 0.3541925251483917,
|
|
"grad_norm": 33.791534423828125,
|
|
"learning_rate": 2.3423053240837514e-08,
|
|
"logits/chosen": -0.43104076385498047,
|
|
"logits/rejected": -0.4429478645324707,
|
|
"logps/chosen": -359.1304931640625,
|
|
"logps/ref_chosen": -57.26078796386719,
|
|
"logps/ref_rejected": -100.6937255859375,
|
|
"logps/rejected": -661.049560546875,
|
|
"loss": 0.9329,
|
|
"margin_dpo/margin_mean": 258.4861755371094,
|
|
"margin_dpo/margin_std": 268.7314758300781,
|
|
"step": 597
|
|
},
|
|
{
|
|
"epoch": 0.8781204111600588,
|
|
"fcm_dpo/beta": 0.002603329950943589,
|
|
"fcm_dpo/delta": 0.007749959826469421,
|
|
"fcm_dpo/margin": 227.01988220214844,
|
|
"fcm_dpo/q_t": 0.36725500226020813,
|
|
"grad_norm": 40.18560791015625,
|
|
"learning_rate": 2.2883682045119062e-08,
|
|
"logits/chosen": -0.5023446083068848,
|
|
"logits/rejected": -0.4874638319015503,
|
|
"logps/chosen": -370.9491882324219,
|
|
"logps/ref_chosen": -52.51850509643555,
|
|
"logps/ref_rejected": -89.44385528564453,
|
|
"logps/rejected": -634.8944091796875,
|
|
"loss": 0.9904,
|
|
"margin_dpo/margin_mean": 227.01986694335938,
|
|
"margin_dpo/margin_std": 255.60830688476562,
|
|
"step": 598
|
|
},
|
|
{
|
|
"epoch": 0.8795888399412628,
|
|
"fcm_dpo/beta": 0.002633871277794242,
|
|
"fcm_dpo/delta": 0.049679867923259735,
|
|
"fcm_dpo/margin": 209.84701538085938,
|
|
"fcm_dpo/q_t": 0.3737853169441223,
|
|
"grad_norm": 29.854141235351562,
|
|
"learning_rate": 2.2350295575598367e-08,
|
|
"logits/chosen": -0.4743267297744751,
|
|
"logits/rejected": -0.46898123621940613,
|
|
"logps/chosen": -339.2664794921875,
|
|
"logps/ref_chosen": -49.802677154541016,
|
|
"logps/ref_rejected": -82.978515625,
|
|
"logps/rejected": -582.2893676757812,
|
|
"loss": 0.9804,
|
|
"margin_dpo/margin_mean": 209.8470458984375,
|
|
"margin_dpo/margin_std": 206.9398193359375,
|
|
"step": 599
|
|
},
|
|
{
|
|
"epoch": 0.8810572687224669,
|
|
"fcm_dpo/beta": 0.0026514409109950066,
|
|
"fcm_dpo/delta": 0.004689887166023254,
|
|
"fcm_dpo/margin": 224.59609985351562,
|
|
"fcm_dpo/q_t": 0.3712194561958313,
|
|
"grad_norm": 36.80059051513672,
|
|
"learning_rate": 2.1822907887504932e-08,
|
|
"logits/chosen": -0.5273237824440002,
|
|
"logits/rejected": -0.5121384263038635,
|
|
"logps/chosen": -400.91802978515625,
|
|
"logps/ref_chosen": -66.43487548828125,
|
|
"logps/ref_rejected": -85.45649719238281,
|
|
"logps/rejected": -644.5357666015625,
|
|
"loss": 0.998,
|
|
"margin_dpo/margin_mean": 224.59609985351562,
|
|
"margin_dpo/margin_std": 278.3258056640625,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8810572687224669,
|
|
"eval_fcm_dpo/beta": 0.002657976234331727,
|
|
"eval_logits/chosen": -0.5060350298881531,
|
|
"eval_logits/rejected": -0.4877537786960602,
|
|
"eval_logps/chosen": -436.8470153808594,
|
|
"eval_logps/ref_chosen": -79.05104064941406,
|
|
"eval_logps/ref_rejected": -86.79793548583984,
|
|
"eval_logps/rejected": -597.329345703125,
|
|
"eval_loss": 0.569061279296875,
|
|
"eval_margin_dpo/margin_mean": 152.7354736328125,
|
|
"eval_margin_dpo/margin_std": 265.09954833984375,
|
|
"eval_runtime": 39.2811,
|
|
"eval_samples_per_second": 59.545,
|
|
"eval_steps_per_second": 1.884,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.882525697503671,
|
|
"fcm_dpo/beta": 0.002635272918269038,
|
|
"fcm_dpo/delta": -0.03896784782409668,
|
|
"fcm_dpo/margin": 241.43040466308594,
|
|
"fcm_dpo/q_t": 0.3611557185649872,
|
|
"grad_norm": 41.90599822998047,
|
|
"learning_rate": 2.1301532877994742e-08,
|
|
"logits/chosen": -0.47748884558677673,
|
|
"logits/rejected": -0.46581321954727173,
|
|
"logps/chosen": -405.7626953125,
|
|
"logps/ref_chosen": -59.13361358642578,
|
|
"logps/ref_rejected": -94.69093322753906,
|
|
"logps/rejected": -682.7504272460938,
|
|
"loss": 0.9629,
|
|
"margin_dpo/margin_mean": 241.43038940429688,
|
|
"margin_dpo/margin_std": 275.83514404296875,
|
|
"step": 601
|
|
},
|
|
{
|
|
"epoch": 0.8839941262848752,
|
|
"fcm_dpo/beta": 0.002600977662950754,
|
|
"fcm_dpo/delta": -0.12249189615249634,
|
|
"fcm_dpo/margin": 274.4678955078125,
|
|
"fcm_dpo/q_t": 0.34391018748283386,
|
|
"grad_norm": 34.37300491333008,
|
|
"learning_rate": 2.0786184285784298e-08,
|
|
"logits/chosen": -0.4986855983734131,
|
|
"logits/rejected": -0.4927312731742859,
|
|
"logps/chosen": -303.8367919921875,
|
|
"logps/ref_chosen": -48.59352111816406,
|
|
"logps/ref_rejected": -87.6685562133789,
|
|
"logps/rejected": -617.3796997070312,
|
|
"loss": 0.8916,
|
|
"margin_dpo/margin_mean": 274.4678955078125,
|
|
"margin_dpo/margin_std": 258.61602783203125,
|
|
"step": 602
|
|
},
|
|
{
|
|
"epoch": 0.8854625550660793,
|
|
"fcm_dpo/beta": 0.002573822159320116,
|
|
"fcm_dpo/delta": -0.00591304013505578,
|
|
"fcm_dpo/margin": 235.269287109375,
|
|
"fcm_dpo/q_t": 0.3697740435600281,
|
|
"grad_norm": 44.532958984375,
|
|
"learning_rate": 2.0276875690788204e-08,
|
|
"logits/chosen": -0.5375059843063354,
|
|
"logits/rejected": -0.5155242681503296,
|
|
"logps/chosen": -360.972900390625,
|
|
"logps/ref_chosen": -70.41461944580078,
|
|
"logps/ref_rejected": -100.32559967041016,
|
|
"logps/rejected": -626.1531982421875,
|
|
"loss": 0.9847,
|
|
"margin_dpo/margin_mean": 235.269287109375,
|
|
"margin_dpo/margin_std": 282.26904296875,
|
|
"step": 603
|
|
},
|
|
{
|
|
"epoch": 0.8869309838472834,
|
|
"fcm_dpo/beta": 0.002521086484193802,
|
|
"fcm_dpo/delta": -0.11240590363740921,
|
|
"fcm_dpo/margin": 279.26800537109375,
|
|
"fcm_dpo/q_t": 0.34469854831695557,
|
|
"grad_norm": 37.802711486816406,
|
|
"learning_rate": 1.977362051376158e-08,
|
|
"logits/chosen": -0.4898557662963867,
|
|
"logits/rejected": -0.4911998510360718,
|
|
"logps/chosen": -313.27166748046875,
|
|
"logps/ref_chosen": -46.45808029174805,
|
|
"logps/ref_rejected": -91.8544921875,
|
|
"logps/rejected": -637.9360961914062,
|
|
"loss": 0.9107,
|
|
"margin_dpo/margin_mean": 279.26800537109375,
|
|
"margin_dpo/margin_std": 277.7120056152344,
|
|
"step": 604
|
|
},
|
|
{
|
|
"epoch": 0.8883994126284875,
|
|
"fcm_dpo/beta": 0.0025093178264796734,
|
|
"fcm_dpo/delta": 0.017898954451084137,
|
|
"fcm_dpo/margin": 232.27915954589844,
|
|
"fcm_dpo/q_t": 0.37571078538894653,
|
|
"grad_norm": 30.63051414489746,
|
|
"learning_rate": 1.9276432015946446e-08,
|
|
"logits/chosen": -0.5367037057876587,
|
|
"logits/rejected": -0.5347219109535217,
|
|
"logps/chosen": -390.2449645996094,
|
|
"logps/ref_chosen": -66.24933624267578,
|
|
"logps/ref_rejected": -102.30496978759766,
|
|
"logps/rejected": -658.5797119140625,
|
|
"loss": 0.9934,
|
|
"margin_dpo/margin_mean": 232.27914428710938,
|
|
"margin_dpo/margin_std": 289.82879638671875,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.8898678414096917,
|
|
"fcm_dpo/beta": 0.002527826000005007,
|
|
"fcm_dpo/delta": -0.03301798552274704,
|
|
"fcm_dpo/margin": 249.30398559570312,
|
|
"fcm_dpo/q_t": 0.36113643646240234,
|
|
"grad_norm": 36.33147430419922,
|
|
"learning_rate": 1.8785323298722093e-08,
|
|
"logits/chosen": -0.5017037391662598,
|
|
"logits/rejected": -0.4989688992500305,
|
|
"logps/chosen": -350.18060302734375,
|
|
"logps/ref_chosen": -54.819122314453125,
|
|
"logps/ref_rejected": -98.37146759033203,
|
|
"logps/rejected": -643.0369873046875,
|
|
"loss": 0.9437,
|
|
"margin_dpo/margin_mean": 249.30398559570312,
|
|
"margin_dpo/margin_std": 251.91043090820312,
|
|
"step": 606
|
|
},
|
|
{
|
|
"epoch": 0.8913362701908958,
|
|
"fcm_dpo/beta": 0.002542435657233,
|
|
"fcm_dpo/delta": 0.07603956758975983,
|
|
"fcm_dpo/margin": 207.49200439453125,
|
|
"fcm_dpo/q_t": 0.382717490196228,
|
|
"grad_norm": 28.992673873901367,
|
|
"learning_rate": 1.8300307303259904e-08,
|
|
"logits/chosen": -0.5028606057167053,
|
|
"logits/rejected": -0.48178744316101074,
|
|
"logps/chosen": -371.967041015625,
|
|
"logps/ref_chosen": -58.08403778076172,
|
|
"logps/ref_rejected": -79.777099609375,
|
|
"logps/rejected": -601.1521606445312,
|
|
"loss": 1.0122,
|
|
"margin_dpo/margin_mean": 207.4920196533203,
|
|
"margin_dpo/margin_std": 238.70074462890625,
|
|
"step": 607
|
|
},
|
|
{
|
|
"epoch": 0.8928046989720999,
|
|
"fcm_dpo/beta": 0.0025535556487739086,
|
|
"fcm_dpo/delta": 0.010348204523324966,
|
|
"fcm_dpo/margin": 231.10964965820312,
|
|
"fcm_dpo/q_t": 0.3670482039451599,
|
|
"grad_norm": 27.258190155029297,
|
|
"learning_rate": 1.7821396810182437e-08,
|
|
"logits/chosen": -0.536124587059021,
|
|
"logits/rejected": -0.5197678208351135,
|
|
"logps/chosen": -358.027099609375,
|
|
"logps/ref_chosen": -57.450836181640625,
|
|
"logps/ref_rejected": -94.77339172363281,
|
|
"logps/rejected": -626.4592895507812,
|
|
"loss": 0.9573,
|
|
"margin_dpo/margin_mean": 231.10964965820312,
|
|
"margin_dpo/margin_std": 227.2630615234375,
|
|
"step": 608
|
|
},
|
|
{
|
|
"epoch": 0.8942731277533039,
|
|
"fcm_dpo/beta": 0.0025187768042087555,
|
|
"fcm_dpo/delta": -0.08650342375040054,
|
|
"fcm_dpo/margin": 270.2357177734375,
|
|
"fcm_dpo/q_t": 0.3506140112876892,
|
|
"grad_norm": 41.33991241455078,
|
|
"learning_rate": 1.7348604439226617e-08,
|
|
"logits/chosen": -0.5273410677909851,
|
|
"logits/rejected": -0.516228199005127,
|
|
"logps/chosen": -351.49822998046875,
|
|
"logps/ref_chosen": -58.805355072021484,
|
|
"logps/ref_rejected": -88.81600952148438,
|
|
"logps/rejected": -651.74462890625,
|
|
"loss": 0.9151,
|
|
"margin_dpo/margin_mean": 270.2357177734375,
|
|
"margin_dpo/margin_std": 274.7491760253906,
|
|
"step": 609
|
|
},
|
|
{
|
|
"epoch": 0.895741556534508,
|
|
"fcm_dpo/beta": 0.002538463566452265,
|
|
"fcm_dpo/delta": 0.07885053008794785,
|
|
"fcm_dpo/margin": 206.88258361816406,
|
|
"fcm_dpo/q_t": 0.3827626705169678,
|
|
"grad_norm": 40.440711975097656,
|
|
"learning_rate": 1.6881942648911074e-08,
|
|
"logits/chosen": -0.5288474559783936,
|
|
"logits/rejected": -0.4975678324699402,
|
|
"logps/chosen": -368.1615905761719,
|
|
"logps/ref_chosen": -65.69503784179688,
|
|
"logps/ref_rejected": -83.40538787841797,
|
|
"logps/rejected": -592.7545166015625,
|
|
"loss": 1.0173,
|
|
"margin_dpo/margin_mean": 206.88258361816406,
|
|
"margin_dpo/margin_std": 242.43508911132812,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.8972099853157122,
|
|
"fcm_dpo/beta": 0.002536176238209009,
|
|
"fcm_dpo/delta": -0.05313975363969803,
|
|
"fcm_dpo/margin": 256.1167907714844,
|
|
"fcm_dpo/q_t": 0.35713207721710205,
|
|
"grad_norm": 35.82498550415039,
|
|
"learning_rate": 1.6421423736208e-08,
|
|
"logits/chosen": -0.5013567209243774,
|
|
"logits/rejected": -0.5022546052932739,
|
|
"logps/chosen": -356.3680725097656,
|
|
"logps/ref_chosen": -52.59946823120117,
|
|
"logps/ref_rejected": -86.33099365234375,
|
|
"logps/rejected": -646.2163696289062,
|
|
"loss": 0.9377,
|
|
"margin_dpo/margin_mean": 256.1167907714844,
|
|
"margin_dpo/margin_std": 261.8753662109375,
|
|
"step": 611
|
|
},
|
|
{
|
|
"epoch": 0.8986784140969163,
|
|
"fcm_dpo/beta": 0.0024938895367085934,
|
|
"fcm_dpo/delta": -0.07635810971260071,
|
|
"fcm_dpo/margin": 269.17584228515625,
|
|
"fcm_dpo/q_t": 0.3497973084449768,
|
|
"grad_norm": 25.489120483398438,
|
|
"learning_rate": 1.5967059836219042e-08,
|
|
"logits/chosen": -0.49843016266822815,
|
|
"logits/rejected": -0.48337119817733765,
|
|
"logps/chosen": -366.25396728515625,
|
|
"logps/ref_chosen": -59.32372283935547,
|
|
"logps/ref_rejected": -88.31239318847656,
|
|
"logps/rejected": -664.41845703125,
|
|
"loss": 0.9038,
|
|
"margin_dpo/margin_mean": 269.17584228515625,
|
|
"margin_dpo/margin_std": 241.643310546875,
|
|
"step": 612
|
|
},
|
|
{
|
|
"epoch": 0.9001468428781204,
|
|
"fcm_dpo/beta": 0.0024755061604082584,
|
|
"fcm_dpo/delta": -0.025314215570688248,
|
|
"fcm_dpo/margin": 251.95530700683594,
|
|
"fcm_dpo/q_t": 0.3586321473121643,
|
|
"grad_norm": 35.185272216796875,
|
|
"learning_rate": 1.551886292185553e-08,
|
|
"logits/chosen": -0.5439824461936951,
|
|
"logits/rejected": -0.5609041452407837,
|
|
"logps/chosen": -351.8963317871094,
|
|
"logps/ref_chosen": -59.72996520996094,
|
|
"logps/ref_rejected": -105.10752868652344,
|
|
"logps/rejected": -649.229248046875,
|
|
"loss": 0.9211,
|
|
"margin_dpo/margin_mean": 251.95529174804688,
|
|
"margin_dpo/margin_std": 217.81700134277344,
|
|
"step": 613
|
|
},
|
|
{
|
|
"epoch": 0.9016152716593245,
|
|
"fcm_dpo/beta": 0.002444417215883732,
|
|
"fcm_dpo/delta": -0.04796981438994408,
|
|
"fcm_dpo/margin": 263.7240905761719,
|
|
"fcm_dpo/q_t": 0.35601553320884705,
|
|
"grad_norm": 34.938133239746094,
|
|
"learning_rate": 1.507684480352292e-08,
|
|
"logits/chosen": -0.4674204885959625,
|
|
"logits/rejected": -0.49593475461006165,
|
|
"logps/chosen": -351.73431396484375,
|
|
"logps/ref_chosen": -52.93898010253906,
|
|
"logps/ref_rejected": -104.67938232421875,
|
|
"logps/rejected": -667.1987915039062,
|
|
"loss": 0.932,
|
|
"margin_dpo/margin_mean": 263.7240905761719,
|
|
"margin_dpo/margin_std": 258.8504638671875,
|
|
"step": 614
|
|
},
|
|
{
|
|
"epoch": 0.9030837004405287,
|
|
"fcm_dpo/beta": 0.0024889023043215275,
|
|
"fcm_dpo/delta": 0.09434535354375839,
|
|
"fcm_dpo/margin": 204.8232421875,
|
|
"fcm_dpo/q_t": 0.38655292987823486,
|
|
"grad_norm": 31.439695358276367,
|
|
"learning_rate": 1.4641017128809801e-08,
|
|
"logits/chosen": -0.5380944609642029,
|
|
"logits/rejected": -0.5339647531509399,
|
|
"logps/chosen": -367.2992248535156,
|
|
"logps/ref_chosen": -65.81727600097656,
|
|
"logps/ref_rejected": -95.17749786376953,
|
|
"logps/rejected": -601.482666015625,
|
|
"loss": 1.0194,
|
|
"margin_dpo/margin_mean": 204.8232421875,
|
|
"margin_dpo/margin_std": 236.53875732421875,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.9045521292217328,
|
|
"fcm_dpo/beta": 0.002577736508101225,
|
|
"fcm_dpo/delta": 0.2006855607032776,
|
|
"fcm_dpo/margin": 157.602294921875,
|
|
"fcm_dpo/q_t": 0.4117230772972107,
|
|
"grad_norm": 39.64345932006836,
|
|
"learning_rate": 1.4211391382180637e-08,
|
|
"logits/chosen": -0.5193255543708801,
|
|
"logits/rejected": -0.4902590811252594,
|
|
"logps/chosen": -436.3590393066406,
|
|
"logps/ref_chosen": -65.13285827636719,
|
|
"logps/ref_rejected": -74.70050048828125,
|
|
"logps/rejected": -603.5289916992188,
|
|
"loss": 1.1199,
|
|
"margin_dpo/margin_mean": 157.602294921875,
|
|
"margin_dpo/margin_std": 252.67433166503906,
|
|
"step": 616
|
|
},
|
|
{
|
|
"epoch": 0.9060205580029369,
|
|
"fcm_dpo/beta": 0.0026704384945333004,
|
|
"fcm_dpo/delta": 0.1679898351430893,
|
|
"fcm_dpo/margin": 164.12026977539062,
|
|
"fcm_dpo/q_t": 0.4053176939487457,
|
|
"grad_norm": 46.60618209838867,
|
|
"learning_rate": 1.378797888467345e-08,
|
|
"logits/chosen": -0.5051152110099792,
|
|
"logits/rejected": -0.47557491064071655,
|
|
"logps/chosen": -400.8250732421875,
|
|
"logps/ref_chosen": -63.005550384521484,
|
|
"logps/ref_rejected": -64.234130859375,
|
|
"logps/rejected": -566.1739501953125,
|
|
"loss": 1.0936,
|
|
"margin_dpo/margin_mean": 164.12026977539062,
|
|
"margin_dpo/margin_std": 247.26060485839844,
|
|
"step": 617
|
|
},
|
|
{
|
|
"epoch": 0.9074889867841409,
|
|
"fcm_dpo/beta": 0.002672924194484949,
|
|
"fcm_dpo/delta": -0.059617504477500916,
|
|
"fcm_dpo/margin": 245.20562744140625,
|
|
"fcm_dpo/q_t": 0.3566618263721466,
|
|
"grad_norm": 40.86361312866211,
|
|
"learning_rate": 1.3370790793601371e-08,
|
|
"logits/chosen": -0.4829975962638855,
|
|
"logits/rejected": -0.4715951681137085,
|
|
"logps/chosen": -403.0682373046875,
|
|
"logps/ref_chosen": -67.10134887695312,
|
|
"logps/ref_rejected": -92.15340423583984,
|
|
"logps/rejected": -673.325927734375,
|
|
"loss": 0.9513,
|
|
"margin_dpo/margin_mean": 245.2056427001953,
|
|
"margin_dpo/margin_std": 266.8808898925781,
|
|
"step": 618
|
|
},
|
|
{
|
|
"epoch": 0.908957415565345,
|
|
"fcm_dpo/beta": 0.0026871771551668644,
|
|
"fcm_dpo/delta": 0.07144356518983841,
|
|
"fcm_dpo/margin": 197.93942260742188,
|
|
"fcm_dpo/q_t": 0.38919079303741455,
|
|
"grad_norm": 40.54890441894531,
|
|
"learning_rate": 1.2959838102258535e-08,
|
|
"logits/chosen": -0.5091539025306702,
|
|
"logits/rejected": -0.5012193918228149,
|
|
"logps/chosen": -386.0754699707031,
|
|
"logps/ref_chosen": -55.978233337402344,
|
|
"logps/ref_rejected": -93.1854019165039,
|
|
"logps/rejected": -621.2220458984375,
|
|
"loss": 1.0637,
|
|
"margin_dpo/margin_mean": 197.93943786621094,
|
|
"margin_dpo/margin_std": 297.71466064453125,
|
|
"step": 619
|
|
},
|
|
{
|
|
"epoch": 0.9104258443465492,
|
|
"fcm_dpo/beta": 0.0026975993532687426,
|
|
"fcm_dpo/delta": 0.0638522133231163,
|
|
"fcm_dpo/margin": 199.90643310546875,
|
|
"fcm_dpo/q_t": 0.3846666216850281,
|
|
"grad_norm": 46.63349914550781,
|
|
"learning_rate": 1.2555131639630567e-08,
|
|
"logits/chosen": -0.5148506164550781,
|
|
"logits/rejected": -0.4951644539833069,
|
|
"logps/chosen": -366.35540771484375,
|
|
"logps/ref_chosen": -59.79750061035156,
|
|
"logps/ref_rejected": -78.41075134277344,
|
|
"logps/rejected": -584.8751220703125,
|
|
"loss": 1.0283,
|
|
"margin_dpo/margin_mean": 199.90643310546875,
|
|
"margin_dpo/margin_std": 263.07427978515625,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.9118942731277533,
|
|
"fcm_dpo/beta": 0.0026699635200202465,
|
|
"fcm_dpo/delta": -0.17680245637893677,
|
|
"fcm_dpo/margin": 285.97552490234375,
|
|
"fcm_dpo/q_t": 0.33413994312286377,
|
|
"grad_norm": 35.8352165222168,
|
|
"learning_rate": 1.2156682070109086e-08,
|
|
"logits/chosen": -0.5005359649658203,
|
|
"logits/rejected": -0.5020206570625305,
|
|
"logps/chosen": -341.9903564453125,
|
|
"logps/ref_chosen": -53.93375778198242,
|
|
"logps/ref_rejected": -88.36951446533203,
|
|
"logps/rejected": -662.401611328125,
|
|
"loss": 0.8752,
|
|
"margin_dpo/margin_mean": 285.97552490234375,
|
|
"margin_dpo/margin_std": 268.96636962890625,
|
|
"step": 621
|
|
},
|
|
{
|
|
"epoch": 0.9133627019089574,
|
|
"fcm_dpo/beta": 0.0026113735511898994,
|
|
"fcm_dpo/delta": 0.0012344121932983398,
|
|
"fcm_dpo/margin": 229.01025390625,
|
|
"fcm_dpo/q_t": 0.3693368434906006,
|
|
"grad_norm": 33.05309295654297,
|
|
"learning_rate": 1.1764499893210878e-08,
|
|
"logits/chosen": -0.48245564103126526,
|
|
"logits/rejected": -0.4655565917491913,
|
|
"logps/chosen": -363.42919921875,
|
|
"logps/ref_chosen": -60.28582000732422,
|
|
"logps/ref_rejected": -85.51873779296875,
|
|
"logps/rejected": -617.67236328125,
|
|
"loss": 0.9679,
|
|
"margin_dpo/margin_mean": 229.01025390625,
|
|
"margin_dpo/margin_std": 250.83233642578125,
|
|
"step": 622
|
|
},
|
|
{
|
|
"epoch": 0.9148311306901615,
|
|
"fcm_dpo/beta": 0.002670406363904476,
|
|
"fcm_dpo/delta": 0.12260958552360535,
|
|
"fcm_dpo/margin": 181.0537109375,
|
|
"fcm_dpo/q_t": 0.39631474018096924,
|
|
"grad_norm": 40.9214973449707,
|
|
"learning_rate": 1.1378595443300998e-08,
|
|
"logits/chosen": -0.48193132877349854,
|
|
"logits/rejected": -0.4616912603378296,
|
|
"logps/chosen": -400.8602294921875,
|
|
"logps/ref_chosen": -64.1569595336914,
|
|
"logps/ref_rejected": -85.08304595947266,
|
|
"logps/rejected": -602.840087890625,
|
|
"loss": 1.0693,
|
|
"margin_dpo/margin_mean": 181.05372619628906,
|
|
"margin_dpo/margin_std": 264.8545227050781,
|
|
"step": 623
|
|
},
|
|
{
|
|
"epoch": 0.9162995594713657,
|
|
"fcm_dpo/beta": 0.0026587401516735554,
|
|
"fcm_dpo/delta": -0.08800569176673889,
|
|
"fcm_dpo/margin": 256.51275634765625,
|
|
"fcm_dpo/q_t": 0.34991389513015747,
|
|
"grad_norm": 39.98982620239258,
|
|
"learning_rate": 1.0998978889320582e-08,
|
|
"logits/chosen": -0.5772846937179565,
|
|
"logits/rejected": -0.5503325462341309,
|
|
"logps/chosen": -415.0184020996094,
|
|
"logps/ref_chosen": -71.91862487792969,
|
|
"logps/ref_rejected": -97.13203430175781,
|
|
"logps/rejected": -696.7445678710938,
|
|
"loss": 0.9264,
|
|
"margin_dpo/margin_mean": 256.51275634765625,
|
|
"margin_dpo/margin_std": 264.0021057128906,
|
|
"step": 624
|
|
},
|
|
{
|
|
"epoch": 0.9177679882525698,
|
|
"fcm_dpo/beta": 0.0026332507841289043,
|
|
"fcm_dpo/delta": -0.023220915347337723,
|
|
"fcm_dpo/margin": 236.09141540527344,
|
|
"fcm_dpo/q_t": 0.36515945196151733,
|
|
"grad_norm": 42.99685287475586,
|
|
"learning_rate": 1.0625660234518913e-08,
|
|
"logits/chosen": -0.4856789708137512,
|
|
"logits/rejected": -0.468561589717865,
|
|
"logps/chosen": -366.0799560546875,
|
|
"logps/ref_chosen": -58.342071533203125,
|
|
"logps/ref_rejected": -86.09038543701172,
|
|
"logps/rejected": -629.919677734375,
|
|
"loss": 0.9517,
|
|
"margin_dpo/margin_mean": 236.09140014648438,
|
|
"margin_dpo/margin_std": 255.3983154296875,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.9192364170337739,
|
|
"fcm_dpo/beta": 0.0026790527626872063,
|
|
"fcm_dpo/delta": 0.09501801431179047,
|
|
"fcm_dpo/margin": 190.15899658203125,
|
|
"fcm_dpo/q_t": 0.3896622657775879,
|
|
"grad_norm": 35.053749084472656,
|
|
"learning_rate": 1.0258649316189721e-08,
|
|
"logits/chosen": -0.516830325126648,
|
|
"logits/rejected": -0.4992465078830719,
|
|
"logps/chosen": -417.03143310546875,
|
|
"logps/ref_chosen": -75.11260986328125,
|
|
"logps/ref_rejected": -99.188720703125,
|
|
"logps/rejected": -631.2666015625,
|
|
"loss": 1.0376,
|
|
"margin_dpo/margin_mean": 190.15899658203125,
|
|
"margin_dpo/margin_std": 242.58265686035156,
|
|
"step": 626
|
|
},
|
|
{
|
|
"epoch": 0.920704845814978,
|
|
"fcm_dpo/beta": 0.0026376736350357533,
|
|
"fcm_dpo/delta": -0.13061486184597015,
|
|
"fcm_dpo/margin": 273.473388671875,
|
|
"fcm_dpo/q_t": 0.35445448756217957,
|
|
"grad_norm": 37.661746978759766,
|
|
"learning_rate": 9.897955805412e-09,
|
|
"logits/chosen": -0.4934673309326172,
|
|
"logits/rejected": -0.5177198648452759,
|
|
"logps/chosen": -314.9818115234375,
|
|
"logps/ref_chosen": -47.74314880371094,
|
|
"logps/ref_rejected": -106.75448608398438,
|
|
"logps/rejected": -647.466552734375,
|
|
"loss": 0.9501,
|
|
"margin_dpo/margin_mean": 273.4734191894531,
|
|
"margin_dpo/margin_std": 342.03204345703125,
|
|
"step": 627
|
|
},
|
|
{
|
|
"epoch": 0.922173274596182,
|
|
"fcm_dpo/beta": 0.0025675692595541477,
|
|
"fcm_dpo/delta": -0.09277446568012238,
|
|
"fcm_dpo/margin": 267.0002746582031,
|
|
"fcm_dpo/q_t": 0.34974387288093567,
|
|
"grad_norm": 32.36410140991211,
|
|
"learning_rate": 9.543589206795238e-09,
|
|
"logits/chosen": -0.5133162140846252,
|
|
"logits/rejected": -0.5017916560173035,
|
|
"logps/chosen": -382.24267578125,
|
|
"logps/ref_chosen": -60.182945251464844,
|
|
"logps/ref_rejected": -101.55467224121094,
|
|
"logps/rejected": -690.6146240234375,
|
|
"loss": 0.9174,
|
|
"margin_dpo/margin_mean": 267.0002746582031,
|
|
"margin_dpo/margin_std": 266.8199768066406,
|
|
"step": 628
|
|
},
|
|
{
|
|
"epoch": 0.9236417033773862,
|
|
"fcm_dpo/beta": 0.002568417927250266,
|
|
"fcm_dpo/delta": 0.013333270326256752,
|
|
"fcm_dpo/margin": 228.7070770263672,
|
|
"fcm_dpo/q_t": 0.3673741817474365,
|
|
"grad_norm": 39.50351333618164,
|
|
"learning_rate": 9.19555885822887e-09,
|
|
"logits/chosen": -0.4918622374534607,
|
|
"logits/rejected": -0.47989362478256226,
|
|
"logps/chosen": -393.0993957519531,
|
|
"logps/ref_chosen": -64.21354675292969,
|
|
"logps/ref_rejected": -91.65367126464844,
|
|
"logps/rejected": -649.24658203125,
|
|
"loss": 0.9515,
|
|
"margin_dpo/margin_mean": 228.70706176757812,
|
|
"margin_dpo/margin_std": 213.66384887695312,
|
|
"step": 629
|
|
},
|
|
{
|
|
"epoch": 0.9251101321585903,
|
|
"fcm_dpo/beta": 0.0026454501785337925,
|
|
"fcm_dpo/delta": 0.16232366859912872,
|
|
"fcm_dpo/margin": 167.9291534423828,
|
|
"fcm_dpo/q_t": 0.4042319357395172,
|
|
"grad_norm": 37.71936798095703,
|
|
"learning_rate": 8.85387393063622e-09,
|
|
"logits/chosen": -0.5329476594924927,
|
|
"logits/rejected": -0.5061321258544922,
|
|
"logps/chosen": -354.07684326171875,
|
|
"logps/ref_chosen": -59.29100036621094,
|
|
"logps/ref_rejected": -83.59829711914062,
|
|
"logps/rejected": -546.3132934570312,
|
|
"loss": 1.0856,
|
|
"margin_dpo/margin_mean": 167.9291534423828,
|
|
"margin_dpo/margin_std": 248.06668090820312,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.9265785609397944,
|
|
"fcm_dpo/beta": 0.002692385343834758,
|
|
"fcm_dpo/delta": 0.05998341366648674,
|
|
"fcm_dpo/margin": 201.62759399414062,
|
|
"fcm_dpo/q_t": 0.3826253414154053,
|
|
"grad_norm": 39.2678337097168,
|
|
"learning_rate": 8.518543427732949e-09,
|
|
"logits/chosen": -0.5022942423820496,
|
|
"logits/rejected": -0.48305743932724,
|
|
"logps/chosen": -390.76043701171875,
|
|
"logps/ref_chosen": -59.45360565185547,
|
|
"logps/ref_rejected": -80.95156860351562,
|
|
"logps/rejected": -613.885986328125,
|
|
"loss": 1.0448,
|
|
"margin_dpo/margin_mean": 201.62759399414062,
|
|
"margin_dpo/margin_std": 280.7120361328125,
|
|
"step": 631
|
|
},
|
|
{
|
|
"epoch": 0.9280469897209985,
|
|
"fcm_dpo/beta": 0.0027071628719568253,
|
|
"fcm_dpo/delta": 0.05815380439162254,
|
|
"fcm_dpo/margin": 201.3222198486328,
|
|
"fcm_dpo/q_t": 0.3803539276123047,
|
|
"grad_norm": 38.19340133666992,
|
|
"learning_rate": 8.189576185789637e-09,
|
|
"logits/chosen": -0.4966999888420105,
|
|
"logits/rejected": -0.47419142723083496,
|
|
"logps/chosen": -389.7144775390625,
|
|
"logps/ref_chosen": -61.35155487060547,
|
|
"logps/ref_rejected": -86.16017150878906,
|
|
"logps/rejected": -615.8453369140625,
|
|
"loss": 1.0424,
|
|
"margin_dpo/margin_mean": 201.3222198486328,
|
|
"margin_dpo/margin_std": 276.93914794921875,
|
|
"step": 632
|
|
},
|
|
{
|
|
"epoch": 0.9295154185022027,
|
|
"fcm_dpo/beta": 0.0027928436174988747,
|
|
"fcm_dpo/delta": 0.12002822011709213,
|
|
"fcm_dpo/margin": 173.24588012695312,
|
|
"fcm_dpo/q_t": 0.39484164118766785,
|
|
"grad_norm": 57.099609375,
|
|
"learning_rate": 7.866980873399015e-09,
|
|
"logits/chosen": -0.5218715667724609,
|
|
"logits/rejected": -0.5239961743354797,
|
|
"logps/chosen": -393.3670349121094,
|
|
"logps/ref_chosen": -57.27816390991211,
|
|
"logps/ref_rejected": -91.58395385742188,
|
|
"logps/rejected": -600.918701171875,
|
|
"loss": 1.0778,
|
|
"margin_dpo/margin_mean": 173.24588012695312,
|
|
"margin_dpo/margin_std": 253.44149780273438,
|
|
"step": 633
|
|
},
|
|
{
|
|
"epoch": 0.9309838472834068,
|
|
"fcm_dpo/beta": 0.002813713625073433,
|
|
"fcm_dpo/delta": 0.08731327205896378,
|
|
"fcm_dpo/margin": 183.8138885498047,
|
|
"fcm_dpo/q_t": 0.3883476257324219,
|
|
"grad_norm": 39.00724411010742,
|
|
"learning_rate": 7.550765991247654e-09,
|
|
"logits/chosen": -0.5172115564346313,
|
|
"logits/rejected": -0.5073580741882324,
|
|
"logps/chosen": -427.5379333496094,
|
|
"logps/ref_chosen": -66.61896514892578,
|
|
"logps/ref_rejected": -107.12564849853516,
|
|
"logps/rejected": -651.8585205078125,
|
|
"loss": 1.0418,
|
|
"margin_dpo/margin_mean": 183.8138885498047,
|
|
"margin_dpo/margin_std": 243.1580047607422,
|
|
"step": 634
|
|
},
|
|
{
|
|
"epoch": 0.9324522760646109,
|
|
"fcm_dpo/beta": 0.0028590408619493246,
|
|
"fcm_dpo/delta": 0.05354948714375496,
|
|
"fcm_dpo/margin": 192.16888427734375,
|
|
"fcm_dpo/q_t": 0.38193368911743164,
|
|
"grad_norm": 41.086822509765625,
|
|
"learning_rate": 7.240939871891699e-09,
|
|
"logits/chosen": -0.5438894629478455,
|
|
"logits/rejected": -0.5025356411933899,
|
|
"logps/chosen": -382.0107421875,
|
|
"logps/ref_chosen": -73.95551300048828,
|
|
"logps/ref_rejected": -82.50045776367188,
|
|
"logps/rejected": -582.724609375,
|
|
"loss": 1.0099,
|
|
"margin_dpo/margin_mean": 192.16888427734375,
|
|
"margin_dpo/margin_std": 238.52110290527344,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.933920704845815,
|
|
"fcm_dpo/beta": 0.0028688067104667425,
|
|
"fcm_dpo/delta": -0.04967202991247177,
|
|
"fcm_dpo/margin": 225.1034698486328,
|
|
"fcm_dpo/q_t": 0.35912269353866577,
|
|
"grad_norm": 42.04436492919922,
|
|
"learning_rate": 6.937510679537628e-09,
|
|
"logits/chosen": -0.4912013113498688,
|
|
"logits/rejected": -0.4742831587791443,
|
|
"logps/chosen": -358.6704406738281,
|
|
"logps/ref_chosen": -59.628910064697266,
|
|
"logps/ref_rejected": -81.97883605957031,
|
|
"logps/rejected": -606.1238403320312,
|
|
"loss": 0.9427,
|
|
"margin_dpo/margin_mean": 225.10345458984375,
|
|
"margin_dpo/margin_std": 237.44015502929688,
|
|
"step": 636
|
|
},
|
|
{
|
|
"epoch": 0.9353891336270191,
|
|
"fcm_dpo/beta": 0.002806810662150383,
|
|
"fcm_dpo/delta": -0.06823452562093735,
|
|
"fcm_dpo/margin": 236.39791870117188,
|
|
"fcm_dpo/q_t": 0.3558032512664795,
|
|
"grad_norm": 40.245174407958984,
|
|
"learning_rate": 6.640486409826785e-09,
|
|
"logits/chosen": -0.4979478716850281,
|
|
"logits/rejected": -0.496917724609375,
|
|
"logps/chosen": -365.4517822265625,
|
|
"logps/ref_chosen": -49.652687072753906,
|
|
"logps/ref_rejected": -98.40513610839844,
|
|
"logps/rejected": -650.6021728515625,
|
|
"loss": 0.9336,
|
|
"margin_dpo/margin_mean": 236.39791870117188,
|
|
"margin_dpo/margin_std": 248.04531860351562,
|
|
"step": 637
|
|
},
|
|
{
|
|
"epoch": 0.9368575624082232,
|
|
"fcm_dpo/beta": 0.0028112411964684725,
|
|
"fcm_dpo/delta": -0.03967977315187454,
|
|
"fcm_dpo/margin": 226.1969757080078,
|
|
"fcm_dpo/q_t": 0.357632577419281,
|
|
"grad_norm": 33.084903717041016,
|
|
"learning_rate": 6.349874889624962e-09,
|
|
"logits/chosen": -0.4676443338394165,
|
|
"logits/rejected": -0.4407140016555786,
|
|
"logps/chosen": -347.86865234375,
|
|
"logps/ref_chosen": -58.156639099121094,
|
|
"logps/ref_rejected": -79.3014907836914,
|
|
"logps/rejected": -595.2105102539062,
|
|
"loss": 0.9258,
|
|
"margin_dpo/margin_mean": 226.1969757080078,
|
|
"margin_dpo/margin_std": 198.74960327148438,
|
|
"step": 638
|
|
},
|
|
{
|
|
"epoch": 0.9383259911894273,
|
|
"fcm_dpo/beta": 0.0028432803228497505,
|
|
"fcm_dpo/delta": 0.1298457235097885,
|
|
"fcm_dpo/margin": 167.3184356689453,
|
|
"fcm_dpo/q_t": 0.3954669237136841,
|
|
"grad_norm": 51.73390197753906,
|
|
"learning_rate": 6.065683776815933e-09,
|
|
"logits/chosen": -0.4740576148033142,
|
|
"logits/rejected": -0.434129536151886,
|
|
"logps/chosen": -424.7391357421875,
|
|
"logps/ref_chosen": -72.32319641113281,
|
|
"logps/ref_rejected": -74.2749252319336,
|
|
"logps/rejected": -594.00927734375,
|
|
"loss": 1.0733,
|
|
"margin_dpo/margin_mean": 167.3184356689453,
|
|
"margin_dpo/margin_std": 239.7473602294922,
|
|
"step": 639
|
|
},
|
|
{
|
|
"epoch": 0.9397944199706314,
|
|
"fcm_dpo/beta": 0.002794375643134117,
|
|
"fcm_dpo/delta": -0.18512314558029175,
|
|
"fcm_dpo/margin": 275.9643249511719,
|
|
"fcm_dpo/q_t": 0.3381561040878296,
|
|
"grad_norm": 32.32111358642578,
|
|
"learning_rate": 5.7879205600998296e-09,
|
|
"logits/chosen": -0.4964972138404846,
|
|
"logits/rejected": -0.4891301393508911,
|
|
"logps/chosen": -369.9661560058594,
|
|
"logps/ref_chosen": -56.13436508178711,
|
|
"logps/ref_rejected": -108.60014343261719,
|
|
"logps/rejected": -698.396240234375,
|
|
"loss": 0.8868,
|
|
"margin_dpo/margin_mean": 275.9643249511719,
|
|
"margin_dpo/margin_std": 286.1241149902344,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.9412628487518355,
|
|
"fcm_dpo/beta": 0.002765617100521922,
|
|
"fcm_dpo/delta": 0.06337472051382065,
|
|
"fcm_dpo/margin": 195.2603759765625,
|
|
"fcm_dpo/q_t": 0.38529226183891296,
|
|
"grad_norm": 54.78212356567383,
|
|
"learning_rate": 5.516592558795746e-09,
|
|
"logits/chosen": -0.49673527479171753,
|
|
"logits/rejected": -0.4844779968261719,
|
|
"logps/chosen": -419.86749267578125,
|
|
"logps/ref_chosen": -64.99689483642578,
|
|
"logps/ref_rejected": -86.99232482910156,
|
|
"logps/rejected": -637.123291015625,
|
|
"loss": 1.0383,
|
|
"margin_dpo/margin_mean": 195.26036071777344,
|
|
"margin_dpo/margin_std": 274.37164306640625,
|
|
"step": 641
|
|
},
|
|
{
|
|
"epoch": 0.9427312775330396,
|
|
"fcm_dpo/beta": 0.002786886878311634,
|
|
"fcm_dpo/delta": -0.06086990237236023,
|
|
"fcm_dpo/margin": 235.2342987060547,
|
|
"fcm_dpo/q_t": 0.3679524064064026,
|
|
"grad_norm": 43.42346954345703,
|
|
"learning_rate": 5.251706922648868e-09,
|
|
"logits/chosen": -0.5038265585899353,
|
|
"logits/rejected": -0.5031158328056335,
|
|
"logps/chosen": -400.65997314453125,
|
|
"logps/ref_chosen": -65.68924713134766,
|
|
"logps/ref_rejected": -110.24205017089844,
|
|
"logps/rejected": -680.4470825195312,
|
|
"loss": 0.9975,
|
|
"margin_dpo/margin_mean": 235.23428344726562,
|
|
"margin_dpo/margin_std": 318.88861083984375,
|
|
"step": 642
|
|
},
|
|
{
|
|
"epoch": 0.9441997063142438,
|
|
"fcm_dpo/beta": 0.002727922983467579,
|
|
"fcm_dpo/delta": 0.0013057217001914978,
|
|
"fcm_dpo/margin": 218.98019409179688,
|
|
"fcm_dpo/q_t": 0.36833906173706055,
|
|
"grad_norm": 39.67738342285156,
|
|
"learning_rate": 4.993270631642038e-09,
|
|
"logits/chosen": -0.5181025266647339,
|
|
"logits/rejected": -0.5136141180992126,
|
|
"logps/chosen": -359.638427734375,
|
|
"logps/ref_chosen": -51.94999694824219,
|
|
"logps/ref_rejected": -87.46833801269531,
|
|
"logps/rejected": -614.136962890625,
|
|
"loss": 0.9763,
|
|
"margin_dpo/margin_mean": 218.98019409179688,
|
|
"margin_dpo/margin_std": 238.99154663085938,
|
|
"step": 643
|
|
},
|
|
{
|
|
"epoch": 0.9456681350954479,
|
|
"fcm_dpo/beta": 0.0027766618877649307,
|
|
"fcm_dpo/delta": 0.07305951416492462,
|
|
"fcm_dpo/margin": 191.19378662109375,
|
|
"fcm_dpo/q_t": 0.38764306902885437,
|
|
"grad_norm": 49.11812210083008,
|
|
"learning_rate": 4.741290495811873e-09,
|
|
"logits/chosen": -0.511053204536438,
|
|
"logits/rejected": -0.5009315013885498,
|
|
"logps/chosen": -381.0382080078125,
|
|
"logps/ref_chosen": -59.017662048339844,
|
|
"logps/ref_rejected": -87.13668823242188,
|
|
"logps/rejected": -600.3509521484375,
|
|
"loss": 1.0642,
|
|
"margin_dpo/margin_mean": 191.19378662109375,
|
|
"margin_dpo/margin_std": 282.96331787109375,
|
|
"step": 644
|
|
},
|
|
{
|
|
"epoch": 0.947136563876652,
|
|
"fcm_dpo/beta": 0.0028436644934117794,
|
|
"fcm_dpo/delta": 0.16144075989723206,
|
|
"fcm_dpo/margin": 156.83892822265625,
|
|
"fcm_dpo/q_t": 0.40858685970306396,
|
|
"grad_norm": 49.89131546020508,
|
|
"learning_rate": 4.495773155069299e-09,
|
|
"logits/chosen": -0.4825410842895508,
|
|
"logits/rejected": -0.47511184215545654,
|
|
"logps/chosen": -378.183349609375,
|
|
"logps/ref_chosen": -55.87602233886719,
|
|
"logps/ref_rejected": -97.78080749511719,
|
|
"logps/rejected": -576.9270629882812,
|
|
"loss": 1.1329,
|
|
"margin_dpo/margin_mean": 156.83892822265625,
|
|
"margin_dpo/margin_std": 280.7100524902344,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.9486049926578561,
|
|
"fcm_dpo/beta": 0.002883891575038433,
|
|
"fcm_dpo/delta": 0.029555249959230423,
|
|
"fcm_dpo/margin": 198.30703735351562,
|
|
"fcm_dpo/q_t": 0.3727361261844635,
|
|
"grad_norm": 39.009521484375,
|
|
"learning_rate": 4.256725079024553e-09,
|
|
"logits/chosen": -0.48562538623809814,
|
|
"logits/rejected": -0.46007299423217773,
|
|
"logps/chosen": -364.4145812988281,
|
|
"logps/ref_chosen": -61.275787353515625,
|
|
"logps/ref_rejected": -77.50580596923828,
|
|
"logps/rejected": -578.95166015625,
|
|
"loss": 0.9785,
|
|
"margin_dpo/margin_mean": 198.30703735351562,
|
|
"margin_dpo/margin_std": 211.89578247070312,
|
|
"step": 646
|
|
},
|
|
{
|
|
"epoch": 0.9500734214390602,
|
|
"fcm_dpo/beta": 0.002909876639023423,
|
|
"fcm_dpo/delta": 0.007972326129674911,
|
|
"fcm_dpo/margin": 203.5854949951172,
|
|
"fcm_dpo/q_t": 0.3704141676425934,
|
|
"grad_norm": 34.949928283691406,
|
|
"learning_rate": 4.024152566816791e-09,
|
|
"logits/chosen": -0.4611695408821106,
|
|
"logits/rejected": -0.4655107855796814,
|
|
"logps/chosen": -345.0274353027344,
|
|
"logps/ref_chosen": -54.8524169921875,
|
|
"logps/ref_rejected": -93.5194091796875,
|
|
"logps/rejected": -587.2799072265625,
|
|
"loss": 0.9958,
|
|
"margin_dpo/margin_mean": 203.5854949951172,
|
|
"margin_dpo/margin_std": 246.224853515625,
|
|
"step": 647
|
|
},
|
|
{
|
|
"epoch": 0.9515418502202643,
|
|
"fcm_dpo/beta": 0.002797810360789299,
|
|
"fcm_dpo/delta": -0.25719988346099854,
|
|
"fcm_dpo/margin": 298.71856689453125,
|
|
"fcm_dpo/q_t": 0.3337755799293518,
|
|
"grad_norm": 35.38324737548828,
|
|
"learning_rate": 3.798061746947995e-09,
|
|
"logits/chosen": -0.4994831681251526,
|
|
"logits/rejected": -0.5130796432495117,
|
|
"logps/chosen": -361.3632507324219,
|
|
"logps/ref_chosen": -54.17146682739258,
|
|
"logps/ref_rejected": -98.7127914428711,
|
|
"logps/rejected": -704.6231689453125,
|
|
"loss": 0.8967,
|
|
"margin_dpo/margin_mean": 298.71856689453125,
|
|
"margin_dpo/margin_std": 349.7194519042969,
|
|
"step": 648
|
|
},
|
|
{
|
|
"epoch": 0.9530102790014684,
|
|
"fcm_dpo/beta": 0.002828485332429409,
|
|
"fcm_dpo/delta": 0.1316825896501541,
|
|
"fcm_dpo/margin": 167.37564086914062,
|
|
"fcm_dpo/q_t": 0.40139663219451904,
|
|
"grad_norm": 42.975502014160156,
|
|
"learning_rate": 3.5784585771215235e-09,
|
|
"logits/chosen": -0.5309435725212097,
|
|
"logits/rejected": -0.5060242414474487,
|
|
"logps/chosen": -379.24853515625,
|
|
"logps/ref_chosen": -62.480350494384766,
|
|
"logps/ref_rejected": -80.07717895507812,
|
|
"logps/rejected": -564.2210083007812,
|
|
"loss": 1.09,
|
|
"margin_dpo/margin_mean": 167.37564086914062,
|
|
"margin_dpo/margin_std": 259.5540466308594,
|
|
"step": 649
|
|
},
|
|
{
|
|
"epoch": 0.9544787077826725,
|
|
"fcm_dpo/beta": 0.002783937146887183,
|
|
"fcm_dpo/delta": -0.10079372674226761,
|
|
"fcm_dpo/margin": 249.080322265625,
|
|
"fcm_dpo/q_t": 0.3533610701560974,
|
|
"grad_norm": 34.299888610839844,
|
|
"learning_rate": 3.3653488440851253e-09,
|
|
"logits/chosen": -0.48052090406417847,
|
|
"logits/rejected": -0.47680503129959106,
|
|
"logps/chosen": -385.91802978515625,
|
|
"logps/ref_chosen": -56.09281921386719,
|
|
"logps/ref_rejected": -98.26483917236328,
|
|
"logps/rejected": -677.1703491210938,
|
|
"loss": 0.9576,
|
|
"margin_dpo/margin_mean": 249.08029174804688,
|
|
"margin_dpo/margin_std": 299.54718017578125,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.9559471365638766,
|
|
"fcm_dpo/beta": 0.0027128136716783047,
|
|
"fcm_dpo/delta": -0.12499568611383438,
|
|
"fcm_dpo/margin": 263.5880126953125,
|
|
"fcm_dpo/q_t": 0.34237995743751526,
|
|
"grad_norm": 38.067604064941406,
|
|
"learning_rate": 3.158738163478475e-09,
|
|
"logits/chosen": -0.5105220079421997,
|
|
"logits/rejected": -0.5287940502166748,
|
|
"logps/chosen": -301.7446594238281,
|
|
"logps/ref_chosen": -43.42544937133789,
|
|
"logps/ref_rejected": -99.95791625976562,
|
|
"logps/rejected": -621.8651123046875,
|
|
"loss": 0.8879,
|
|
"margin_dpo/margin_mean": 263.5880126953125,
|
|
"margin_dpo/margin_std": 240.21612548828125,
|
|
"step": 651
|
|
},
|
|
{
|
|
"epoch": 0.9574155653450808,
|
|
"fcm_dpo/beta": 0.0027055356185883284,
|
|
"fcm_dpo/delta": -0.019109947606921196,
|
|
"fcm_dpo/margin": 228.38198852539062,
|
|
"fcm_dpo/q_t": 0.36907005310058594,
|
|
"grad_norm": 36.77167510986328,
|
|
"learning_rate": 2.9586319796851555e-09,
|
|
"logits/chosen": -0.5113554000854492,
|
|
"logits/rejected": -0.504870593547821,
|
|
"logps/chosen": -353.9315185546875,
|
|
"logps/ref_chosen": -62.57680892944336,
|
|
"logps/ref_rejected": -111.76779174804688,
|
|
"logps/rejected": -631.5044555664062,
|
|
"loss": 0.9818,
|
|
"margin_dpo/margin_mean": 228.38198852539062,
|
|
"margin_dpo/margin_std": 280.3888244628906,
|
|
"step": 652
|
|
},
|
|
{
|
|
"epoch": 0.9588839941262849,
|
|
"fcm_dpo/beta": 0.0026833987794816494,
|
|
"fcm_dpo/delta": -0.041953183710575104,
|
|
"fcm_dpo/margin": 238.23910522460938,
|
|
"fcm_dpo/q_t": 0.3634333610534668,
|
|
"grad_norm": 42.57564163208008,
|
|
"learning_rate": 2.7650355656892166e-09,
|
|
"logits/chosen": -0.5182297825813293,
|
|
"logits/rejected": -0.517985999584198,
|
|
"logps/chosen": -397.2801818847656,
|
|
"logps/ref_chosen": -61.11295700073242,
|
|
"logps/ref_rejected": -103.24960327148438,
|
|
"logps/rejected": -677.6558837890625,
|
|
"loss": 0.9644,
|
|
"margin_dpo/margin_mean": 238.23910522460938,
|
|
"margin_dpo/margin_std": 277.93377685546875,
|
|
"step": 653
|
|
},
|
|
{
|
|
"epoch": 0.960352422907489,
|
|
"fcm_dpo/beta": 0.002681623911485076,
|
|
"fcm_dpo/delta": 0.03697919845581055,
|
|
"fcm_dpo/margin": 210.70355224609375,
|
|
"fcm_dpo/q_t": 0.3804095983505249,
|
|
"grad_norm": 35.654144287109375,
|
|
"learning_rate": 2.577954022936174e-09,
|
|
"logits/chosen": -0.49398699402809143,
|
|
"logits/rejected": -0.4834080934524536,
|
|
"logps/chosen": -394.8516845703125,
|
|
"logps/ref_chosen": -61.7281379699707,
|
|
"logps/ref_rejected": -98.7738037109375,
|
|
"logps/rejected": -642.600830078125,
|
|
"loss": 1.0183,
|
|
"margin_dpo/margin_mean": 210.70355224609375,
|
|
"margin_dpo/margin_std": 280.0821533203125,
|
|
"step": 654
|
|
},
|
|
{
|
|
"epoch": 0.9618208516886931,
|
|
"fcm_dpo/beta": 0.002694209571927786,
|
|
"fcm_dpo/delta": 0.017087338492274284,
|
|
"fcm_dpo/margin": 216.68463134765625,
|
|
"fcm_dpo/q_t": 0.3784465789794922,
|
|
"grad_norm": 33.220149993896484,
|
|
"learning_rate": 2.397392281198729e-09,
|
|
"logits/chosen": -0.45762312412261963,
|
|
"logits/rejected": -0.4692486524581909,
|
|
"logps/chosen": -356.33929443359375,
|
|
"logps/ref_chosen": -49.576812744140625,
|
|
"logps/ref_rejected": -98.29183197021484,
|
|
"logps/rejected": -621.7389526367188,
|
|
"loss": 1.0201,
|
|
"margin_dpo/margin_mean": 216.6846466064453,
|
|
"margin_dpo/margin_std": 295.111083984375,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.9632892804698973,
|
|
"fcm_dpo/beta": 0.002611855510622263,
|
|
"fcm_dpo/delta": -0.25648045539855957,
|
|
"fcm_dpo/margin": 319.9178771972656,
|
|
"fcm_dpo/q_t": 0.3181704878807068,
|
|
"grad_norm": 54.623451232910156,
|
|
"learning_rate": 2.223355098446622e-09,
|
|
"logits/chosen": -0.4365518093109131,
|
|
"logits/rejected": -0.44737708568573,
|
|
"logps/chosen": -353.51763916015625,
|
|
"logps/ref_chosen": -52.54943084716797,
|
|
"logps/ref_rejected": -113.67464447021484,
|
|
"logps/rejected": -734.560791015625,
|
|
"loss": 0.8148,
|
|
"margin_dpo/margin_mean": 319.91790771484375,
|
|
"margin_dpo/margin_std": 263.46356201171875,
|
|
"step": 656
|
|
},
|
|
{
|
|
"epoch": 0.9647577092511013,
|
|
"fcm_dpo/beta": 0.002502152230590582,
|
|
"fcm_dpo/delta": -0.15946225821971893,
|
|
"fcm_dpo/margin": 298.534423828125,
|
|
"fcm_dpo/q_t": 0.33715760707855225,
|
|
"grad_norm": 33.79457473754883,
|
|
"learning_rate": 2.055847060721566e-09,
|
|
"logits/chosen": -0.5489492416381836,
|
|
"logits/rejected": -0.5547805428504944,
|
|
"logps/chosen": -332.32037353515625,
|
|
"logps/ref_chosen": -46.700538635253906,
|
|
"logps/ref_rejected": -97.91487121582031,
|
|
"logps/rejected": -682.0692138671875,
|
|
"loss": 0.8675,
|
|
"margin_dpo/margin_mean": 298.534423828125,
|
|
"margin_dpo/margin_std": 273.04095458984375,
|
|
"step": 657
|
|
},
|
|
{
|
|
"epoch": 0.9662261380323054,
|
|
"fcm_dpo/beta": 0.0024999987799674273,
|
|
"fcm_dpo/delta": 0.03055243194103241,
|
|
"fcm_dpo/margin": 228.47642517089844,
|
|
"fcm_dpo/q_t": 0.37107935547828674,
|
|
"grad_norm": 32.88031768798828,
|
|
"learning_rate": 1.8948725820160662e-09,
|
|
"logits/chosen": -0.4807060956954956,
|
|
"logits/rejected": -0.4754422903060913,
|
|
"logps/chosen": -388.13995361328125,
|
|
"logps/ref_chosen": -60.95820999145508,
|
|
"logps/ref_rejected": -95.93949127197266,
|
|
"logps/rejected": -651.59765625,
|
|
"loss": 0.9682,
|
|
"margin_dpo/margin_mean": 228.4764404296875,
|
|
"margin_dpo/margin_std": 229.4976806640625,
|
|
"step": 658
|
|
},
|
|
{
|
|
"epoch": 0.9676945668135095,
|
|
"fcm_dpo/beta": 0.002503772731870413,
|
|
"fcm_dpo/delta": 0.029256850481033325,
|
|
"fcm_dpo/margin": 228.52227783203125,
|
|
"fcm_dpo/q_t": 0.37436485290527344,
|
|
"grad_norm": 36.82291030883789,
|
|
"learning_rate": 1.7404359041573723e-09,
|
|
"logits/chosen": -0.509990394115448,
|
|
"logits/rejected": -0.4739938974380493,
|
|
"logps/chosen": -371.61309814453125,
|
|
"logps/ref_chosen": -76.74298095703125,
|
|
"logps/ref_rejected": -87.4709701538086,
|
|
"logps/rejected": -610.8634033203125,
|
|
"loss": 0.9845,
|
|
"margin_dpo/margin_mean": 228.52227783203125,
|
|
"margin_dpo/margin_std": 254.27716064453125,
|
|
"step": 659
|
|
},
|
|
{
|
|
"epoch": 0.9691629955947136,
|
|
"fcm_dpo/beta": 0.002496623434126377,
|
|
"fcm_dpo/delta": -0.0633494034409523,
|
|
"fcm_dpo/margin": 264.0384521484375,
|
|
"fcm_dpo/q_t": 0.35319751501083374,
|
|
"grad_norm": 33.35771942138672,
|
|
"learning_rate": 1.592541096695571e-09,
|
|
"logits/chosen": -0.49027347564697266,
|
|
"logits/rejected": -0.4790500998497009,
|
|
"logps/chosen": -362.43536376953125,
|
|
"logps/ref_chosen": -59.04788589477539,
|
|
"logps/ref_rejected": -75.96005249023438,
|
|
"logps/rejected": -643.385986328125,
|
|
"loss": 0.9142,
|
|
"margin_dpo/margin_mean": 264.0384521484375,
|
|
"margin_dpo/margin_std": 247.07907104492188,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.9706314243759178,
|
|
"fcm_dpo/beta": 0.0024961913004517555,
|
|
"fcm_dpo/delta": -0.03317072242498398,
|
|
"fcm_dpo/margin": 252.34422302246094,
|
|
"fcm_dpo/q_t": 0.3640047013759613,
|
|
"grad_norm": 30.544677734375,
|
|
"learning_rate": 1.4511920567963908e-09,
|
|
"logits/chosen": -0.5381925702095032,
|
|
"logits/rejected": -0.519443154335022,
|
|
"logps/chosen": -326.90234375,
|
|
"logps/ref_chosen": -50.673973083496094,
|
|
"logps/ref_rejected": -86.00569152832031,
|
|
"logps/rejected": -614.5782470703125,
|
|
"loss": 0.9623,
|
|
"margin_dpo/margin_mean": 252.34423828125,
|
|
"margin_dpo/margin_std": 285.3419189453125,
|
|
"step": 661
|
|
},
|
|
{
|
|
"epoch": 0.9720998531571219,
|
|
"fcm_dpo/beta": 0.0025062900967895985,
|
|
"fcm_dpo/delta": 0.08712703734636307,
|
|
"fcm_dpo/margin": 206.3653106689453,
|
|
"fcm_dpo/q_t": 0.3861844539642334,
|
|
"grad_norm": 34.15724563598633,
|
|
"learning_rate": 1.3163925091384532e-09,
|
|
"logits/chosen": -0.5257574319839478,
|
|
"logits/rejected": -0.5023434162139893,
|
|
"logps/chosen": -401.38140869140625,
|
|
"logps/ref_chosen": -69.26106262207031,
|
|
"logps/ref_rejected": -89.05593872070312,
|
|
"logps/rejected": -627.5416259765625,
|
|
"loss": 1.0401,
|
|
"margin_dpo/margin_mean": 206.36529541015625,
|
|
"margin_dpo/margin_std": 271.6153564453125,
|
|
"step": 662
|
|
},
|
|
{
|
|
"epoch": 0.973568281938326,
|
|
"fcm_dpo/beta": 0.002501696813851595,
|
|
"fcm_dpo/delta": -0.016493409872055054,
|
|
"fcm_dpo/margin": 245.9669189453125,
|
|
"fcm_dpo/q_t": 0.3665703535079956,
|
|
"grad_norm": 30.06166648864746,
|
|
"learning_rate": 1.1881460058152382e-09,
|
|
"logits/chosen": -0.5630896687507629,
|
|
"logits/rejected": -0.5610999464988708,
|
|
"logps/chosen": -358.4510498046875,
|
|
"logps/ref_chosen": -64.87890625,
|
|
"logps/ref_rejected": -113.92536926269531,
|
|
"logps/rejected": -653.4644165039062,
|
|
"loss": 0.969,
|
|
"margin_dpo/margin_mean": 245.9669189453125,
|
|
"margin_dpo/margin_std": 283.71075439453125,
|
|
"step": 663
|
|
},
|
|
{
|
|
"epoch": 0.9750367107195301,
|
|
"fcm_dpo/beta": 0.0024649640545248985,
|
|
"fcm_dpo/delta": -0.09488554298877716,
|
|
"fcm_dpo/margin": 279.1371765136719,
|
|
"fcm_dpo/q_t": 0.35125353932380676,
|
|
"grad_norm": 34.83463668823242,
|
|
"learning_rate": 1.066455926241383e-09,
|
|
"logits/chosen": -0.4723320007324219,
|
|
"logits/rejected": -0.4694727063179016,
|
|
"logps/chosen": -382.6121826171875,
|
|
"logps/ref_chosen": -60.88847351074219,
|
|
"logps/ref_rejected": -105.521728515625,
|
|
"logps/rejected": -706.382568359375,
|
|
"loss": 0.9418,
|
|
"margin_dpo/margin_mean": 279.1371765136719,
|
|
"margin_dpo/margin_std": 316.78741455078125,
|
|
"step": 664
|
|
},
|
|
{
|
|
"epoch": 0.9765051395007343,
|
|
"fcm_dpo/beta": 0.0024584224447607994,
|
|
"fcm_dpo/delta": 0.006961781997233629,
|
|
"fcm_dpo/margin": 241.38986206054688,
|
|
"fcm_dpo/q_t": 0.3676924705505371,
|
|
"grad_norm": 32.79265594482422,
|
|
"learning_rate": 9.513254770636137e-10,
|
|
"logits/chosen": -0.4960812032222748,
|
|
"logits/rejected": -0.4816419780254364,
|
|
"logps/chosen": -358.48394775390625,
|
|
"logps/ref_chosen": -60.56413269042969,
|
|
"logps/ref_rejected": -84.80882263183594,
|
|
"logps/rejected": -624.1184692382812,
|
|
"loss": 0.9606,
|
|
"margin_dpo/margin_mean": 241.38986206054688,
|
|
"margin_dpo/margin_std": 244.98712158203125,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.9779735682819384,
|
|
"fcm_dpo/beta": 0.002456535119563341,
|
|
"fcm_dpo/delta": 0.005466273054480553,
|
|
"fcm_dpo/margin": 242.0909423828125,
|
|
"fcm_dpo/q_t": 0.36501848697662354,
|
|
"grad_norm": 38.639163970947266,
|
|
"learning_rate": 8.427576920763956e-10,
|
|
"logits/chosen": -0.4671856760978699,
|
|
"logits/rejected": -0.4504069983959198,
|
|
"logps/chosen": -372.948486328125,
|
|
"logps/ref_chosen": -64.41996002197266,
|
|
"logps/ref_rejected": -95.8916244506836,
|
|
"logps/rejected": -646.5111083984375,
|
|
"loss": 0.9541,
|
|
"margin_dpo/margin_mean": 242.0909423828125,
|
|
"margin_dpo/margin_std": 233.2517852783203,
|
|
"step": 666
|
|
},
|
|
{
|
|
"epoch": 0.9794419970631424,
|
|
"fcm_dpo/beta": 0.002473963890224695,
|
|
"fcm_dpo/delta": 0.02551562525331974,
|
|
"fcm_dpo/margin": 232.80560302734375,
|
|
"fcm_dpo/q_t": 0.3697671890258789,
|
|
"grad_norm": 38.66985321044922,
|
|
"learning_rate": 7.407554321417764e-10,
|
|
"logits/chosen": -0.47391679883003235,
|
|
"logits/rejected": -0.4410842955112457,
|
|
"logps/chosen": -402.3163146972656,
|
|
"logps/ref_chosen": -69.27702331542969,
|
|
"logps/ref_rejected": -87.83549499511719,
|
|
"logps/rejected": -653.680419921875,
|
|
"loss": 0.9638,
|
|
"margin_dpo/margin_mean": 232.80560302734375,
|
|
"margin_dpo/margin_std": 229.169677734375,
|
|
"step": 667
|
|
},
|
|
{
|
|
"epoch": 0.9809104258443465,
|
|
"fcm_dpo/beta": 0.00251585291698575,
|
|
"fcm_dpo/delta": 0.06850365549325943,
|
|
"fcm_dpo/margin": 212.354736328125,
|
|
"fcm_dpo/q_t": 0.3838024139404297,
|
|
"grad_norm": 52.99601745605469,
|
|
"learning_rate": 6.453213851142225e-10,
|
|
"logits/chosen": -0.5375820398330688,
|
|
"logits/rejected": -0.529564380645752,
|
|
"logps/chosen": -411.6194763183594,
|
|
"logps/ref_chosen": -72.60400390625,
|
|
"logps/ref_rejected": -103.73905944824219,
|
|
"logps/rejected": -655.1092529296875,
|
|
"loss": 1.0277,
|
|
"margin_dpo/margin_mean": 212.354736328125,
|
|
"margin_dpo/margin_std": 273.1512145996094,
|
|
"step": 668
|
|
},
|
|
{
|
|
"epoch": 0.9823788546255506,
|
|
"fcm_dpo/beta": 0.0024861646816134453,
|
|
"fcm_dpo/delta": -0.06441953778266907,
|
|
"fcm_dpo/margin": 265.5224609375,
|
|
"fcm_dpo/q_t": 0.35320982336997986,
|
|
"grad_norm": 32.41237258911133,
|
|
"learning_rate": 5.564580657695939e-10,
|
|
"logits/chosen": -0.49402615427970886,
|
|
"logits/rejected": -0.4743247926235199,
|
|
"logps/chosen": -311.386962890625,
|
|
"logps/ref_chosen": -46.116416931152344,
|
|
"logps/ref_rejected": -77.92434692382812,
|
|
"logps/rejected": -608.7173461914062,
|
|
"loss": 0.9308,
|
|
"margin_dpo/margin_mean": 265.5224609375,
|
|
"margin_dpo/margin_std": 272.043701171875,
|
|
"step": 669
|
|
},
|
|
{
|
|
"epoch": 0.9838472834067548,
|
|
"fcm_dpo/beta": 0.0024540587328374386,
|
|
"fcm_dpo/delta": -0.09894061833620071,
|
|
"fcm_dpo/margin": 282.0305480957031,
|
|
"fcm_dpo/q_t": 0.3481866717338562,
|
|
"grad_norm": 27.054269790649414,
|
|
"learning_rate": 4.741678157389739e-10,
|
|
"logits/chosen": -0.4560254216194153,
|
|
"logits/rejected": -0.4461694657802582,
|
|
"logps/chosen": -328.38433837890625,
|
|
"logps/ref_chosen": -62.34575271606445,
|
|
"logps/ref_rejected": -96.9405517578125,
|
|
"logps/rejected": -645.0096435546875,
|
|
"loss": 0.9109,
|
|
"margin_dpo/margin_mean": 282.0305480957031,
|
|
"margin_dpo/margin_std": 275.9473876953125,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.9853157121879589,
|
|
"fcm_dpo/beta": 0.0024509113281965256,
|
|
"fcm_dpo/delta": 0.019564248621463776,
|
|
"fcm_dpo/margin": 237.01344299316406,
|
|
"fcm_dpo/q_t": 0.3713776171207428,
|
|
"grad_norm": 38.91750717163086,
|
|
"learning_rate": 3.9845280344705245e-10,
|
|
"logits/chosen": -0.47259026765823364,
|
|
"logits/rejected": -0.46925845742225647,
|
|
"logps/chosen": -369.33929443359375,
|
|
"logps/ref_chosen": -48.00010681152344,
|
|
"logps/ref_rejected": -83.81932067871094,
|
|
"logps/rejected": -642.1719970703125,
|
|
"loss": 0.9848,
|
|
"margin_dpo/margin_mean": 237.013427734375,
|
|
"margin_dpo/margin_std": 265.4302673339844,
|
|
"step": 671
|
|
},
|
|
{
|
|
"epoch": 0.986784140969163,
|
|
"fcm_dpo/beta": 0.0024676902685314417,
|
|
"fcm_dpo/delta": 0.07117144763469696,
|
|
"fcm_dpo/margin": 215.81497192382812,
|
|
"fcm_dpo/q_t": 0.3837113380432129,
|
|
"grad_norm": 31.851991653442383,
|
|
"learning_rate": 3.293150240547549e-10,
|
|
"logits/chosen": -0.5248799920082092,
|
|
"logits/rejected": -0.5065342783927917,
|
|
"logps/chosen": -408.06689453125,
|
|
"logps/ref_chosen": -58.58328628540039,
|
|
"logps/ref_rejected": -93.14015197753906,
|
|
"logps/rejected": -658.438720703125,
|
|
"loss": 1.0327,
|
|
"margin_dpo/margin_mean": 215.81497192382812,
|
|
"margin_dpo/margin_std": 279.0846252441406,
|
|
"step": 672
|
|
},
|
|
{
|
|
"epoch": 0.9882525697503671,
|
|
"fcm_dpo/beta": 0.0024534822441637516,
|
|
"fcm_dpo/delta": -0.05598102882504463,
|
|
"fcm_dpo/margin": 265.86773681640625,
|
|
"fcm_dpo/q_t": 0.3518218994140625,
|
|
"grad_norm": 31.22127914428711,
|
|
"learning_rate": 2.6675629940689504e-10,
|
|
"logits/chosen": -0.4801812767982483,
|
|
"logits/rejected": -0.4714806079864502,
|
|
"logps/chosen": -353.5696105957031,
|
|
"logps/ref_chosen": -46.72320556640625,
|
|
"logps/ref_rejected": -85.29623413085938,
|
|
"logps/rejected": -658.0103759765625,
|
|
"loss": 0.9048,
|
|
"margin_dpo/margin_mean": 265.86773681640625,
|
|
"margin_dpo/margin_std": 224.6098175048828,
|
|
"step": 673
|
|
},
|
|
{
|
|
"epoch": 0.9897209985315712,
|
|
"fcm_dpo/beta": 0.002430280437693,
|
|
"fcm_dpo/delta": -0.05990447849035263,
|
|
"fcm_dpo/margin": 269.932861328125,
|
|
"fcm_dpo/q_t": 0.3552195131778717,
|
|
"grad_norm": 27.59811019897461,
|
|
"learning_rate": 2.1077827798404725e-10,
|
|
"logits/chosen": -0.45021355152130127,
|
|
"logits/rejected": -0.43945032358169556,
|
|
"logps/chosen": -328.8429870605469,
|
|
"logps/ref_chosen": -45.445526123046875,
|
|
"logps/ref_rejected": -70.04593658447266,
|
|
"logps/rejected": -623.3762817382812,
|
|
"loss": 0.9216,
|
|
"margin_dpo/margin_mean": 269.932861328125,
|
|
"margin_dpo/margin_std": 263.62127685546875,
|
|
"step": 674
|
|
},
|
|
{
|
|
"epoch": 0.9911894273127754,
|
|
"fcm_dpo/beta": 0.0023772665299475193,
|
|
"fcm_dpo/delta": -0.08401606976985931,
|
|
"fcm_dpo/margin": 285.0059814453125,
|
|
"fcm_dpo/q_t": 0.3517681956291199,
|
|
"grad_norm": 29.97161865234375,
|
|
"learning_rate": 1.6138243485910863e-10,
|
|
"logits/chosen": -0.4851476550102234,
|
|
"logits/rejected": -0.47465410828590393,
|
|
"logps/chosen": -329.2435607910156,
|
|
"logps/ref_chosen": -44.17628479003906,
|
|
"logps/ref_rejected": -74.09197998046875,
|
|
"logps/rejected": -644.165283203125,
|
|
"loss": 0.9136,
|
|
"margin_dpo/margin_mean": 285.0060119628906,
|
|
"margin_dpo/margin_std": 279.90576171875,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.9926578560939795,
|
|
"fcm_dpo/beta": 0.0023738746531307697,
|
|
"fcm_dpo/delta": 0.0118600744754076,
|
|
"fcm_dpo/margin": 247.96432495117188,
|
|
"fcm_dpo/q_t": 0.3671633303165436,
|
|
"grad_norm": 31.170412063598633,
|
|
"learning_rate": 1.1857007165852472e-10,
|
|
"logits/chosen": -0.5280739068984985,
|
|
"logits/rejected": -0.5125676393508911,
|
|
"logps/chosen": -399.6715087890625,
|
|
"logps/ref_chosen": -71.39852905273438,
|
|
"logps/ref_rejected": -88.3587646484375,
|
|
"logps/rejected": -664.5960693359375,
|
|
"loss": 0.9577,
|
|
"margin_dpo/margin_mean": 247.96432495117188,
|
|
"margin_dpo/margin_std": 239.91629028320312,
|
|
"step": 676
|
|
},
|
|
{
|
|
"epoch": 0.9941262848751835,
|
|
"fcm_dpo/beta": 0.0024010459892451763,
|
|
"fcm_dpo/delta": 0.033741872757673264,
|
|
"fcm_dpo/margin": 236.56593322753906,
|
|
"fcm_dpo/q_t": 0.37002164125442505,
|
|
"grad_norm": 29.955596923828125,
|
|
"learning_rate": 8.23423165278725e-11,
|
|
"logits/chosen": -0.4702920615673065,
|
|
"logits/rejected": -0.43533623218536377,
|
|
"logps/chosen": -371.1539306640625,
|
|
"logps/ref_chosen": -56.527435302734375,
|
|
"logps/ref_rejected": -78.22654724121094,
|
|
"logps/rejected": -629.4189453125,
|
|
"loss": 0.9604,
|
|
"margin_dpo/margin_mean": 236.56593322753906,
|
|
"margin_dpo/margin_std": 216.29861450195312,
|
|
"step": 677
|
|
},
|
|
{
|
|
"epoch": 0.9955947136563876,
|
|
"fcm_dpo/beta": 0.0023945937864482403,
|
|
"fcm_dpo/delta": -0.025561237707734108,
|
|
"fcm_dpo/margin": 260.56817626953125,
|
|
"fcm_dpo/q_t": 0.36279577016830444,
|
|
"grad_norm": 30.772653579711914,
|
|
"learning_rate": 5.270012410216185e-11,
|
|
"logits/chosen": -0.4658196270465851,
|
|
"logits/rejected": -0.4596235156059265,
|
|
"logps/chosen": -331.82763671875,
|
|
"logps/ref_chosen": -46.13447570800781,
|
|
"logps/ref_rejected": -80.60462951660156,
|
|
"logps/rejected": -626.865966796875,
|
|
"loss": 0.96,
|
|
"margin_dpo/margin_mean": 260.5681457519531,
|
|
"margin_dpo/margin_std": 285.284423828125,
|
|
"step": 678
|
|
},
|
|
{
|
|
"epoch": 0.9970631424375918,
|
|
"fcm_dpo/beta": 0.002417163457721472,
|
|
"fcm_dpo/delta": 0.0805218368768692,
|
|
"fcm_dpo/margin": 216.64544677734375,
|
|
"fcm_dpo/q_t": 0.3846209943294525,
|
|
"grad_norm": 31.97832489013672,
|
|
"learning_rate": 2.9644275480772416e-11,
|
|
"logits/chosen": -0.498549222946167,
|
|
"logits/rejected": -0.4737178087234497,
|
|
"logps/chosen": -354.89007568359375,
|
|
"logps/ref_chosen": -50.294921875,
|
|
"logps/ref_rejected": -76.59813690185547,
|
|
"logps/rejected": -597.8387451171875,
|
|
"loss": 1.0171,
|
|
"margin_dpo/margin_mean": 216.6454315185547,
|
|
"margin_dpo/margin_std": 261.185546875,
|
|
"step": 679
|
|
},
|
|
{
|
|
"epoch": 0.9985315712187959,
|
|
"fcm_dpo/beta": 0.002414638176560402,
|
|
"fcm_dpo/delta": -0.01086747832596302,
|
|
"fcm_dpo/margin": 252.61300659179688,
|
|
"fcm_dpo/q_t": 0.36631903052330017,
|
|
"grad_norm": 35.76852798461914,
|
|
"learning_rate": 1.31753782067201e-11,
|
|
"logits/chosen": -0.5048670768737793,
|
|
"logits/rejected": -0.5000746250152588,
|
|
"logps/chosen": -395.61883544921875,
|
|
"logps/ref_chosen": -76.91569519042969,
|
|
"logps/ref_rejected": -112.384765625,
|
|
"logps/rejected": -683.700927734375,
|
|
"loss": 0.9757,
|
|
"margin_dpo/margin_mean": 252.61300659179688,
|
|
"margin_dpo/margin_std": 294.8744201660156,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"fcm_dpo/beta": 0.002459340263158083,
|
|
"fcm_dpo/delta": 0.04384385794401169,
|
|
"fcm_dpo/margin": 226.318603515625,
|
|
"fcm_dpo/q_t": 0.3781166672706604,
|
|
"grad_norm": 32.83413314819336,
|
|
"learning_rate": 3.2938662507808745e-12,
|
|
"logits/chosen": -0.53011155128479,
|
|
"logits/rejected": -0.5230584740638733,
|
|
"logps/chosen": -388.7578125,
|
|
"logps/ref_chosen": -60.957279205322266,
|
|
"logps/ref_rejected": -88.55797576904297,
|
|
"logps/rejected": -642.6771240234375,
|
|
"loss": 1.0058,
|
|
"margin_dpo/margin_mean": 226.318603515625,
|
|
"margin_dpo/margin_std": 260.7975769042969,
|
|
"step": 681
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 681,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.9836619676647382,
|
|
"train_runtime": 1868.3255,
|
|
"train_samples_per_second": 23.335,
|
|
"train_steps_per_second": 0.364
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 681,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|