ed60227d3e
Model: jackf857/qwen3-8b-base-margin-dpo-hh-helpful-4xh200-batch-64-20260423-233948 Source: Original Platform
10355 lines
368 KiB
JSON
10355 lines
368 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 100,
|
|
"global_step": 681,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0014684287812041115,
|
|
"grad_norm": 38.81923294067383,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.1428939700126648,
|
|
"logits/rejected": 0.2641817033290863,
|
|
"logps/chosen": -93.47865295410156,
|
|
"logps/ref_chosen": -93.52638244628906,
|
|
"logps/ref_rejected": -87.86244201660156,
|
|
"logps/rejected": -87.99055480957031,
|
|
"loss": 1.3807,
|
|
"margin_dpo/margin_mean": 0.17585086822509766,
|
|
"margin_dpo/margin_std": 0.4164121747016907,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.002936857562408223,
|
|
"grad_norm": 29.758581161499023,
|
|
"learning_rate": 7.246376811594203e-09,
|
|
"logits/chosen": 0.512961745262146,
|
|
"logits/rejected": 0.6081655621528625,
|
|
"logps/chosen": -73.25503540039062,
|
|
"logps/ref_chosen": -73.14814758300781,
|
|
"logps/ref_rejected": -89.64044952392578,
|
|
"logps/rejected": -89.52894592285156,
|
|
"loss": 1.3992,
|
|
"margin_dpo/margin_mean": -0.21839451789855957,
|
|
"margin_dpo/margin_std": 0.35581544041633606,
|
|
"step": 2
|
|
},
|
|
{
|
|
"epoch": 0.004405286343612335,
|
|
"grad_norm": 26.006439208984375,
|
|
"learning_rate": 1.4492753623188406e-08,
|
|
"logits/chosen": 0.19662049412727356,
|
|
"logits/rejected": 0.37471911311149597,
|
|
"logps/chosen": -87.74197387695312,
|
|
"logps/ref_chosen": -87.6579818725586,
|
|
"logps/ref_rejected": -79.18247985839844,
|
|
"logps/rejected": -79.21536254882812,
|
|
"loss": 1.3837,
|
|
"margin_dpo/margin_mean": -0.05109751224517822,
|
|
"margin_dpo/margin_std": 0.4686650037765503,
|
|
"step": 3
|
|
},
|
|
{
|
|
"epoch": 0.005873715124816446,
|
|
"grad_norm": 32.8250846862793,
|
|
"learning_rate": 2.1739130434782606e-08,
|
|
"logits/chosen": 0.915730357170105,
|
|
"logits/rejected": 1.033768653869629,
|
|
"logps/chosen": -105.30754089355469,
|
|
"logps/ref_chosen": -105.203125,
|
|
"logps/ref_rejected": -102.10517120361328,
|
|
"logps/rejected": -102.12972259521484,
|
|
"loss": 1.3874,
|
|
"margin_dpo/margin_mean": -0.0798501968383789,
|
|
"margin_dpo/margin_std": 0.3638986647129059,
|
|
"step": 4
|
|
},
|
|
{
|
|
"epoch": 0.007342143906020558,
|
|
"grad_norm": 35.19728469848633,
|
|
"learning_rate": 2.898550724637681e-08,
|
|
"logits/chosen": 0.7488752007484436,
|
|
"logits/rejected": 1.0472900867462158,
|
|
"logps/chosen": -100.4297103881836,
|
|
"logps/ref_chosen": -100.44102478027344,
|
|
"logps/ref_rejected": -85.82490539550781,
|
|
"logps/rejected": -85.85408020019531,
|
|
"loss": 1.3839,
|
|
"margin_dpo/margin_mean": 0.04048585891723633,
|
|
"margin_dpo/margin_std": 0.3670598566532135,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.00881057268722467,
|
|
"grad_norm": 36.038734436035156,
|
|
"learning_rate": 3.6231884057971014e-08,
|
|
"logits/chosen": 0.8637112379074097,
|
|
"logits/rejected": 1.1790170669555664,
|
|
"logps/chosen": -80.59754943847656,
|
|
"logps/ref_chosen": -80.54146575927734,
|
|
"logps/ref_rejected": -98.20770263671875,
|
|
"logps/rejected": -98.21786499023438,
|
|
"loss": 1.3876,
|
|
"margin_dpo/margin_mean": -0.04591703414916992,
|
|
"margin_dpo/margin_std": 0.4173058867454529,
|
|
"step": 6
|
|
},
|
|
{
|
|
"epoch": 0.010279001468428781,
|
|
"grad_norm": 37.97282791137695,
|
|
"learning_rate": 4.347826086956521e-08,
|
|
"logits/chosen": 0.5081789493560791,
|
|
"logits/rejected": 0.3699849247932434,
|
|
"logps/chosen": -73.5165786743164,
|
|
"logps/ref_chosen": -73.49665832519531,
|
|
"logps/ref_rejected": -140.68423461914062,
|
|
"logps/rejected": -140.79434204101562,
|
|
"loss": 1.3861,
|
|
"margin_dpo/margin_mean": 0.09017288684844971,
|
|
"margin_dpo/margin_std": 0.35818028450012207,
|
|
"step": 7
|
|
},
|
|
{
|
|
"epoch": 0.011747430249632892,
|
|
"grad_norm": 32.7470588684082,
|
|
"learning_rate": 5.0724637681159424e-08,
|
|
"logits/chosen": 0.4470105469226837,
|
|
"logits/rejected": 0.7369768619537354,
|
|
"logps/chosen": -98.04469299316406,
|
|
"logps/ref_chosen": -98.0727310180664,
|
|
"logps/ref_rejected": -81.36228942871094,
|
|
"logps/rejected": -81.3835678100586,
|
|
"loss": 1.3838,
|
|
"margin_dpo/margin_mean": 0.049311161041259766,
|
|
"margin_dpo/margin_std": 0.30581894516944885,
|
|
"step": 8
|
|
},
|
|
{
|
|
"epoch": 0.013215859030837005,
|
|
"grad_norm": 35.992156982421875,
|
|
"learning_rate": 5.797101449275362e-08,
|
|
"logits/chosen": 0.950916051864624,
|
|
"logits/rejected": 1.1352921724319458,
|
|
"logps/chosen": -107.1036376953125,
|
|
"logps/ref_chosen": -107.1068115234375,
|
|
"logps/ref_rejected": -116.85135650634766,
|
|
"logps/rejected": -116.99057006835938,
|
|
"loss": 1.3824,
|
|
"margin_dpo/margin_mean": 0.14239072799682617,
|
|
"margin_dpo/margin_std": 0.3212544918060303,
|
|
"step": 9
|
|
},
|
|
{
|
|
"epoch": 0.014684287812041116,
|
|
"grad_norm": 33.56389617919922,
|
|
"learning_rate": 6.521739130434782e-08,
|
|
"logits/chosen": 0.49613165855407715,
|
|
"logits/rejected": 0.59611976146698,
|
|
"logps/chosen": -70.4057388305664,
|
|
"logps/ref_chosen": -70.45864868164062,
|
|
"logps/ref_rejected": -82.8171615600586,
|
|
"logps/rejected": -82.73629760742188,
|
|
"loss": 1.3932,
|
|
"margin_dpo/margin_mean": -0.027948379516601562,
|
|
"margin_dpo/margin_std": 0.4402916133403778,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.016152716593245228,
|
|
"grad_norm": 36.561405181884766,
|
|
"learning_rate": 7.246376811594203e-08,
|
|
"logits/chosen": 0.7130615711212158,
|
|
"logits/rejected": 0.9787681102752686,
|
|
"logps/chosen": -98.91744232177734,
|
|
"logps/ref_chosen": -98.9473876953125,
|
|
"logps/ref_rejected": -64.17475128173828,
|
|
"logps/rejected": -64.2801742553711,
|
|
"loss": 1.3795,
|
|
"margin_dpo/margin_mean": 0.1353701949119568,
|
|
"margin_dpo/margin_std": 0.31776195764541626,
|
|
"step": 11
|
|
},
|
|
{
|
|
"epoch": 0.01762114537444934,
|
|
"grad_norm": 39.813255310058594,
|
|
"learning_rate": 7.971014492753623e-08,
|
|
"logits/chosen": -0.26005640625953674,
|
|
"logits/rejected": -0.09152386337518692,
|
|
"logps/chosen": -104.32192993164062,
|
|
"logps/ref_chosen": -104.27352142333984,
|
|
"logps/ref_rejected": -128.4927215576172,
|
|
"logps/rejected": -128.5462646484375,
|
|
"loss": 1.3757,
|
|
"margin_dpo/margin_mean": 0.005125999450683594,
|
|
"margin_dpo/margin_std": 0.4299471974372864,
|
|
"step": 12
|
|
},
|
|
{
|
|
"epoch": 0.01908957415565345,
|
|
"grad_norm": 41.7731819152832,
|
|
"learning_rate": 8.695652173913042e-08,
|
|
"logits/chosen": 0.3225553631782532,
|
|
"logits/rejected": 0.6485546827316284,
|
|
"logps/chosen": -101.96672058105469,
|
|
"logps/ref_chosen": -102.03691101074219,
|
|
"logps/ref_rejected": -78.48355102539062,
|
|
"logps/rejected": -78.45487976074219,
|
|
"loss": 1.382,
|
|
"margin_dpo/margin_mean": 0.041529059410095215,
|
|
"margin_dpo/margin_std": 0.49482065439224243,
|
|
"step": 13
|
|
},
|
|
{
|
|
"epoch": 0.020558002936857563,
|
|
"grad_norm": 35.894309997558594,
|
|
"learning_rate": 9.420289855072464e-08,
|
|
"logits/chosen": 0.40504711866378784,
|
|
"logits/rejected": 0.5168009400367737,
|
|
"logps/chosen": -119.18898010253906,
|
|
"logps/ref_chosen": -119.02677917480469,
|
|
"logps/ref_rejected": -117.97889709472656,
|
|
"logps/rejected": -117.81944274902344,
|
|
"loss": 1.3945,
|
|
"margin_dpo/margin_mean": -0.3216536045074463,
|
|
"margin_dpo/margin_std": 0.47595691680908203,
|
|
"step": 14
|
|
},
|
|
{
|
|
"epoch": 0.022026431718061675,
|
|
"grad_norm": 31.745962142944336,
|
|
"learning_rate": 1.0144927536231885e-07,
|
|
"logits/chosen": 0.22957265377044678,
|
|
"logits/rejected": 0.4173096716403961,
|
|
"logps/chosen": -74.27200317382812,
|
|
"logps/ref_chosen": -74.2003173828125,
|
|
"logps/ref_rejected": -95.664306640625,
|
|
"logps/rejected": -95.67192077636719,
|
|
"loss": 1.3854,
|
|
"margin_dpo/margin_mean": -0.06406116485595703,
|
|
"margin_dpo/margin_std": 0.37379056215286255,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.023494860499265784,
|
|
"grad_norm": 34.3470344543457,
|
|
"learning_rate": 1.0869565217391303e-07,
|
|
"logits/chosen": -0.015835046768188477,
|
|
"logits/rejected": 0.32137343287467957,
|
|
"logps/chosen": -111.2186279296875,
|
|
"logps/ref_chosen": -111.30982208251953,
|
|
"logps/ref_rejected": -87.0030517578125,
|
|
"logps/rejected": -87.1607666015625,
|
|
"loss": 1.3769,
|
|
"margin_dpo/margin_mean": 0.24891865253448486,
|
|
"margin_dpo/margin_std": 0.6235331296920776,
|
|
"step": 16
|
|
},
|
|
{
|
|
"epoch": 0.024963289280469897,
|
|
"grad_norm": 35.640289306640625,
|
|
"learning_rate": 1.1594202898550725e-07,
|
|
"logits/chosen": 0.13770782947540283,
|
|
"logits/rejected": 0.37006980180740356,
|
|
"logps/chosen": -104.69229125976562,
|
|
"logps/ref_chosen": -104.84930419921875,
|
|
"logps/ref_rejected": -103.08859252929688,
|
|
"logps/rejected": -103.0672607421875,
|
|
"loss": 1.3844,
|
|
"margin_dpo/margin_mean": 0.13567650318145752,
|
|
"margin_dpo/margin_std": 0.3899996876716614,
|
|
"step": 17
|
|
},
|
|
{
|
|
"epoch": 0.02643171806167401,
|
|
"grad_norm": 41.77239227294922,
|
|
"learning_rate": 1.2318840579710146e-07,
|
|
"logits/chosen": 0.07230012118816376,
|
|
"logits/rejected": 0.59023118019104,
|
|
"logps/chosen": -95.5035400390625,
|
|
"logps/ref_chosen": -95.57725524902344,
|
|
"logps/ref_rejected": -106.00630187988281,
|
|
"logps/rejected": -105.9694595336914,
|
|
"loss": 1.3876,
|
|
"margin_dpo/margin_mean": 0.03688061237335205,
|
|
"margin_dpo/margin_std": 0.48090770840644836,
|
|
"step": 18
|
|
},
|
|
{
|
|
"epoch": 0.027900146842878122,
|
|
"grad_norm": 31.308109283447266,
|
|
"learning_rate": 1.3043478260869563e-07,
|
|
"logits/chosen": 0.2451936900615692,
|
|
"logits/rejected": 0.5206657648086548,
|
|
"logps/chosen": -81.89685821533203,
|
|
"logps/ref_chosen": -81.85494995117188,
|
|
"logps/ref_rejected": -86.16612243652344,
|
|
"logps/rejected": -86.17457580566406,
|
|
"loss": 1.3845,
|
|
"margin_dpo/margin_mean": -0.03345954418182373,
|
|
"margin_dpo/margin_std": 0.2865228056907654,
|
|
"step": 19
|
|
},
|
|
{
|
|
"epoch": 0.02936857562408223,
|
|
"grad_norm": 32.505680084228516,
|
|
"learning_rate": 1.3768115942028986e-07,
|
|
"logits/chosen": 0.35453853011131287,
|
|
"logits/rejected": 0.6151764988899231,
|
|
"logps/chosen": -94.32595825195312,
|
|
"logps/ref_chosen": -94.21621704101562,
|
|
"logps/ref_rejected": -83.03634643554688,
|
|
"logps/rejected": -83.14512634277344,
|
|
"loss": 1.3841,
|
|
"margin_dpo/margin_mean": -0.0009757280349731445,
|
|
"margin_dpo/margin_std": 0.45821622014045715,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.030837004405286344,
|
|
"grad_norm": 31.37238883972168,
|
|
"learning_rate": 1.4492753623188405e-07,
|
|
"logits/chosen": 0.6882296800613403,
|
|
"logits/rejected": 1.0381821393966675,
|
|
"logps/chosen": -98.95984649658203,
|
|
"logps/ref_chosen": -99.03758239746094,
|
|
"logps/ref_rejected": -67.93568420410156,
|
|
"logps/rejected": -67.98387908935547,
|
|
"loss": 1.3781,
|
|
"margin_dpo/margin_mean": 0.12594342231750488,
|
|
"margin_dpo/margin_std": 0.3411893844604492,
|
|
"step": 21
|
|
},
|
|
{
|
|
"epoch": 0.032305433186490456,
|
|
"grad_norm": 42.97276306152344,
|
|
"learning_rate": 1.5217391304347825e-07,
|
|
"logits/chosen": 0.5317803621292114,
|
|
"logits/rejected": 0.4500496983528137,
|
|
"logps/chosen": -74.99850463867188,
|
|
"logps/ref_chosen": -74.9967041015625,
|
|
"logps/ref_rejected": -128.09115600585938,
|
|
"logps/rejected": -128.13372802734375,
|
|
"loss": 1.3815,
|
|
"margin_dpo/margin_mean": 0.040738582611083984,
|
|
"margin_dpo/margin_std": 0.3939288258552551,
|
|
"step": 22
|
|
},
|
|
{
|
|
"epoch": 0.033773861967694566,
|
|
"grad_norm": 31.378528594970703,
|
|
"learning_rate": 1.5942028985507245e-07,
|
|
"logits/chosen": 0.5977481007575989,
|
|
"logits/rejected": 0.6081861257553101,
|
|
"logps/chosen": -81.11967468261719,
|
|
"logps/ref_chosen": -81.07363891601562,
|
|
"logps/ref_rejected": -110.53385925292969,
|
|
"logps/rejected": -110.63468933105469,
|
|
"loss": 1.3858,
|
|
"margin_dpo/margin_mean": 0.05478942394256592,
|
|
"margin_dpo/margin_std": 0.2976340651512146,
|
|
"step": 23
|
|
},
|
|
{
|
|
"epoch": 0.03524229074889868,
|
|
"grad_norm": 44.402305603027344,
|
|
"learning_rate": 1.6666666666666665e-07,
|
|
"logits/chosen": 0.6498656272888184,
|
|
"logits/rejected": 0.7695809602737427,
|
|
"logps/chosen": -65.41180419921875,
|
|
"logps/ref_chosen": -65.40289306640625,
|
|
"logps/ref_rejected": -64.838623046875,
|
|
"logps/rejected": -64.81765747070312,
|
|
"loss": 1.3832,
|
|
"margin_dpo/margin_mean": -0.02987229824066162,
|
|
"margin_dpo/margin_std": 0.22926175594329834,
|
|
"step": 24
|
|
},
|
|
{
|
|
"epoch": 0.03671071953010279,
|
|
"grad_norm": 36.98027038574219,
|
|
"learning_rate": 1.7391304347826085e-07,
|
|
"logits/chosen": 0.8376168012619019,
|
|
"logits/rejected": 0.9103662967681885,
|
|
"logps/chosen": -85.75971984863281,
|
|
"logps/ref_chosen": -85.73371887207031,
|
|
"logps/ref_rejected": -121.51497650146484,
|
|
"logps/rejected": -121.53792572021484,
|
|
"loss": 1.3793,
|
|
"margin_dpo/margin_mean": -0.003041386604309082,
|
|
"margin_dpo/margin_std": 0.5337319374084473,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.0381791483113069,
|
|
"grad_norm": 41.38254165649414,
|
|
"learning_rate": 1.8115942028985507e-07,
|
|
"logits/chosen": 0.19763419032096863,
|
|
"logits/rejected": 0.4370805025100708,
|
|
"logps/chosen": -81.58695220947266,
|
|
"logps/ref_chosen": -81.50300598144531,
|
|
"logps/ref_rejected": -150.1811981201172,
|
|
"logps/rejected": -150.1844482421875,
|
|
"loss": 1.3965,
|
|
"margin_dpo/margin_mean": -0.0806952714920044,
|
|
"margin_dpo/margin_std": 0.3255520761013031,
|
|
"step": 26
|
|
},
|
|
{
|
|
"epoch": 0.039647577092511016,
|
|
"grad_norm": 44.32342529296875,
|
|
"learning_rate": 1.8840579710144927e-07,
|
|
"logits/chosen": -0.14630630612373352,
|
|
"logits/rejected": 0.150857076048851,
|
|
"logps/chosen": -80.22408294677734,
|
|
"logps/ref_chosen": -80.2718505859375,
|
|
"logps/ref_rejected": -113.22959899902344,
|
|
"logps/rejected": -113.37527465820312,
|
|
"loss": 1.3714,
|
|
"margin_dpo/margin_mean": 0.19344651699066162,
|
|
"margin_dpo/margin_std": 0.43172699213027954,
|
|
"step": 27
|
|
},
|
|
{
|
|
"epoch": 0.041116005873715125,
|
|
"grad_norm": 36.64478302001953,
|
|
"learning_rate": 1.9565217391304347e-07,
|
|
"logits/chosen": 0.3271695375442505,
|
|
"logits/rejected": 0.4185149073600769,
|
|
"logps/chosen": -92.43882751464844,
|
|
"logps/ref_chosen": -92.57330322265625,
|
|
"logps/ref_rejected": -98.44024658203125,
|
|
"logps/rejected": -98.50408935546875,
|
|
"loss": 1.3747,
|
|
"margin_dpo/margin_mean": 0.19830501079559326,
|
|
"margin_dpo/margin_std": 0.31042030453681946,
|
|
"step": 28
|
|
},
|
|
{
|
|
"epoch": 0.042584434654919234,
|
|
"grad_norm": 42.460182189941406,
|
|
"learning_rate": 2.028985507246377e-07,
|
|
"logits/chosen": 0.35313108563423157,
|
|
"logits/rejected": 0.45420438051223755,
|
|
"logps/chosen": -79.79510498046875,
|
|
"logps/ref_chosen": -79.78430938720703,
|
|
"logps/ref_rejected": -96.3348159790039,
|
|
"logps/rejected": -96.49360656738281,
|
|
"loss": 1.3787,
|
|
"margin_dpo/margin_mean": 0.1480005383491516,
|
|
"margin_dpo/margin_std": 0.3144315779209137,
|
|
"step": 29
|
|
},
|
|
{
|
|
"epoch": 0.04405286343612335,
|
|
"grad_norm": 39.358245849609375,
|
|
"learning_rate": 2.1014492753623187e-07,
|
|
"logits/chosen": 0.7991341948509216,
|
|
"logits/rejected": 0.8507763147354126,
|
|
"logps/chosen": -115.38392639160156,
|
|
"logps/ref_chosen": -115.44339752197266,
|
|
"logps/ref_rejected": -118.73816680908203,
|
|
"logps/rejected": -118.85655975341797,
|
|
"loss": 1.3812,
|
|
"margin_dpo/margin_mean": 0.17785942554473877,
|
|
"margin_dpo/margin_std": 0.42279767990112305,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.04552129221732746,
|
|
"grad_norm": 31.34886360168457,
|
|
"learning_rate": 2.1739130434782607e-07,
|
|
"logits/chosen": 0.7612583637237549,
|
|
"logits/rejected": 0.9483832120895386,
|
|
"logps/chosen": -80.2682113647461,
|
|
"logps/ref_chosen": -80.27281188964844,
|
|
"logps/ref_rejected": -72.05465698242188,
|
|
"logps/rejected": -72.02093505859375,
|
|
"loss": 1.376,
|
|
"margin_dpo/margin_mean": -0.029139041900634766,
|
|
"margin_dpo/margin_std": 0.2328224629163742,
|
|
"step": 31
|
|
},
|
|
{
|
|
"epoch": 0.04698972099853157,
|
|
"grad_norm": 35.19465255737305,
|
|
"learning_rate": 2.2463768115942027e-07,
|
|
"logits/chosen": 0.9847223162651062,
|
|
"logits/rejected": 1.10117506980896,
|
|
"logps/chosen": -106.95022583007812,
|
|
"logps/ref_chosen": -106.981201171875,
|
|
"logps/ref_rejected": -101.810302734375,
|
|
"logps/rejected": -101.90875244140625,
|
|
"loss": 1.3753,
|
|
"margin_dpo/margin_mean": 0.1294306516647339,
|
|
"margin_dpo/margin_std": 0.5172077417373657,
|
|
"step": 32
|
|
},
|
|
{
|
|
"epoch": 0.048458149779735685,
|
|
"grad_norm": 33.5123291015625,
|
|
"learning_rate": 2.318840579710145e-07,
|
|
"logits/chosen": 0.8942906856536865,
|
|
"logits/rejected": 1.0839948654174805,
|
|
"logps/chosen": -92.93179321289062,
|
|
"logps/ref_chosen": -92.88273620605469,
|
|
"logps/ref_rejected": -107.86016082763672,
|
|
"logps/rejected": -108.07872009277344,
|
|
"loss": 1.373,
|
|
"margin_dpo/margin_mean": 0.16950154304504395,
|
|
"margin_dpo/margin_std": 0.31402474641799927,
|
|
"step": 33
|
|
},
|
|
{
|
|
"epoch": 0.049926578560939794,
|
|
"grad_norm": 40.969905853271484,
|
|
"learning_rate": 2.391304347826087e-07,
|
|
"logits/chosen": 0.3540779948234558,
|
|
"logits/rejected": 0.8411053419113159,
|
|
"logps/chosen": -96.10871887207031,
|
|
"logps/ref_chosen": -96.13287353515625,
|
|
"logps/ref_rejected": -111.3116683959961,
|
|
"logps/rejected": -111.40603637695312,
|
|
"loss": 1.3738,
|
|
"margin_dpo/margin_mean": 0.11852216720581055,
|
|
"margin_dpo/margin_std": 0.2863542437553406,
|
|
"step": 34
|
|
},
|
|
{
|
|
"epoch": 0.0513950073421439,
|
|
"grad_norm": 43.0073127746582,
|
|
"learning_rate": 2.463768115942029e-07,
|
|
"logits/chosen": 0.4877297878265381,
|
|
"logits/rejected": 0.7324085235595703,
|
|
"logps/chosen": -68.8739013671875,
|
|
"logps/ref_chosen": -68.71487426757812,
|
|
"logps/ref_rejected": -110.0155029296875,
|
|
"logps/rejected": -110.1253662109375,
|
|
"loss": 1.3743,
|
|
"margin_dpo/margin_mean": -0.04916191101074219,
|
|
"margin_dpo/margin_std": 0.3702870309352875,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.05286343612334802,
|
|
"grad_norm": 32.48810577392578,
|
|
"learning_rate": 2.536231884057971e-07,
|
|
"logits/chosen": 0.265766441822052,
|
|
"logits/rejected": 0.3960006833076477,
|
|
"logps/chosen": -65.26568603515625,
|
|
"logps/ref_chosen": -65.30921936035156,
|
|
"logps/ref_rejected": -72.58026885986328,
|
|
"logps/rejected": -72.68403625488281,
|
|
"loss": 1.3695,
|
|
"margin_dpo/margin_mean": 0.1472916603088379,
|
|
"margin_dpo/margin_std": 0.4905526340007782,
|
|
"step": 36
|
|
},
|
|
{
|
|
"epoch": 0.05433186490455213,
|
|
"grad_norm": 35.842777252197266,
|
|
"learning_rate": 2.6086956521739126e-07,
|
|
"logits/chosen": 0.3955993056297302,
|
|
"logits/rejected": 0.6936404705047607,
|
|
"logps/chosen": -113.5171127319336,
|
|
"logps/ref_chosen": -113.48770141601562,
|
|
"logps/ref_rejected": -84.58368682861328,
|
|
"logps/rejected": -84.8321304321289,
|
|
"loss": 1.3602,
|
|
"margin_dpo/margin_mean": 0.21903866529464722,
|
|
"margin_dpo/margin_std": 0.4852628707885742,
|
|
"step": 37
|
|
},
|
|
{
|
|
"epoch": 0.055800293685756244,
|
|
"grad_norm": 36.82903289794922,
|
|
"learning_rate": 2.681159420289855e-07,
|
|
"logits/chosen": 0.6250002384185791,
|
|
"logits/rejected": 0.7215943336486816,
|
|
"logps/chosen": -92.1820297241211,
|
|
"logps/ref_chosen": -92.26864624023438,
|
|
"logps/ref_rejected": -146.35977172851562,
|
|
"logps/rejected": -146.4608917236328,
|
|
"loss": 1.3655,
|
|
"margin_dpo/margin_mean": 0.187727689743042,
|
|
"margin_dpo/margin_std": 0.7143464088439941,
|
|
"step": 38
|
|
},
|
|
{
|
|
"epoch": 0.05726872246696035,
|
|
"grad_norm": 36.069664001464844,
|
|
"learning_rate": 2.753623188405797e-07,
|
|
"logits/chosen": 0.7301809191703796,
|
|
"logits/rejected": 1.0973470211029053,
|
|
"logps/chosen": -81.108154296875,
|
|
"logps/ref_chosen": -81.08113098144531,
|
|
"logps/ref_rejected": -83.06800842285156,
|
|
"logps/rejected": -83.49285888671875,
|
|
"loss": 1.3574,
|
|
"margin_dpo/margin_mean": 0.39781689643859863,
|
|
"margin_dpo/margin_std": 0.4662584066390991,
|
|
"step": 39
|
|
},
|
|
{
|
|
"epoch": 0.05873715124816446,
|
|
"grad_norm": 29.052732467651367,
|
|
"learning_rate": 2.8260869565217386e-07,
|
|
"logits/chosen": 0.8642468452453613,
|
|
"logits/rejected": 0.9313390254974365,
|
|
"logps/chosen": -76.96479797363281,
|
|
"logps/ref_chosen": -76.87438201904297,
|
|
"logps/ref_rejected": -74.98965454101562,
|
|
"logps/rejected": -75.19844055175781,
|
|
"loss": 1.3609,
|
|
"margin_dpo/margin_mean": 0.1183696985244751,
|
|
"margin_dpo/margin_std": 0.33317673206329346,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.06020558002936858,
|
|
"grad_norm": 33.42491149902344,
|
|
"learning_rate": 2.898550724637681e-07,
|
|
"logits/chosen": 0.7419478893280029,
|
|
"logits/rejected": 0.8945612907409668,
|
|
"logps/chosen": -113.77964782714844,
|
|
"logps/ref_chosen": -113.75234985351562,
|
|
"logps/ref_rejected": -106.27317810058594,
|
|
"logps/rejected": -106.6886215209961,
|
|
"loss": 1.3571,
|
|
"margin_dpo/margin_mean": 0.3881516456604004,
|
|
"margin_dpo/margin_std": 0.5789889693260193,
|
|
"step": 41
|
|
},
|
|
{
|
|
"epoch": 0.06167400881057269,
|
|
"grad_norm": 40.86435317993164,
|
|
"learning_rate": 2.971014492753623e-07,
|
|
"logits/chosen": 0.7604937553405762,
|
|
"logits/rejected": 0.8403340578079224,
|
|
"logps/chosen": -93.4481430053711,
|
|
"logps/ref_chosen": -93.46589660644531,
|
|
"logps/ref_rejected": -137.63717651367188,
|
|
"logps/rejected": -138.0484619140625,
|
|
"loss": 1.3456,
|
|
"margin_dpo/margin_mean": 0.4290355443954468,
|
|
"margin_dpo/margin_std": 0.652177631855011,
|
|
"step": 42
|
|
},
|
|
{
|
|
"epoch": 0.0631424375917768,
|
|
"grad_norm": 38.23468017578125,
|
|
"learning_rate": 3.043478260869565e-07,
|
|
"logits/chosen": 0.5066407322883606,
|
|
"logits/rejected": 0.628272294998169,
|
|
"logps/chosen": -87.23843383789062,
|
|
"logps/ref_chosen": -87.20645141601562,
|
|
"logps/ref_rejected": -86.70765686035156,
|
|
"logps/rejected": -87.05091094970703,
|
|
"loss": 1.3532,
|
|
"margin_dpo/margin_mean": 0.3112701177597046,
|
|
"margin_dpo/margin_std": 0.5115389227867126,
|
|
"step": 43
|
|
},
|
|
{
|
|
"epoch": 0.06461086637298091,
|
|
"grad_norm": 34.67744445800781,
|
|
"learning_rate": 3.115942028985507e-07,
|
|
"logits/chosen": 0.7453466057777405,
|
|
"logits/rejected": 0.9643809795379639,
|
|
"logps/chosen": -65.89019775390625,
|
|
"logps/ref_chosen": -65.7902603149414,
|
|
"logps/ref_rejected": -115.73516082763672,
|
|
"logps/rejected": -116.15614318847656,
|
|
"loss": 1.3572,
|
|
"margin_dpo/margin_mean": 0.32104432582855225,
|
|
"margin_dpo/margin_std": 0.7572367191314697,
|
|
"step": 44
|
|
},
|
|
{
|
|
"epoch": 0.06607929515418502,
|
|
"grad_norm": 24.18403434753418,
|
|
"learning_rate": 3.188405797101449e-07,
|
|
"logits/chosen": 0.4455117881298065,
|
|
"logits/rejected": 0.5920794010162354,
|
|
"logps/chosen": -85.85913848876953,
|
|
"logps/ref_chosen": -85.71604919433594,
|
|
"logps/ref_rejected": -82.59597778320312,
|
|
"logps/rejected": -83.0451889038086,
|
|
"loss": 1.3571,
|
|
"margin_dpo/margin_mean": 0.3061286211013794,
|
|
"margin_dpo/margin_std": 0.5986896753311157,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.06754772393538913,
|
|
"grad_norm": 33.199153900146484,
|
|
"learning_rate": 3.260869565217391e-07,
|
|
"logits/chosen": 0.5929129719734192,
|
|
"logits/rejected": 0.8907681703567505,
|
|
"logps/chosen": -95.22297668457031,
|
|
"logps/ref_chosen": -95.29031372070312,
|
|
"logps/ref_rejected": -113.86479949951172,
|
|
"logps/rejected": -114.26651000976562,
|
|
"loss": 1.3424,
|
|
"margin_dpo/margin_mean": 0.4690542221069336,
|
|
"margin_dpo/margin_std": 0.6764923334121704,
|
|
"step": 46
|
|
},
|
|
{
|
|
"epoch": 0.06901615271659324,
|
|
"grad_norm": 34.175819396972656,
|
|
"learning_rate": 3.333333333333333e-07,
|
|
"logits/chosen": -0.16016048192977905,
|
|
"logits/rejected": 0.03261171281337738,
|
|
"logps/chosen": -73.86705780029297,
|
|
"logps/ref_chosen": -73.91566467285156,
|
|
"logps/ref_rejected": -74.19608306884766,
|
|
"logps/rejected": -74.57892608642578,
|
|
"loss": 1.3475,
|
|
"margin_dpo/margin_mean": 0.4314463138580322,
|
|
"margin_dpo/margin_std": 0.887498140335083,
|
|
"step": 47
|
|
},
|
|
{
|
|
"epoch": 0.07048458149779736,
|
|
"grad_norm": 39.072574615478516,
|
|
"learning_rate": 3.4057971014492755e-07,
|
|
"logits/chosen": 0.28998929262161255,
|
|
"logits/rejected": 0.39170825481414795,
|
|
"logps/chosen": -104.47331237792969,
|
|
"logps/ref_chosen": -104.229248046875,
|
|
"logps/ref_rejected": -107.5794677734375,
|
|
"logps/rejected": -108.28083801269531,
|
|
"loss": 1.3388,
|
|
"margin_dpo/margin_mean": 0.4573037624359131,
|
|
"margin_dpo/margin_std": 1.0886731147766113,
|
|
"step": 48
|
|
},
|
|
{
|
|
"epoch": 0.07195301027900147,
|
|
"grad_norm": 39.3463134765625,
|
|
"learning_rate": 3.478260869565217e-07,
|
|
"logits/chosen": 0.07694488763809204,
|
|
"logits/rejected": 0.623125433921814,
|
|
"logps/chosen": -81.1698226928711,
|
|
"logps/ref_chosen": -81.32856750488281,
|
|
"logps/ref_rejected": -74.75869750976562,
|
|
"logps/rejected": -75.13246154785156,
|
|
"loss": 1.3129,
|
|
"margin_dpo/margin_mean": 0.5325143337249756,
|
|
"margin_dpo/margin_std": 0.7589560747146606,
|
|
"step": 49
|
|
},
|
|
{
|
|
"epoch": 0.07342143906020558,
|
|
"grad_norm": 33.31324768066406,
|
|
"learning_rate": 3.5507246376811595e-07,
|
|
"logits/chosen": 0.8446512222290039,
|
|
"logits/rejected": 1.1388261318206787,
|
|
"logps/chosen": -104.20743560791016,
|
|
"logps/ref_chosen": -103.97581481933594,
|
|
"logps/ref_rejected": -122.77733612060547,
|
|
"logps/rejected": -123.37358093261719,
|
|
"loss": 1.3292,
|
|
"margin_dpo/margin_mean": 0.3646160364151001,
|
|
"margin_dpo/margin_std": 0.785099983215332,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.07488986784140969,
|
|
"grad_norm": 30.007966995239258,
|
|
"learning_rate": 3.6231884057971015e-07,
|
|
"logits/chosen": 0.6785503029823303,
|
|
"logits/rejected": 1.0297834873199463,
|
|
"logps/chosen": -110.92190551757812,
|
|
"logps/ref_chosen": -110.60376739501953,
|
|
"logps/ref_rejected": -89.77001953125,
|
|
"logps/rejected": -90.43896484375,
|
|
"loss": 1.33,
|
|
"margin_dpo/margin_mean": 0.3508089780807495,
|
|
"margin_dpo/margin_std": 1.0862066745758057,
|
|
"step": 51
|
|
},
|
|
{
|
|
"epoch": 0.0763582966226138,
|
|
"grad_norm": 32.3361701965332,
|
|
"learning_rate": 3.695652173913043e-07,
|
|
"logits/chosen": 0.3316473960876465,
|
|
"logits/rejected": 0.5727195739746094,
|
|
"logps/chosen": -117.17269134521484,
|
|
"logps/ref_chosen": -116.8155517578125,
|
|
"logps/ref_rejected": -82.47869873046875,
|
|
"logps/rejected": -83.43679809570312,
|
|
"loss": 1.3089,
|
|
"margin_dpo/margin_mean": 0.6009612083435059,
|
|
"margin_dpo/margin_std": 0.7621315717697144,
|
|
"step": 52
|
|
},
|
|
{
|
|
"epoch": 0.07782672540381791,
|
|
"grad_norm": 39.464393615722656,
|
|
"learning_rate": 3.7681159420289855e-07,
|
|
"logits/chosen": 0.769700288772583,
|
|
"logits/rejected": 0.8581574559211731,
|
|
"logps/chosen": -97.04790496826172,
|
|
"logps/ref_chosen": -97.222412109375,
|
|
"logps/ref_rejected": -115.93175506591797,
|
|
"logps/rejected": -117.12537384033203,
|
|
"loss": 1.2854,
|
|
"margin_dpo/margin_mean": 1.3681297302246094,
|
|
"margin_dpo/margin_std": 1.9412078857421875,
|
|
"step": 53
|
|
},
|
|
{
|
|
"epoch": 0.07929515418502203,
|
|
"grad_norm": 40.05606460571289,
|
|
"learning_rate": 3.8405797101449274e-07,
|
|
"logits/chosen": 0.8848798871040344,
|
|
"logits/rejected": 0.9821081757545471,
|
|
"logps/chosen": -68.86314392089844,
|
|
"logps/ref_chosen": -68.70967864990234,
|
|
"logps/ref_rejected": -86.82157135009766,
|
|
"logps/rejected": -87.76393127441406,
|
|
"loss": 1.2888,
|
|
"margin_dpo/margin_mean": 0.7888990640640259,
|
|
"margin_dpo/margin_std": 0.9890120029449463,
|
|
"step": 54
|
|
},
|
|
{
|
|
"epoch": 0.08076358296622614,
|
|
"grad_norm": 35.246360778808594,
|
|
"learning_rate": 3.9130434782608694e-07,
|
|
"logits/chosen": -0.12803569436073303,
|
|
"logits/rejected": 0.09957672655582428,
|
|
"logps/chosen": -98.69027709960938,
|
|
"logps/ref_chosen": -99.13316345214844,
|
|
"logps/ref_rejected": -112.1254653930664,
|
|
"logps/rejected": -113.6478271484375,
|
|
"loss": 1.2802,
|
|
"margin_dpo/margin_mean": 1.9652525186538696,
|
|
"margin_dpo/margin_std": 1.6538578271865845,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.08223201174743025,
|
|
"grad_norm": 33.27520751953125,
|
|
"learning_rate": 3.9855072463768114e-07,
|
|
"logits/chosen": -0.09207522869110107,
|
|
"logits/rejected": 0.29723483324050903,
|
|
"logps/chosen": -98.00480651855469,
|
|
"logps/ref_chosen": -97.85421752929688,
|
|
"logps/ref_rejected": -83.46979522705078,
|
|
"logps/rejected": -84.39903259277344,
|
|
"loss": 1.2933,
|
|
"margin_dpo/margin_mean": 0.7786517143249512,
|
|
"margin_dpo/margin_std": 1.203643798828125,
|
|
"step": 56
|
|
},
|
|
{
|
|
"epoch": 0.08370044052863436,
|
|
"grad_norm": 33.76715850830078,
|
|
"learning_rate": 4.057971014492754e-07,
|
|
"logits/chosen": 0.17622031271457672,
|
|
"logits/rejected": 0.25724083185195923,
|
|
"logps/chosen": -99.81153869628906,
|
|
"logps/ref_chosen": -100.10478973388672,
|
|
"logps/ref_rejected": -130.69100952148438,
|
|
"logps/rejected": -131.81729125976562,
|
|
"loss": 1.2831,
|
|
"margin_dpo/margin_mean": 1.4195455312728882,
|
|
"margin_dpo/margin_std": 1.264824390411377,
|
|
"step": 57
|
|
},
|
|
{
|
|
"epoch": 0.08516886930983847,
|
|
"grad_norm": 38.04386901855469,
|
|
"learning_rate": 4.1304347826086954e-07,
|
|
"logits/chosen": -0.1125476062297821,
|
|
"logits/rejected": 0.2727716565132141,
|
|
"logps/chosen": -109.82270812988281,
|
|
"logps/ref_chosen": -109.95325469970703,
|
|
"logps/ref_rejected": -70.22386169433594,
|
|
"logps/rejected": -71.33671569824219,
|
|
"loss": 1.251,
|
|
"margin_dpo/margin_mean": 1.2433971166610718,
|
|
"margin_dpo/margin_std": 1.3449275493621826,
|
|
"step": 58
|
|
},
|
|
{
|
|
"epoch": 0.08663729809104258,
|
|
"grad_norm": 33.97794723510742,
|
|
"learning_rate": 4.2028985507246374e-07,
|
|
"logits/chosen": 0.39747869968414307,
|
|
"logits/rejected": 0.5710791349411011,
|
|
"logps/chosen": -100.960693359375,
|
|
"logps/ref_chosen": -100.9923095703125,
|
|
"logps/ref_rejected": -110.70699310302734,
|
|
"logps/rejected": -112.17672729492188,
|
|
"loss": 1.2692,
|
|
"margin_dpo/margin_mean": 1.5013514757156372,
|
|
"margin_dpo/margin_std": 1.5745304822921753,
|
|
"step": 59
|
|
},
|
|
{
|
|
"epoch": 0.0881057268722467,
|
|
"grad_norm": 28.536787033081055,
|
|
"learning_rate": 4.2753623188405794e-07,
|
|
"logits/chosen": 0.35590529441833496,
|
|
"logits/rejected": 0.8095076680183411,
|
|
"logps/chosen": -107.55908203125,
|
|
"logps/ref_chosen": -108.16279602050781,
|
|
"logps/ref_rejected": -68.67430877685547,
|
|
"logps/rejected": -69.468017578125,
|
|
"loss": 1.2738,
|
|
"margin_dpo/margin_mean": 1.3974266052246094,
|
|
"margin_dpo/margin_std": 1.2194509506225586,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.08957415565345081,
|
|
"grad_norm": 28.542030334472656,
|
|
"learning_rate": 4.3478260869565214e-07,
|
|
"logits/chosen": -0.4731886386871338,
|
|
"logits/rejected": -0.2491273283958435,
|
|
"logps/chosen": -80.26652526855469,
|
|
"logps/ref_chosen": -80.13163757324219,
|
|
"logps/ref_rejected": -83.95764923095703,
|
|
"logps/rejected": -85.35874938964844,
|
|
"loss": 1.2856,
|
|
"margin_dpo/margin_mean": 1.2662101984024048,
|
|
"margin_dpo/margin_std": 1.339787483215332,
|
|
"step": 61
|
|
},
|
|
{
|
|
"epoch": 0.09104258443465492,
|
|
"grad_norm": 28.514225006103516,
|
|
"learning_rate": 4.420289855072464e-07,
|
|
"logits/chosen": 0.680814802646637,
|
|
"logits/rejected": 0.6677084565162659,
|
|
"logps/chosen": -50.487548828125,
|
|
"logps/ref_chosen": -50.332794189453125,
|
|
"logps/ref_rejected": -72.85269927978516,
|
|
"logps/rejected": -73.86290740966797,
|
|
"loss": 1.285,
|
|
"margin_dpo/margin_mean": 0.8554513454437256,
|
|
"margin_dpo/margin_std": 1.4367563724517822,
|
|
"step": 62
|
|
},
|
|
{
|
|
"epoch": 0.09251101321585903,
|
|
"grad_norm": 34.94499206542969,
|
|
"learning_rate": 4.4927536231884053e-07,
|
|
"logits/chosen": 0.4011261463165283,
|
|
"logits/rejected": 0.6790079474449158,
|
|
"logps/chosen": -121.78221893310547,
|
|
"logps/ref_chosen": -121.70402526855469,
|
|
"logps/ref_rejected": -81.41825866699219,
|
|
"logps/rejected": -82.76856994628906,
|
|
"loss": 1.246,
|
|
"margin_dpo/margin_mean": 1.2721160650253296,
|
|
"margin_dpo/margin_std": 1.2849652767181396,
|
|
"step": 63
|
|
},
|
|
{
|
|
"epoch": 0.09397944199706314,
|
|
"grad_norm": 34.87194061279297,
|
|
"learning_rate": 4.5652173913043473e-07,
|
|
"logits/chosen": 0.7611849308013916,
|
|
"logits/rejected": 0.9642089605331421,
|
|
"logps/chosen": -80.07738494873047,
|
|
"logps/ref_chosen": -79.740234375,
|
|
"logps/ref_rejected": -104.4854736328125,
|
|
"logps/rejected": -105.55560302734375,
|
|
"loss": 1.2345,
|
|
"margin_dpo/margin_mean": 0.7329847812652588,
|
|
"margin_dpo/margin_std": 1.2104127407073975,
|
|
"step": 64
|
|
},
|
|
{
|
|
"epoch": 0.09544787077826726,
|
|
"grad_norm": 38.41434860229492,
|
|
"learning_rate": 4.63768115942029e-07,
|
|
"logits/chosen": 0.254086971282959,
|
|
"logits/rejected": 0.1695535033941269,
|
|
"logps/chosen": -78.44050598144531,
|
|
"logps/ref_chosen": -78.41619873046875,
|
|
"logps/ref_rejected": -135.5757293701172,
|
|
"logps/rejected": -137.61776733398438,
|
|
"loss": 1.2297,
|
|
"margin_dpo/margin_mean": 2.0177080631256104,
|
|
"margin_dpo/margin_std": 2.553192615509033,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.09691629955947137,
|
|
"grad_norm": 25.3912410736084,
|
|
"learning_rate": 4.7101449275362313e-07,
|
|
"logits/chosen": -0.09305822104215622,
|
|
"logits/rejected": 0.15512987971305847,
|
|
"logps/chosen": -65.99453735351562,
|
|
"logps/ref_chosen": -65.68583679199219,
|
|
"logps/ref_rejected": -76.33775329589844,
|
|
"logps/rejected": -77.69013214111328,
|
|
"loss": 1.2646,
|
|
"margin_dpo/margin_mean": 1.0436724424362183,
|
|
"margin_dpo/margin_std": 1.7997262477874756,
|
|
"step": 66
|
|
},
|
|
{
|
|
"epoch": 0.09838472834067548,
|
|
"grad_norm": 24.43991470336914,
|
|
"learning_rate": 4.782608695652174e-07,
|
|
"logits/chosen": 0.40889042615890503,
|
|
"logits/rejected": 0.5336402654647827,
|
|
"logps/chosen": -74.0006103515625,
|
|
"logps/ref_chosen": -73.65798950195312,
|
|
"logps/ref_rejected": -70.01232147216797,
|
|
"logps/rejected": -71.95913696289062,
|
|
"loss": 1.3012,
|
|
"margin_dpo/margin_mean": 1.6041903495788574,
|
|
"margin_dpo/margin_std": 2.0201072692871094,
|
|
"step": 67
|
|
},
|
|
{
|
|
"epoch": 0.09985315712187959,
|
|
"grad_norm": 28.16604995727539,
|
|
"learning_rate": 4.855072463768116e-07,
|
|
"logits/chosen": -0.8838789463043213,
|
|
"logits/rejected": -0.6690104007720947,
|
|
"logps/chosen": -85.18283081054688,
|
|
"logps/ref_chosen": -84.98939514160156,
|
|
"logps/ref_rejected": -94.47724914550781,
|
|
"logps/rejected": -97.47535705566406,
|
|
"loss": 1.2043,
|
|
"margin_dpo/margin_mean": 2.804673910140991,
|
|
"margin_dpo/margin_std": 2.7214131355285645,
|
|
"step": 68
|
|
},
|
|
{
|
|
"epoch": 0.1013215859030837,
|
|
"grad_norm": 30.81968116760254,
|
|
"learning_rate": 4.927536231884058e-07,
|
|
"logits/chosen": 0.12748196721076965,
|
|
"logits/rejected": 0.1792227327823639,
|
|
"logps/chosen": -86.99532318115234,
|
|
"logps/ref_chosen": -85.89825439453125,
|
|
"logps/ref_rejected": -118.52421569824219,
|
|
"logps/rejected": -120.83209228515625,
|
|
"loss": 1.2047,
|
|
"margin_dpo/margin_mean": 1.2108116149902344,
|
|
"margin_dpo/margin_std": 2.8251242637634277,
|
|
"step": 69
|
|
},
|
|
{
|
|
"epoch": 0.1027900146842878,
|
|
"grad_norm": 29.18440818786621,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": -0.12536486983299255,
|
|
"logits/rejected": 0.061571136116981506,
|
|
"logps/chosen": -125.14912414550781,
|
|
"logps/ref_chosen": -124.22589111328125,
|
|
"logps/ref_rejected": -99.41220092773438,
|
|
"logps/rejected": -102.61834716796875,
|
|
"loss": 1.1769,
|
|
"margin_dpo/margin_mean": 2.2829203605651855,
|
|
"margin_dpo/margin_std": 4.698617458343506,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.10425844346549193,
|
|
"grad_norm": 34.87384796142578,
|
|
"learning_rate": 4.999967061337492e-07,
|
|
"logits/chosen": 0.3338947892189026,
|
|
"logits/rejected": 0.5460184812545776,
|
|
"logps/chosen": -80.64649963378906,
|
|
"logps/ref_chosen": -80.65885925292969,
|
|
"logps/ref_rejected": -89.29473876953125,
|
|
"logps/rejected": -92.06094360351562,
|
|
"loss": 1.0958,
|
|
"margin_dpo/margin_mean": 2.7785582542419434,
|
|
"margin_dpo/margin_std": 2.691640853881836,
|
|
"step": 71
|
|
},
|
|
{
|
|
"epoch": 0.10572687224669604,
|
|
"grad_norm": 29.42046356201172,
|
|
"learning_rate": 4.999868246217933e-07,
|
|
"logits/chosen": 0.11558493971824646,
|
|
"logits/rejected": 0.336860328912735,
|
|
"logps/chosen": -91.0280990600586,
|
|
"logps/ref_chosen": -90.32836151123047,
|
|
"logps/ref_rejected": -87.39801025390625,
|
|
"logps/rejected": -90.02912902832031,
|
|
"loss": 1.1477,
|
|
"margin_dpo/margin_mean": 1.9313820600509644,
|
|
"margin_dpo/margin_std": 3.1832776069641113,
|
|
"step": 72
|
|
},
|
|
{
|
|
"epoch": 0.10719530102790015,
|
|
"grad_norm": 27.79156494140625,
|
|
"learning_rate": 4.999703557245192e-07,
|
|
"logits/chosen": -1.0717291831970215,
|
|
"logits/rejected": -0.48407894372940063,
|
|
"logps/chosen": -113.90251159667969,
|
|
"logps/ref_chosen": -112.08250427246094,
|
|
"logps/ref_rejected": -96.43185424804688,
|
|
"logps/rejected": -100.67330169677734,
|
|
"loss": 1.1568,
|
|
"margin_dpo/margin_mean": 2.4214420318603516,
|
|
"margin_dpo/margin_std": 4.514315605163574,
|
|
"step": 73
|
|
},
|
|
{
|
|
"epoch": 0.10866372980910426,
|
|
"grad_norm": 31.334726333618164,
|
|
"learning_rate": 4.999472998758977e-07,
|
|
"logits/chosen": -0.442888081073761,
|
|
"logits/rejected": -0.4210275411605835,
|
|
"logps/chosen": -84.89183044433594,
|
|
"logps/ref_chosen": -84.09466552734375,
|
|
"logps/ref_rejected": -118.03408813476562,
|
|
"logps/rejected": -123.54209899902344,
|
|
"loss": 1.1118,
|
|
"margin_dpo/margin_mean": 4.710856914520264,
|
|
"margin_dpo/margin_std": 4.988119125366211,
|
|
"step": 74
|
|
},
|
|
{
|
|
"epoch": 0.11013215859030837,
|
|
"grad_norm": 32.42177963256836,
|
|
"learning_rate": 4.999176576834721e-07,
|
|
"logits/chosen": -0.5751562714576721,
|
|
"logits/rejected": 0.15103864669799805,
|
|
"logps/chosen": -89.07445526123047,
|
|
"logps/ref_chosen": -88.38374328613281,
|
|
"logps/ref_rejected": -112.27578735351562,
|
|
"logps/rejected": -117.11609649658203,
|
|
"loss": 1.0438,
|
|
"margin_dpo/margin_mean": 4.149603366851807,
|
|
"margin_dpo/margin_std": 4.267527103424072,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.11160058737151249,
|
|
"grad_norm": 23.008657455444336,
|
|
"learning_rate": 4.998814299283415e-07,
|
|
"logits/chosen": -1.5539774894714355,
|
|
"logits/rejected": -1.1704045534133911,
|
|
"logps/chosen": -101.99429321289062,
|
|
"logps/ref_chosen": -100.0120849609375,
|
|
"logps/ref_rejected": -88.49131774902344,
|
|
"logps/rejected": -93.37272644042969,
|
|
"loss": 1.1557,
|
|
"margin_dpo/margin_mean": 2.8992037773132324,
|
|
"margin_dpo/margin_std": 4.499299049377441,
|
|
"step": 76
|
|
},
|
|
{
|
|
"epoch": 0.1130690161527166,
|
|
"grad_norm": 27.028614044189453,
|
|
"learning_rate": 4.998386175651409e-07,
|
|
"logits/chosen": -0.5113037824630737,
|
|
"logits/rejected": -0.39275413751602173,
|
|
"logps/chosen": -74.51853942871094,
|
|
"logps/ref_chosen": -73.30534362792969,
|
|
"logps/ref_rejected": -102.59431457519531,
|
|
"logps/rejected": -107.80052185058594,
|
|
"loss": 1.1272,
|
|
"margin_dpo/margin_mean": 3.9930152893066406,
|
|
"margin_dpo/margin_std": 3.8414201736450195,
|
|
"step": 77
|
|
},
|
|
{
|
|
"epoch": 0.1145374449339207,
|
|
"grad_norm": 25.387378692626953,
|
|
"learning_rate": 4.997892217220159e-07,
|
|
"logits/chosen": 0.6262469291687012,
|
|
"logits/rejected": 0.7459644079208374,
|
|
"logps/chosen": -72.970703125,
|
|
"logps/ref_chosen": -72.40596008300781,
|
|
"logps/ref_rejected": -116.26104736328125,
|
|
"logps/rejected": -121.18896484375,
|
|
"loss": 1.0955,
|
|
"margin_dpo/margin_mean": 4.363163948059082,
|
|
"margin_dpo/margin_std": 4.845981597900391,
|
|
"step": 78
|
|
},
|
|
{
|
|
"epoch": 0.11600587371512482,
|
|
"grad_norm": 27.026376724243164,
|
|
"learning_rate": 4.997332437005931e-07,
|
|
"logits/chosen": -0.08271521329879761,
|
|
"logits/rejected": 0.5097367763519287,
|
|
"logps/chosen": -103.35887145996094,
|
|
"logps/ref_chosen": -101.26777648925781,
|
|
"logps/ref_rejected": -88.68028259277344,
|
|
"logps/rejected": -96.11433410644531,
|
|
"loss": 1.0431,
|
|
"margin_dpo/margin_mean": 5.342948913574219,
|
|
"margin_dpo/margin_std": 7.4267473220825195,
|
|
"step": 79
|
|
},
|
|
{
|
|
"epoch": 0.11747430249632893,
|
|
"grad_norm": 25.63165855407715,
|
|
"learning_rate": 4.996706849759452e-07,
|
|
"logits/chosen": -0.6737807393074036,
|
|
"logits/rejected": 0.1990889012813568,
|
|
"logps/chosen": -117.60404968261719,
|
|
"logps/ref_chosen": -114.87706756591797,
|
|
"logps/ref_rejected": -71.03803253173828,
|
|
"logps/rejected": -74.76948547363281,
|
|
"loss": 1.1065,
|
|
"margin_dpo/margin_mean": 1.004472255706787,
|
|
"margin_dpo/margin_std": 3.0686776638031006,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.11894273127753303,
|
|
"grad_norm": 27.555042266845703,
|
|
"learning_rate": 4.996015471965529e-07,
|
|
"logits/chosen": -0.7485864162445068,
|
|
"logits/rejected": -0.1424974799156189,
|
|
"logps/chosen": -91.39581298828125,
|
|
"logps/ref_chosen": -90.15489196777344,
|
|
"logps/ref_rejected": -153.15872192382812,
|
|
"logps/rejected": -158.87777709960938,
|
|
"loss": 1.0254,
|
|
"margin_dpo/margin_mean": 4.478157043457031,
|
|
"margin_dpo/margin_std": 5.330191612243652,
|
|
"step": 81
|
|
},
|
|
{
|
|
"epoch": 0.12041116005873716,
|
|
"grad_norm": 25.869966506958008,
|
|
"learning_rate": 4.995258321842611e-07,
|
|
"logits/chosen": -0.13274694979190826,
|
|
"logits/rejected": 0.08919930458068848,
|
|
"logps/chosen": -66.34426879882812,
|
|
"logps/ref_chosen": -65.0645751953125,
|
|
"logps/ref_rejected": -79.4079818725586,
|
|
"logps/rejected": -85.55947875976562,
|
|
"loss": 1.1125,
|
|
"margin_dpo/margin_mean": 4.871798992156982,
|
|
"margin_dpo/margin_std": 4.440828323364258,
|
|
"step": 82
|
|
},
|
|
{
|
|
"epoch": 0.12187958883994127,
|
|
"grad_norm": 27.123699188232422,
|
|
"learning_rate": 4.994435419342304e-07,
|
|
"logits/chosen": -0.7392994165420532,
|
|
"logits/rejected": -0.6463851928710938,
|
|
"logps/chosen": -100.03043365478516,
|
|
"logps/ref_chosen": -97.5862045288086,
|
|
"logps/ref_rejected": -144.89761352539062,
|
|
"logps/rejected": -152.01702880859375,
|
|
"loss": 1.1083,
|
|
"margin_dpo/margin_mean": 4.6751909255981445,
|
|
"margin_dpo/margin_std": 5.925076007843018,
|
|
"step": 83
|
|
},
|
|
{
|
|
"epoch": 0.12334801762114538,
|
|
"grad_norm": 28.914873123168945,
|
|
"learning_rate": 4.993546786148857e-07,
|
|
"logits/chosen": -1.024448275566101,
|
|
"logits/rejected": -0.6994166970252991,
|
|
"logps/chosen": -107.0411148071289,
|
|
"logps/ref_chosen": -104.22205352783203,
|
|
"logps/ref_rejected": -110.47735595703125,
|
|
"logps/rejected": -115.09510040283203,
|
|
"loss": 1.2085,
|
|
"margin_dpo/margin_mean": 1.7986866235733032,
|
|
"margin_dpo/margin_std": 4.0745134353637695,
|
|
"step": 84
|
|
},
|
|
{
|
|
"epoch": 0.12481644640234948,
|
|
"grad_norm": 27.426593780517578,
|
|
"learning_rate": 4.992592445678582e-07,
|
|
"logits/chosen": 0.10627250373363495,
|
|
"logits/rejected": 0.41944852471351624,
|
|
"logps/chosen": -67.73141479492188,
|
|
"logps/ref_chosen": -65.42636108398438,
|
|
"logps/ref_rejected": -77.69692993164062,
|
|
"logps/rejected": -84.1421127319336,
|
|
"loss": 1.1649,
|
|
"margin_dpo/margin_mean": 4.140119552612305,
|
|
"margin_dpo/margin_std": 8.340166091918945,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.1262848751835536,
|
|
"grad_norm": 22.034379959106445,
|
|
"learning_rate": 4.991572423079235e-07,
|
|
"logits/chosen": -0.6100976467132568,
|
|
"logits/rejected": -0.47021234035491943,
|
|
"logps/chosen": -71.08971405029297,
|
|
"logps/ref_chosen": -68.32093811035156,
|
|
"logps/ref_rejected": -81.26939392089844,
|
|
"logps/rejected": -87.36192321777344,
|
|
"loss": 1.0992,
|
|
"margin_dpo/margin_mean": 3.323758125305176,
|
|
"margin_dpo/margin_std": 5.560352325439453,
|
|
"step": 86
|
|
},
|
|
{
|
|
"epoch": 0.1277533039647577,
|
|
"grad_norm": 27.28897476196289,
|
|
"learning_rate": 4.990486745229364e-07,
|
|
"logits/chosen": -1.008190631866455,
|
|
"logits/rejected": -0.7083594799041748,
|
|
"logps/chosen": -109.26034545898438,
|
|
"logps/ref_chosen": -107.66572570800781,
|
|
"logps/ref_rejected": -116.20704650878906,
|
|
"logps/rejected": -121.74065399169922,
|
|
"loss": 1.0929,
|
|
"margin_dpo/margin_mean": 3.938999652862549,
|
|
"margin_dpo/margin_std": 4.811038017272949,
|
|
"step": 87
|
|
},
|
|
{
|
|
"epoch": 0.12922173274596183,
|
|
"grad_norm": 24.714147567749023,
|
|
"learning_rate": 4.989335440737586e-07,
|
|
"logits/chosen": -0.6558359861373901,
|
|
"logits/rejected": -0.6540915966033936,
|
|
"logps/chosen": -97.98976135253906,
|
|
"logps/ref_chosen": -93.07302856445312,
|
|
"logps/ref_rejected": -109.19470977783203,
|
|
"logps/rejected": -117.9246826171875,
|
|
"loss": 1.1388,
|
|
"margin_dpo/margin_mean": 3.813239097595215,
|
|
"margin_dpo/margin_std": 7.841992378234863,
|
|
"step": 88
|
|
},
|
|
{
|
|
"epoch": 0.13069016152716592,
|
|
"grad_norm": 24.070466995239258,
|
|
"learning_rate": 4.988118539941847e-07,
|
|
"logits/chosen": 0.05120593309402466,
|
|
"logits/rejected": 0.2133779525756836,
|
|
"logps/chosen": -78.58067321777344,
|
|
"logps/ref_chosen": -76.54256439208984,
|
|
"logps/ref_rejected": -91.92141723632812,
|
|
"logps/rejected": -98.58912658691406,
|
|
"loss": 1.1054,
|
|
"margin_dpo/margin_mean": 4.629606246948242,
|
|
"margin_dpo/margin_std": 4.11317253112793,
|
|
"step": 89
|
|
},
|
|
{
|
|
"epoch": 0.13215859030837004,
|
|
"grad_norm": 26.12279510498047,
|
|
"learning_rate": 4.986836074908615e-07,
|
|
"logits/chosen": -0.6932872533798218,
|
|
"logits/rejected": -0.4962659478187561,
|
|
"logps/chosen": -85.04541015625,
|
|
"logps/ref_chosen": -83.1722640991211,
|
|
"logps/ref_rejected": -109.89913940429688,
|
|
"logps/rejected": -116.90010070800781,
|
|
"loss": 1.0281,
|
|
"margin_dpo/margin_mean": 5.127821445465088,
|
|
"margin_dpo/margin_std": 6.227309226989746,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.13362701908957417,
|
|
"grad_norm": 26.667461395263672,
|
|
"learning_rate": 4.985488079432037e-07,
|
|
"logits/chosen": -0.5161850452423096,
|
|
"logits/rejected": -0.007033824920654297,
|
|
"logps/chosen": -76.52804565429688,
|
|
"logps/ref_chosen": -75.09698486328125,
|
|
"logps/ref_rejected": -94.42585754394531,
|
|
"logps/rejected": -102.70915222167969,
|
|
"loss": 1.0875,
|
|
"margin_dpo/margin_mean": 6.852234363555908,
|
|
"margin_dpo/margin_std": 7.726470947265625,
|
|
"step": 91
|
|
},
|
|
{
|
|
"epoch": 0.13509544787077826,
|
|
"grad_norm": 23.602184295654297,
|
|
"learning_rate": 4.984074589033043e-07,
|
|
"logits/chosen": -1.3876515626907349,
|
|
"logits/rejected": -1.1129705905914307,
|
|
"logps/chosen": -83.0078125,
|
|
"logps/ref_chosen": -79.81562805175781,
|
|
"logps/ref_rejected": -77.73824310302734,
|
|
"logps/rejected": -83.26744842529297,
|
|
"loss": 1.1189,
|
|
"margin_dpo/margin_mean": 2.3370065689086914,
|
|
"margin_dpo/margin_std": 6.140504837036133,
|
|
"step": 92
|
|
},
|
|
{
|
|
"epoch": 0.13656387665198239,
|
|
"grad_norm": 27.450332641601562,
|
|
"learning_rate": 4.982595640958425e-07,
|
|
"logits/chosen": -1.3145904541015625,
|
|
"logits/rejected": -0.6349297761917114,
|
|
"logps/chosen": -94.47523498535156,
|
|
"logps/ref_chosen": -90.42925262451172,
|
|
"logps/ref_rejected": -99.89634704589844,
|
|
"logps/rejected": -108.31857299804688,
|
|
"loss": 1.1517,
|
|
"margin_dpo/margin_mean": 4.376245498657227,
|
|
"margin_dpo/margin_std": 6.5292510986328125,
|
|
"step": 93
|
|
},
|
|
{
|
|
"epoch": 0.13803230543318648,
|
|
"grad_norm": 27.18844223022461,
|
|
"learning_rate": 4.98105127417984e-07,
|
|
"logits/chosen": -0.6020889282226562,
|
|
"logits/rejected": -0.1479286551475525,
|
|
"logps/chosen": -87.33253479003906,
|
|
"logps/ref_chosen": -84.00141906738281,
|
|
"logps/ref_rejected": -62.051231384277344,
|
|
"logps/rejected": -68.11126708984375,
|
|
"loss": 1.0268,
|
|
"margin_dpo/margin_mean": 2.7289280891418457,
|
|
"margin_dpo/margin_std": 6.002341270446777,
|
|
"step": 94
|
|
},
|
|
{
|
|
"epoch": 0.1395007342143906,
|
|
"grad_norm": 22.77342987060547,
|
|
"learning_rate": 4.979441529392784e-07,
|
|
"logits/chosen": -1.6398792266845703,
|
|
"logits/rejected": -0.8445466160774231,
|
|
"logps/chosen": -85.50116729736328,
|
|
"logps/ref_chosen": -83.24620056152344,
|
|
"logps/ref_rejected": -112.14282989501953,
|
|
"logps/rejected": -120.98870849609375,
|
|
"loss": 1.0895,
|
|
"margin_dpo/margin_mean": 6.590925693511963,
|
|
"margin_dpo/margin_std": 7.665090560913086,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.14096916299559473,
|
|
"grad_norm": 23.296894073486328,
|
|
"learning_rate": 4.977766449015534e-07,
|
|
"logits/chosen": -0.34511518478393555,
|
|
"logits/rejected": -0.08061528205871582,
|
|
"logps/chosen": -123.06785583496094,
|
|
"logps/ref_chosen": -120.4042739868164,
|
|
"logps/ref_rejected": -105.736328125,
|
|
"logps/rejected": -115.217041015625,
|
|
"loss": 1.0107,
|
|
"margin_dpo/margin_mean": 6.817150115966797,
|
|
"margin_dpo/margin_std": 9.70801067352295,
|
|
"step": 96
|
|
},
|
|
{
|
|
"epoch": 0.14243759177679882,
|
|
"grad_norm": 27.008302688598633,
|
|
"learning_rate": 4.976026077188012e-07,
|
|
"logits/chosen": -0.5496343374252319,
|
|
"logits/rejected": 0.04007942974567413,
|
|
"logps/chosen": -92.0294189453125,
|
|
"logps/ref_chosen": -89.40409851074219,
|
|
"logps/ref_rejected": -72.7824478149414,
|
|
"logps/rejected": -81.84813690185547,
|
|
"loss": 1.0255,
|
|
"margin_dpo/margin_mean": 6.44036865234375,
|
|
"margin_dpo/margin_std": 6.640345096588135,
|
|
"step": 97
|
|
},
|
|
{
|
|
"epoch": 0.14390602055800295,
|
|
"grad_norm": 24.74591636657715,
|
|
"learning_rate": 4.974220459770639e-07,
|
|
"logits/chosen": -0.39552319049835205,
|
|
"logits/rejected": -0.35785946249961853,
|
|
"logps/chosen": -79.35165405273438,
|
|
"logps/ref_chosen": -77.14939880371094,
|
|
"logps/ref_rejected": -102.4090576171875,
|
|
"logps/rejected": -111.64566802978516,
|
|
"loss": 0.9763,
|
|
"margin_dpo/margin_mean": 7.034348964691162,
|
|
"margin_dpo/margin_std": 5.715366363525391,
|
|
"step": 98
|
|
},
|
|
{
|
|
"epoch": 0.14537444933920704,
|
|
"grad_norm": 23.71422004699707,
|
|
"learning_rate": 4.972349644343108e-07,
|
|
"logits/chosen": -0.6945886611938477,
|
|
"logits/rejected": -0.29277852177619934,
|
|
"logps/chosen": -69.09884643554688,
|
|
"logps/ref_chosen": -67.58863067626953,
|
|
"logps/ref_rejected": -80.49214172363281,
|
|
"logps/rejected": -87.75712585449219,
|
|
"loss": 0.9865,
|
|
"margin_dpo/margin_mean": 5.754771709442139,
|
|
"margin_dpo/margin_std": 6.445099353790283,
|
|
"step": 99
|
|
},
|
|
{
|
|
"epoch": 0.14684287812041116,
|
|
"grad_norm": 22.417694091796875,
|
|
"learning_rate": 4.970413680203148e-07,
|
|
"logits/chosen": -1.128528356552124,
|
|
"logits/rejected": -0.9991703033447266,
|
|
"logps/chosen": -73.37350463867188,
|
|
"logps/ref_chosen": -70.40953826904297,
|
|
"logps/ref_rejected": -77.42765808105469,
|
|
"logps/rejected": -83.2952880859375,
|
|
"loss": 1.0426,
|
|
"margin_dpo/margin_mean": 2.903665065765381,
|
|
"margin_dpo/margin_std": 4.343265056610107,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.14684287812041116,
|
|
"eval_logits/chosen": -0.05655202642083168,
|
|
"eval_logits/rejected": 0.19737417995929718,
|
|
"eval_logps/chosen": -105.63566589355469,
|
|
"eval_logps/ref_chosen": -101.88616943359375,
|
|
"eval_logps/ref_rejected": -92.68607330322266,
|
|
"eval_logps/rejected": -99.1901626586914,
|
|
"eval_loss": 0.6031284928321838,
|
|
"eval_margin_dpo/margin_mean": 2.754598617553711,
|
|
"eval_margin_dpo/margin_std": 5.884398937225342,
|
|
"eval_runtime": 44.1021,
|
|
"eval_samples_per_second": 53.036,
|
|
"eval_steps_per_second": 1.678,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.14831130690161526,
|
|
"grad_norm": 28.25216293334961,
|
|
"learning_rate": 4.968412618365215e-07,
|
|
"logits/chosen": -0.1628236174583435,
|
|
"logits/rejected": -0.21538278460502625,
|
|
"logps/chosen": -128.88844299316406,
|
|
"logps/ref_chosen": -123.95951843261719,
|
|
"logps/ref_rejected": -124.9482421875,
|
|
"logps/rejected": -136.82745361328125,
|
|
"loss": 1.1382,
|
|
"margin_dpo/margin_mean": 6.950290203094482,
|
|
"margin_dpo/margin_std": 8.697531700134277,
|
|
"step": 101
|
|
},
|
|
{
|
|
"epoch": 0.14977973568281938,
|
|
"grad_norm": 35.67359924316406,
|
|
"learning_rate": 4.966346511559149e-07,
|
|
"logits/chosen": -0.657694935798645,
|
|
"logits/rejected": -0.1648797243833542,
|
|
"logps/chosen": -133.60629272460938,
|
|
"logps/ref_chosen": -128.27081298828125,
|
|
"logps/ref_rejected": -83.74739074707031,
|
|
"logps/rejected": -90.74542236328125,
|
|
"loss": 1.1861,
|
|
"margin_dpo/margin_mean": 1.6625572443008423,
|
|
"margin_dpo/margin_std": 7.926374912261963,
|
|
"step": 102
|
|
},
|
|
{
|
|
"epoch": 0.1512481644640235,
|
|
"grad_norm": 26.774639129638672,
|
|
"learning_rate": 4.964215414228785e-07,
|
|
"logits/chosen": -0.28504350781440735,
|
|
"logits/rejected": 0.10399870574474335,
|
|
"logps/chosen": -82.16940307617188,
|
|
"logps/ref_chosen": -79.08724212646484,
|
|
"logps/ref_rejected": -58.38372802734375,
|
|
"logps/rejected": -63.93667984008789,
|
|
"loss": 0.997,
|
|
"margin_dpo/margin_mean": 2.47078800201416,
|
|
"margin_dpo/margin_std": 4.759175777435303,
|
|
"step": 103
|
|
},
|
|
{
|
|
"epoch": 0.1527165932452276,
|
|
"grad_norm": 22.17858123779297,
|
|
"learning_rate": 4.96201938253052e-07,
|
|
"logits/chosen": -1.632993459701538,
|
|
"logits/rejected": -0.9502384066581726,
|
|
"logps/chosen": -81.82193756103516,
|
|
"logps/ref_chosen": -79.506103515625,
|
|
"logps/ref_rejected": -77.93867492675781,
|
|
"logps/rejected": -85.17577362060547,
|
|
"loss": 0.9781,
|
|
"margin_dpo/margin_mean": 4.921270847320557,
|
|
"margin_dpo/margin_std": 6.682742118835449,
|
|
"step": 104
|
|
},
|
|
{
|
|
"epoch": 0.15418502202643172,
|
|
"grad_norm": 26.013565063476562,
|
|
"learning_rate": 4.959758474331832e-07,
|
|
"logits/chosen": -1.6654616594314575,
|
|
"logits/rejected": -1.016417384147644,
|
|
"logps/chosen": -129.20193481445312,
|
|
"logps/ref_chosen": -123.54203796386719,
|
|
"logps/ref_rejected": -97.36883544921875,
|
|
"logps/rejected": -104.8051528930664,
|
|
"loss": 0.9886,
|
|
"margin_dpo/margin_mean": 1.7764201164245605,
|
|
"margin_dpo/margin_std": 7.046723365783691,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.15565345080763582,
|
|
"grad_norm": 25.039297103881836,
|
|
"learning_rate": 4.957432749209755e-07,
|
|
"logits/chosen": -0.7109205722808838,
|
|
"logits/rejected": -0.1984243392944336,
|
|
"logps/chosen": -68.43580627441406,
|
|
"logps/ref_chosen": -65.17315673828125,
|
|
"logps/ref_rejected": -73.81947326660156,
|
|
"logps/rejected": -82.34314727783203,
|
|
"loss": 1.0052,
|
|
"margin_dpo/margin_mean": 5.261022090911865,
|
|
"margin_dpo/margin_std": 6.1609601974487305,
|
|
"step": 106
|
|
},
|
|
{
|
|
"epoch": 0.15712187958883994,
|
|
"grad_norm": 25.406858444213867,
|
|
"learning_rate": 4.955042268449307e-07,
|
|
"logits/chosen": -0.551948070526123,
|
|
"logits/rejected": -0.09590325504541397,
|
|
"logps/chosen": -80.75108337402344,
|
|
"logps/ref_chosen": -77.79078674316406,
|
|
"logps/ref_rejected": -82.89598083496094,
|
|
"logps/rejected": -88.46271514892578,
|
|
"loss": 0.9866,
|
|
"margin_dpo/margin_mean": 2.6064443588256836,
|
|
"margin_dpo/margin_std": 5.637351036071777,
|
|
"step": 107
|
|
},
|
|
{
|
|
"epoch": 0.15859030837004406,
|
|
"grad_norm": 24.171070098876953,
|
|
"learning_rate": 4.952587095041881e-07,
|
|
"logits/chosen": -0.9575413465499878,
|
|
"logits/rejected": -0.5017733573913574,
|
|
"logps/chosen": -132.8676300048828,
|
|
"logps/ref_chosen": -128.1168212890625,
|
|
"logps/ref_rejected": -125.18524932861328,
|
|
"logps/rejected": -132.35462951660156,
|
|
"loss": 0.9856,
|
|
"margin_dpo/margin_mean": 2.4185736179351807,
|
|
"margin_dpo/margin_std": 7.777304649353027,
|
|
"step": 108
|
|
},
|
|
{
|
|
"epoch": 0.16005873715124816,
|
|
"grad_norm": 23.65253257751465,
|
|
"learning_rate": 4.95006729368358e-07,
|
|
"logits/chosen": -1.0863566398620605,
|
|
"logits/rejected": -1.1219680309295654,
|
|
"logps/chosen": -79.58955383300781,
|
|
"logps/ref_chosen": -77.46507263183594,
|
|
"logps/ref_rejected": -105.97697448730469,
|
|
"logps/rejected": -115.61421203613281,
|
|
"loss": 0.9014,
|
|
"margin_dpo/margin_mean": 7.51275634765625,
|
|
"margin_dpo/margin_std": 6.889702320098877,
|
|
"step": 109
|
|
},
|
|
{
|
|
"epoch": 0.16152716593245228,
|
|
"grad_norm": 27.846046447753906,
|
|
"learning_rate": 4.947482930773511e-07,
|
|
"logits/chosen": -0.27717405557632446,
|
|
"logits/rejected": 0.07007038593292236,
|
|
"logps/chosen": -121.49076843261719,
|
|
"logps/ref_chosen": -118.46145629882812,
|
|
"logps/ref_rejected": -89.87979888916016,
|
|
"logps/rejected": -97.38516235351562,
|
|
"loss": 1.071,
|
|
"margin_dpo/margin_mean": 4.476062774658203,
|
|
"margin_dpo/margin_std": 6.243172645568848,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.16299559471365638,
|
|
"grad_norm": 22.43236541748047,
|
|
"learning_rate": 4.944834074412042e-07,
|
|
"logits/chosen": -1.0582770109176636,
|
|
"logits/rejected": -1.1601223945617676,
|
|
"logps/chosen": -85.39334106445312,
|
|
"logps/ref_chosen": -82.5454330444336,
|
|
"logps/ref_rejected": -126.61203002929688,
|
|
"logps/rejected": -140.41445922851562,
|
|
"loss": 0.8494,
|
|
"margin_dpo/margin_mean": 10.954520225524902,
|
|
"margin_dpo/margin_std": 10.141668319702148,
|
|
"step": 111
|
|
},
|
|
{
|
|
"epoch": 0.1644640234948605,
|
|
"grad_norm": 25.659700393676758,
|
|
"learning_rate": 4.942120794399002e-07,
|
|
"logits/chosen": -0.23621593415737152,
|
|
"logits/rejected": -0.1482967883348465,
|
|
"logps/chosen": -64.30489349365234,
|
|
"logps/ref_chosen": -60.22978973388672,
|
|
"logps/ref_rejected": -87.82637023925781,
|
|
"logps/rejected": -96.92811584472656,
|
|
"loss": 1.0748,
|
|
"margin_dpo/margin_mean": 5.026644706726074,
|
|
"margin_dpo/margin_std": 6.663045883178711,
|
|
"step": 112
|
|
},
|
|
{
|
|
"epoch": 0.16593245227606462,
|
|
"grad_norm": 28.988935470581055,
|
|
"learning_rate": 4.939343162231841e-07,
|
|
"logits/chosen": -0.25600025057792664,
|
|
"logits/rejected": -0.23611587285995483,
|
|
"logps/chosen": -79.95890808105469,
|
|
"logps/ref_chosen": -76.71855926513672,
|
|
"logps/ref_rejected": -109.30464172363281,
|
|
"logps/rejected": -118.75503540039062,
|
|
"loss": 1.0397,
|
|
"margin_dpo/margin_mean": 6.210036277770996,
|
|
"margin_dpo/margin_std": 6.891805648803711,
|
|
"step": 113
|
|
},
|
|
{
|
|
"epoch": 0.16740088105726872,
|
|
"grad_norm": 29.809431076049805,
|
|
"learning_rate": 4.936501251103751e-07,
|
|
"logits/chosen": -1.982461929321289,
|
|
"logits/rejected": -1.0373061895370483,
|
|
"logps/chosen": -117.84416198730469,
|
|
"logps/ref_chosen": -114.472412109375,
|
|
"logps/ref_rejected": -98.01756286621094,
|
|
"logps/rejected": -111.39344787597656,
|
|
"loss": 0.9951,
|
|
"margin_dpo/margin_mean": 10.004129409790039,
|
|
"margin_dpo/margin_std": 10.323932647705078,
|
|
"step": 114
|
|
},
|
|
{
|
|
"epoch": 0.16886930983847284,
|
|
"grad_norm": 25.537490844726562,
|
|
"learning_rate": 4.933595135901732e-07,
|
|
"logits/chosen": -0.7669371962547302,
|
|
"logits/rejected": -0.15004056692123413,
|
|
"logps/chosen": -80.99656677246094,
|
|
"logps/ref_chosen": -79.65080261230469,
|
|
"logps/ref_rejected": -74.55479431152344,
|
|
"logps/rejected": -83.04512023925781,
|
|
"loss": 0.9494,
|
|
"margin_dpo/margin_mean": 7.144563674926758,
|
|
"margin_dpo/margin_std": 6.993829727172852,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.17033773861967694,
|
|
"grad_norm": 22.20758628845215,
|
|
"learning_rate": 4.930624893204624e-07,
|
|
"logits/chosen": -0.7407585382461548,
|
|
"logits/rejected": -0.43094000220298767,
|
|
"logps/chosen": -74.30891418457031,
|
|
"logps/ref_chosen": -70.655517578125,
|
|
"logps/ref_rejected": -57.157615661621094,
|
|
"logps/rejected": -63.68486404418945,
|
|
"loss": 0.9878,
|
|
"margin_dpo/margin_mean": 2.8738439083099365,
|
|
"margin_dpo/margin_std": 4.748990058898926,
|
|
"step": 116
|
|
},
|
|
{
|
|
"epoch": 0.17180616740088106,
|
|
"grad_norm": 36.41432571411133,
|
|
"learning_rate": 4.927590601281083e-07,
|
|
"logits/chosen": -1.0424773693084717,
|
|
"logits/rejected": -0.5706847906112671,
|
|
"logps/chosen": -87.08619689941406,
|
|
"logps/ref_chosen": -82.48936462402344,
|
|
"logps/ref_rejected": -76.40602111816406,
|
|
"logps/rejected": -85.58804321289062,
|
|
"loss": 1.0785,
|
|
"margin_dpo/margin_mean": 4.585183620452881,
|
|
"margin_dpo/margin_std": 8.649291038513184,
|
|
"step": 117
|
|
},
|
|
{
|
|
"epoch": 0.17327459618208516,
|
|
"grad_norm": 21.416038513183594,
|
|
"learning_rate": 4.924492340087524e-07,
|
|
"logits/chosen": -0.19199329614639282,
|
|
"logits/rejected": 0.20044465363025665,
|
|
"logps/chosen": -90.31568908691406,
|
|
"logps/ref_chosen": -89.27349853515625,
|
|
"logps/ref_rejected": -82.73481750488281,
|
|
"logps/rejected": -90.73196411132812,
|
|
"loss": 0.9264,
|
|
"margin_dpo/margin_mean": 6.954972743988037,
|
|
"margin_dpo/margin_std": 6.001160144805908,
|
|
"step": 118
|
|
},
|
|
{
|
|
"epoch": 0.17474302496328928,
|
|
"grad_norm": 28.66317367553711,
|
|
"learning_rate": 4.92133019126601e-07,
|
|
"logits/chosen": -0.8395548462867737,
|
|
"logits/rejected": -0.40599122643470764,
|
|
"logps/chosen": -137.56922912597656,
|
|
"logps/ref_chosen": -135.48094177246094,
|
|
"logps/ref_rejected": -134.4525909423828,
|
|
"logps/rejected": -144.9943389892578,
|
|
"loss": 0.9982,
|
|
"margin_dpo/margin_mean": 8.453452110290527,
|
|
"margin_dpo/margin_std": 9.262985229492188,
|
|
"step": 119
|
|
},
|
|
{
|
|
"epoch": 0.1762114537444934,
|
|
"grad_norm": 35.21387481689453,
|
|
"learning_rate": 4.918104238142103e-07,
|
|
"logits/chosen": -1.461838722229004,
|
|
"logits/rejected": -0.18029293417930603,
|
|
"logps/chosen": -149.92428588867188,
|
|
"logps/ref_chosen": -145.44580078125,
|
|
"logps/ref_rejected": -104.62899780273438,
|
|
"logps/rejected": -118.67092895507812,
|
|
"loss": 0.921,
|
|
"margin_dpo/margin_mean": 9.563423156738281,
|
|
"margin_dpo/margin_std": 11.283243179321289,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.1776798825256975,
|
|
"grad_norm": 24.13612937927246,
|
|
"learning_rate": 4.91481456572267e-07,
|
|
"logits/chosen": -0.112807996571064,
|
|
"logits/rejected": 0.10981197655200958,
|
|
"logps/chosen": -79.19808959960938,
|
|
"logps/ref_chosen": -77.02085876464844,
|
|
"logps/ref_rejected": -99.70750427246094,
|
|
"logps/rejected": -108.17426300048828,
|
|
"loss": 0.8526,
|
|
"margin_dpo/margin_mean": 6.289523124694824,
|
|
"margin_dpo/margin_std": 6.245270252227783,
|
|
"step": 121
|
|
},
|
|
{
|
|
"epoch": 0.17914831130690162,
|
|
"grad_norm": 23.817363739013672,
|
|
"learning_rate": 4.911461260693638e-07,
|
|
"logits/chosen": -0.4354201555252075,
|
|
"logits/rejected": -0.6057559251785278,
|
|
"logps/chosen": -77.3744888305664,
|
|
"logps/ref_chosen": -73.7746353149414,
|
|
"logps/ref_rejected": -106.66211700439453,
|
|
"logps/rejected": -117.64198303222656,
|
|
"loss": 0.836,
|
|
"margin_dpo/margin_mean": 7.380002975463867,
|
|
"margin_dpo/margin_std": 8.832366943359375,
|
|
"step": 122
|
|
},
|
|
{
|
|
"epoch": 0.18061674008810572,
|
|
"grad_norm": 23.779541015625,
|
|
"learning_rate": 4.908044411417711e-07,
|
|
"logits/chosen": -0.24536287784576416,
|
|
"logits/rejected": -0.17662711441516876,
|
|
"logps/chosen": -82.35337829589844,
|
|
"logps/ref_chosen": -79.58645629882812,
|
|
"logps/ref_rejected": -91.20085144042969,
|
|
"logps/rejected": -100.98405456542969,
|
|
"loss": 0.9648,
|
|
"margin_dpo/margin_mean": 7.016263484954834,
|
|
"margin_dpo/margin_std": 4.6288957595825195,
|
|
"step": 123
|
|
},
|
|
{
|
|
"epoch": 0.18208516886930984,
|
|
"grad_norm": 25.46894073486328,
|
|
"learning_rate": 4.904564107932048e-07,
|
|
"logits/chosen": -1.0531779527664185,
|
|
"logits/rejected": -0.9245602488517761,
|
|
"logps/chosen": -107.60164642333984,
|
|
"logps/ref_chosen": -105.69091796875,
|
|
"logps/ref_rejected": -128.0390625,
|
|
"logps/rejected": -138.65370178222656,
|
|
"loss": 0.9621,
|
|
"margin_dpo/margin_mean": 8.70390510559082,
|
|
"margin_dpo/margin_std": 12.833571434020996,
|
|
"step": 124
|
|
},
|
|
{
|
|
"epoch": 0.18355359765051396,
|
|
"grad_norm": 29.539167404174805,
|
|
"learning_rate": 4.90102044194588e-07,
|
|
"logits/chosen": -0.7907294631004333,
|
|
"logits/rejected": -0.3655990660190582,
|
|
"logps/chosen": -73.57737731933594,
|
|
"logps/ref_chosen": -68.22820281982422,
|
|
"logps/ref_rejected": -78.79598236083984,
|
|
"logps/rejected": -87.29751586914062,
|
|
"loss": 1.0325,
|
|
"margin_dpo/margin_mean": 3.152360200881958,
|
|
"margin_dpo/margin_std": 8.048112869262695,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.18502202643171806,
|
|
"grad_norm": 23.945877075195312,
|
|
"learning_rate": 4.897413506838102e-07,
|
|
"logits/chosen": -1.0596747398376465,
|
|
"logits/rejected": -0.6876899600028992,
|
|
"logps/chosen": -129.45509338378906,
|
|
"logps/ref_chosen": -126.91588592529297,
|
|
"logps/ref_rejected": -133.4127655029297,
|
|
"logps/rejected": -144.79702758789062,
|
|
"loss": 0.8182,
|
|
"margin_dpo/margin_mean": 8.845060348510742,
|
|
"margin_dpo/margin_std": 8.391151428222656,
|
|
"step": 126
|
|
},
|
|
{
|
|
"epoch": 0.18649045521292218,
|
|
"grad_norm": 27.48787498474121,
|
|
"learning_rate": 4.89374339765481e-07,
|
|
"logits/chosen": -1.344987392425537,
|
|
"logits/rejected": -0.998265266418457,
|
|
"logps/chosen": -109.96749877929688,
|
|
"logps/ref_chosen": -106.9218978881836,
|
|
"logps/ref_rejected": -73.36544036865234,
|
|
"logps/rejected": -81.34496307373047,
|
|
"loss": 0.992,
|
|
"margin_dpo/margin_mean": 4.933928966522217,
|
|
"margin_dpo/margin_std": 8.037854194641113,
|
|
"step": 127
|
|
},
|
|
{
|
|
"epoch": 0.18795888399412627,
|
|
"grad_norm": 30.776973724365234,
|
|
"learning_rate": 4.890010211106795e-07,
|
|
"logits/chosen": -1.3177127838134766,
|
|
"logits/rejected": -1.1496309041976929,
|
|
"logps/chosen": -71.75405883789062,
|
|
"logps/ref_chosen": -68.51551055908203,
|
|
"logps/ref_rejected": -81.37910461425781,
|
|
"logps/rejected": -92.6358642578125,
|
|
"loss": 1.016,
|
|
"margin_dpo/margin_mean": 8.018218994140625,
|
|
"margin_dpo/margin_std": 7.071664810180664,
|
|
"step": 128
|
|
},
|
|
{
|
|
"epoch": 0.1894273127753304,
|
|
"grad_norm": 25.891700744628906,
|
|
"learning_rate": 4.88621404556699e-07,
|
|
"logits/chosen": -0.7863626480102539,
|
|
"logits/rejected": -0.49298733472824097,
|
|
"logps/chosen": -72.28971099853516,
|
|
"logps/ref_chosen": -67.88934326171875,
|
|
"logps/ref_rejected": -81.64795684814453,
|
|
"logps/rejected": -92.97059631347656,
|
|
"loss": 0.9865,
|
|
"margin_dpo/margin_mean": 6.922273635864258,
|
|
"margin_dpo/margin_std": 9.410476684570312,
|
|
"step": 129
|
|
},
|
|
{
|
|
"epoch": 0.19089574155653452,
|
|
"grad_norm": 21.4912109375,
|
|
"learning_rate": 4.882355001067891e-07,
|
|
"logits/chosen": -1.364875078201294,
|
|
"logits/rejected": -1.045945644378662,
|
|
"logps/chosen": -81.25286102294922,
|
|
"logps/ref_chosen": -78.04290008544922,
|
|
"logps/ref_rejected": -87.63352966308594,
|
|
"logps/rejected": -101.17716979980469,
|
|
"loss": 0.8669,
|
|
"margin_dpo/margin_mean": 10.333681106567383,
|
|
"margin_dpo/margin_std": 10.306255340576172,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.19236417033773862,
|
|
"grad_norm": 22.531719207763672,
|
|
"learning_rate": 4.878433179298909e-07,
|
|
"logits/chosen": -0.6210666298866272,
|
|
"logits/rejected": -0.29343628883361816,
|
|
"logps/chosen": -95.02635192871094,
|
|
"logps/ref_chosen": -92.93765258789062,
|
|
"logps/ref_rejected": -105.63180541992188,
|
|
"logps/rejected": -113.58657836914062,
|
|
"loss": 0.9294,
|
|
"margin_dpo/margin_mean": 5.866049289703369,
|
|
"margin_dpo/margin_std": 11.821134567260742,
|
|
"step": 131
|
|
},
|
|
{
|
|
"epoch": 0.19383259911894274,
|
|
"grad_norm": 28.748842239379883,
|
|
"learning_rate": 4.874448683603694e-07,
|
|
"logits/chosen": -0.5462979674339294,
|
|
"logits/rejected": -0.1083882749080658,
|
|
"logps/chosen": -109.67427062988281,
|
|
"logps/ref_chosen": -106.78312683105469,
|
|
"logps/ref_rejected": -98.28593444824219,
|
|
"logps/rejected": -109.65825653076172,
|
|
"loss": 0.8907,
|
|
"margin_dpo/margin_mean": 8.481182098388672,
|
|
"margin_dpo/margin_std": 8.121637344360352,
|
|
"step": 132
|
|
},
|
|
{
|
|
"epoch": 0.19530102790014683,
|
|
"grad_norm": 23.444847106933594,
|
|
"learning_rate": 4.870401618977415e-07,
|
|
"logits/chosen": -1.1360797882080078,
|
|
"logits/rejected": -0.8285423517227173,
|
|
"logps/chosen": -103.25596618652344,
|
|
"logps/ref_chosen": -100.59583282470703,
|
|
"logps/ref_rejected": -97.97810363769531,
|
|
"logps/rejected": -107.94853210449219,
|
|
"loss": 0.8704,
|
|
"margin_dpo/margin_mean": 7.310290336608887,
|
|
"margin_dpo/margin_std": 9.262811660766602,
|
|
"step": 133
|
|
},
|
|
{
|
|
"epoch": 0.19676945668135096,
|
|
"grad_norm": 21.67630958557129,
|
|
"learning_rate": 4.866292092063986e-07,
|
|
"logits/chosen": -0.6046867370605469,
|
|
"logits/rejected": -0.4987502992153168,
|
|
"logps/chosen": -100.99070739746094,
|
|
"logps/ref_chosen": -95.42115783691406,
|
|
"logps/ref_rejected": -100.67765808105469,
|
|
"logps/rejected": -112.1727523803711,
|
|
"loss": 0.8261,
|
|
"margin_dpo/margin_mean": 5.9255475997924805,
|
|
"margin_dpo/margin_std": 8.01714038848877,
|
|
"step": 134
|
|
},
|
|
{
|
|
"epoch": 0.19823788546255505,
|
|
"grad_norm": 29.390384674072266,
|
|
"learning_rate": 4.862120211153265e-07,
|
|
"logits/chosen": 0.34038400650024414,
|
|
"logits/rejected": 0.10170932114124298,
|
|
"logps/chosen": -74.57313537597656,
|
|
"logps/ref_chosen": -71.50714111328125,
|
|
"logps/ref_rejected": -195.5370330810547,
|
|
"logps/rejected": -207.61270141601562,
|
|
"loss": 0.9108,
|
|
"margin_dpo/margin_mean": 9.009686470031738,
|
|
"margin_dpo/margin_std": 8.271293640136719,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.19970631424375918,
|
|
"grad_norm": 22.794597625732422,
|
|
"learning_rate": 4.857886086178193e-07,
|
|
"logits/chosen": -0.5607945919036865,
|
|
"logits/rejected": -0.4255688488483429,
|
|
"logps/chosen": -98.30638122558594,
|
|
"logps/ref_chosen": -93.96214294433594,
|
|
"logps/ref_rejected": -106.4173812866211,
|
|
"logps/rejected": -118.60725402832031,
|
|
"loss": 0.8187,
|
|
"margin_dpo/margin_mean": 7.845623016357422,
|
|
"margin_dpo/margin_std": 5.256319999694824,
|
|
"step": 136
|
|
},
|
|
{
|
|
"epoch": 0.2011747430249633,
|
|
"grad_norm": 22.39464569091797,
|
|
"learning_rate": 4.853589828711902e-07,
|
|
"logits/chosen": -1.5836092233657837,
|
|
"logits/rejected": -1.3331845998764038,
|
|
"logps/chosen": -78.58103942871094,
|
|
"logps/ref_chosen": -79.13609313964844,
|
|
"logps/ref_rejected": -118.01248168945312,
|
|
"logps/rejected": -131.11453247070312,
|
|
"loss": 0.6806,
|
|
"margin_dpo/margin_mean": 13.657103538513184,
|
|
"margin_dpo/margin_std": 11.201322555541992,
|
|
"step": 137
|
|
},
|
|
{
|
|
"epoch": 0.2026431718061674,
|
|
"grad_norm": 22.287464141845703,
|
|
"learning_rate": 4.849231551964771e-07,
|
|
"logits/chosen": -0.6701608300209045,
|
|
"logits/rejected": -0.45578303933143616,
|
|
"logps/chosen": -107.3237533569336,
|
|
"logps/ref_chosen": -104.25080871582031,
|
|
"logps/ref_rejected": -103.565185546875,
|
|
"logps/rejected": -113.90208435058594,
|
|
"loss": 0.8695,
|
|
"margin_dpo/margin_mean": 7.2639617919921875,
|
|
"margin_dpo/margin_std": 8.118474960327148,
|
|
"step": 138
|
|
},
|
|
{
|
|
"epoch": 0.20411160058737152,
|
|
"grad_norm": 23.708158493041992,
|
|
"learning_rate": 4.844811370781446e-07,
|
|
"logits/chosen": -1.3766424655914307,
|
|
"logits/rejected": -0.7842209935188293,
|
|
"logps/chosen": -94.10173797607422,
|
|
"logps/ref_chosen": -91.9404296875,
|
|
"logps/ref_rejected": -84.98133850097656,
|
|
"logps/rejected": -97.67942810058594,
|
|
"loss": 0.8457,
|
|
"margin_dpo/margin_mean": 10.536781311035156,
|
|
"margin_dpo/margin_std": 13.584081649780273,
|
|
"step": 139
|
|
},
|
|
{
|
|
"epoch": 0.2055800293685756,
|
|
"grad_norm": 22.01458168029785,
|
|
"learning_rate": 4.840329401637809e-07,
|
|
"logits/chosen": -1.1145687103271484,
|
|
"logits/rejected": -0.8994247913360596,
|
|
"logps/chosen": -83.47078704833984,
|
|
"logps/ref_chosen": -79.87215423583984,
|
|
"logps/ref_rejected": -98.67296600341797,
|
|
"logps/rejected": -112.2000732421875,
|
|
"loss": 0.8478,
|
|
"margin_dpo/margin_mean": 9.92848014831543,
|
|
"margin_dpo/margin_std": 8.664599418640137,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20704845814977973,
|
|
"grad_norm": 27.232759475708008,
|
|
"learning_rate": 4.83578576263792e-07,
|
|
"logits/chosen": -1.0526609420776367,
|
|
"logits/rejected": -0.5129063129425049,
|
|
"logps/chosen": -140.94210815429688,
|
|
"logps/ref_chosen": -133.52174377441406,
|
|
"logps/ref_rejected": -111.38741302490234,
|
|
"logps/rejected": -125.99818420410156,
|
|
"loss": 0.7886,
|
|
"margin_dpo/margin_mean": 7.190412521362305,
|
|
"margin_dpo/margin_std": 9.38884162902832,
|
|
"step": 141
|
|
},
|
|
{
|
|
"epoch": 0.20851688693098386,
|
|
"grad_norm": 22.663490295410156,
|
|
"learning_rate": 4.83118057351089e-07,
|
|
"logits/chosen": -1.2602531909942627,
|
|
"logits/rejected": -1.1595778465270996,
|
|
"logps/chosen": -106.17414093017578,
|
|
"logps/ref_chosen": -105.53807067871094,
|
|
"logps/ref_rejected": -120.66288757324219,
|
|
"logps/rejected": -135.26641845703125,
|
|
"loss": 0.819,
|
|
"margin_dpo/margin_mean": 13.967466354370117,
|
|
"margin_dpo/margin_std": 12.330192565917969,
|
|
"step": 142
|
|
},
|
|
{
|
|
"epoch": 0.20998531571218795,
|
|
"grad_norm": 23.751354217529297,
|
|
"learning_rate": 4.826513955607734e-07,
|
|
"logits/chosen": -0.888142466545105,
|
|
"logits/rejected": -0.4934452772140503,
|
|
"logps/chosen": -92.56117248535156,
|
|
"logps/ref_chosen": -89.25540161132812,
|
|
"logps/ref_rejected": -85.94788360595703,
|
|
"logps/rejected": -96.44986724853516,
|
|
"loss": 0.9112,
|
|
"margin_dpo/margin_mean": 7.196202278137207,
|
|
"margin_dpo/margin_std": 9.4541015625,
|
|
"step": 143
|
|
},
|
|
{
|
|
"epoch": 0.21145374449339208,
|
|
"grad_norm": 31.30705451965332,
|
|
"learning_rate": 4.821786031898176e-07,
|
|
"logits/chosen": -0.9406213760375977,
|
|
"logits/rejected": -0.4508327841758728,
|
|
"logps/chosen": -101.48098754882812,
|
|
"logps/ref_chosen": -100.95454406738281,
|
|
"logps/ref_rejected": -88.343017578125,
|
|
"logps/rejected": -100.29680633544922,
|
|
"loss": 0.9176,
|
|
"margin_dpo/margin_mean": 11.427347183227539,
|
|
"margin_dpo/margin_std": 12.994205474853516,
|
|
"step": 144
|
|
},
|
|
{
|
|
"epoch": 0.21292217327459617,
|
|
"grad_norm": 23.50806999206543,
|
|
"learning_rate": 4.816996926967401e-07,
|
|
"logits/chosen": -0.6580193042755127,
|
|
"logits/rejected": 0.0010183602571487427,
|
|
"logps/chosen": -116.13957977294922,
|
|
"logps/ref_chosen": -115.84953308105469,
|
|
"logps/ref_rejected": -90.881591796875,
|
|
"logps/rejected": -103.59181213378906,
|
|
"loss": 0.8195,
|
|
"margin_dpo/margin_mean": 12.420181274414062,
|
|
"margin_dpo/margin_std": 8.770981788635254,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.2143906020558003,
|
|
"grad_norm": 25.116533279418945,
|
|
"learning_rate": 4.812146767012779e-07,
|
|
"logits/chosen": -0.485411673784256,
|
|
"logits/rejected": -0.17881183326244354,
|
|
"logps/chosen": -81.39533233642578,
|
|
"logps/ref_chosen": -78.54154968261719,
|
|
"logps/ref_rejected": -75.02702331542969,
|
|
"logps/rejected": -88.35206604003906,
|
|
"loss": 0.7418,
|
|
"margin_dpo/margin_mean": 10.471261978149414,
|
|
"margin_dpo/margin_std": 8.270082473754883,
|
|
"step": 146
|
|
},
|
|
{
|
|
"epoch": 0.21585903083700442,
|
|
"grad_norm": 21.721435546875,
|
|
"learning_rate": 4.807235679840536e-07,
|
|
"logits/chosen": -0.47551432251930237,
|
|
"logits/rejected": 0.09487782418727875,
|
|
"logps/chosen": -82.86421203613281,
|
|
"logps/ref_chosen": -83.60895538330078,
|
|
"logps/ref_rejected": -65.389404296875,
|
|
"logps/rejected": -79.1151123046875,
|
|
"loss": 0.7268,
|
|
"margin_dpo/margin_mean": 14.47046184539795,
|
|
"margin_dpo/margin_std": 9.606327056884766,
|
|
"step": 147
|
|
},
|
|
{
|
|
"epoch": 0.2173274596182085,
|
|
"grad_norm": 24.2994327545166,
|
|
"learning_rate": 4.802263794862384e-07,
|
|
"logits/chosen": -1.1637771129608154,
|
|
"logits/rejected": -0.457122266292572,
|
|
"logps/chosen": -120.93125915527344,
|
|
"logps/ref_chosen": -122.75179290771484,
|
|
"logps/ref_rejected": -121.82928466796875,
|
|
"logps/rejected": -132.29440307617188,
|
|
"loss": 0.86,
|
|
"margin_dpo/margin_mean": 12.285654067993164,
|
|
"margin_dpo/margin_std": 12.449949264526367,
|
|
"step": 148
|
|
},
|
|
{
|
|
"epoch": 0.21879588839941264,
|
|
"grad_norm": 25.9832820892334,
|
|
"learning_rate": 4.797231243092118e-07,
|
|
"logits/chosen": -0.9811546802520752,
|
|
"logits/rejected": -0.9098988771438599,
|
|
"logps/chosen": -75.59249877929688,
|
|
"logps/ref_chosen": -71.0545425415039,
|
|
"logps/ref_rejected": -72.17367553710938,
|
|
"logps/rejected": -85.99839782714844,
|
|
"loss": 0.8852,
|
|
"margin_dpo/margin_mean": 9.286754608154297,
|
|
"margin_dpo/margin_std": 11.353466033935547,
|
|
"step": 149
|
|
},
|
|
{
|
|
"epoch": 0.22026431718061673,
|
|
"grad_norm": 24.666427612304688,
|
|
"learning_rate": 4.792138157142157e-07,
|
|
"logits/chosen": -1.180204153060913,
|
|
"logits/rejected": -0.681179404258728,
|
|
"logps/chosen": -99.05314636230469,
|
|
"logps/ref_chosen": -97.1973876953125,
|
|
"logps/ref_rejected": -126.63013458251953,
|
|
"logps/rejected": -143.3655548095703,
|
|
"loss": 0.8209,
|
|
"margin_dpo/margin_mean": 14.87967586517334,
|
|
"margin_dpo/margin_std": 16.285690307617188,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.22173274596182085,
|
|
"grad_norm": 30.53975486755371,
|
|
"learning_rate": 4.786984671220053e-07,
|
|
"logits/chosen": -2.2579073905944824,
|
|
"logits/rejected": -1.7060633897781372,
|
|
"logps/chosen": -122.10612487792969,
|
|
"logps/ref_chosen": -118.28496551513672,
|
|
"logps/ref_rejected": -105.15388488769531,
|
|
"logps/rejected": -112.59619140625,
|
|
"loss": 0.8771,
|
|
"margin_dpo/margin_mean": 3.6211354732513428,
|
|
"margin_dpo/margin_std": 8.112462043762207,
|
|
"step": 151
|
|
},
|
|
{
|
|
"epoch": 0.22320117474302498,
|
|
"grad_norm": 22.90477180480957,
|
|
"learning_rate": 4.78177092112495e-07,
|
|
"logits/chosen": -0.7480742335319519,
|
|
"logits/rejected": -0.3700064718723297,
|
|
"logps/chosen": -98.88221740722656,
|
|
"logps/ref_chosen": -99.09291076660156,
|
|
"logps/ref_rejected": -128.00595092773438,
|
|
"logps/rejected": -141.12022399902344,
|
|
"loss": 0.7394,
|
|
"margin_dpo/margin_mean": 13.324972152709961,
|
|
"margin_dpo/margin_std": 13.08827018737793,
|
|
"step": 152
|
|
},
|
|
{
|
|
"epoch": 0.22466960352422907,
|
|
"grad_norm": 25.02080535888672,
|
|
"learning_rate": 4.776497044244016e-07,
|
|
"logits/chosen": -1.737720251083374,
|
|
"logits/rejected": -1.291832447052002,
|
|
"logps/chosen": -97.09506225585938,
|
|
"logps/ref_chosen": -93.71240997314453,
|
|
"logps/ref_rejected": -89.57848358154297,
|
|
"logps/rejected": -99.30720520019531,
|
|
"loss": 0.939,
|
|
"margin_dpo/margin_mean": 6.3460798263549805,
|
|
"margin_dpo/margin_std": 9.05051040649414,
|
|
"step": 153
|
|
},
|
|
{
|
|
"epoch": 0.2261380323054332,
|
|
"grad_norm": 21.28890037536621,
|
|
"learning_rate": 4.771163179548808e-07,
|
|
"logits/chosen": -1.4931442737579346,
|
|
"logits/rejected": -1.3525810241699219,
|
|
"logps/chosen": -71.58004760742188,
|
|
"logps/ref_chosen": -70.46749877929688,
|
|
"logps/ref_rejected": -105.41502380371094,
|
|
"logps/rejected": -121.78628540039062,
|
|
"loss": 0.6519,
|
|
"margin_dpo/margin_mean": 15.258707046508789,
|
|
"margin_dpo/margin_std": 9.662094116210938,
|
|
"step": 154
|
|
},
|
|
{
|
|
"epoch": 0.2276064610866373,
|
|
"grad_norm": 35.946075439453125,
|
|
"learning_rate": 4.7657694675916247e-07,
|
|
"logits/chosen": -1.0313202142715454,
|
|
"logits/rejected": -0.722881019115448,
|
|
"logps/chosen": -120.5129165649414,
|
|
"logps/ref_chosen": -114.00595092773438,
|
|
"logps/ref_rejected": -78.87548828125,
|
|
"logps/rejected": -86.8126220703125,
|
|
"loss": 0.8807,
|
|
"margin_dpo/margin_mean": 1.4301615953445435,
|
|
"margin_dpo/margin_std": 11.984375953674316,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.2290748898678414,
|
|
"grad_norm": 31.770828247070312,
|
|
"learning_rate": 4.7603160505017893e-07,
|
|
"logits/chosen": -1.7006816864013672,
|
|
"logits/rejected": -1.1611135005950928,
|
|
"logps/chosen": -107.15304565429688,
|
|
"logps/ref_chosen": -102.44667053222656,
|
|
"logps/ref_rejected": -89.52784729003906,
|
|
"logps/rejected": -102.72535705566406,
|
|
"loss": 0.8866,
|
|
"margin_dpo/margin_mean": 8.491141319274902,
|
|
"margin_dpo/margin_std": 11.84402847290039,
|
|
"step": 156
|
|
},
|
|
{
|
|
"epoch": 0.2305433186490455,
|
|
"grad_norm": 23.618078231811523,
|
|
"learning_rate": 4.7548030719819154e-07,
|
|
"logits/chosen": -1.8400967121124268,
|
|
"logits/rejected": -1.6669890880584717,
|
|
"logps/chosen": -82.84764099121094,
|
|
"logps/ref_chosen": -79.52685546875,
|
|
"logps/ref_rejected": -141.90887451171875,
|
|
"logps/rejected": -159.43484497070312,
|
|
"loss": 0.7208,
|
|
"margin_dpo/margin_mean": 14.205196380615234,
|
|
"margin_dpo/margin_std": 9.605611801147461,
|
|
"step": 157
|
|
},
|
|
{
|
|
"epoch": 0.23201174743024963,
|
|
"grad_norm": 23.659147262573242,
|
|
"learning_rate": 4.7492306773041136e-07,
|
|
"logits/chosen": -1.324110746383667,
|
|
"logits/rejected": -0.9022949934005737,
|
|
"logps/chosen": -78.40304565429688,
|
|
"logps/ref_chosen": -77.00485229492188,
|
|
"logps/ref_rejected": -72.17926788330078,
|
|
"logps/rejected": -84.44666290283203,
|
|
"loss": 0.6906,
|
|
"margin_dpo/margin_mean": 10.869192123413086,
|
|
"margin_dpo/margin_std": 9.30504035949707,
|
|
"step": 158
|
|
},
|
|
{
|
|
"epoch": 0.23348017621145375,
|
|
"grad_norm": 23.302608489990234,
|
|
"learning_rate": 4.743599013306165e-07,
|
|
"logits/chosen": -0.2675594389438629,
|
|
"logits/rejected": 0.02939712256193161,
|
|
"logps/chosen": -138.03421020507812,
|
|
"logps/ref_chosen": -132.81875610351562,
|
|
"logps/ref_rejected": -109.44320678710938,
|
|
"logps/rejected": -123.77915954589844,
|
|
"loss": 0.7792,
|
|
"margin_dpo/margin_mean": 9.12051010131836,
|
|
"margin_dpo/margin_std": 10.716907501220703,
|
|
"step": 159
|
|
},
|
|
{
|
|
"epoch": 0.23494860499265785,
|
|
"grad_norm": 21.932205200195312,
|
|
"learning_rate": 4.737908228387656e-07,
|
|
"logits/chosen": -1.7152812480926514,
|
|
"logits/rejected": -1.0649853944778442,
|
|
"logps/chosen": -81.76644897460938,
|
|
"logps/ref_chosen": -81.04146575927734,
|
|
"logps/ref_rejected": -98.09086608886719,
|
|
"logps/rejected": -109.38851165771484,
|
|
"loss": 0.7741,
|
|
"margin_dpo/margin_mean": 10.57265567779541,
|
|
"margin_dpo/margin_std": 7.167086601257324,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23641703377386197,
|
|
"grad_norm": 27.166719436645508,
|
|
"learning_rate": 4.7321584725060594e-07,
|
|
"logits/chosen": -1.2614675760269165,
|
|
"logits/rejected": -0.5976088643074036,
|
|
"logps/chosen": -120.11067962646484,
|
|
"logps/ref_chosen": -116.38681030273438,
|
|
"logps/ref_rejected": -101.4004898071289,
|
|
"logps/rejected": -117.86997985839844,
|
|
"loss": 0.837,
|
|
"margin_dpo/margin_mean": 12.745627403259277,
|
|
"margin_dpo/margin_std": 12.207100868225098,
|
|
"step": 161
|
|
},
|
|
{
|
|
"epoch": 0.23788546255506607,
|
|
"grad_norm": 24.060861587524414,
|
|
"learning_rate": 4.7263498971727905e-07,
|
|
"logits/chosen": -0.5426618456840515,
|
|
"logits/rejected": -0.5506966710090637,
|
|
"logps/chosen": -56.9135627746582,
|
|
"logps/ref_chosen": -56.23395538330078,
|
|
"logps/ref_rejected": -102.3882827758789,
|
|
"logps/rejected": -114.80542755126953,
|
|
"loss": 0.7991,
|
|
"margin_dpo/margin_mean": 11.737544059753418,
|
|
"margin_dpo/margin_std": 10.131451606750488,
|
|
"step": 162
|
|
},
|
|
{
|
|
"epoch": 0.2393538913362702,
|
|
"grad_norm": 19.28577995300293,
|
|
"learning_rate": 4.720482655449212e-07,
|
|
"logits/chosen": -1.3950588703155518,
|
|
"logits/rejected": -0.9398083686828613,
|
|
"logps/chosen": -68.87108612060547,
|
|
"logps/ref_chosen": -66.62980651855469,
|
|
"logps/ref_rejected": -79.65003967285156,
|
|
"logps/rejected": -96.35603332519531,
|
|
"loss": 0.6908,
|
|
"margin_dpo/margin_mean": 14.464705467224121,
|
|
"margin_dpo/margin_std": 12.065017700195312,
|
|
"step": 163
|
|
},
|
|
{
|
|
"epoch": 0.24082232011747431,
|
|
"grad_norm": 24.83587074279785,
|
|
"learning_rate": 4.714556901942599e-07,
|
|
"logits/chosen": -1.0690147876739502,
|
|
"logits/rejected": -0.7840194702148438,
|
|
"logps/chosen": -104.05577087402344,
|
|
"logps/ref_chosen": -100.38748168945312,
|
|
"logps/ref_rejected": -91.67731475830078,
|
|
"logps/rejected": -103.01776885986328,
|
|
"loss": 0.8817,
|
|
"margin_dpo/margin_mean": 7.672161102294922,
|
|
"margin_dpo/margin_std": 8.832603454589844,
|
|
"step": 164
|
|
},
|
|
{
|
|
"epoch": 0.2422907488986784,
|
|
"grad_norm": 21.194198608398438,
|
|
"learning_rate": 4.708572792802069e-07,
|
|
"logits/chosen": -1.1250362396240234,
|
|
"logits/rejected": -0.4985249936580658,
|
|
"logps/chosen": -108.24383544921875,
|
|
"logps/ref_chosen": -107.4050064086914,
|
|
"logps/ref_rejected": -47.73858642578125,
|
|
"logps/rejected": -60.22450256347656,
|
|
"loss": 0.7184,
|
|
"margin_dpo/margin_mean": 11.6470947265625,
|
|
"margin_dpo/margin_std": 7.4804840087890625,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.24375917767988253,
|
|
"grad_norm": 23.84546661376953,
|
|
"learning_rate": 4.702530485714461e-07,
|
|
"logits/chosen": -2.16715669631958,
|
|
"logits/rejected": -2.0556259155273438,
|
|
"logps/chosen": -75.3694839477539,
|
|
"logps/ref_chosen": -71.99931335449219,
|
|
"logps/ref_rejected": -110.64549255371094,
|
|
"logps/rejected": -129.89505004882812,
|
|
"loss": 0.7701,
|
|
"margin_dpo/margin_mean": 15.879393577575684,
|
|
"margin_dpo/margin_std": 12.976381301879883,
|
|
"step": 166
|
|
},
|
|
{
|
|
"epoch": 0.24522760646108663,
|
|
"grad_norm": 22.239116668701172,
|
|
"learning_rate": 4.6964301399001877e-07,
|
|
"logits/chosen": 0.32823115587234497,
|
|
"logits/rejected": 0.6900070905685425,
|
|
"logps/chosen": -105.99115753173828,
|
|
"logps/ref_chosen": -101.20901489257812,
|
|
"logps/ref_rejected": -129.06129455566406,
|
|
"logps/rejected": -150.244384765625,
|
|
"loss": 0.636,
|
|
"margin_dpo/margin_mean": 16.40095329284668,
|
|
"margin_dpo/margin_std": 10.954397201538086,
|
|
"step": 167
|
|
},
|
|
{
|
|
"epoch": 0.24669603524229075,
|
|
"grad_norm": 26.696422576904297,
|
|
"learning_rate": 4.690271916109034e-07,
|
|
"logits/chosen": -2.4106054306030273,
|
|
"logits/rejected": -1.815399408340454,
|
|
"logps/chosen": -88.74671173095703,
|
|
"logps/ref_chosen": -82.77108764648438,
|
|
"logps/ref_rejected": -73.81690216064453,
|
|
"logps/rejected": -85.89750671386719,
|
|
"loss": 0.7792,
|
|
"margin_dpo/margin_mean": 6.10498046875,
|
|
"margin_dpo/margin_std": 10.449946403503418,
|
|
"step": 168
|
|
},
|
|
{
|
|
"epoch": 0.24816446402349487,
|
|
"grad_norm": 22.59729766845703,
|
|
"learning_rate": 4.6840559766159235e-07,
|
|
"logits/chosen": -1.7961640357971191,
|
|
"logits/rejected": -1.697147250175476,
|
|
"logps/chosen": -68.48163604736328,
|
|
"logps/ref_chosen": -66.73175811767578,
|
|
"logps/ref_rejected": -91.14268493652344,
|
|
"logps/rejected": -102.66637420654297,
|
|
"loss": 0.7176,
|
|
"margin_dpo/margin_mean": 9.773816108703613,
|
|
"margin_dpo/margin_std": 11.327061653137207,
|
|
"step": 169
|
|
},
|
|
{
|
|
"epoch": 0.24963289280469897,
|
|
"grad_norm": 24.59431266784668,
|
|
"learning_rate": 4.6777824852166437e-07,
|
|
"logits/chosen": -1.0736992359161377,
|
|
"logits/rejected": -1.0651642084121704,
|
|
"logps/chosen": -76.36054992675781,
|
|
"logps/ref_chosen": -70.11228942871094,
|
|
"logps/ref_rejected": -82.50209045410156,
|
|
"logps/rejected": -96.03549194335938,
|
|
"loss": 0.7313,
|
|
"margin_dpo/margin_mean": 7.2851338386535645,
|
|
"margin_dpo/margin_std": 7.817892074584961,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.2511013215859031,
|
|
"grad_norm": 30.791086196899414,
|
|
"learning_rate": 4.6714516072235273e-07,
|
|
"logits/chosen": -2.3488950729370117,
|
|
"logits/rejected": -1.7639861106872559,
|
|
"logps/chosen": -129.69264221191406,
|
|
"logps/ref_chosen": -124.22677612304688,
|
|
"logps/ref_rejected": -82.39506530761719,
|
|
"logps/rejected": -96.64595031738281,
|
|
"loss": 0.7744,
|
|
"margin_dpo/margin_mean": 8.785017967224121,
|
|
"margin_dpo/margin_std": 11.808923721313477,
|
|
"step": 171
|
|
},
|
|
{
|
|
"epoch": 0.2525697503671072,
|
|
"grad_norm": 19.004745483398438,
|
|
"learning_rate": 4.6650635094610966e-07,
|
|
"logits/chosen": -0.8665769100189209,
|
|
"logits/rejected": -0.5177565813064575,
|
|
"logps/chosen": -91.90443420410156,
|
|
"logps/ref_chosen": -85.85418701171875,
|
|
"logps/ref_rejected": -84.93639373779297,
|
|
"logps/rejected": -100.33929443359375,
|
|
"loss": 0.6408,
|
|
"margin_dpo/margin_mean": 9.352657318115234,
|
|
"margin_dpo/margin_std": 8.445768356323242,
|
|
"step": 172
|
|
},
|
|
{
|
|
"epoch": 0.2540381791483113,
|
|
"grad_norm": 22.988880157470703,
|
|
"learning_rate": 4.6586183602616687e-07,
|
|
"logits/chosen": -1.054530382156372,
|
|
"logits/rejected": -0.6302087903022766,
|
|
"logps/chosen": -93.0565185546875,
|
|
"logps/ref_chosen": -85.85157012939453,
|
|
"logps/ref_rejected": -71.90559387207031,
|
|
"logps/rejected": -88.41697692871094,
|
|
"loss": 0.8184,
|
|
"margin_dpo/margin_mean": 9.306428909301758,
|
|
"margin_dpo/margin_std": 10.248741149902344,
|
|
"step": 173
|
|
},
|
|
{
|
|
"epoch": 0.2555066079295154,
|
|
"grad_norm": 25.447433471679688,
|
|
"learning_rate": 4.652116329460919e-07,
|
|
"logits/chosen": -1.548011064529419,
|
|
"logits/rejected": -1.144951343536377,
|
|
"logps/chosen": -81.47247314453125,
|
|
"logps/ref_chosen": -74.92919921875,
|
|
"logps/ref_rejected": -56.715511322021484,
|
|
"logps/rejected": -75.88140869140625,
|
|
"loss": 0.7049,
|
|
"margin_dpo/margin_mean": 12.622618675231934,
|
|
"margin_dpo/margin_std": 9.480951309204102,
|
|
"step": 174
|
|
},
|
|
{
|
|
"epoch": 0.25697503671071953,
|
|
"grad_norm": 27.344615936279297,
|
|
"learning_rate": 4.645557588393406e-07,
|
|
"logits/chosen": -1.1176164150238037,
|
|
"logits/rejected": -0.8251014351844788,
|
|
"logps/chosen": -121.8826904296875,
|
|
"logps/ref_chosen": -114.08724212646484,
|
|
"logps/ref_rejected": -119.46188354492188,
|
|
"logps/rejected": -142.9967041015625,
|
|
"loss": 0.67,
|
|
"margin_dpo/margin_mean": 15.739368438720703,
|
|
"margin_dpo/margin_std": 11.752717018127441,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.25844346549192365,
|
|
"grad_norm": 22.77707862854004,
|
|
"learning_rate": 4.638942309888058e-07,
|
|
"logits/chosen": -0.36084672808647156,
|
|
"logits/rejected": -0.19117483496665955,
|
|
"logps/chosen": -79.84271240234375,
|
|
"logps/ref_chosen": -75.32672882080078,
|
|
"logps/ref_rejected": -117.2403335571289,
|
|
"logps/rejected": -136.30145263671875,
|
|
"loss": 0.7795,
|
|
"margin_dpo/margin_mean": 14.545129776000977,
|
|
"margin_dpo/margin_std": 12.101448059082031,
|
|
"step": 176
|
|
},
|
|
{
|
|
"epoch": 0.2599118942731278,
|
|
"grad_norm": 20.781431198120117,
|
|
"learning_rate": 4.6322706682636137e-07,
|
|
"logits/chosen": -1.7568106651306152,
|
|
"logits/rejected": -1.8467406034469604,
|
|
"logps/chosen": -86.03660583496094,
|
|
"logps/ref_chosen": -80.876953125,
|
|
"logps/ref_rejected": -119.15406799316406,
|
|
"logps/rejected": -141.73297119140625,
|
|
"loss": 0.638,
|
|
"margin_dpo/margin_mean": 17.41925621032715,
|
|
"margin_dpo/margin_std": 16.072580337524414,
|
|
"step": 177
|
|
},
|
|
{
|
|
"epoch": 0.26138032305433184,
|
|
"grad_norm": 23.510520935058594,
|
|
"learning_rate": 4.6255428393240354e-07,
|
|
"logits/chosen": -0.6462529301643372,
|
|
"logits/rejected": -0.716810405254364,
|
|
"logps/chosen": -67.99021911621094,
|
|
"logps/ref_chosen": -59.280879974365234,
|
|
"logps/ref_rejected": -85.19041442871094,
|
|
"logps/rejected": -107.30775451660156,
|
|
"loss": 0.5335,
|
|
"margin_dpo/margin_mean": 13.408007621765137,
|
|
"margin_dpo/margin_std": 8.432206153869629,
|
|
"step": 178
|
|
},
|
|
{
|
|
"epoch": 0.26284875183553597,
|
|
"grad_norm": 33.201324462890625,
|
|
"learning_rate": 4.6187590003538724e-07,
|
|
"logits/chosen": -1.1230896711349487,
|
|
"logits/rejected": -0.8792758584022522,
|
|
"logps/chosen": -67.34712219238281,
|
|
"logps/ref_chosen": -59.78905487060547,
|
|
"logps/ref_rejected": -66.46996307373047,
|
|
"logps/rejected": -85.9849853515625,
|
|
"loss": 0.8597,
|
|
"margin_dpo/margin_mean": 11.95695686340332,
|
|
"margin_dpo/margin_std": 12.786809921264648,
|
|
"step": 179
|
|
},
|
|
{
|
|
"epoch": 0.2643171806167401,
|
|
"grad_norm": 26.94002914428711,
|
|
"learning_rate": 4.611919330113591e-07,
|
|
"logits/chosen": -1.9314937591552734,
|
|
"logits/rejected": -1.3737207651138306,
|
|
"logps/chosen": -120.11798095703125,
|
|
"logps/ref_chosen": -112.93758392333984,
|
|
"logps/ref_rejected": -105.24171447753906,
|
|
"logps/rejected": -130.20762634277344,
|
|
"loss": 0.6352,
|
|
"margin_dpo/margin_mean": 17.785518646240234,
|
|
"margin_dpo/margin_std": 19.370147705078125,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.2657856093979442,
|
|
"grad_norm": 28.8751277923584,
|
|
"learning_rate": 4.605024008834863e-07,
|
|
"logits/chosen": -0.8084018230438232,
|
|
"logits/rejected": -0.6283854842185974,
|
|
"logps/chosen": -67.87879180908203,
|
|
"logps/ref_chosen": -62.6356201171875,
|
|
"logps/ref_rejected": -62.49972152709961,
|
|
"logps/rejected": -77.07058715820312,
|
|
"loss": 0.7728,
|
|
"margin_dpo/margin_mean": 9.327696800231934,
|
|
"margin_dpo/margin_std": 9.290848731994629,
|
|
"step": 181
|
|
},
|
|
{
|
|
"epoch": 0.26725403817914833,
|
|
"grad_norm": 23.121599197387695,
|
|
"learning_rate": 4.598073218215817e-07,
|
|
"logits/chosen": -0.31414175033569336,
|
|
"logits/rejected": -0.38997751474380493,
|
|
"logps/chosen": -78.23188018798828,
|
|
"logps/ref_chosen": -68.78814697265625,
|
|
"logps/ref_rejected": -137.93020629882812,
|
|
"logps/rejected": -168.03782653808594,
|
|
"loss": 0.7281,
|
|
"margin_dpo/margin_mean": 20.663890838623047,
|
|
"margin_dpo/margin_std": 18.779584884643555,
|
|
"step": 182
|
|
},
|
|
{
|
|
"epoch": 0.2687224669603524,
|
|
"grad_norm": 25.60964584350586,
|
|
"learning_rate": 4.5910671414162484e-07,
|
|
"logits/chosen": -1.1187982559204102,
|
|
"logits/rejected": -0.5141847133636475,
|
|
"logps/chosen": -115.06136322021484,
|
|
"logps/ref_chosen": -104.33895874023438,
|
|
"logps/ref_rejected": -97.7330093383789,
|
|
"logps/rejected": -124.12959289550781,
|
|
"loss": 0.6346,
|
|
"margin_dpo/margin_mean": 15.674171447753906,
|
|
"margin_dpo/margin_std": 16.124521255493164,
|
|
"step": 183
|
|
},
|
|
{
|
|
"epoch": 0.2701908957415565,
|
|
"grad_norm": 32.20849609375,
|
|
"learning_rate": 4.5840059630527985e-07,
|
|
"logits/chosen": -1.0942208766937256,
|
|
"logits/rejected": -0.936927855014801,
|
|
"logps/chosen": -107.64266967773438,
|
|
"logps/ref_chosen": -97.3797607421875,
|
|
"logps/ref_rejected": -100.01255798339844,
|
|
"logps/rejected": -123.9334487915039,
|
|
"loss": 0.9811,
|
|
"margin_dpo/margin_mean": 13.65797233581543,
|
|
"margin_dpo/margin_std": 14.047473907470703,
|
|
"step": 184
|
|
},
|
|
{
|
|
"epoch": 0.27165932452276065,
|
|
"grad_norm": 35.5129508972168,
|
|
"learning_rate": 4.5768898691940836e-07,
|
|
"logits/chosen": -1.4037067890167236,
|
|
"logits/rejected": -0.8602940440177917,
|
|
"logps/chosen": -97.39491271972656,
|
|
"logps/ref_chosen": -82.86443328857422,
|
|
"logps/ref_rejected": -84.46593475341797,
|
|
"logps/rejected": -108.24913024902344,
|
|
"loss": 0.8675,
|
|
"margin_dpo/margin_mean": 9.252706527709961,
|
|
"margin_dpo/margin_std": 9.427391052246094,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.27312775330396477,
|
|
"grad_norm": 32.151607513427734,
|
|
"learning_rate": 4.5697190473557947e-07,
|
|
"logits/chosen": -1.355446696281433,
|
|
"logits/rejected": -0.5260564088821411,
|
|
"logps/chosen": -150.05572509765625,
|
|
"logps/ref_chosen": -139.9113311767578,
|
|
"logps/ref_rejected": -107.13050842285156,
|
|
"logps/rejected": -135.98916625976562,
|
|
"loss": 0.7012,
|
|
"margin_dpo/margin_mean": 18.71424102783203,
|
|
"margin_dpo/margin_std": 14.531794548034668,
|
|
"step": 186
|
|
},
|
|
{
|
|
"epoch": 0.2745961820851689,
|
|
"grad_norm": 26.140470504760742,
|
|
"learning_rate": 4.5624936864957555e-07,
|
|
"logits/chosen": -1.562740445137024,
|
|
"logits/rejected": -1.1477710008621216,
|
|
"logps/chosen": -97.55638122558594,
|
|
"logps/ref_chosen": -91.39136505126953,
|
|
"logps/ref_rejected": -94.65269470214844,
|
|
"logps/rejected": -114.19329833984375,
|
|
"loss": 0.8602,
|
|
"margin_dpo/margin_mean": 13.375587463378906,
|
|
"margin_dpo/margin_std": 13.763025283813477,
|
|
"step": 187
|
|
},
|
|
{
|
|
"epoch": 0.27606461086637296,
|
|
"grad_norm": 26.123018264770508,
|
|
"learning_rate": 4.5552139770089454e-07,
|
|
"logits/chosen": -1.5542670488357544,
|
|
"logits/rejected": -1.3543874025344849,
|
|
"logps/chosen": -91.50485229492188,
|
|
"logps/ref_chosen": -81.19329833984375,
|
|
"logps/ref_rejected": -98.7778091430664,
|
|
"logps/rejected": -125.56822204589844,
|
|
"loss": 0.7258,
|
|
"margin_dpo/margin_mean": 16.47885513305664,
|
|
"margin_dpo/margin_std": 8.776860237121582,
|
|
"step": 188
|
|
},
|
|
{
|
|
"epoch": 0.2775330396475771,
|
|
"grad_norm": 22.942975997924805,
|
|
"learning_rate": 4.5478801107224794e-07,
|
|
"logits/chosen": -1.4397048950195312,
|
|
"logits/rejected": -0.5799304246902466,
|
|
"logps/chosen": -94.46961975097656,
|
|
"logps/ref_chosen": -88.11688232421875,
|
|
"logps/ref_rejected": -78.47306060791016,
|
|
"logps/rejected": -101.44779968261719,
|
|
"loss": 0.7501,
|
|
"margin_dpo/margin_mean": 16.621992111206055,
|
|
"margin_dpo/margin_std": 16.106063842773438,
|
|
"step": 189
|
|
},
|
|
{
|
|
"epoch": 0.2790014684287812,
|
|
"grad_norm": 22.044652938842773,
|
|
"learning_rate": 4.5404922808905543e-07,
|
|
"logits/chosen": -0.8703410625457764,
|
|
"logits/rejected": -0.3017834424972534,
|
|
"logps/chosen": -114.73606872558594,
|
|
"logps/ref_chosen": -104.41696166992188,
|
|
"logps/ref_rejected": -111.64453125,
|
|
"logps/rejected": -136.51731872558594,
|
|
"loss": 0.585,
|
|
"margin_dpo/margin_mean": 14.553679466247559,
|
|
"margin_dpo/margin_std": 9.547571182250977,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.28046989720998533,
|
|
"grad_norm": 25.596473693847656,
|
|
"learning_rate": 4.5330506821893565e-07,
|
|
"logits/chosen": -1.403781533241272,
|
|
"logits/rejected": -1.2404292821884155,
|
|
"logps/chosen": -111.43516540527344,
|
|
"logps/ref_chosen": -104.78724670410156,
|
|
"logps/ref_rejected": -146.44696044921875,
|
|
"logps/rejected": -176.109130859375,
|
|
"loss": 0.5893,
|
|
"margin_dpo/margin_mean": 23.01424789428711,
|
|
"margin_dpo/margin_std": 19.176429748535156,
|
|
"step": 191
|
|
},
|
|
{
|
|
"epoch": 0.28193832599118945,
|
|
"grad_norm": 33.474796295166016,
|
|
"learning_rate": 4.5255555107119336e-07,
|
|
"logits/chosen": -0.6243493556976318,
|
|
"logits/rejected": -0.2121521681547165,
|
|
"logps/chosen": -105.75337982177734,
|
|
"logps/ref_chosen": -95.80691528320312,
|
|
"logps/ref_rejected": -108.17986297607422,
|
|
"logps/rejected": -127.128173828125,
|
|
"loss": 0.8404,
|
|
"margin_dpo/margin_mean": 9.001851081848145,
|
|
"margin_dpo/margin_std": 13.27437973022461,
|
|
"step": 192
|
|
},
|
|
{
|
|
"epoch": 0.2834067547723935,
|
|
"grad_norm": 37.02301788330078,
|
|
"learning_rate": 4.5180069639630236e-07,
|
|
"logits/chosen": -1.5451886653900146,
|
|
"logits/rejected": -0.5159762501716614,
|
|
"logps/chosen": -126.4687271118164,
|
|
"logps/ref_chosen": -116.82717895507812,
|
|
"logps/ref_rejected": -69.49932861328125,
|
|
"logps/rejected": -90.38545989990234,
|
|
"loss": 0.967,
|
|
"margin_dpo/margin_mean": 11.244586944580078,
|
|
"margin_dpo/margin_std": 15.227866172790527,
|
|
"step": 193
|
|
},
|
|
{
|
|
"epoch": 0.28487518355359764,
|
|
"grad_norm": 25.955747604370117,
|
|
"learning_rate": 4.510405240853854e-07,
|
|
"logits/chosen": -2.664726734161377,
|
|
"logits/rejected": -1.99815034866333,
|
|
"logps/chosen": -87.97075653076172,
|
|
"logps/ref_chosen": -80.39901733398438,
|
|
"logps/ref_rejected": -78.88285064697266,
|
|
"logps/rejected": -98.36553955078125,
|
|
"loss": 0.8125,
|
|
"margin_dpo/margin_mean": 11.910947799682617,
|
|
"margin_dpo/margin_std": 10.079092025756836,
|
|
"step": 194
|
|
},
|
|
{
|
|
"epoch": 0.28634361233480177,
|
|
"grad_norm": 25.988468170166016,
|
|
"learning_rate": 4.5027505416968985e-07,
|
|
"logits/chosen": -1.6600630283355713,
|
|
"logits/rejected": -1.396432638168335,
|
|
"logps/chosen": -96.83953857421875,
|
|
"logps/ref_chosen": -89.2958984375,
|
|
"logps/ref_rejected": -106.6185302734375,
|
|
"logps/rejected": -132.700927734375,
|
|
"loss": 0.7439,
|
|
"margin_dpo/margin_mean": 18.538780212402344,
|
|
"margin_dpo/margin_std": 10.213254928588867,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.2878120411160059,
|
|
"grad_norm": 20.28085708618164,
|
|
"learning_rate": 4.495043068200599e-07,
|
|
"logits/chosen": -0.3120614290237427,
|
|
"logits/rejected": -0.17643268406391144,
|
|
"logps/chosen": -83.5776596069336,
|
|
"logps/ref_chosen": -72.89015197753906,
|
|
"logps/ref_rejected": -109.44696044921875,
|
|
"logps/rejected": -139.74281311035156,
|
|
"loss": 0.6272,
|
|
"margin_dpo/margin_mean": 19.608352661132812,
|
|
"margin_dpo/margin_std": 10.485124588012695,
|
|
"step": 196
|
|
},
|
|
{
|
|
"epoch": 0.28928046989721,
|
|
"grad_norm": 27.453264236450195,
|
|
"learning_rate": 4.4872830234640493e-07,
|
|
"logits/chosen": -0.7792481780052185,
|
|
"logits/rejected": -0.7744120359420776,
|
|
"logps/chosen": -69.80078887939453,
|
|
"logps/ref_chosen": -61.45612716674805,
|
|
"logps/ref_rejected": -73.83760070800781,
|
|
"logps/rejected": -95.51461791992188,
|
|
"loss": 0.8937,
|
|
"margin_dpo/margin_mean": 13.332344055175781,
|
|
"margin_dpo/margin_std": 11.496066093444824,
|
|
"step": 197
|
|
},
|
|
{
|
|
"epoch": 0.2907488986784141,
|
|
"grad_norm": 29.672889709472656,
|
|
"learning_rate": 4.479470611971645e-07,
|
|
"logits/chosen": -1.0126028060913086,
|
|
"logits/rejected": -1.1434699296951294,
|
|
"logps/chosen": -121.82696533203125,
|
|
"logps/ref_chosen": -107.06954956054688,
|
|
"logps/ref_rejected": -159.97869873046875,
|
|
"logps/rejected": -187.76712036132812,
|
|
"loss": 0.696,
|
|
"margin_dpo/margin_mean": 13.031010627746582,
|
|
"margin_dpo/margin_std": 15.836978912353516,
|
|
"step": 198
|
|
},
|
|
{
|
|
"epoch": 0.2922173274596182,
|
|
"grad_norm": 27.473636627197266,
|
|
"learning_rate": 4.471606039587695e-07,
|
|
"logits/chosen": -1.398844838142395,
|
|
"logits/rejected": -0.5553488731384277,
|
|
"logps/chosen": -127.450439453125,
|
|
"logps/ref_chosen": -113.38455963134766,
|
|
"logps/ref_rejected": -75.62179565429688,
|
|
"logps/rejected": -101.59065246582031,
|
|
"loss": 0.7567,
|
|
"margin_dpo/margin_mean": 11.902961730957031,
|
|
"margin_dpo/margin_std": 13.907926559448242,
|
|
"step": 199
|
|
},
|
|
{
|
|
"epoch": 0.2936857562408223,
|
|
"grad_norm": 23.272537231445312,
|
|
"learning_rate": 4.4636895135509966e-07,
|
|
"logits/chosen": -0.4032726287841797,
|
|
"logits/rejected": -0.5059705972671509,
|
|
"logps/chosen": -64.51741027832031,
|
|
"logps/ref_chosen": -50.07917785644531,
|
|
"logps/ref_rejected": -104.77871704101562,
|
|
"logps/rejected": -134.1025390625,
|
|
"loss": 0.7622,
|
|
"margin_dpo/margin_mean": 14.885597229003906,
|
|
"margin_dpo/margin_std": 16.44847869873047,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2936857562408223,
|
|
"eval_logits/chosen": -1.120969533920288,
|
|
"eval_logits/rejected": -0.845413327217102,
|
|
"eval_logps/chosen": -113.375244140625,
|
|
"eval_logps/ref_chosen": -101.88616943359375,
|
|
"eval_logps/ref_rejected": -92.68607330322266,
|
|
"eval_logps/rejected": -114.73187255859375,
|
|
"eval_loss": 0.4643746018409729,
|
|
"eval_margin_dpo/margin_mean": 10.556744575500488,
|
|
"eval_margin_dpo/margin_std": 12.665112495422363,
|
|
"eval_runtime": 43.9772,
|
|
"eval_samples_per_second": 53.187,
|
|
"eval_steps_per_second": 1.683,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.29515418502202645,
|
|
"grad_norm": 31.166837692260742,
|
|
"learning_rate": 4.455721242469372e-07,
|
|
"logits/chosen": -1.3415249586105347,
|
|
"logits/rejected": -0.7633596658706665,
|
|
"logps/chosen": -132.33038330078125,
|
|
"logps/ref_chosen": -125.86758422851562,
|
|
"logps/ref_rejected": -137.8953094482422,
|
|
"logps/rejected": -152.9993438720703,
|
|
"loss": 0.8579,
|
|
"margin_dpo/margin_mean": 8.6412353515625,
|
|
"margin_dpo/margin_std": 15.760478973388672,
|
|
"step": 201
|
|
},
|
|
{
|
|
"epoch": 0.2966226138032305,
|
|
"grad_norm": 25.604938507080078,
|
|
"learning_rate": 4.4477014363141755e-07,
|
|
"logits/chosen": -1.2493139505386353,
|
|
"logits/rejected": -1.0067428350448608,
|
|
"logps/chosen": -73.95713806152344,
|
|
"logps/ref_chosen": -62.47052001953125,
|
|
"logps/ref_rejected": -76.64714813232422,
|
|
"logps/rejected": -104.02867126464844,
|
|
"loss": 0.8075,
|
|
"margin_dpo/margin_mean": 15.89489459991455,
|
|
"margin_dpo/margin_std": 15.220779418945312,
|
|
"step": 202
|
|
},
|
|
{
|
|
"epoch": 0.29809104258443464,
|
|
"grad_norm": 22.339271545410156,
|
|
"learning_rate": 4.439630306414758e-07,
|
|
"logits/chosen": -1.5672788619995117,
|
|
"logits/rejected": -1.3719830513000488,
|
|
"logps/chosen": -92.4307861328125,
|
|
"logps/ref_chosen": -85.48118591308594,
|
|
"logps/ref_rejected": -96.69866943359375,
|
|
"logps/rejected": -116.5356674194336,
|
|
"loss": 0.6844,
|
|
"margin_dpo/margin_mean": 12.887399673461914,
|
|
"margin_dpo/margin_std": 15.519386291503906,
|
|
"step": 203
|
|
},
|
|
{
|
|
"epoch": 0.29955947136563876,
|
|
"grad_norm": 34.63078308105469,
|
|
"learning_rate": 4.431508065452897e-07,
|
|
"logits/chosen": -1.294830083847046,
|
|
"logits/rejected": -0.8463597893714905,
|
|
"logps/chosen": -99.44182586669922,
|
|
"logps/ref_chosen": -85.38599395751953,
|
|
"logps/ref_rejected": -87.58256530761719,
|
|
"logps/rejected": -111.54623413085938,
|
|
"loss": 0.8576,
|
|
"margin_dpo/margin_mean": 9.907830238342285,
|
|
"margin_dpo/margin_std": 11.625691413879395,
|
|
"step": 204
|
|
},
|
|
{
|
|
"epoch": 0.3010279001468429,
|
|
"grad_norm": 19.02329444885254,
|
|
"learning_rate": 4.4233349274571974e-07,
|
|
"logits/chosen": -0.9210774898529053,
|
|
"logits/rejected": -0.40849876403808594,
|
|
"logps/chosen": -127.82294464111328,
|
|
"logps/ref_chosen": -121.29222869873047,
|
|
"logps/ref_rejected": -83.04063415527344,
|
|
"logps/rejected": -105.96640014648438,
|
|
"loss": 0.5644,
|
|
"margin_dpo/margin_mean": 16.39504623413086,
|
|
"margin_dpo/margin_std": 11.657992362976074,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.302496328928047,
|
|
"grad_norm": 27.465503692626953,
|
|
"learning_rate": 4.415111107797445e-07,
|
|
"logits/chosen": -1.0976223945617676,
|
|
"logits/rejected": -0.7354283332824707,
|
|
"logps/chosen": -92.56300354003906,
|
|
"logps/ref_chosen": -81.04837799072266,
|
|
"logps/ref_rejected": -107.19715118408203,
|
|
"logps/rejected": -131.0432891845703,
|
|
"loss": 0.6035,
|
|
"margin_dpo/margin_mean": 12.331515312194824,
|
|
"margin_dpo/margin_std": 9.227275848388672,
|
|
"step": 206
|
|
},
|
|
{
|
|
"epoch": 0.3039647577092511,
|
|
"grad_norm": 39.439029693603516,
|
|
"learning_rate": 4.4068368231789365e-07,
|
|
"logits/chosen": -2.2387092113494873,
|
|
"logits/rejected": -1.860313892364502,
|
|
"logps/chosen": -117.16340637207031,
|
|
"logps/ref_chosen": -102.81378173828125,
|
|
"logps/ref_rejected": -78.59852600097656,
|
|
"logps/rejected": -104.43505859375,
|
|
"loss": 0.7898,
|
|
"margin_dpo/margin_mean": 11.486899375915527,
|
|
"margin_dpo/margin_std": 15.101633071899414,
|
|
"step": 207
|
|
},
|
|
{
|
|
"epoch": 0.3054331864904552,
|
|
"grad_norm": 23.772550582885742,
|
|
"learning_rate": 4.398512291636768e-07,
|
|
"logits/chosen": -1.4251892566680908,
|
|
"logits/rejected": -0.9521887302398682,
|
|
"logps/chosen": -79.19771575927734,
|
|
"logps/ref_chosen": -70.0069580078125,
|
|
"logps/ref_rejected": -79.3374252319336,
|
|
"logps/rejected": -103.45726776123047,
|
|
"loss": 0.6505,
|
|
"margin_dpo/margin_mean": 14.929088592529297,
|
|
"margin_dpo/margin_std": 9.954263687133789,
|
|
"step": 208
|
|
},
|
|
{
|
|
"epoch": 0.3069016152716593,
|
|
"grad_norm": 24.988706588745117,
|
|
"learning_rate": 4.3901377325300857e-07,
|
|
"logits/chosen": -1.2561299800872803,
|
|
"logits/rejected": -0.9694398641586304,
|
|
"logps/chosen": -78.55195617675781,
|
|
"logps/ref_chosen": -74.00470733642578,
|
|
"logps/ref_rejected": -87.94189453125,
|
|
"logps/rejected": -113.91144561767578,
|
|
"loss": 0.7762,
|
|
"margin_dpo/margin_mean": 21.42230224609375,
|
|
"margin_dpo/margin_std": 15.412055969238281,
|
|
"step": 209
|
|
},
|
|
{
|
|
"epoch": 0.30837004405286345,
|
|
"grad_norm": 24.69098472595215,
|
|
"learning_rate": 4.381713366536311e-07,
|
|
"logits/chosen": -1.796557068824768,
|
|
"logits/rejected": -1.1550593376159668,
|
|
"logps/chosen": -85.33075714111328,
|
|
"logps/ref_chosen": -76.31346130371094,
|
|
"logps/ref_rejected": -63.925392150878906,
|
|
"logps/rejected": -86.75428771972656,
|
|
"loss": 0.733,
|
|
"margin_dpo/margin_mean": 13.81161117553711,
|
|
"margin_dpo/margin_std": 13.800355911254883,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.30983847283406757,
|
|
"grad_norm": 27.62305450439453,
|
|
"learning_rate": 4.373239415645323e-07,
|
|
"logits/chosen": -1.5198208093643188,
|
|
"logits/rejected": -0.88636314868927,
|
|
"logps/chosen": -150.07386779785156,
|
|
"logps/ref_chosen": -140.85391235351562,
|
|
"logps/ref_rejected": -107.42497253417969,
|
|
"logps/rejected": -136.897705078125,
|
|
"loss": 0.7325,
|
|
"margin_dpo/margin_mean": 20.25276756286621,
|
|
"margin_dpo/margin_std": 15.092392921447754,
|
|
"step": 211
|
|
},
|
|
{
|
|
"epoch": 0.31130690161527164,
|
|
"grad_norm": 19.40737533569336,
|
|
"learning_rate": 4.3647161031536086e-07,
|
|
"logits/chosen": -1.8869035243988037,
|
|
"logits/rejected": -1.2525444030761719,
|
|
"logps/chosen": -132.343017578125,
|
|
"logps/ref_chosen": -125.50788116455078,
|
|
"logps/ref_rejected": -108.29771423339844,
|
|
"logps/rejected": -132.68862915039062,
|
|
"loss": 0.5226,
|
|
"margin_dpo/margin_mean": 17.555767059326172,
|
|
"margin_dpo/margin_std": 19.352325439453125,
|
|
"step": 212
|
|
},
|
|
{
|
|
"epoch": 0.31277533039647576,
|
|
"grad_norm": 32.36695861816406,
|
|
"learning_rate": 4.3561436536583774e-07,
|
|
"logits/chosen": -1.4449195861816406,
|
|
"logits/rejected": -1.2709022760391235,
|
|
"logps/chosen": -85.28340911865234,
|
|
"logps/ref_chosen": -75.74522399902344,
|
|
"logps/ref_rejected": -122.3311996459961,
|
|
"logps/rejected": -149.52574157714844,
|
|
"loss": 0.6238,
|
|
"margin_dpo/margin_mean": 17.656354904174805,
|
|
"margin_dpo/margin_std": 11.100046157836914,
|
|
"step": 213
|
|
},
|
|
{
|
|
"epoch": 0.3142437591776799,
|
|
"grad_norm": 27.299671173095703,
|
|
"learning_rate": 4.3475222930516473e-07,
|
|
"logits/chosen": -1.442638874053955,
|
|
"logits/rejected": -1.1300638914108276,
|
|
"logps/chosen": -73.31192779541016,
|
|
"logps/ref_chosen": -64.04418182373047,
|
|
"logps/ref_rejected": -50.824100494384766,
|
|
"logps/rejected": -68.72589874267578,
|
|
"loss": 0.8735,
|
|
"margin_dpo/margin_mean": 8.634056091308594,
|
|
"margin_dpo/margin_std": 14.054182052612305,
|
|
"step": 214
|
|
},
|
|
{
|
|
"epoch": 0.315712187958884,
|
|
"grad_norm": 21.764801025390625,
|
|
"learning_rate": 4.3388522485142885e-07,
|
|
"logits/chosen": -1.2465263605117798,
|
|
"logits/rejected": -0.9289161562919617,
|
|
"logps/chosen": -80.82921600341797,
|
|
"logps/ref_chosen": -70.8616714477539,
|
|
"logps/ref_rejected": -92.91224670410156,
|
|
"logps/rejected": -125.51627349853516,
|
|
"loss": 0.5661,
|
|
"margin_dpo/margin_mean": 22.63648796081543,
|
|
"margin_dpo/margin_std": 13.089027404785156,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.31718061674008813,
|
|
"grad_norm": 21.630605697631836,
|
|
"learning_rate": 4.330133748510036e-07,
|
|
"logits/chosen": -1.3383195400238037,
|
|
"logits/rejected": -1.1601924896240234,
|
|
"logps/chosen": -97.20977020263672,
|
|
"logps/ref_chosen": -85.44164276123047,
|
|
"logps/ref_rejected": -98.54600524902344,
|
|
"logps/rejected": -124.43745422363281,
|
|
"loss": 0.6578,
|
|
"margin_dpo/margin_mean": 14.123327255249023,
|
|
"margin_dpo/margin_std": 9.03171443939209,
|
|
"step": 216
|
|
},
|
|
{
|
|
"epoch": 0.3186490455212922,
|
|
"grad_norm": 31.357202529907227,
|
|
"learning_rate": 4.3213670227794757e-07,
|
|
"logits/chosen": -1.8061637878417969,
|
|
"logits/rejected": -1.6783373355865479,
|
|
"logps/chosen": -93.33474731445312,
|
|
"logps/ref_chosen": -83.99208068847656,
|
|
"logps/ref_rejected": -123.9173583984375,
|
|
"logps/rejected": -151.92092895507812,
|
|
"loss": 0.7858,
|
|
"margin_dpo/margin_mean": 18.660911560058594,
|
|
"margin_dpo/margin_std": 14.71579360961914,
|
|
"step": 217
|
|
},
|
|
{
|
|
"epoch": 0.3201174743024963,
|
|
"grad_norm": 32.019493103027344,
|
|
"learning_rate": 4.3125523023339815e-07,
|
|
"logits/chosen": -0.8639114499092102,
|
|
"logits/rejected": -0.6436069011688232,
|
|
"logps/chosen": -80.2852783203125,
|
|
"logps/ref_chosen": -67.8966293334961,
|
|
"logps/ref_rejected": -85.52115631103516,
|
|
"logps/rejected": -115.44291687011719,
|
|
"loss": 0.7321,
|
|
"margin_dpo/margin_mean": 17.53311538696289,
|
|
"margin_dpo/margin_std": 12.562833786010742,
|
|
"step": 218
|
|
},
|
|
{
|
|
"epoch": 0.32158590308370044,
|
|
"grad_norm": 45.11946105957031,
|
|
"learning_rate": 4.303689819449636e-07,
|
|
"logits/chosen": -1.2486598491668701,
|
|
"logits/rejected": -0.9495463371276855,
|
|
"logps/chosen": -111.53719329833984,
|
|
"logps/ref_chosen": -97.2021484375,
|
|
"logps/ref_rejected": -134.10055541992188,
|
|
"logps/rejected": -167.79376220703125,
|
|
"loss": 0.7715,
|
|
"margin_dpo/margin_mean": 19.358152389526367,
|
|
"margin_dpo/margin_std": 19.34707260131836,
|
|
"step": 219
|
|
},
|
|
{
|
|
"epoch": 0.32305433186490456,
|
|
"grad_norm": 28.063581466674805,
|
|
"learning_rate": 4.2947798076611047e-07,
|
|
"logits/chosen": -1.6841022968292236,
|
|
"logits/rejected": -1.6491317749023438,
|
|
"logps/chosen": -86.5882339477539,
|
|
"logps/ref_chosen": -74.71533966064453,
|
|
"logps/ref_rejected": -92.06246185302734,
|
|
"logps/rejected": -117.34354400634766,
|
|
"loss": 0.7639,
|
|
"margin_dpo/margin_mean": 13.408201217651367,
|
|
"margin_dpo/margin_std": 13.679563522338867,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.3245227606461087,
|
|
"grad_norm": 22.508024215698242,
|
|
"learning_rate": 4.285822501755485e-07,
|
|
"logits/chosen": -0.7385995388031006,
|
|
"logits/rejected": -0.8880366683006287,
|
|
"logps/chosen": -69.58158111572266,
|
|
"logps/ref_chosen": -59.28809356689453,
|
|
"logps/ref_rejected": -111.22705841064453,
|
|
"logps/rejected": -140.26364135742188,
|
|
"loss": 0.5435,
|
|
"margin_dpo/margin_mean": 18.743091583251953,
|
|
"margin_dpo/margin_std": 17.367952346801758,
|
|
"step": 221
|
|
},
|
|
{
|
|
"epoch": 0.32599118942731276,
|
|
"grad_norm": 48.555931091308594,
|
|
"learning_rate": 4.276818137766118e-07,
|
|
"logits/chosen": -1.3711744546890259,
|
|
"logits/rejected": -0.945267915725708,
|
|
"logps/chosen": -102.54623413085938,
|
|
"logps/ref_chosen": -90.29017639160156,
|
|
"logps/ref_rejected": -102.741455078125,
|
|
"logps/rejected": -132.53857421875,
|
|
"loss": 0.7982,
|
|
"margin_dpo/margin_mean": 17.54104995727539,
|
|
"margin_dpo/margin_std": 18.724212646484375,
|
|
"step": 222
|
|
},
|
|
{
|
|
"epoch": 0.3274596182085169,
|
|
"grad_norm": 25.106897354125977,
|
|
"learning_rate": 4.2677669529663686e-07,
|
|
"logits/chosen": -1.2980246543884277,
|
|
"logits/rejected": -1.0237699747085571,
|
|
"logps/chosen": -96.40541076660156,
|
|
"logps/ref_chosen": -80.65748596191406,
|
|
"logps/ref_rejected": -96.07778930664062,
|
|
"logps/rejected": -119.77986145019531,
|
|
"loss": 0.8083,
|
|
"margin_dpo/margin_mean": 7.954143524169922,
|
|
"margin_dpo/margin_std": 10.7671480178833,
|
|
"step": 223
|
|
},
|
|
{
|
|
"epoch": 0.328928046989721,
|
|
"grad_norm": 27.507966995239258,
|
|
"learning_rate": 4.2586691858633747e-07,
|
|
"logits/chosen": -1.3987040519714355,
|
|
"logits/rejected": -0.79180908203125,
|
|
"logps/chosen": -102.76864624023438,
|
|
"logps/ref_chosen": -91.31893920898438,
|
|
"logps/ref_rejected": -79.40377044677734,
|
|
"logps/rejected": -105.58963012695312,
|
|
"loss": 0.6666,
|
|
"margin_dpo/margin_mean": 14.736160278320312,
|
|
"margin_dpo/margin_std": 14.218803405761719,
|
|
"step": 224
|
|
},
|
|
{
|
|
"epoch": 0.3303964757709251,
|
|
"grad_norm": 25.836376190185547,
|
|
"learning_rate": 4.249525076191759e-07,
|
|
"logits/chosen": -2.027069091796875,
|
|
"logits/rejected": -1.221968650817871,
|
|
"logps/chosen": -123.97574615478516,
|
|
"logps/ref_chosen": -113.84062194824219,
|
|
"logps/ref_rejected": -112.13330078125,
|
|
"logps/rejected": -142.740966796875,
|
|
"loss": 0.6625,
|
|
"margin_dpo/margin_mean": 20.47254180908203,
|
|
"margin_dpo/margin_std": 17.923603057861328,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.33186490455212925,
|
|
"grad_norm": 24.863615036010742,
|
|
"learning_rate": 4.2403348649073167e-07,
|
|
"logits/chosen": -1.66581130027771,
|
|
"logits/rejected": -0.9718565344810486,
|
|
"logps/chosen": -111.22610473632812,
|
|
"logps/ref_chosen": -99.33114624023438,
|
|
"logps/ref_rejected": -44.584197998046875,
|
|
"logps/rejected": -71.60173797607422,
|
|
"loss": 0.6747,
|
|
"margin_dpo/margin_mean": 15.122581481933594,
|
|
"margin_dpo/margin_std": 14.921210289001465,
|
|
"step": 226
|
|
},
|
|
{
|
|
"epoch": 0.3333333333333333,
|
|
"grad_norm": 31.727184295654297,
|
|
"learning_rate": 4.2310987941806615e-07,
|
|
"logits/chosen": -0.6366788148880005,
|
|
"logits/rejected": -0.558417558670044,
|
|
"logps/chosen": -92.5580062866211,
|
|
"logps/ref_chosen": -78.06465148925781,
|
|
"logps/ref_rejected": -98.39476013183594,
|
|
"logps/rejected": -127.857177734375,
|
|
"loss": 0.7772,
|
|
"margin_dpo/margin_mean": 14.969067573547363,
|
|
"margin_dpo/margin_std": 11.369447708129883,
|
|
"step": 227
|
|
},
|
|
{
|
|
"epoch": 0.33480176211453744,
|
|
"grad_norm": 24.629833221435547,
|
|
"learning_rate": 4.2218171073908463e-07,
|
|
"logits/chosen": -1.5679694414138794,
|
|
"logits/rejected": -1.2253010272979736,
|
|
"logps/chosen": -133.501220703125,
|
|
"logps/ref_chosen": -118.2425537109375,
|
|
"logps/ref_rejected": -149.5880889892578,
|
|
"logps/rejected": -177.65139770507812,
|
|
"loss": 0.7634,
|
|
"margin_dpo/margin_mean": 12.804656982421875,
|
|
"margin_dpo/margin_std": 13.355108261108398,
|
|
"step": 228
|
|
},
|
|
{
|
|
"epoch": 0.33627019089574156,
|
|
"grad_norm": 22.838153839111328,
|
|
"learning_rate": 4.212490049118951e-07,
|
|
"logits/chosen": -1.934976577758789,
|
|
"logits/rejected": -1.0155363082885742,
|
|
"logps/chosen": -93.84185791015625,
|
|
"logps/ref_chosen": -86.05354309082031,
|
|
"logps/ref_rejected": -84.62686157226562,
|
|
"logps/rejected": -119.24227142333984,
|
|
"loss": 0.5845,
|
|
"margin_dpo/margin_mean": 26.82710075378418,
|
|
"margin_dpo/margin_std": 14.36768913269043,
|
|
"step": 229
|
|
},
|
|
{
|
|
"epoch": 0.3377386196769457,
|
|
"grad_norm": 22.581356048583984,
|
|
"learning_rate": 4.203117865141635e-07,
|
|
"logits/chosen": -2.0330729484558105,
|
|
"logits/rejected": -1.621316909790039,
|
|
"logps/chosen": -82.906005859375,
|
|
"logps/ref_chosen": -72.93336486816406,
|
|
"logps/ref_rejected": -98.11904907226562,
|
|
"logps/rejected": -126.56108856201172,
|
|
"loss": 0.5216,
|
|
"margin_dpo/margin_mean": 18.46939468383789,
|
|
"margin_dpo/margin_std": 15.664244651794434,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.3392070484581498,
|
|
"grad_norm": 22.018823623657227,
|
|
"learning_rate": 4.1937008024246625e-07,
|
|
"logits/chosen": -2.015583038330078,
|
|
"logits/rejected": -1.279665470123291,
|
|
"logps/chosen": -111.63320922851562,
|
|
"logps/ref_chosen": -100.19026947021484,
|
|
"logps/ref_rejected": -91.82012939453125,
|
|
"logps/rejected": -123.36331176757812,
|
|
"loss": 0.6761,
|
|
"margin_dpo/margin_mean": 20.100221633911133,
|
|
"margin_dpo/margin_std": 17.13475799560547,
|
|
"step": 231
|
|
},
|
|
{
|
|
"epoch": 0.3406754772393539,
|
|
"grad_norm": 28.21176528930664,
|
|
"learning_rate": 4.1842391091163933e-07,
|
|
"logits/chosen": -0.8941325545310974,
|
|
"logits/rejected": -0.42558372020721436,
|
|
"logps/chosen": -122.58344268798828,
|
|
"logps/ref_chosen": -110.88046264648438,
|
|
"logps/ref_rejected": -81.28340148925781,
|
|
"logps/rejected": -104.9071044921875,
|
|
"loss": 0.7972,
|
|
"margin_dpo/margin_mean": 11.920722007751465,
|
|
"margin_dpo/margin_std": 17.419721603393555,
|
|
"step": 232
|
|
},
|
|
{
|
|
"epoch": 0.342143906020558,
|
|
"grad_norm": 25.46039390563965,
|
|
"learning_rate": 4.174733034541245e-07,
|
|
"logits/chosen": -0.8610257506370544,
|
|
"logits/rejected": -0.8550558090209961,
|
|
"logps/chosen": -106.99727630615234,
|
|
"logps/ref_chosen": -89.96463775634766,
|
|
"logps/ref_rejected": -108.56855773925781,
|
|
"logps/rejected": -140.77781677246094,
|
|
"loss": 0.6343,
|
|
"margin_dpo/margin_mean": 15.176610946655273,
|
|
"margin_dpo/margin_std": 17.110292434692383,
|
|
"step": 233
|
|
},
|
|
{
|
|
"epoch": 0.3436123348017621,
|
|
"grad_norm": 25.88790512084961,
|
|
"learning_rate": 4.165182829193126e-07,
|
|
"logits/chosen": -1.1169289350509644,
|
|
"logits/rejected": -0.8724891543388367,
|
|
"logps/chosen": -85.27423858642578,
|
|
"logps/ref_chosen": -73.45465850830078,
|
|
"logps/ref_rejected": -119.467041015625,
|
|
"logps/rejected": -149.45150756835938,
|
|
"loss": 0.6698,
|
|
"margin_dpo/margin_mean": 18.164871215820312,
|
|
"margin_dpo/margin_std": 13.520463943481445,
|
|
"step": 234
|
|
},
|
|
{
|
|
"epoch": 0.34508076358296624,
|
|
"grad_norm": 23.897993087768555,
|
|
"learning_rate": 4.1555887447288255e-07,
|
|
"logits/chosen": -0.2996028959751129,
|
|
"logits/rejected": 0.14718373119831085,
|
|
"logps/chosen": -116.7324447631836,
|
|
"logps/ref_chosen": -102.08454132080078,
|
|
"logps/ref_rejected": -99.83527374267578,
|
|
"logps/rejected": -128.42771911621094,
|
|
"loss": 0.6736,
|
|
"margin_dpo/margin_mean": 13.944540977478027,
|
|
"margin_dpo/margin_std": 15.060174942016602,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.3465491923641703,
|
|
"grad_norm": 22.317142486572266,
|
|
"learning_rate": 4.1459510339613946e-07,
|
|
"logits/chosen": -1.0241249799728394,
|
|
"logits/rejected": -1.0809072256088257,
|
|
"logps/chosen": -93.7081298828125,
|
|
"logps/ref_chosen": -80.65806579589844,
|
|
"logps/ref_rejected": -149.62136840820312,
|
|
"logps/rejected": -181.28709411621094,
|
|
"loss": 0.6689,
|
|
"margin_dpo/margin_mean": 18.615659713745117,
|
|
"margin_dpo/margin_std": 21.68730926513672,
|
|
"step": 236
|
|
},
|
|
{
|
|
"epoch": 0.34801762114537443,
|
|
"grad_norm": 28.688703536987305,
|
|
"learning_rate": 4.136269950853473e-07,
|
|
"logits/chosen": -0.570310115814209,
|
|
"logits/rejected": -0.23448389768600464,
|
|
"logps/chosen": -90.6567611694336,
|
|
"logps/ref_chosen": -80.41928100585938,
|
|
"logps/ref_rejected": -78.44154357910156,
|
|
"logps/rejected": -106.99150848388672,
|
|
"loss": 0.8527,
|
|
"margin_dpo/margin_mean": 18.312477111816406,
|
|
"margin_dpo/margin_std": 15.394723892211914,
|
|
"step": 237
|
|
},
|
|
{
|
|
"epoch": 0.34948604992657856,
|
|
"grad_norm": 23.128028869628906,
|
|
"learning_rate": 4.126545750510605e-07,
|
|
"logits/chosen": -1.2925560474395752,
|
|
"logits/rejected": -1.2870701551437378,
|
|
"logps/chosen": -77.17804718017578,
|
|
"logps/ref_chosen": -60.57853698730469,
|
|
"logps/ref_rejected": -97.90704345703125,
|
|
"logps/rejected": -129.80294799804688,
|
|
"loss": 0.7124,
|
|
"margin_dpo/margin_mean": 15.296379089355469,
|
|
"margin_dpo/margin_std": 14.916626930236816,
|
|
"step": 238
|
|
},
|
|
{
|
|
"epoch": 0.3509544787077827,
|
|
"grad_norm": 27.944942474365234,
|
|
"learning_rate": 4.116778689174514e-07,
|
|
"logits/chosen": -1.2913211584091187,
|
|
"logits/rejected": -0.8684008121490479,
|
|
"logps/chosen": -107.7053451538086,
|
|
"logps/ref_chosen": -97.148681640625,
|
|
"logps/ref_rejected": -113.69243621826172,
|
|
"logps/rejected": -142.38980102539062,
|
|
"loss": 0.7431,
|
|
"margin_dpo/margin_mean": 18.140701293945312,
|
|
"margin_dpo/margin_std": 14.662839889526367,
|
|
"step": 239
|
|
},
|
|
{
|
|
"epoch": 0.3524229074889868,
|
|
"grad_norm": 33.86201858520508,
|
|
"learning_rate": 4.106969024216348e-07,
|
|
"logits/chosen": -1.348999261856079,
|
|
"logits/rejected": -1.2273541688919067,
|
|
"logps/chosen": -87.25590515136719,
|
|
"logps/ref_chosen": -69.32797241210938,
|
|
"logps/ref_rejected": -81.43118286132812,
|
|
"logps/rejected": -111.20504760742188,
|
|
"loss": 0.7521,
|
|
"margin_dpo/margin_mean": 11.845929145812988,
|
|
"margin_dpo/margin_std": 12.4403076171875,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.35389133627019087,
|
|
"grad_norm": 22.594024658203125,
|
|
"learning_rate": 4.097117014129903e-07,
|
|
"logits/chosen": -1.7714438438415527,
|
|
"logits/rejected": -0.992931604385376,
|
|
"logps/chosen": -92.8592529296875,
|
|
"logps/ref_chosen": -82.93885803222656,
|
|
"logps/ref_rejected": -73.07276916503906,
|
|
"logps/rejected": -107.22079467773438,
|
|
"loss": 0.4697,
|
|
"margin_dpo/margin_mean": 24.22762107849121,
|
|
"margin_dpo/margin_std": 16.218782424926758,
|
|
"step": 241
|
|
},
|
|
{
|
|
"epoch": 0.355359765051395,
|
|
"grad_norm": 26.068859100341797,
|
|
"learning_rate": 4.087222918524807e-07,
|
|
"logits/chosen": -1.198895812034607,
|
|
"logits/rejected": -0.9665822982788086,
|
|
"logps/chosen": -113.93301391601562,
|
|
"logps/ref_chosen": -98.63720703125,
|
|
"logps/ref_rejected": -84.78510284423828,
|
|
"logps/rejected": -116.67051696777344,
|
|
"loss": 0.6153,
|
|
"margin_dpo/margin_mean": 16.589599609375,
|
|
"margin_dpo/margin_std": 15.04800796508789,
|
|
"step": 242
|
|
},
|
|
{
|
|
"epoch": 0.3568281938325991,
|
|
"grad_norm": 27.26268768310547,
|
|
"learning_rate": 4.07728699811968e-07,
|
|
"logits/chosen": -1.2558754682540894,
|
|
"logits/rejected": -0.5668581128120422,
|
|
"logps/chosen": -118.21045684814453,
|
|
"logps/ref_chosen": -104.37299346923828,
|
|
"logps/ref_rejected": -81.6190185546875,
|
|
"logps/rejected": -112.95948791503906,
|
|
"loss": 0.7338,
|
|
"margin_dpo/margin_mean": 17.503009796142578,
|
|
"margin_dpo/margin_std": 15.984872817993164,
|
|
"step": 243
|
|
},
|
|
{
|
|
"epoch": 0.35829662261380324,
|
|
"grad_norm": 22.77792739868164,
|
|
"learning_rate": 4.067309514735267e-07,
|
|
"logits/chosen": -2.0644044876098633,
|
|
"logits/rejected": -1.902282953262329,
|
|
"logps/chosen": -103.87003326416016,
|
|
"logps/ref_chosen": -93.71182250976562,
|
|
"logps/ref_rejected": -128.01568603515625,
|
|
"logps/rejected": -156.72738647460938,
|
|
"loss": 0.6778,
|
|
"margin_dpo/margin_mean": 18.553510665893555,
|
|
"margin_dpo/margin_std": 11.52467155456543,
|
|
"step": 244
|
|
},
|
|
{
|
|
"epoch": 0.35976505139500736,
|
|
"grad_norm": 22.12187385559082,
|
|
"learning_rate": 4.057290731287531e-07,
|
|
"logits/chosen": -1.0430703163146973,
|
|
"logits/rejected": -0.8094485998153687,
|
|
"logps/chosen": -109.80570983886719,
|
|
"logps/ref_chosen": -101.46330261230469,
|
|
"logps/ref_rejected": -103.5881118774414,
|
|
"logps/rejected": -134.34341430664062,
|
|
"loss": 0.5495,
|
|
"margin_dpo/margin_mean": 22.41289710998535,
|
|
"margin_dpo/margin_std": 14.135283470153809,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.36123348017621143,
|
|
"grad_norm": 32.46879959106445,
|
|
"learning_rate": 4.047230911780736e-07,
|
|
"logits/chosen": -1.5383753776550293,
|
|
"logits/rejected": -1.2452689409255981,
|
|
"logps/chosen": -95.5453872680664,
|
|
"logps/ref_chosen": -79.95723724365234,
|
|
"logps/ref_rejected": -111.7950439453125,
|
|
"logps/rejected": -149.63845825195312,
|
|
"loss": 0.7436,
|
|
"margin_dpo/margin_mean": 22.255264282226562,
|
|
"margin_dpo/margin_std": 13.090051651000977,
|
|
"step": 246
|
|
},
|
|
{
|
|
"epoch": 0.36270190895741555,
|
|
"grad_norm": 20.374637603759766,
|
|
"learning_rate": 4.0371303213004814e-07,
|
|
"logits/chosen": -2.6092662811279297,
|
|
"logits/rejected": -2.0955309867858887,
|
|
"logps/chosen": -100.3311996459961,
|
|
"logps/ref_chosen": -87.5809326171875,
|
|
"logps/ref_rejected": -107.0278549194336,
|
|
"logps/rejected": -140.61538696289062,
|
|
"loss": 0.5841,
|
|
"margin_dpo/margin_mean": 20.837257385253906,
|
|
"margin_dpo/margin_std": 12.746124267578125,
|
|
"step": 247
|
|
},
|
|
{
|
|
"epoch": 0.3641703377386197,
|
|
"grad_norm": 31.063600540161133,
|
|
"learning_rate": 4.0269892260067197e-07,
|
|
"logits/chosen": -1.4017457962036133,
|
|
"logits/rejected": -1.265455961227417,
|
|
"logps/chosen": -100.98844909667969,
|
|
"logps/ref_chosen": -83.37393188476562,
|
|
"logps/ref_rejected": -113.40422821044922,
|
|
"logps/rejected": -142.08447265625,
|
|
"loss": 0.7611,
|
|
"margin_dpo/margin_mean": 11.065750122070312,
|
|
"margin_dpo/margin_std": 18.710220336914062,
|
|
"step": 248
|
|
},
|
|
{
|
|
"epoch": 0.3656387665198238,
|
|
"grad_norm": 34.479984283447266,
|
|
"learning_rate": 4.0168078931267426e-07,
|
|
"logits/chosen": -1.5276246070861816,
|
|
"logits/rejected": -1.1095179319381714,
|
|
"logps/chosen": -110.28913116455078,
|
|
"logps/ref_chosen": -90.19677734375,
|
|
"logps/ref_rejected": -80.43144226074219,
|
|
"logps/rejected": -107.76251983642578,
|
|
"loss": 0.8498,
|
|
"margin_dpo/margin_mean": 7.238720893859863,
|
|
"margin_dpo/margin_std": 15.718103408813477,
|
|
"step": 249
|
|
},
|
|
{
|
|
"epoch": 0.3671071953010279,
|
|
"grad_norm": 41.039459228515625,
|
|
"learning_rate": 4.006586590948141e-07,
|
|
"logits/chosen": -1.7807164192199707,
|
|
"logits/rejected": -0.9707603454589844,
|
|
"logps/chosen": -154.2344207763672,
|
|
"logps/ref_chosen": -138.57211303710938,
|
|
"logps/ref_rejected": -77.57225799560547,
|
|
"logps/rejected": -106.68568420410156,
|
|
"loss": 0.7792,
|
|
"margin_dpo/margin_mean": 13.451122283935547,
|
|
"margin_dpo/margin_std": 14.689545631408691,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.368575624082232,
|
|
"grad_norm": 27.502151489257812,
|
|
"learning_rate": 3.9963255888117325e-07,
|
|
"logits/chosen": -1.5643404722213745,
|
|
"logits/rejected": -0.5259659290313721,
|
|
"logps/chosen": -115.2924575805664,
|
|
"logps/ref_chosen": -104.47010803222656,
|
|
"logps/ref_rejected": -72.63395690917969,
|
|
"logps/rejected": -100.4027099609375,
|
|
"loss": 0.8432,
|
|
"margin_dpo/margin_mean": 16.946407318115234,
|
|
"margin_dpo/margin_std": 17.334667205810547,
|
|
"step": 251
|
|
},
|
|
{
|
|
"epoch": 0.3700440528634361,
|
|
"grad_norm": 26.42395782470703,
|
|
"learning_rate": 3.9860251571044666e-07,
|
|
"logits/chosen": -1.0975241661071777,
|
|
"logits/rejected": -0.6452772617340088,
|
|
"logps/chosen": -110.46723937988281,
|
|
"logps/ref_chosen": -97.77006530761719,
|
|
"logps/ref_rejected": -87.78140258789062,
|
|
"logps/rejected": -118.4449234008789,
|
|
"loss": 0.7427,
|
|
"margin_dpo/margin_mean": 17.966344833374023,
|
|
"margin_dpo/margin_std": 16.002155303955078,
|
|
"step": 252
|
|
},
|
|
{
|
|
"epoch": 0.37151248164464024,
|
|
"grad_norm": 28.601953506469727,
|
|
"learning_rate": 3.9756855672522986e-07,
|
|
"logits/chosen": -0.999688982963562,
|
|
"logits/rejected": -0.5330642461776733,
|
|
"logps/chosen": -106.92183685302734,
|
|
"logps/ref_chosen": -89.15538024902344,
|
|
"logps/ref_rejected": -91.28717041015625,
|
|
"logps/rejected": -125.74440002441406,
|
|
"loss": 0.7583,
|
|
"margin_dpo/margin_mean": 16.69076919555664,
|
|
"margin_dpo/margin_std": 8.515735626220703,
|
|
"step": 253
|
|
},
|
|
{
|
|
"epoch": 0.37298091042584436,
|
|
"grad_norm": 24.010168075561523,
|
|
"learning_rate": 3.965307091713037e-07,
|
|
"logits/chosen": -0.826606035232544,
|
|
"logits/rejected": -0.84417724609375,
|
|
"logps/chosen": -120.03327941894531,
|
|
"logps/ref_chosen": -104.02809143066406,
|
|
"logps/ref_rejected": -117.66902160644531,
|
|
"logps/rejected": -143.052978515625,
|
|
"loss": 0.7741,
|
|
"margin_dpo/margin_mean": 9.378756523132324,
|
|
"margin_dpo/margin_std": 13.781966209411621,
|
|
"step": 254
|
|
},
|
|
{
|
|
"epoch": 0.3744493392070485,
|
|
"grad_norm": 27.174829483032227,
|
|
"learning_rate": 3.954890003969163e-07,
|
|
"logits/chosen": -1.2953753471374512,
|
|
"logits/rejected": -0.8916399478912354,
|
|
"logps/chosen": -160.41029357910156,
|
|
"logps/ref_chosen": -140.6939697265625,
|
|
"logps/ref_rejected": -141.0518341064453,
|
|
"logps/rejected": -173.4158935546875,
|
|
"loss": 0.66,
|
|
"margin_dpo/margin_mean": 12.647738456726074,
|
|
"margin_dpo/margin_std": 11.482951164245605,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.37591776798825255,
|
|
"grad_norm": 28.97931671142578,
|
|
"learning_rate": 3.944434578520628e-07,
|
|
"logits/chosen": -1.5570547580718994,
|
|
"logits/rejected": -1.3356791734695435,
|
|
"logps/chosen": -94.7815170288086,
|
|
"logps/ref_chosen": -77.7818374633789,
|
|
"logps/ref_rejected": -105.81022644042969,
|
|
"logps/rejected": -141.99551391601562,
|
|
"loss": 0.6885,
|
|
"margin_dpo/margin_mean": 19.18560028076172,
|
|
"margin_dpo/margin_std": 17.2878360748291,
|
|
"step": 256
|
|
},
|
|
{
|
|
"epoch": 0.37738619676945667,
|
|
"grad_norm": 27.675180435180664,
|
|
"learning_rate": 3.933941090877615e-07,
|
|
"logits/chosen": -1.4617829322814941,
|
|
"logits/rejected": -0.874555230140686,
|
|
"logps/chosen": -120.56830596923828,
|
|
"logps/ref_chosen": -104.2494888305664,
|
|
"logps/ref_rejected": -103.968505859375,
|
|
"logps/rejected": -140.97515869140625,
|
|
"loss": 0.6644,
|
|
"margin_dpo/margin_mean": 20.687824249267578,
|
|
"margin_dpo/margin_std": 18.404783248901367,
|
|
"step": 257
|
|
},
|
|
{
|
|
"epoch": 0.3788546255506608,
|
|
"grad_norm": 26.982419967651367,
|
|
"learning_rate": 3.923409817553284e-07,
|
|
"logits/chosen": -1.7992323637008667,
|
|
"logits/rejected": -1.3448323011398315,
|
|
"logps/chosen": -111.3430404663086,
|
|
"logps/ref_chosen": -95.29330444335938,
|
|
"logps/ref_rejected": -93.0291748046875,
|
|
"logps/rejected": -121.9286880493164,
|
|
"loss": 0.6151,
|
|
"margin_dpo/margin_mean": 12.84976863861084,
|
|
"margin_dpo/margin_std": 11.585992813110352,
|
|
"step": 258
|
|
},
|
|
{
|
|
"epoch": 0.3803230543318649,
|
|
"grad_norm": 28.79665756225586,
|
|
"learning_rate": 3.9128410360564793e-07,
|
|
"logits/chosen": -0.6006170511245728,
|
|
"logits/rejected": 0.08246561884880066,
|
|
"logps/chosen": -117.34747314453125,
|
|
"logps/ref_chosen": -96.55043029785156,
|
|
"logps/ref_rejected": -79.44844818115234,
|
|
"logps/rejected": -114.84416198730469,
|
|
"loss": 0.817,
|
|
"margin_dpo/margin_mean": 14.59865951538086,
|
|
"margin_dpo/margin_std": 21.104887008666992,
|
|
"step": 259
|
|
},
|
|
{
|
|
"epoch": 0.38179148311306904,
|
|
"grad_norm": 31.739402770996094,
|
|
"learning_rate": 3.9022350248844246e-07,
|
|
"logits/chosen": -0.8702751398086548,
|
|
"logits/rejected": -0.7750843167304993,
|
|
"logps/chosen": -89.928466796875,
|
|
"logps/ref_chosen": -69.2818603515625,
|
|
"logps/ref_rejected": -107.59248352050781,
|
|
"logps/rejected": -143.739013671875,
|
|
"loss": 0.937,
|
|
"margin_dpo/margin_mean": 15.499946594238281,
|
|
"margin_dpo/margin_std": 15.068811416625977,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3832599118942731,
|
|
"grad_norm": 24.90912628173828,
|
|
"learning_rate": 3.891592063515376e-07,
|
|
"logits/chosen": -1.9554228782653809,
|
|
"logits/rejected": -1.618184208869934,
|
|
"logps/chosen": -109.2607421875,
|
|
"logps/ref_chosen": -93.40982055664062,
|
|
"logps/ref_rejected": -122.90621948242188,
|
|
"logps/rejected": -155.48861694335938,
|
|
"loss": 0.6509,
|
|
"margin_dpo/margin_mean": 16.731483459472656,
|
|
"margin_dpo/margin_std": 16.480255126953125,
|
|
"step": 261
|
|
},
|
|
{
|
|
"epoch": 0.38472834067547723,
|
|
"grad_norm": 24.286285400390625,
|
|
"learning_rate": 3.880912432401264e-07,
|
|
"logits/chosen": -1.1788181066513062,
|
|
"logits/rejected": -0.6409732699394226,
|
|
"logps/chosen": -110.82711791992188,
|
|
"logps/ref_chosen": -94.37785339355469,
|
|
"logps/ref_rejected": -128.30076599121094,
|
|
"logps/rejected": -155.30270385742188,
|
|
"loss": 0.7006,
|
|
"margin_dpo/margin_mean": 10.552679061889648,
|
|
"margin_dpo/margin_std": 14.37498664855957,
|
|
"step": 262
|
|
},
|
|
{
|
|
"epoch": 0.38619676945668135,
|
|
"grad_norm": 26.123411178588867,
|
|
"learning_rate": 3.870196412960302e-07,
|
|
"logits/chosen": -2.209285259246826,
|
|
"logits/rejected": -1.72807776927948,
|
|
"logps/chosen": -125.52157592773438,
|
|
"logps/ref_chosen": -112.25532531738281,
|
|
"logps/ref_rejected": -118.7103042602539,
|
|
"logps/rejected": -153.01513671875,
|
|
"loss": 0.5653,
|
|
"margin_dpo/margin_mean": 21.03856658935547,
|
|
"margin_dpo/margin_std": 13.661331176757812,
|
|
"step": 263
|
|
},
|
|
{
|
|
"epoch": 0.3876651982378855,
|
|
"grad_norm": 26.471071243286133,
|
|
"learning_rate": 3.8594442875695665e-07,
|
|
"logits/chosen": -2.094125747680664,
|
|
"logits/rejected": -1.5296189785003662,
|
|
"logps/chosen": -88.62034606933594,
|
|
"logps/ref_chosen": -74.47600555419922,
|
|
"logps/ref_rejected": -82.83815002441406,
|
|
"logps/rejected": -115.3791732788086,
|
|
"loss": 0.6855,
|
|
"margin_dpo/margin_mean": 18.396671295166016,
|
|
"margin_dpo/margin_std": 14.122834205627441,
|
|
"step": 264
|
|
},
|
|
{
|
|
"epoch": 0.3891336270190896,
|
|
"grad_norm": 24.215213775634766,
|
|
"learning_rate": 3.848656339557562e-07,
|
|
"logits/chosen": -1.6657154560089111,
|
|
"logits/rejected": -0.7729620933532715,
|
|
"logps/chosen": -117.15290069580078,
|
|
"logps/ref_chosen": -104.01231384277344,
|
|
"logps/ref_rejected": -84.9332275390625,
|
|
"logps/rejected": -115.11080932617188,
|
|
"loss": 0.7431,
|
|
"margin_dpo/margin_mean": 17.0369873046875,
|
|
"margin_dpo/margin_std": 18.63312530517578,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.39060205580029367,
|
|
"grad_norm": 33.143089294433594,
|
|
"learning_rate": 3.8378328531967507e-07,
|
|
"logits/chosen": -1.3864130973815918,
|
|
"logits/rejected": -0.2755950391292572,
|
|
"logps/chosen": -141.586669921875,
|
|
"logps/ref_chosen": -127.02003479003906,
|
|
"logps/ref_rejected": -63.359657287597656,
|
|
"logps/rejected": -94.93106079101562,
|
|
"loss": 0.7461,
|
|
"margin_dpo/margin_mean": 17.004764556884766,
|
|
"margin_dpo/margin_std": 11.14462661743164,
|
|
"step": 266
|
|
},
|
|
{
|
|
"epoch": 0.3920704845814978,
|
|
"grad_norm": 23.171924591064453,
|
|
"learning_rate": 3.8269741136960646e-07,
|
|
"logits/chosen": -1.4743266105651855,
|
|
"logits/rejected": -0.8582184910774231,
|
|
"logps/chosen": -113.98542785644531,
|
|
"logps/ref_chosen": -100.46978759765625,
|
|
"logps/ref_rejected": -86.45939636230469,
|
|
"logps/rejected": -115.53654479980469,
|
|
"loss": 0.5557,
|
|
"margin_dpo/margin_mean": 15.561500549316406,
|
|
"margin_dpo/margin_std": 13.366697311401367,
|
|
"step": 267
|
|
},
|
|
{
|
|
"epoch": 0.3935389133627019,
|
|
"grad_norm": 23.90070152282715,
|
|
"learning_rate": 3.8160804071933894e-07,
|
|
"logits/chosen": -0.6755940914154053,
|
|
"logits/rejected": -0.30946046113967896,
|
|
"logps/chosen": -104.34091186523438,
|
|
"logps/ref_chosen": -87.36892700195312,
|
|
"logps/ref_rejected": -75.56564331054688,
|
|
"logps/rejected": -104.53038787841797,
|
|
"loss": 0.6504,
|
|
"margin_dpo/margin_mean": 11.992761611938477,
|
|
"margin_dpo/margin_std": 12.630704879760742,
|
|
"step": 268
|
|
},
|
|
{
|
|
"epoch": 0.39500734214390604,
|
|
"grad_norm": 23.326845169067383,
|
|
"learning_rate": 3.8051520207480204e-07,
|
|
"logits/chosen": -1.7525832653045654,
|
|
"logits/rejected": -1.138352632522583,
|
|
"logps/chosen": -113.57240295410156,
|
|
"logps/ref_chosen": -105.39424133300781,
|
|
"logps/ref_rejected": -102.73477172851562,
|
|
"logps/rejected": -131.50343322753906,
|
|
"loss": 0.5895,
|
|
"margin_dpo/margin_mean": 20.590484619140625,
|
|
"margin_dpo/margin_std": 11.655923843383789,
|
|
"step": 269
|
|
},
|
|
{
|
|
"epoch": 0.3964757709251101,
|
|
"grad_norm": 28.63389015197754,
|
|
"learning_rate": 3.794189242333106e-07,
|
|
"logits/chosen": -2.346863269805908,
|
|
"logits/rejected": -1.4268286228179932,
|
|
"logps/chosen": -135.16775512695312,
|
|
"logps/ref_chosen": -117.72895812988281,
|
|
"logps/ref_rejected": -108.10507202148438,
|
|
"logps/rejected": -145.209716796875,
|
|
"loss": 0.7485,
|
|
"margin_dpo/margin_mean": 19.665857315063477,
|
|
"margin_dpo/margin_std": 18.711822509765625,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.39794419970631423,
|
|
"grad_norm": 25.63298988342285,
|
|
"learning_rate": 3.7831923608280514e-07,
|
|
"logits/chosen": -1.5109490156173706,
|
|
"logits/rejected": -0.8422449827194214,
|
|
"logps/chosen": -134.63824462890625,
|
|
"logps/ref_chosen": -121.63641357421875,
|
|
"logps/ref_rejected": -107.82986450195312,
|
|
"logps/rejected": -140.81619262695312,
|
|
"loss": 0.6551,
|
|
"margin_dpo/margin_mean": 19.984500885009766,
|
|
"margin_dpo/margin_std": 19.420936584472656,
|
|
"step": 271
|
|
},
|
|
{
|
|
"epoch": 0.39941262848751835,
|
|
"grad_norm": 27.825122833251953,
|
|
"learning_rate": 3.772161666010912e-07,
|
|
"logits/chosen": -1.984161138534546,
|
|
"logits/rejected": -1.790366768836975,
|
|
"logps/chosen": -64.66868591308594,
|
|
"logps/ref_chosen": -57.89815902709961,
|
|
"logps/ref_rejected": -91.71653747558594,
|
|
"logps/rejected": -118.36727905273438,
|
|
"loss": 0.6546,
|
|
"margin_dpo/margin_mean": 19.88021469116211,
|
|
"margin_dpo/margin_std": 20.506275177001953,
|
|
"step": 272
|
|
},
|
|
{
|
|
"epoch": 0.4008810572687225,
|
|
"grad_norm": 27.83376121520996,
|
|
"learning_rate": 3.761097448550755e-07,
|
|
"logits/chosen": -1.444991111755371,
|
|
"logits/rejected": -0.9423968195915222,
|
|
"logps/chosen": -132.13973999023438,
|
|
"logps/ref_chosen": -116.63589477539062,
|
|
"logps/ref_rejected": -116.1893081665039,
|
|
"logps/rejected": -151.94931030273438,
|
|
"loss": 0.7343,
|
|
"margin_dpo/margin_mean": 20.25616455078125,
|
|
"margin_dpo/margin_std": 20.952659606933594,
|
|
"step": 273
|
|
},
|
|
{
|
|
"epoch": 0.4023494860499266,
|
|
"grad_norm": 26.02191925048828,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": -1.2731891870498657,
|
|
"logits/rejected": -1.1204042434692383,
|
|
"logps/chosen": -81.43106079101562,
|
|
"logps/ref_chosen": -64.8055419921875,
|
|
"logps/ref_rejected": -85.86842346191406,
|
|
"logps/rejected": -118.44586181640625,
|
|
"loss": 0.6414,
|
|
"margin_dpo/margin_mean": 15.951923370361328,
|
|
"margin_dpo/margin_std": 12.496143341064453,
|
|
"step": 274
|
|
},
|
|
{
|
|
"epoch": 0.40381791483113066,
|
|
"grad_norm": 28.154870986938477,
|
|
"learning_rate": 3.738869612786737e-07,
|
|
"logits/chosen": -2.217825412750244,
|
|
"logits/rejected": -1.8745087385177612,
|
|
"logps/chosen": -75.03510284423828,
|
|
"logps/ref_chosen": -66.54554748535156,
|
|
"logps/ref_rejected": -83.95608520507812,
|
|
"logps/rejected": -110.03489685058594,
|
|
"loss": 0.6737,
|
|
"margin_dpo/margin_mean": 17.58925437927246,
|
|
"margin_dpo/margin_std": 12.368159294128418,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.4052863436123348,
|
|
"grad_norm": 27.130475997924805,
|
|
"learning_rate": 3.7277065802070204e-07,
|
|
"logits/chosen": -1.3086066246032715,
|
|
"logits/rejected": -1.1000663042068481,
|
|
"logps/chosen": -88.32952880859375,
|
|
"logps/ref_chosen": -73.30899047851562,
|
|
"logps/ref_rejected": -80.19606018066406,
|
|
"logps/rejected": -112.3787612915039,
|
|
"loss": 0.6958,
|
|
"margin_dpo/margin_mean": 17.16216278076172,
|
|
"margin_dpo/margin_std": 11.365278244018555,
|
|
"step": 276
|
|
},
|
|
{
|
|
"epoch": 0.4067547723935389,
|
|
"grad_norm": 23.917224884033203,
|
|
"learning_rate": 3.71651119641714e-07,
|
|
"logits/chosen": -1.4923615455627441,
|
|
"logits/rejected": -1.091668963432312,
|
|
"logps/chosen": -91.10482025146484,
|
|
"logps/ref_chosen": -73.8440933227539,
|
|
"logps/ref_rejected": -124.34796905517578,
|
|
"logps/rejected": -157.8816680908203,
|
|
"loss": 0.6035,
|
|
"margin_dpo/margin_mean": 16.272974014282227,
|
|
"margin_dpo/margin_std": 13.451412200927734,
|
|
"step": 277
|
|
},
|
|
{
|
|
"epoch": 0.40822320117474303,
|
|
"grad_norm": 27.672937393188477,
|
|
"learning_rate": 3.705283756425872e-07,
|
|
"logits/chosen": -1.5108946561813354,
|
|
"logits/rejected": -1.3069382905960083,
|
|
"logps/chosen": -78.12380981445312,
|
|
"logps/ref_chosen": -64.88898468017578,
|
|
"logps/ref_rejected": -91.60508728027344,
|
|
"logps/rejected": -124.02960205078125,
|
|
"loss": 0.7411,
|
|
"margin_dpo/margin_mean": 19.189693450927734,
|
|
"margin_dpo/margin_std": 15.39527702331543,
|
|
"step": 278
|
|
},
|
|
{
|
|
"epoch": 0.40969162995594716,
|
|
"grad_norm": 27.276323318481445,
|
|
"learning_rate": 3.6940245560867e-07,
|
|
"logits/chosen": -1.1005783081054688,
|
|
"logits/rejected": -0.8380391597747803,
|
|
"logps/chosen": -113.02943420410156,
|
|
"logps/ref_chosen": -100.27659606933594,
|
|
"logps/ref_rejected": -109.22632598876953,
|
|
"logps/rejected": -133.98086547851562,
|
|
"loss": 0.7615,
|
|
"margin_dpo/margin_mean": 12.001691818237305,
|
|
"margin_dpo/margin_std": 11.654304504394531,
|
|
"step": 279
|
|
},
|
|
{
|
|
"epoch": 0.4111600587371512,
|
|
"grad_norm": 23.190258026123047,
|
|
"learning_rate": 3.6827338920900253e-07,
|
|
"logits/chosen": -2.4184060096740723,
|
|
"logits/rejected": -2.72609281539917,
|
|
"logps/chosen": -83.09910583496094,
|
|
"logps/ref_chosen": -65.84967041015625,
|
|
"logps/ref_rejected": -122.72234344482422,
|
|
"logps/rejected": -159.41729736328125,
|
|
"loss": 0.5692,
|
|
"margin_dpo/margin_mean": 19.445505142211914,
|
|
"margin_dpo/margin_std": 17.48896598815918,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.41262848751835535,
|
|
"grad_norm": 21.81446647644043,
|
|
"learning_rate": 3.6714120619553435e-07,
|
|
"logits/chosen": -1.2980034351348877,
|
|
"logits/rejected": -0.9670097231864929,
|
|
"logps/chosen": -73.10958862304688,
|
|
"logps/ref_chosen": -57.2689208984375,
|
|
"logps/ref_rejected": -67.501220703125,
|
|
"logps/rejected": -100.85271453857422,
|
|
"loss": 0.6119,
|
|
"margin_dpo/margin_mean": 17.510818481445312,
|
|
"margin_dpo/margin_std": 18.7252197265625,
|
|
"step": 281
|
|
},
|
|
{
|
|
"epoch": 0.41409691629955947,
|
|
"grad_norm": 20.058345794677734,
|
|
"learning_rate": 3.660059364023408e-07,
|
|
"logits/chosen": -1.7840750217437744,
|
|
"logits/rejected": -1.2310118675231934,
|
|
"logps/chosen": -94.27513122558594,
|
|
"logps/ref_chosen": -85.69962310791016,
|
|
"logps/ref_rejected": -66.8458480834961,
|
|
"logps/rejected": -100.76190185546875,
|
|
"loss": 0.4488,
|
|
"margin_dpo/margin_mean": 25.340524673461914,
|
|
"margin_dpo/margin_std": 17.682575225830078,
|
|
"step": 282
|
|
},
|
|
{
|
|
"epoch": 0.4155653450807636,
|
|
"grad_norm": 39.15544509887695,
|
|
"learning_rate": 3.6486760974483685e-07,
|
|
"logits/chosen": -2.0153682231903076,
|
|
"logits/rejected": -1.574284553527832,
|
|
"logps/chosen": -101.2388916015625,
|
|
"logps/ref_chosen": -85.58895111083984,
|
|
"logps/ref_rejected": -84.03312683105469,
|
|
"logps/rejected": -112.30723571777344,
|
|
"loss": 0.8681,
|
|
"margin_dpo/margin_mean": 12.62417221069336,
|
|
"margin_dpo/margin_std": 15.997934341430664,
|
|
"step": 283
|
|
},
|
|
{
|
|
"epoch": 0.4170337738619677,
|
|
"grad_norm": 17.253101348876953,
|
|
"learning_rate": 3.6372625621898863e-07,
|
|
"logits/chosen": -1.7107484340667725,
|
|
"logits/rejected": -1.297828197479248,
|
|
"logps/chosen": -107.02427673339844,
|
|
"logps/ref_chosen": -93.25978088378906,
|
|
"logps/ref_rejected": -70.50126647949219,
|
|
"logps/rejected": -102.14917755126953,
|
|
"loss": 0.4147,
|
|
"margin_dpo/margin_mean": 17.883419036865234,
|
|
"margin_dpo/margin_std": 9.083778381347656,
|
|
"step": 284
|
|
},
|
|
{
|
|
"epoch": 0.4185022026431718,
|
|
"grad_norm": 30.760282516479492,
|
|
"learning_rate": 3.625819059005228e-07,
|
|
"logits/chosen": -1.5096139907836914,
|
|
"logits/rejected": -1.7117888927459717,
|
|
"logps/chosen": -70.15441131591797,
|
|
"logps/ref_chosen": -51.80888366699219,
|
|
"logps/ref_rejected": -89.65523529052734,
|
|
"logps/rejected": -124.66044616699219,
|
|
"loss": 0.6078,
|
|
"margin_dpo/margin_mean": 16.659687042236328,
|
|
"margin_dpo/margin_std": 12.118792533874512,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.4199706314243759,
|
|
"grad_norm": 26.206777572631836,
|
|
"learning_rate": 3.614345889441346e-07,
|
|
"logits/chosen": -2.420334577560425,
|
|
"logits/rejected": -1.927842617034912,
|
|
"logps/chosen": -116.84445190429688,
|
|
"logps/ref_chosen": -100.83771514892578,
|
|
"logps/ref_rejected": -72.48152923583984,
|
|
"logps/rejected": -104.88263702392578,
|
|
"loss": 0.59,
|
|
"margin_dpo/margin_mean": 16.394378662109375,
|
|
"margin_dpo/margin_std": 9.293878555297852,
|
|
"step": 286
|
|
},
|
|
{
|
|
"epoch": 0.42143906020558003,
|
|
"grad_norm": 35.31901168823242,
|
|
"learning_rate": 3.6028433558269275e-07,
|
|
"logits/chosen": -1.129586100578308,
|
|
"logits/rejected": -0.4217372238636017,
|
|
"logps/chosen": -124.8189697265625,
|
|
"logps/ref_chosen": -108.39360809326172,
|
|
"logps/ref_rejected": -76.55667114257812,
|
|
"logps/rejected": -108.34898376464844,
|
|
"loss": 0.8528,
|
|
"margin_dpo/margin_mean": 15.366945266723633,
|
|
"margin_dpo/margin_std": 17.302759170532227,
|
|
"step": 287
|
|
},
|
|
{
|
|
"epoch": 0.42290748898678415,
|
|
"grad_norm": 25.598308563232422,
|
|
"learning_rate": 3.5913117612644327e-07,
|
|
"logits/chosen": -1.43570876121521,
|
|
"logits/rejected": -1.31893789768219,
|
|
"logps/chosen": -97.67936706542969,
|
|
"logps/ref_chosen": -81.42936706542969,
|
|
"logps/ref_rejected": -91.9423599243164,
|
|
"logps/rejected": -120.51500701904297,
|
|
"loss": 0.6139,
|
|
"margin_dpo/margin_mean": 12.322646141052246,
|
|
"margin_dpo/margin_std": 16.771167755126953,
|
|
"step": 288
|
|
},
|
|
{
|
|
"epoch": 0.4243759177679883,
|
|
"grad_norm": 22.585121154785156,
|
|
"learning_rate": 3.5797514096221024e-07,
|
|
"logits/chosen": -1.5673437118530273,
|
|
"logits/rejected": -1.1167399883270264,
|
|
"logps/chosen": -103.3167724609375,
|
|
"logps/ref_chosen": -91.00823974609375,
|
|
"logps/ref_rejected": -105.28831481933594,
|
|
"logps/rejected": -147.36233520507812,
|
|
"loss": 0.6594,
|
|
"margin_dpo/margin_mean": 29.76548957824707,
|
|
"margin_dpo/margin_std": 26.266082763671875,
|
|
"step": 289
|
|
},
|
|
{
|
|
"epoch": 0.42584434654919234,
|
|
"grad_norm": 27.944913864135742,
|
|
"learning_rate": 3.568162605525952e-07,
|
|
"logits/chosen": -1.0414509773254395,
|
|
"logits/rejected": -1.1000698804855347,
|
|
"logps/chosen": -106.29632568359375,
|
|
"logps/ref_chosen": -86.29432678222656,
|
|
"logps/ref_rejected": -118.67485046386719,
|
|
"logps/rejected": -159.3505859375,
|
|
"loss": 0.6342,
|
|
"margin_dpo/margin_mean": 20.673748016357422,
|
|
"margin_dpo/margin_std": 14.754283905029297,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.42731277533039647,
|
|
"grad_norm": 33.19970703125,
|
|
"learning_rate": 3.5565456543517485e-07,
|
|
"logits/chosen": -1.560898780822754,
|
|
"logits/rejected": -1.653581142425537,
|
|
"logps/chosen": -85.19589233398438,
|
|
"logps/ref_chosen": -66.42658233642578,
|
|
"logps/ref_rejected": -90.32164001464844,
|
|
"logps/rejected": -126.43409729003906,
|
|
"loss": 0.781,
|
|
"margin_dpo/margin_mean": 17.343158721923828,
|
|
"margin_dpo/margin_std": 13.000414848327637,
|
|
"step": 291
|
|
},
|
|
{
|
|
"epoch": 0.4287812041116006,
|
|
"grad_norm": 21.21723747253418,
|
|
"learning_rate": 3.5449008622169583e-07,
|
|
"logits/chosen": -1.6994218826293945,
|
|
"logits/rejected": -1.175065040588379,
|
|
"logps/chosen": -115.83615112304688,
|
|
"logps/ref_chosen": -101.82511901855469,
|
|
"logps/ref_rejected": -100.87593078613281,
|
|
"logps/rejected": -134.69090270996094,
|
|
"loss": 0.4858,
|
|
"margin_dpo/margin_mean": 19.803937911987305,
|
|
"margin_dpo/margin_std": 12.30525016784668,
|
|
"step": 292
|
|
},
|
|
{
|
|
"epoch": 0.4302496328928047,
|
|
"grad_norm": 26.848459243774414,
|
|
"learning_rate": 3.5332285359726846e-07,
|
|
"logits/chosen": -1.7359650135040283,
|
|
"logits/rejected": -1.7631828784942627,
|
|
"logps/chosen": -84.04584503173828,
|
|
"logps/ref_chosen": -68.95051574707031,
|
|
"logps/ref_rejected": -82.94426727294922,
|
|
"logps/rejected": -109.67572784423828,
|
|
"loss": 0.8721,
|
|
"margin_dpo/margin_mean": 11.636128425598145,
|
|
"margin_dpo/margin_std": 17.83340072631836,
|
|
"step": 293
|
|
},
|
|
{
|
|
"epoch": 0.43171806167400884,
|
|
"grad_norm": 28.9974308013916,
|
|
"learning_rate": 3.5215289831955786e-07,
|
|
"logits/chosen": -1.5406566858291626,
|
|
"logits/rejected": -1.1875331401824951,
|
|
"logps/chosen": -102.3502197265625,
|
|
"logps/ref_chosen": -88.47676086425781,
|
|
"logps/ref_rejected": -70.6723861694336,
|
|
"logps/rejected": -98.13722229003906,
|
|
"loss": 0.8365,
|
|
"margin_dpo/margin_mean": 13.591376304626465,
|
|
"margin_dpo/margin_std": 13.403884887695312,
|
|
"step": 294
|
|
},
|
|
{
|
|
"epoch": 0.4331864904552129,
|
|
"grad_norm": 25.117948532104492,
|
|
"learning_rate": 3.509802512179737e-07,
|
|
"logits/chosen": -1.3463271856307983,
|
|
"logits/rejected": -0.9588865637779236,
|
|
"logps/chosen": -99.4218978881836,
|
|
"logps/ref_chosen": -82.94652557373047,
|
|
"logps/ref_rejected": -80.15540313720703,
|
|
"logps/rejected": -114.45523071289062,
|
|
"loss": 0.6613,
|
|
"margin_dpo/margin_mean": 17.824451446533203,
|
|
"margin_dpo/margin_std": 8.08942985534668,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.434654919236417,
|
|
"grad_norm": 27.37685775756836,
|
|
"learning_rate": 3.498049431928577e-07,
|
|
"logits/chosen": -2.3768234252929688,
|
|
"logits/rejected": -1.7210721969604492,
|
|
"logps/chosen": -138.28659057617188,
|
|
"logps/ref_chosen": -118.51683807373047,
|
|
"logps/ref_rejected": -119.9332504272461,
|
|
"logps/rejected": -159.16708374023438,
|
|
"loss": 0.6912,
|
|
"margin_dpo/margin_mean": 19.46407127380371,
|
|
"margin_dpo/margin_std": 28.34971046447754,
|
|
"step": 296
|
|
},
|
|
{
|
|
"epoch": 0.43612334801762115,
|
|
"grad_norm": 24.673267364501953,
|
|
"learning_rate": 3.486270052146694e-07,
|
|
"logits/chosen": -0.6197730302810669,
|
|
"logits/rejected": -0.3352678418159485,
|
|
"logps/chosen": -105.87931823730469,
|
|
"logps/ref_chosen": -86.62548065185547,
|
|
"logps/ref_rejected": -93.621826171875,
|
|
"logps/rejected": -133.82965087890625,
|
|
"loss": 0.6463,
|
|
"margin_dpo/margin_mean": 20.95398712158203,
|
|
"margin_dpo/margin_std": 13.207864761352539,
|
|
"step": 297
|
|
},
|
|
{
|
|
"epoch": 0.43759177679882527,
|
|
"grad_norm": 26.703006744384766,
|
|
"learning_rate": 3.474464683231698e-07,
|
|
"logits/chosen": -2.5179362297058105,
|
|
"logits/rejected": -2.055812358856201,
|
|
"logps/chosen": -141.28953552246094,
|
|
"logps/ref_chosen": -125.80955505371094,
|
|
"logps/ref_rejected": -98.05117797851562,
|
|
"logps/rejected": -129.42611694335938,
|
|
"loss": 0.6232,
|
|
"margin_dpo/margin_mean": 15.894968032836914,
|
|
"margin_dpo/margin_std": 15.374116897583008,
|
|
"step": 298
|
|
},
|
|
{
|
|
"epoch": 0.4390602055800294,
|
|
"grad_norm": 25.000356674194336,
|
|
"learning_rate": 3.462633636266041e-07,
|
|
"logits/chosen": -0.8845536708831787,
|
|
"logits/rejected": -0.7745420932769775,
|
|
"logps/chosen": -76.33744812011719,
|
|
"logps/ref_chosen": -60.100074768066406,
|
|
"logps/ref_rejected": -95.97373962402344,
|
|
"logps/rejected": -133.07330322265625,
|
|
"loss": 0.5611,
|
|
"margin_dpo/margin_mean": 20.862197875976562,
|
|
"margin_dpo/margin_std": 13.313728332519531,
|
|
"step": 299
|
|
},
|
|
{
|
|
"epoch": 0.44052863436123346,
|
|
"grad_norm": 29.056922912597656,
|
|
"learning_rate": 3.4507772230088147e-07,
|
|
"logits/chosen": -1.2322702407836914,
|
|
"logits/rejected": -1.1211085319519043,
|
|
"logps/chosen": -86.75079345703125,
|
|
"logps/ref_chosen": -63.767547607421875,
|
|
"logps/ref_rejected": -94.31537628173828,
|
|
"logps/rejected": -129.37054443359375,
|
|
"loss": 0.7508,
|
|
"margin_dpo/margin_mean": 12.071918487548828,
|
|
"margin_dpo/margin_std": 16.181241989135742,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.44052863436123346,
|
|
"eval_logits/chosen": -1.2686725854873657,
|
|
"eval_logits/rejected": -0.9791460633277893,
|
|
"eval_logps/chosen": -121.85298156738281,
|
|
"eval_logps/ref_chosen": -101.88616943359375,
|
|
"eval_logps/ref_rejected": -92.68607330322266,
|
|
"eval_logps/rejected": -126.02144622802734,
|
|
"eval_loss": 0.43480244278907776,
|
|
"eval_margin_dpo/margin_mean": 13.368559837341309,
|
|
"eval_margin_dpo/margin_std": 14.839603424072266,
|
|
"eval_runtime": 44.0016,
|
|
"eval_samples_per_second": 53.157,
|
|
"eval_steps_per_second": 1.682,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.4419970631424376,
|
|
"grad_norm": 23.944438934326172,
|
|
"learning_rate": 3.4388957558875316e-07,
|
|
"logits/chosen": -1.3114334344863892,
|
|
"logits/rejected": -0.5625029802322388,
|
|
"logps/chosen": -120.26191711425781,
|
|
"logps/ref_chosen": -103.51814270019531,
|
|
"logps/ref_rejected": -110.81497192382812,
|
|
"logps/rejected": -150.2152099609375,
|
|
"loss": 0.5739,
|
|
"margin_dpo/margin_mean": 22.65645980834961,
|
|
"margin_dpo/margin_std": 16.655263900756836,
|
|
"step": 301
|
|
},
|
|
{
|
|
"epoch": 0.4434654919236417,
|
|
"grad_norm": 29.19167137145996,
|
|
"learning_rate": 3.426989547989902e-07,
|
|
"logits/chosen": -1.2964709997177124,
|
|
"logits/rejected": -0.9586907625198364,
|
|
"logps/chosen": -113.50164794921875,
|
|
"logps/ref_chosen": -97.01749420166016,
|
|
"logps/ref_rejected": -118.47299194335938,
|
|
"logps/rejected": -145.63783264160156,
|
|
"loss": 0.7385,
|
|
"margin_dpo/margin_mean": 10.680692672729492,
|
|
"margin_dpo/margin_std": 13.586481094360352,
|
|
"step": 302
|
|
},
|
|
{
|
|
"epoch": 0.44493392070484583,
|
|
"grad_norm": 23.38942527770996,
|
|
"learning_rate": 3.4150589130555773e-07,
|
|
"logits/chosen": -0.9705812931060791,
|
|
"logits/rejected": -1.0122885704040527,
|
|
"logps/chosen": -96.07485961914062,
|
|
"logps/ref_chosen": -77.64624786376953,
|
|
"logps/ref_rejected": -90.76347351074219,
|
|
"logps/rejected": -125.125,
|
|
"loss": 0.5719,
|
|
"margin_dpo/margin_mean": 15.932912826538086,
|
|
"margin_dpo/margin_std": 12.196745872497559,
|
|
"step": 303
|
|
},
|
|
{
|
|
"epoch": 0.44640234948604995,
|
|
"grad_norm": 29.73624038696289,
|
|
"learning_rate": 3.403104165467883e-07,
|
|
"logits/chosen": -1.6866213083267212,
|
|
"logits/rejected": -1.4702863693237305,
|
|
"logps/chosen": -131.42453002929688,
|
|
"logps/ref_chosen": -111.68305206298828,
|
|
"logps/ref_rejected": -106.78506469726562,
|
|
"logps/rejected": -145.26040649414062,
|
|
"loss": 0.6441,
|
|
"margin_dpo/margin_mean": 18.733875274658203,
|
|
"margin_dpo/margin_std": 14.239300727844238,
|
|
"step": 304
|
|
},
|
|
{
|
|
"epoch": 0.447870778267254,
|
|
"grad_norm": 25.76927947998047,
|
|
"learning_rate": 3.391125620245535e-07,
|
|
"logits/chosen": -1.6310691833496094,
|
|
"logits/rejected": -0.7466331124305725,
|
|
"logps/chosen": -156.75167846679688,
|
|
"logps/ref_chosen": -140.52206420898438,
|
|
"logps/ref_rejected": -94.84587097167969,
|
|
"logps/rejected": -133.16110229492188,
|
|
"loss": 0.6036,
|
|
"margin_dpo/margin_mean": 22.085636138916016,
|
|
"margin_dpo/margin_std": 14.780380249023438,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.44933920704845814,
|
|
"grad_norm": 17.852848052978516,
|
|
"learning_rate": 3.3791235930343417e-07,
|
|
"logits/chosen": -1.5937268733978271,
|
|
"logits/rejected": -1.1459739208221436,
|
|
"logps/chosen": -124.42718505859375,
|
|
"logps/ref_chosen": -113.0892333984375,
|
|
"logps/ref_rejected": -107.38107299804688,
|
|
"logps/rejected": -143.6051483154297,
|
|
"loss": 0.3552,
|
|
"margin_dpo/margin_mean": 24.886112213134766,
|
|
"margin_dpo/margin_std": 15.110748291015625,
|
|
"step": 306
|
|
},
|
|
{
|
|
"epoch": 0.45080763582966227,
|
|
"grad_norm": 35.94956970214844,
|
|
"learning_rate": 3.367098400098881e-07,
|
|
"logits/chosen": -1.6853301525115967,
|
|
"logits/rejected": -1.5398199558258057,
|
|
"logps/chosen": -138.31626892089844,
|
|
"logps/ref_chosen": -119.14762878417969,
|
|
"logps/ref_rejected": -85.65526580810547,
|
|
"logps/rejected": -114.65570068359375,
|
|
"loss": 0.7146,
|
|
"margin_dpo/margin_mean": 9.831794738769531,
|
|
"margin_dpo/margin_std": 15.621646881103516,
|
|
"step": 307
|
|
},
|
|
{
|
|
"epoch": 0.4522760646108664,
|
|
"grad_norm": 27.655181884765625,
|
|
"learning_rate": 3.355050358314172e-07,
|
|
"logits/chosen": -1.4368455410003662,
|
|
"logits/rejected": -1.1521961688995361,
|
|
"logps/chosen": -127.35299682617188,
|
|
"logps/ref_chosen": -108.64510345458984,
|
|
"logps/ref_rejected": -94.37911224365234,
|
|
"logps/rejected": -127.02978515625,
|
|
"loss": 0.7696,
|
|
"margin_dpo/margin_mean": 13.942778587341309,
|
|
"margin_dpo/margin_std": 18.120010375976562,
|
|
"step": 308
|
|
},
|
|
{
|
|
"epoch": 0.45374449339207046,
|
|
"grad_norm": 25.771364212036133,
|
|
"learning_rate": 3.3429797851573183e-07,
|
|
"logits/chosen": -1.5995187759399414,
|
|
"logits/rejected": -1.3868792057037354,
|
|
"logps/chosen": -78.335205078125,
|
|
"logps/ref_chosen": -59.73954772949219,
|
|
"logps/ref_rejected": -78.94290924072266,
|
|
"logps/rejected": -114.00508117675781,
|
|
"loss": 0.647,
|
|
"margin_dpo/margin_mean": 16.466510772705078,
|
|
"margin_dpo/margin_std": 17.880075454711914,
|
|
"step": 309
|
|
},
|
|
{
|
|
"epoch": 0.4552129221732746,
|
|
"grad_norm": 27.70083999633789,
|
|
"learning_rate": 3.3308869986991487e-07,
|
|
"logits/chosen": -1.3079609870910645,
|
|
"logits/rejected": -1.0223644971847534,
|
|
"logps/chosen": -104.94872283935547,
|
|
"logps/ref_chosen": -88.61974334716797,
|
|
"logps/ref_rejected": -72.84456634521484,
|
|
"logps/rejected": -101.36610412597656,
|
|
"loss": 0.7686,
|
|
"margin_dpo/margin_mean": 12.192560195922852,
|
|
"margin_dpo/margin_std": 16.817516326904297,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.4566813509544787,
|
|
"grad_norm": 27.827804565429688,
|
|
"learning_rate": 3.3187723175958346e-07,
|
|
"logits/chosen": -1.6938467025756836,
|
|
"logits/rejected": -0.8132745623588562,
|
|
"logps/chosen": -112.90548706054688,
|
|
"logps/ref_chosen": -92.47557067871094,
|
|
"logps/ref_rejected": -84.39120483398438,
|
|
"logps/rejected": -126.1024169921875,
|
|
"loss": 0.622,
|
|
"margin_dpo/margin_mean": 21.281295776367188,
|
|
"margin_dpo/margin_std": 19.100196838378906,
|
|
"step": 311
|
|
},
|
|
{
|
|
"epoch": 0.4581497797356828,
|
|
"grad_norm": 24.641746520996094,
|
|
"learning_rate": 3.306636061080487e-07,
|
|
"logits/chosen": -1.377639651298523,
|
|
"logits/rejected": -0.7076698541641235,
|
|
"logps/chosen": -106.80957794189453,
|
|
"logps/ref_chosen": -87.10482788085938,
|
|
"logps/ref_rejected": -99.46177673339844,
|
|
"logps/rejected": -141.55429077148438,
|
|
"loss": 0.6269,
|
|
"margin_dpo/margin_mean": 22.387775421142578,
|
|
"margin_dpo/margin_std": 24.29810905456543,
|
|
"step": 312
|
|
},
|
|
{
|
|
"epoch": 0.45961820851688695,
|
|
"grad_norm": 24.407838821411133,
|
|
"learning_rate": 3.2944785489547537e-07,
|
|
"logits/chosen": -1.6093132495880127,
|
|
"logits/rejected": -1.3407427072525024,
|
|
"logps/chosen": -87.35713195800781,
|
|
"logps/ref_chosen": -71.71583557128906,
|
|
"logps/ref_rejected": -102.05616760253906,
|
|
"logps/rejected": -141.25453186035156,
|
|
"loss": 0.7303,
|
|
"margin_dpo/margin_mean": 23.557071685791016,
|
|
"margin_dpo/margin_std": 21.290372848510742,
|
|
"step": 313
|
|
},
|
|
{
|
|
"epoch": 0.461086637298091,
|
|
"grad_norm": 26.5968074798584,
|
|
"learning_rate": 3.2823001015803857e-07,
|
|
"logits/chosen": -2.9330849647521973,
|
|
"logits/rejected": -2.505680561065674,
|
|
"logps/chosen": -114.797119140625,
|
|
"logps/ref_chosen": -95.0579605102539,
|
|
"logps/ref_rejected": -88.39830017089844,
|
|
"logps/rejected": -120.34668731689453,
|
|
"loss": 0.6617,
|
|
"margin_dpo/margin_mean": 12.209226608276367,
|
|
"margin_dpo/margin_std": 20.931156158447266,
|
|
"step": 314
|
|
},
|
|
{
|
|
"epoch": 0.46255506607929514,
|
|
"grad_norm": 31.087507247924805,
|
|
"learning_rate": 3.270101039870797e-07,
|
|
"logits/chosen": -1.555436134338379,
|
|
"logits/rejected": -1.7317938804626465,
|
|
"logps/chosen": -92.6741943359375,
|
|
"logps/ref_chosen": -74.01944732666016,
|
|
"logps/ref_rejected": -103.11070251464844,
|
|
"logps/rejected": -132.5022735595703,
|
|
"loss": 0.7575,
|
|
"margin_dpo/margin_mean": 10.736818313598633,
|
|
"margin_dpo/margin_std": 16.410961151123047,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.46402349486049926,
|
|
"grad_norm": 29.95094871520996,
|
|
"learning_rate": 3.2578816852826086e-07,
|
|
"logits/chosen": -1.2038843631744385,
|
|
"logits/rejected": -1.3727455139160156,
|
|
"logps/chosen": -93.50067901611328,
|
|
"logps/ref_chosen": -73.40036010742188,
|
|
"logps/ref_rejected": -122.14073181152344,
|
|
"logps/rejected": -161.67385864257812,
|
|
"loss": 0.7261,
|
|
"margin_dpo/margin_mean": 19.432815551757812,
|
|
"margin_dpo/margin_std": 16.819963455200195,
|
|
"step": 316
|
|
},
|
|
{
|
|
"epoch": 0.4654919236417034,
|
|
"grad_norm": 18.27977752685547,
|
|
"learning_rate": 3.2456423598071783e-07,
|
|
"logits/chosen": -1.842048168182373,
|
|
"logits/rejected": -1.331984281539917,
|
|
"logps/chosen": -100.23789978027344,
|
|
"logps/ref_chosen": -86.66116333007812,
|
|
"logps/ref_rejected": -97.44319915771484,
|
|
"logps/rejected": -133.7093505859375,
|
|
"loss": 0.3973,
|
|
"margin_dpo/margin_mean": 22.689420700073242,
|
|
"margin_dpo/margin_std": 17.005748748779297,
|
|
"step": 317
|
|
},
|
|
{
|
|
"epoch": 0.4669603524229075,
|
|
"grad_norm": 23.080183029174805,
|
|
"learning_rate": 3.233383385962115e-07,
|
|
"logits/chosen": -1.1578627824783325,
|
|
"logits/rejected": -1.0486600399017334,
|
|
"logps/chosen": -112.71501159667969,
|
|
"logps/ref_chosen": -92.22932434082031,
|
|
"logps/ref_rejected": -110.58228302001953,
|
|
"logps/rejected": -152.46641540527344,
|
|
"loss": 0.5927,
|
|
"margin_dpo/margin_mean": 21.398456573486328,
|
|
"margin_dpo/margin_std": 14.620698928833008,
|
|
"step": 318
|
|
},
|
|
{
|
|
"epoch": 0.4684287812041116,
|
|
"grad_norm": 22.706588745117188,
|
|
"learning_rate": 3.2211050867827805e-07,
|
|
"logits/chosen": -1.6589261293411255,
|
|
"logits/rejected": -1.2192972898483276,
|
|
"logps/chosen": -91.02305603027344,
|
|
"logps/ref_chosen": -83.20411682128906,
|
|
"logps/ref_rejected": -104.62593078613281,
|
|
"logps/rejected": -138.94976806640625,
|
|
"loss": 0.5608,
|
|
"margin_dpo/margin_mean": 26.50490379333496,
|
|
"margin_dpo/margin_std": 21.008556365966797,
|
|
"step": 319
|
|
},
|
|
{
|
|
"epoch": 0.4698972099853157,
|
|
"grad_norm": 22.778528213500977,
|
|
"learning_rate": 3.208807785813777e-07,
|
|
"logits/chosen": -2.063751220703125,
|
|
"logits/rejected": -1.798086166381836,
|
|
"logps/chosen": -111.82926177978516,
|
|
"logps/ref_chosen": -99.53569030761719,
|
|
"logps/ref_rejected": -103.12991333007812,
|
|
"logps/rejected": -140.8082275390625,
|
|
"loss": 0.6134,
|
|
"margin_dpo/margin_mean": 25.384750366210938,
|
|
"margin_dpo/margin_std": 24.922496795654297,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.4713656387665198,
|
|
"grad_norm": 26.070716857910156,
|
|
"learning_rate": 3.1964918071004217e-07,
|
|
"logits/chosen": -1.1019395589828491,
|
|
"logits/rejected": -0.5719025731086731,
|
|
"logps/chosen": -100.59760284423828,
|
|
"logps/ref_chosen": -83.44072723388672,
|
|
"logps/ref_rejected": -99.94316864013672,
|
|
"logps/rejected": -134.56689453125,
|
|
"loss": 0.6277,
|
|
"margin_dpo/margin_mean": 17.466854095458984,
|
|
"margin_dpo/margin_std": 16.467836380004883,
|
|
"step": 321
|
|
},
|
|
{
|
|
"epoch": 0.47283406754772395,
|
|
"grad_norm": 22.629993438720703,
|
|
"learning_rate": 3.184157475180207e-07,
|
|
"logits/chosen": -1.4932529926300049,
|
|
"logits/rejected": -1.3814778327941895,
|
|
"logps/chosen": -85.35063171386719,
|
|
"logps/ref_chosen": -69.8001708984375,
|
|
"logps/ref_rejected": -94.93486785888672,
|
|
"logps/rejected": -134.5431365966797,
|
|
"loss": 0.5298,
|
|
"margin_dpo/margin_mean": 24.057796478271484,
|
|
"margin_dpo/margin_std": 24.209617614746094,
|
|
"step": 322
|
|
},
|
|
{
|
|
"epoch": 0.47430249632892807,
|
|
"grad_norm": 22.0151309967041,
|
|
"learning_rate": 3.171805115074251e-07,
|
|
"logits/chosen": -1.6108890771865845,
|
|
"logits/rejected": -1.4497694969177246,
|
|
"logps/chosen": -100.01136779785156,
|
|
"logps/ref_chosen": -81.24457550048828,
|
|
"logps/ref_rejected": -93.87039184570312,
|
|
"logps/rejected": -132.82888793945312,
|
|
"loss": 0.5091,
|
|
"margin_dpo/margin_mean": 20.19168472290039,
|
|
"margin_dpo/margin_std": 17.731565475463867,
|
|
"step": 323
|
|
},
|
|
{
|
|
"epoch": 0.47577092511013214,
|
|
"grad_norm": 27.344818115234375,
|
|
"learning_rate": 3.1594350522787295e-07,
|
|
"logits/chosen": -1.890838861465454,
|
|
"logits/rejected": -1.4299018383026123,
|
|
"logps/chosen": -109.41104888916016,
|
|
"logps/ref_chosen": -87.28364562988281,
|
|
"logps/ref_rejected": -114.13909912109375,
|
|
"logps/rejected": -160.6354217529297,
|
|
"loss": 0.4935,
|
|
"margin_dpo/margin_mean": 24.36892318725586,
|
|
"margin_dpo/margin_std": 20.603160858154297,
|
|
"step": 324
|
|
},
|
|
{
|
|
"epoch": 0.47723935389133626,
|
|
"grad_norm": 24.975282669067383,
|
|
"learning_rate": 3.147047612756302e-07,
|
|
"logits/chosen": -1.4415394067764282,
|
|
"logits/rejected": -0.7071768045425415,
|
|
"logps/chosen": -103.07582092285156,
|
|
"logps/ref_chosen": -85.93971252441406,
|
|
"logps/ref_rejected": -58.83104705810547,
|
|
"logps/rejected": -89.58515930175781,
|
|
"loss": 0.6151,
|
|
"margin_dpo/margin_mean": 13.618009567260742,
|
|
"margin_dpo/margin_std": 16.28986167907715,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.4787077826725404,
|
|
"grad_norm": 25.001813888549805,
|
|
"learning_rate": 3.134643122927519e-07,
|
|
"logits/chosen": -2.0634007453918457,
|
|
"logits/rejected": -1.2772150039672852,
|
|
"logps/chosen": -126.6056137084961,
|
|
"logps/ref_chosen": -100.74868774414062,
|
|
"logps/ref_rejected": -88.71658325195312,
|
|
"logps/rejected": -129.34811401367188,
|
|
"loss": 0.5897,
|
|
"margin_dpo/margin_mean": 14.774612426757812,
|
|
"margin_dpo/margin_std": 20.738540649414062,
|
|
"step": 326
|
|
},
|
|
{
|
|
"epoch": 0.4801762114537445,
|
|
"grad_norm": 25.678897857666016,
|
|
"learning_rate": 3.1222219096622264e-07,
|
|
"logits/chosen": -2.5522360801696777,
|
|
"logits/rejected": -2.076728343963623,
|
|
"logps/chosen": -141.49176025390625,
|
|
"logps/ref_chosen": -126.36719512939453,
|
|
"logps/ref_rejected": -162.37741088867188,
|
|
"logps/rejected": -202.36465454101562,
|
|
"loss": 0.5539,
|
|
"margin_dpo/margin_mean": 24.862689971923828,
|
|
"margin_dpo/margin_std": 22.165292739868164,
|
|
"step": 327
|
|
},
|
|
{
|
|
"epoch": 0.48164464023494863,
|
|
"grad_norm": 21.912940979003906,
|
|
"learning_rate": 3.1097843002709427e-07,
|
|
"logits/chosen": -1.0913561582565308,
|
|
"logits/rejected": -0.890350878238678,
|
|
"logps/chosen": -123.70916748046875,
|
|
"logps/ref_chosen": -99.27432250976562,
|
|
"logps/ref_rejected": -93.67889404296875,
|
|
"logps/rejected": -138.63572692871094,
|
|
"loss": 0.5706,
|
|
"margin_dpo/margin_mean": 20.5219783782959,
|
|
"margin_dpo/margin_std": 11.165393829345703,
|
|
"step": 328
|
|
},
|
|
{
|
|
"epoch": 0.4831130690161527,
|
|
"grad_norm": 25.703792572021484,
|
|
"learning_rate": 3.0973306224962437e-07,
|
|
"logits/chosen": -1.7976303100585938,
|
|
"logits/rejected": -1.4397552013397217,
|
|
"logps/chosen": -120.62757873535156,
|
|
"logps/ref_chosen": -102.76374816894531,
|
|
"logps/ref_rejected": -133.03453063964844,
|
|
"logps/rejected": -179.14102172851562,
|
|
"loss": 0.5589,
|
|
"margin_dpo/margin_mean": 28.24266815185547,
|
|
"margin_dpo/margin_std": 21.547605514526367,
|
|
"step": 329
|
|
},
|
|
{
|
|
"epoch": 0.4845814977973568,
|
|
"grad_norm": 20.2358341217041,
|
|
"learning_rate": 3.084861204504122e-07,
|
|
"logits/chosen": -0.47298264503479004,
|
|
"logits/rejected": -0.4694516658782959,
|
|
"logps/chosen": -76.36822509765625,
|
|
"logps/ref_chosen": -58.246788024902344,
|
|
"logps/ref_rejected": -89.55278778076172,
|
|
"logps/rejected": -127.02072143554688,
|
|
"loss": 0.4912,
|
|
"margin_dpo/margin_mean": 19.34650230407715,
|
|
"margin_dpo/margin_std": 12.476625442504883,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.48604992657856094,
|
|
"grad_norm": 26.306129455566406,
|
|
"learning_rate": 3.072376374875335e-07,
|
|
"logits/chosen": -1.67173433303833,
|
|
"logits/rejected": -1.6575570106506348,
|
|
"logps/chosen": -116.8527603149414,
|
|
"logps/ref_chosen": -99.19812774658203,
|
|
"logps/ref_rejected": -123.66153717041016,
|
|
"logps/rejected": -161.8951416015625,
|
|
"loss": 0.5371,
|
|
"margin_dpo/margin_mean": 20.578968048095703,
|
|
"margin_dpo/margin_std": 19.465991973876953,
|
|
"step": 331
|
|
},
|
|
{
|
|
"epoch": 0.48751835535976507,
|
|
"grad_norm": 28.853147506713867,
|
|
"learning_rate": 3.059876462596758e-07,
|
|
"logits/chosen": -2.5498337745666504,
|
|
"logits/rejected": -2.2815890312194824,
|
|
"logps/chosen": -115.20259094238281,
|
|
"logps/ref_chosen": -95.0144271850586,
|
|
"logps/ref_rejected": -104.51777648925781,
|
|
"logps/rejected": -141.555908203125,
|
|
"loss": 0.7089,
|
|
"margin_dpo/margin_mean": 16.849950790405273,
|
|
"margin_dpo/margin_std": 17.792329788208008,
|
|
"step": 332
|
|
},
|
|
{
|
|
"epoch": 0.4889867841409692,
|
|
"grad_norm": 38.133628845214844,
|
|
"learning_rate": 3.0473617970527015e-07,
|
|
"logits/chosen": -1.7919915914535522,
|
|
"logits/rejected": -1.5668425559997559,
|
|
"logps/chosen": -121.86878967285156,
|
|
"logps/ref_chosen": -104.35073852539062,
|
|
"logps/ref_rejected": -137.1612548828125,
|
|
"logps/rejected": -173.7686767578125,
|
|
"loss": 0.775,
|
|
"margin_dpo/margin_mean": 19.08936882019043,
|
|
"margin_dpo/margin_std": 22.122257232666016,
|
|
"step": 333
|
|
},
|
|
{
|
|
"epoch": 0.49045521292217326,
|
|
"grad_norm": 28.414897918701172,
|
|
"learning_rate": 3.034832708016243e-07,
|
|
"logits/chosen": -0.8065652847290039,
|
|
"logits/rejected": -0.567101776599884,
|
|
"logps/chosen": -118.73043823242188,
|
|
"logps/ref_chosen": -94.99000549316406,
|
|
"logps/ref_rejected": -87.60928344726562,
|
|
"logps/rejected": -125.26659393310547,
|
|
"loss": 0.6262,
|
|
"margin_dpo/margin_mean": 13.91687297821045,
|
|
"margin_dpo/margin_std": 11.275923728942871,
|
|
"step": 334
|
|
},
|
|
{
|
|
"epoch": 0.4919236417033774,
|
|
"grad_norm": 33.4683723449707,
|
|
"learning_rate": 3.022289525640531e-07,
|
|
"logits/chosen": -2.3394131660461426,
|
|
"logits/rejected": -1.6820380687713623,
|
|
"logps/chosen": -119.54246520996094,
|
|
"logps/ref_chosen": -98.11556243896484,
|
|
"logps/ref_rejected": -84.25871276855469,
|
|
"logps/rejected": -119.07872772216797,
|
|
"loss": 0.713,
|
|
"margin_dpo/margin_mean": 13.393115043640137,
|
|
"margin_dpo/margin_std": 18.032217025756836,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.4933920704845815,
|
|
"grad_norm": 30.077716827392578,
|
|
"learning_rate": 3.009732580450086e-07,
|
|
"logits/chosen": -0.7926295399665833,
|
|
"logits/rejected": -1.3332951068878174,
|
|
"logps/chosen": -97.50780487060547,
|
|
"logps/ref_chosen": -72.66284942626953,
|
|
"logps/ref_rejected": -142.35540771484375,
|
|
"logps/rejected": -183.529296875,
|
|
"loss": 0.609,
|
|
"margin_dpo/margin_mean": 16.328933715820312,
|
|
"margin_dpo/margin_std": 28.889495849609375,
|
|
"step": 336
|
|
},
|
|
{
|
|
"epoch": 0.4948604992657856,
|
|
"grad_norm": 29.502315521240234,
|
|
"learning_rate": 2.9971622033320914e-07,
|
|
"logits/chosen": -2.5418286323547363,
|
|
"logits/rejected": -2.190610408782959,
|
|
"logps/chosen": -116.11346435546875,
|
|
"logps/ref_chosen": -102.47871398925781,
|
|
"logps/ref_rejected": -137.35902404785156,
|
|
"logps/rejected": -176.83990478515625,
|
|
"loss": 0.5855,
|
|
"margin_dpo/margin_mean": 25.84613800048828,
|
|
"margin_dpo/margin_std": 15.690153121948242,
|
|
"step": 337
|
|
},
|
|
{
|
|
"epoch": 0.49632892804698975,
|
|
"grad_norm": 44.26433181762695,
|
|
"learning_rate": 2.984578725527675e-07,
|
|
"logits/chosen": -1.9211044311523438,
|
|
"logits/rejected": -1.28424870967865,
|
|
"logps/chosen": -147.07164001464844,
|
|
"logps/ref_chosen": -126.42916870117188,
|
|
"logps/ref_rejected": -94.35499572753906,
|
|
"logps/rejected": -133.6391143798828,
|
|
"loss": 0.6668,
|
|
"margin_dpo/margin_mean": 18.641632080078125,
|
|
"margin_dpo/margin_std": 16.844388961791992,
|
|
"step": 338
|
|
},
|
|
{
|
|
"epoch": 0.4977973568281938,
|
|
"grad_norm": 28.031339645385742,
|
|
"learning_rate": 2.9719824786231796e-07,
|
|
"logits/chosen": -2.1791956424713135,
|
|
"logits/rejected": -1.4663095474243164,
|
|
"logps/chosen": -147.836181640625,
|
|
"logps/ref_chosen": -126.5898666381836,
|
|
"logps/ref_rejected": -92.5245361328125,
|
|
"logps/rejected": -132.89358520507812,
|
|
"loss": 0.6793,
|
|
"margin_dpo/margin_mean": 19.122737884521484,
|
|
"margin_dpo/margin_std": 22.091293334960938,
|
|
"step": 339
|
|
},
|
|
{
|
|
"epoch": 0.49926578560939794,
|
|
"grad_norm": 27.09212303161621,
|
|
"learning_rate": 2.959373794541426e-07,
|
|
"logits/chosen": -0.1419842392206192,
|
|
"logits/rejected": 0.0009096264839172363,
|
|
"logps/chosen": -87.25447082519531,
|
|
"logps/ref_chosen": -70.93193054199219,
|
|
"logps/ref_rejected": -81.2646713256836,
|
|
"logps/rejected": -120.7593994140625,
|
|
"loss": 0.6644,
|
|
"margin_dpo/margin_mean": 23.172176361083984,
|
|
"margin_dpo/margin_std": 19.083019256591797,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.5007342143906021,
|
|
"grad_norm": 27.434301376342773,
|
|
"learning_rate": 2.946753005532965e-07,
|
|
"logits/chosen": -0.9750526547431946,
|
|
"logits/rejected": -0.7655836939811707,
|
|
"logps/chosen": -103.8822250366211,
|
|
"logps/ref_chosen": -78.36085510253906,
|
|
"logps/ref_rejected": -118.61456298828125,
|
|
"logps/rejected": -160.45858764648438,
|
|
"loss": 0.5355,
|
|
"margin_dpo/margin_mean": 16.322650909423828,
|
|
"margin_dpo/margin_std": 16.00281524658203,
|
|
"step": 341
|
|
},
|
|
{
|
|
"epoch": 0.5022026431718062,
|
|
"grad_norm": 24.300811767578125,
|
|
"learning_rate": 2.934120444167326e-07,
|
|
"logits/chosen": -1.5221643447875977,
|
|
"logits/rejected": -0.7222243547439575,
|
|
"logps/chosen": -105.76786041259766,
|
|
"logps/ref_chosen": -92.65184020996094,
|
|
"logps/ref_rejected": -92.7236328125,
|
|
"logps/rejected": -133.51319885253906,
|
|
"loss": 0.403,
|
|
"margin_dpo/margin_mean": 27.673545837402344,
|
|
"margin_dpo/margin_std": 10.69522476196289,
|
|
"step": 342
|
|
},
|
|
{
|
|
"epoch": 0.5036710719530103,
|
|
"grad_norm": 22.844953536987305,
|
|
"learning_rate": 2.9214764433242476e-07,
|
|
"logits/chosen": -1.7190932035446167,
|
|
"logits/rejected": -1.3597054481506348,
|
|
"logps/chosen": -94.59676361083984,
|
|
"logps/ref_chosen": -78.70704650878906,
|
|
"logps/ref_rejected": -122.56159973144531,
|
|
"logps/rejected": -157.5832061767578,
|
|
"loss": 0.4424,
|
|
"margin_dpo/margin_mean": 19.131895065307617,
|
|
"margin_dpo/margin_std": 12.232367515563965,
|
|
"step": 343
|
|
},
|
|
{
|
|
"epoch": 0.5051395007342144,
|
|
"grad_norm": 23.992372512817383,
|
|
"learning_rate": 2.9088213361849126e-07,
|
|
"logits/chosen": -1.884738564491272,
|
|
"logits/rejected": -1.7378039360046387,
|
|
"logps/chosen": -86.93094635009766,
|
|
"logps/ref_chosen": -64.14713287353516,
|
|
"logps/ref_rejected": -78.60472869873047,
|
|
"logps/rejected": -116.30023956298828,
|
|
"loss": 0.5859,
|
|
"margin_dpo/margin_mean": 14.911701202392578,
|
|
"margin_dpo/margin_std": 15.901741027832031,
|
|
"step": 344
|
|
},
|
|
{
|
|
"epoch": 0.5066079295154186,
|
|
"grad_norm": 26.966333389282227,
|
|
"learning_rate": 2.896155456223163e-07,
|
|
"logits/chosen": -1.378301978111267,
|
|
"logits/rejected": -0.9885662794113159,
|
|
"logps/chosen": -116.005615234375,
|
|
"logps/ref_chosen": -102.19354248046875,
|
|
"logps/ref_rejected": -102.60723876953125,
|
|
"logps/rejected": -142.3053436279297,
|
|
"loss": 0.6241,
|
|
"margin_dpo/margin_mean": 25.88603401184082,
|
|
"margin_dpo/margin_std": 24.11634063720703,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.5080763582966226,
|
|
"grad_norm": 32.335750579833984,
|
|
"learning_rate": 2.883479137196714e-07,
|
|
"logits/chosen": -1.7958707809448242,
|
|
"logits/rejected": -1.4464631080627441,
|
|
"logps/chosen": -98.60110473632812,
|
|
"logps/ref_chosen": -79.09950256347656,
|
|
"logps/ref_rejected": -101.58006286621094,
|
|
"logps/rejected": -150.30313110351562,
|
|
"loss": 0.7975,
|
|
"margin_dpo/margin_mean": 29.221481323242188,
|
|
"margin_dpo/margin_std": 17.735727310180664,
|
|
"step": 346
|
|
},
|
|
{
|
|
"epoch": 0.5095447870778267,
|
|
"grad_norm": 29.275190353393555,
|
|
"learning_rate": 2.8707927131383614e-07,
|
|
"logits/chosen": -1.1242921352386475,
|
|
"logits/rejected": -0.6852156519889832,
|
|
"logps/chosen": -136.42025756835938,
|
|
"logps/ref_chosen": -105.7974624633789,
|
|
"logps/ref_rejected": -83.19305419921875,
|
|
"logps/rejected": -124.45526123046875,
|
|
"loss": 0.7834,
|
|
"margin_dpo/margin_mean": 10.639400482177734,
|
|
"margin_dpo/margin_std": 14.018272399902344,
|
|
"step": 347
|
|
},
|
|
{
|
|
"epoch": 0.5110132158590308,
|
|
"grad_norm": 26.918550491333008,
|
|
"learning_rate": 2.858096518347179e-07,
|
|
"logits/chosen": -1.8148624897003174,
|
|
"logits/rejected": -1.8900196552276611,
|
|
"logps/chosen": -101.09654235839844,
|
|
"logps/ref_chosen": -80.95931243896484,
|
|
"logps/ref_rejected": -115.63554382324219,
|
|
"logps/rejected": -151.6328887939453,
|
|
"loss": 0.7554,
|
|
"margin_dpo/margin_mean": 15.860109329223633,
|
|
"margin_dpo/margin_std": 15.913261413574219,
|
|
"step": 348
|
|
},
|
|
{
|
|
"epoch": 0.5124816446402349,
|
|
"grad_norm": 28.12725830078125,
|
|
"learning_rate": 2.845390887379706e-07,
|
|
"logits/chosen": -2.0488414764404297,
|
|
"logits/rejected": -2.1182613372802734,
|
|
"logps/chosen": -90.99738311767578,
|
|
"logps/ref_chosen": -69.36511993408203,
|
|
"logps/ref_rejected": -106.0635986328125,
|
|
"logps/rejected": -144.37217712402344,
|
|
"loss": 0.7118,
|
|
"margin_dpo/margin_mean": 16.676326751708984,
|
|
"margin_dpo/margin_std": 14.525110244750977,
|
|
"step": 349
|
|
},
|
|
{
|
|
"epoch": 0.5139500734214391,
|
|
"grad_norm": 30.817626953125,
|
|
"learning_rate": 2.8326761550411346e-07,
|
|
"logits/chosen": -1.3303940296173096,
|
|
"logits/rejected": -1.4277385473251343,
|
|
"logps/chosen": -100.7199478149414,
|
|
"logps/ref_chosen": -76.78309631347656,
|
|
"logps/ref_rejected": -103.20025634765625,
|
|
"logps/rejected": -141.65151977539062,
|
|
"loss": 0.6349,
|
|
"margin_dpo/margin_mean": 14.514402389526367,
|
|
"margin_dpo/margin_std": 12.599069595336914,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.5154185022026432,
|
|
"grad_norm": 27.94927978515625,
|
|
"learning_rate": 2.819952656376487e-07,
|
|
"logits/chosen": -1.4203786849975586,
|
|
"logits/rejected": -1.2766168117523193,
|
|
"logps/chosen": -92.46343994140625,
|
|
"logps/ref_chosen": -72.96452331542969,
|
|
"logps/ref_rejected": -128.21014404296875,
|
|
"logps/rejected": -177.54989624023438,
|
|
"loss": 0.5494,
|
|
"margin_dpo/margin_mean": 29.840824127197266,
|
|
"margin_dpo/margin_std": 14.69981575012207,
|
|
"step": 351
|
|
},
|
|
{
|
|
"epoch": 0.5168869309838473,
|
|
"grad_norm": 31.514175415039062,
|
|
"learning_rate": 2.8072207266617854e-07,
|
|
"logits/chosen": -1.9766767024993896,
|
|
"logits/rejected": -1.4314018487930298,
|
|
"logps/chosen": -121.56500244140625,
|
|
"logps/ref_chosen": -104.49222564697266,
|
|
"logps/ref_rejected": -85.14442443847656,
|
|
"logps/rejected": -120.27680969238281,
|
|
"loss": 0.6142,
|
|
"margin_dpo/margin_mean": 18.059606552124023,
|
|
"margin_dpo/margin_std": 18.000804901123047,
|
|
"step": 352
|
|
},
|
|
{
|
|
"epoch": 0.5183553597650514,
|
|
"grad_norm": 29.008338928222656,
|
|
"learning_rate": 2.794480701395219e-07,
|
|
"logits/chosen": -1.7322311401367188,
|
|
"logits/rejected": -1.5322282314300537,
|
|
"logps/chosen": -96.08810424804688,
|
|
"logps/ref_chosen": -76.10050964355469,
|
|
"logps/ref_rejected": -88.10479736328125,
|
|
"logps/rejected": -126.6584701538086,
|
|
"loss": 0.8245,
|
|
"margin_dpo/margin_mean": 18.566078186035156,
|
|
"margin_dpo/margin_std": 16.642459869384766,
|
|
"step": 353
|
|
},
|
|
{
|
|
"epoch": 0.5198237885462555,
|
|
"grad_norm": 21.19467544555664,
|
|
"learning_rate": 2.781732916288303e-07,
|
|
"logits/chosen": -2.125009536743164,
|
|
"logits/rejected": -1.8005847930908203,
|
|
"logps/chosen": -115.57217407226562,
|
|
"logps/ref_chosen": -98.41739654541016,
|
|
"logps/ref_rejected": -125.88714599609375,
|
|
"logps/rejected": -167.45452880859375,
|
|
"loss": 0.4702,
|
|
"margin_dpo/margin_mean": 24.412607192993164,
|
|
"margin_dpo/margin_std": 19.174697875976562,
|
|
"step": 354
|
|
},
|
|
{
|
|
"epoch": 0.5212922173274597,
|
|
"grad_norm": 28.40703582763672,
|
|
"learning_rate": 2.7689777072570284e-07,
|
|
"logits/chosen": -2.547095775604248,
|
|
"logits/rejected": -2.2789859771728516,
|
|
"logps/chosen": -107.61503601074219,
|
|
"logps/ref_chosen": -88.17337036132812,
|
|
"logps/ref_rejected": -121.17060852050781,
|
|
"logps/rejected": -158.981689453125,
|
|
"loss": 0.6514,
|
|
"margin_dpo/margin_mean": 18.369415283203125,
|
|
"margin_dpo/margin_std": 22.09329605102539,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.5227606461086637,
|
|
"grad_norm": 45.7209587097168,
|
|
"learning_rate": 2.7562154104130176e-07,
|
|
"logits/chosen": -1.3351719379425049,
|
|
"logits/rejected": -0.7877082824707031,
|
|
"logps/chosen": -86.14286804199219,
|
|
"logps/ref_chosen": -66.91217041015625,
|
|
"logps/ref_rejected": -59.56048583984375,
|
|
"logps/rejected": -99.44941711425781,
|
|
"loss": 0.8503,
|
|
"margin_dpo/margin_mean": 20.65822410583496,
|
|
"margin_dpo/margin_std": 12.647079467773438,
|
|
"step": 356
|
|
},
|
|
{
|
|
"epoch": 0.5242290748898678,
|
|
"grad_norm": 19.310302734375,
|
|
"learning_rate": 2.7434463620546594e-07,
|
|
"logits/chosen": -0.9185234308242798,
|
|
"logits/rejected": -0.8068221807479858,
|
|
"logps/chosen": -112.2743911743164,
|
|
"logps/ref_chosen": -94.1016616821289,
|
|
"logps/ref_rejected": -122.81672668457031,
|
|
"logps/rejected": -165.00479125976562,
|
|
"loss": 0.4708,
|
|
"margin_dpo/margin_mean": 24.01534080505371,
|
|
"margin_dpo/margin_std": 11.519631385803223,
|
|
"step": 357
|
|
},
|
|
{
|
|
"epoch": 0.5256975036710719,
|
|
"grad_norm": 27.054889678955078,
|
|
"learning_rate": 2.730670898658255e-07,
|
|
"logits/chosen": -1.6659433841705322,
|
|
"logits/rejected": -1.4695067405700684,
|
|
"logps/chosen": -100.44693756103516,
|
|
"logps/ref_chosen": -85.93841552734375,
|
|
"logps/ref_rejected": -116.45753479003906,
|
|
"logps/rejected": -150.18109130859375,
|
|
"loss": 0.6909,
|
|
"margin_dpo/margin_mean": 19.215017318725586,
|
|
"margin_dpo/margin_std": 14.75373649597168,
|
|
"step": 358
|
|
},
|
|
{
|
|
"epoch": 0.527165932452276,
|
|
"grad_norm": 36.24051284790039,
|
|
"learning_rate": 2.717889356869146e-07,
|
|
"logits/chosen": -2.035553455352783,
|
|
"logits/rejected": -0.9625518321990967,
|
|
"logps/chosen": -134.61293029785156,
|
|
"logps/ref_chosen": -117.40083312988281,
|
|
"logps/ref_rejected": -92.86836242675781,
|
|
"logps/rejected": -135.3282012939453,
|
|
"loss": 0.7865,
|
|
"margin_dpo/margin_mean": 25.24774169921875,
|
|
"margin_dpo/margin_std": 27.104656219482422,
|
|
"step": 359
|
|
},
|
|
{
|
|
"epoch": 0.5286343612334802,
|
|
"grad_norm": 21.825654983520508,
|
|
"learning_rate": 2.7051020734928443e-07,
|
|
"logits/chosen": -1.281875729560852,
|
|
"logits/rejected": -1.1035716533660889,
|
|
"logps/chosen": -100.54425048828125,
|
|
"logps/ref_chosen": -80.67975616455078,
|
|
"logps/ref_rejected": -98.07811737060547,
|
|
"logps/rejected": -139.25164794921875,
|
|
"loss": 0.6295,
|
|
"margin_dpo/margin_mean": 21.309030532836914,
|
|
"margin_dpo/margin_std": 12.134112358093262,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.5301027900146843,
|
|
"grad_norm": 26.63926887512207,
|
|
"learning_rate": 2.6923093854861593e-07,
|
|
"logits/chosen": -1.3344345092773438,
|
|
"logits/rejected": -0.6259992718696594,
|
|
"logps/chosen": -138.5869140625,
|
|
"logps/ref_chosen": -114.17556762695312,
|
|
"logps/ref_rejected": -103.11407470703125,
|
|
"logps/rejected": -145.5262451171875,
|
|
"loss": 0.6894,
|
|
"margin_dpo/margin_mean": 18.000823974609375,
|
|
"margin_dpo/margin_std": 16.170757293701172,
|
|
"step": 361
|
|
},
|
|
{
|
|
"epoch": 0.5315712187958884,
|
|
"grad_norm": 21.635181427001953,
|
|
"learning_rate": 2.679511629948319e-07,
|
|
"logits/chosen": -1.259399175643921,
|
|
"logits/rejected": -0.8324697017669678,
|
|
"logps/chosen": -103.67679595947266,
|
|
"logps/ref_chosen": -78.51634216308594,
|
|
"logps/ref_rejected": -116.00825500488281,
|
|
"logps/rejected": -168.18685913085938,
|
|
"loss": 0.5894,
|
|
"margin_dpo/margin_mean": 27.018146514892578,
|
|
"margin_dpo/margin_std": 23.748802185058594,
|
|
"step": 362
|
|
},
|
|
{
|
|
"epoch": 0.5330396475770925,
|
|
"grad_norm": 20.386882781982422,
|
|
"learning_rate": 2.6667091441120816e-07,
|
|
"logits/chosen": -1.566548466682434,
|
|
"logits/rejected": -1.190373182296753,
|
|
"logps/chosen": -88.85519409179688,
|
|
"logps/ref_chosen": -72.44766235351562,
|
|
"logps/ref_rejected": -82.24168395996094,
|
|
"logps/rejected": -125.80738830566406,
|
|
"loss": 0.478,
|
|
"margin_dpo/margin_mean": 27.158180236816406,
|
|
"margin_dpo/margin_std": 18.305994033813477,
|
|
"step": 363
|
|
},
|
|
{
|
|
"epoch": 0.5345080763582967,
|
|
"grad_norm": 25.959373474121094,
|
|
"learning_rate": 2.6539022653348575e-07,
|
|
"logits/chosen": -1.0531353950500488,
|
|
"logits/rejected": -1.0379638671875,
|
|
"logps/chosen": -97.78792572021484,
|
|
"logps/ref_chosen": -66.97681427001953,
|
|
"logps/ref_rejected": -101.46582794189453,
|
|
"logps/rejected": -141.74819946289062,
|
|
"loss": 0.6582,
|
|
"margin_dpo/margin_mean": 9.471246719360352,
|
|
"margin_dpo/margin_std": 12.190715789794922,
|
|
"step": 364
|
|
},
|
|
{
|
|
"epoch": 0.5359765051395007,
|
|
"grad_norm": 28.187889099121094,
|
|
"learning_rate": 2.641091331089811e-07,
|
|
"logits/chosen": -1.330674171447754,
|
|
"logits/rejected": -1.0664875507354736,
|
|
"logps/chosen": -98.81290435791016,
|
|
"logps/ref_chosen": -79.72026824951172,
|
|
"logps/ref_rejected": -73.94172668457031,
|
|
"logps/rejected": -107.04369354248047,
|
|
"loss": 0.613,
|
|
"margin_dpo/margin_mean": 14.00932502746582,
|
|
"margin_dpo/margin_std": 14.058714866638184,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.5374449339207048,
|
|
"grad_norm": 19.849567413330078,
|
|
"learning_rate": 2.6282766789569736e-07,
|
|
"logits/chosen": -1.6187386512756348,
|
|
"logits/rejected": -1.2361469268798828,
|
|
"logps/chosen": -85.14801788330078,
|
|
"logps/ref_chosen": -62.00611877441406,
|
|
"logps/ref_rejected": -48.86102294921875,
|
|
"logps/rejected": -93.41067504882812,
|
|
"loss": 0.5628,
|
|
"margin_dpo/margin_mean": 21.407752990722656,
|
|
"margin_dpo/margin_std": 21.69418716430664,
|
|
"step": 366
|
|
},
|
|
{
|
|
"epoch": 0.5389133627019089,
|
|
"grad_norm": 22.108535766601562,
|
|
"learning_rate": 2.615458646614349e-07,
|
|
"logits/chosen": -0.6758745908737183,
|
|
"logits/rejected": -0.37606683373451233,
|
|
"logps/chosen": -101.93498229980469,
|
|
"logps/ref_chosen": -86.56153106689453,
|
|
"logps/ref_rejected": -77.52005004882812,
|
|
"logps/rejected": -118.94590759277344,
|
|
"loss": 0.5367,
|
|
"margin_dpo/margin_mean": 26.05239486694336,
|
|
"margin_dpo/margin_std": 20.65791893005371,
|
|
"step": 367
|
|
},
|
|
{
|
|
"epoch": 0.540381791483113,
|
|
"grad_norm": 29.902986526489258,
|
|
"learning_rate": 2.6026375718290083e-07,
|
|
"logits/chosen": -1.2811745405197144,
|
|
"logits/rejected": -1.2634276151657104,
|
|
"logps/chosen": -86.03678894042969,
|
|
"logps/ref_chosen": -58.072303771972656,
|
|
"logps/ref_rejected": -96.70249938964844,
|
|
"logps/rejected": -140.60546875,
|
|
"loss": 0.5946,
|
|
"margin_dpo/margin_mean": 15.938491821289062,
|
|
"margin_dpo/margin_std": 15.870492935180664,
|
|
"step": 368
|
|
},
|
|
{
|
|
"epoch": 0.5418502202643172,
|
|
"grad_norm": 21.808292388916016,
|
|
"learning_rate": 2.589813792448196e-07,
|
|
"logits/chosen": -1.0705976486206055,
|
|
"logits/rejected": -0.9087299704551697,
|
|
"logps/chosen": -99.77658081054688,
|
|
"logps/ref_chosen": -77.49612426757812,
|
|
"logps/ref_rejected": -87.11772918701172,
|
|
"logps/rejected": -124.5665054321289,
|
|
"loss": 0.4996,
|
|
"margin_dpo/margin_mean": 15.168315887451172,
|
|
"margin_dpo/margin_std": 13.686176300048828,
|
|
"step": 369
|
|
},
|
|
{
|
|
"epoch": 0.5433186490455213,
|
|
"grad_norm": 28.55755615234375,
|
|
"learning_rate": 2.5769876463904263e-07,
|
|
"logits/chosen": -1.4109325408935547,
|
|
"logits/rejected": -1.645174503326416,
|
|
"logps/chosen": -103.64090728759766,
|
|
"logps/ref_chosen": -79.18736267089844,
|
|
"logps/ref_rejected": -157.551513671875,
|
|
"logps/rejected": -201.64453125,
|
|
"loss": 0.7399,
|
|
"margin_dpo/margin_mean": 19.639469146728516,
|
|
"margin_dpo/margin_std": 16.816953659057617,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.5447870778267254,
|
|
"grad_norm": 38.25402069091797,
|
|
"learning_rate": 2.5641594716365744e-07,
|
|
"logits/chosen": -1.212741494178772,
|
|
"logits/rejected": -0.6730820536613464,
|
|
"logps/chosen": -119.37736511230469,
|
|
"logps/ref_chosen": -96.76188659667969,
|
|
"logps/ref_rejected": -95.78884887695312,
|
|
"logps/rejected": -138.38714599609375,
|
|
"loss": 0.9139,
|
|
"margin_dpo/margin_mean": 19.98282241821289,
|
|
"margin_dpo/margin_std": 20.218643188476562,
|
|
"step": 371
|
|
},
|
|
{
|
|
"epoch": 0.5462555066079295,
|
|
"grad_norm": 19.106494903564453,
|
|
"learning_rate": 2.551329606220976e-07,
|
|
"logits/chosen": -0.8598968982696533,
|
|
"logits/rejected": -0.258903443813324,
|
|
"logps/chosen": -131.57382202148438,
|
|
"logps/ref_chosen": -111.48440551757812,
|
|
"logps/ref_rejected": -95.45821380615234,
|
|
"logps/rejected": -142.3041229248047,
|
|
"loss": 0.4188,
|
|
"margin_dpo/margin_mean": 26.756494522094727,
|
|
"margin_dpo/margin_std": 18.634157180786133,
|
|
"step": 372
|
|
},
|
|
{
|
|
"epoch": 0.5477239353891337,
|
|
"grad_norm": 25.07010841369629,
|
|
"learning_rate": 2.538498388222517e-07,
|
|
"logits/chosen": -2.0610454082489014,
|
|
"logits/rejected": -1.9063310623168945,
|
|
"logps/chosen": -89.15904235839844,
|
|
"logps/ref_chosen": -69.03050994873047,
|
|
"logps/ref_rejected": -74.32266235351562,
|
|
"logps/rejected": -111.32271575927734,
|
|
"loss": 0.5844,
|
|
"margin_dpo/margin_mean": 16.87152862548828,
|
|
"margin_dpo/margin_std": 10.439271926879883,
|
|
"step": 373
|
|
},
|
|
{
|
|
"epoch": 0.5491923641703378,
|
|
"grad_norm": 19.004375457763672,
|
|
"learning_rate": 2.525666155755725e-07,
|
|
"logits/chosen": -1.6338618993759155,
|
|
"logits/rejected": -1.5218095779418945,
|
|
"logps/chosen": -123.97227478027344,
|
|
"logps/ref_chosen": -103.24068450927734,
|
|
"logps/ref_rejected": -121.46826171875,
|
|
"logps/rejected": -160.615966796875,
|
|
"loss": 0.5226,
|
|
"margin_dpo/margin_mean": 18.416109085083008,
|
|
"margin_dpo/margin_std": 15.674093246459961,
|
|
"step": 374
|
|
},
|
|
{
|
|
"epoch": 0.5506607929515418,
|
|
"grad_norm": 24.541839599609375,
|
|
"learning_rate": 2.512833246961859e-07,
|
|
"logits/chosen": -1.3103591203689575,
|
|
"logits/rejected": -1.2791717052459717,
|
|
"logps/chosen": -100.64812469482422,
|
|
"logps/ref_chosen": -76.98648071289062,
|
|
"logps/ref_rejected": -97.53923034667969,
|
|
"logps/rejected": -141.93963623046875,
|
|
"loss": 0.5928,
|
|
"margin_dpo/margin_mean": 20.738773345947266,
|
|
"margin_dpo/margin_std": 17.845693588256836,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.5521292217327459,
|
|
"grad_norm": 23.439132690429688,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": -1.1369824409484863,
|
|
"logits/rejected": -0.7745121717453003,
|
|
"logps/chosen": -106.30282592773438,
|
|
"logps/ref_chosen": -87.6530990600586,
|
|
"logps/ref_rejected": -97.850830078125,
|
|
"logps/rejected": -135.3538818359375,
|
|
"loss": 0.5365,
|
|
"margin_dpo/margin_mean": 18.85333824157715,
|
|
"margin_dpo/margin_std": 18.811431884765625,
|
|
"step": 376
|
|
},
|
|
{
|
|
"epoch": 0.55359765051395,
|
|
"grad_norm": 32.66826248168945,
|
|
"learning_rate": 2.487166753038141e-07,
|
|
"logits/chosen": -1.4077603816986084,
|
|
"logits/rejected": -0.6672318577766418,
|
|
"logps/chosen": -103.64756774902344,
|
|
"logps/ref_chosen": -83.87545776367188,
|
|
"logps/ref_rejected": -106.24966430664062,
|
|
"logps/rejected": -158.5746307373047,
|
|
"loss": 0.5251,
|
|
"margin_dpo/margin_mean": 32.552860260009766,
|
|
"margin_dpo/margin_std": 22.107181549072266,
|
|
"step": 377
|
|
},
|
|
{
|
|
"epoch": 0.5550660792951542,
|
|
"grad_norm": 25.195751190185547,
|
|
"learning_rate": 2.4743338442442754e-07,
|
|
"logits/chosen": -1.722794771194458,
|
|
"logits/rejected": -1.249429702758789,
|
|
"logps/chosen": -117.63298034667969,
|
|
"logps/ref_chosen": -100.02596282958984,
|
|
"logps/ref_rejected": -122.08744812011719,
|
|
"logps/rejected": -163.1724853515625,
|
|
"loss": 0.5627,
|
|
"margin_dpo/margin_mean": 23.478015899658203,
|
|
"margin_dpo/margin_std": 15.552407264709473,
|
|
"step": 378
|
|
},
|
|
{
|
|
"epoch": 0.5565345080763583,
|
|
"grad_norm": 28.934782028198242,
|
|
"learning_rate": 2.461501611777483e-07,
|
|
"logits/chosen": -2.1368558406829834,
|
|
"logits/rejected": -1.744170904159546,
|
|
"logps/chosen": -139.7799072265625,
|
|
"logps/ref_chosen": -117.1336669921875,
|
|
"logps/ref_rejected": -131.74588012695312,
|
|
"logps/rejected": -178.27560424804688,
|
|
"loss": 0.7004,
|
|
"margin_dpo/margin_mean": 23.883481979370117,
|
|
"margin_dpo/margin_std": 22.75574493408203,
|
|
"step": 379
|
|
},
|
|
{
|
|
"epoch": 0.5580029368575624,
|
|
"grad_norm": 28.92926025390625,
|
|
"learning_rate": 2.4486703937790243e-07,
|
|
"logits/chosen": -1.6153287887573242,
|
|
"logits/rejected": -1.1990188360214233,
|
|
"logps/chosen": -103.89430236816406,
|
|
"logps/ref_chosen": -83.37449645996094,
|
|
"logps/ref_rejected": -103.0969467163086,
|
|
"logps/rejected": -149.7965087890625,
|
|
"loss": 0.6989,
|
|
"margin_dpo/margin_mean": 26.17973518371582,
|
|
"margin_dpo/margin_std": 24.212230682373047,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5594713656387665,
|
|
"grad_norm": 32.6840705871582,
|
|
"learning_rate": 2.435840528363426e-07,
|
|
"logits/chosen": -1.619598388671875,
|
|
"logits/rejected": -1.1338703632354736,
|
|
"logps/chosen": -108.6231918334961,
|
|
"logps/ref_chosen": -94.2698745727539,
|
|
"logps/ref_rejected": -113.51543426513672,
|
|
"logps/rejected": -155.37216186523438,
|
|
"loss": 0.7262,
|
|
"margin_dpo/margin_mean": 27.503414154052734,
|
|
"margin_dpo/margin_std": 16.12104034423828,
|
|
"step": 381
|
|
},
|
|
{
|
|
"epoch": 0.5609397944199707,
|
|
"grad_norm": 24.89579963684082,
|
|
"learning_rate": 2.4230123536095745e-07,
|
|
"logits/chosen": -1.8435094356536865,
|
|
"logits/rejected": -1.3504945039749146,
|
|
"logps/chosen": -131.414306640625,
|
|
"logps/ref_chosen": -118.74935150146484,
|
|
"logps/ref_rejected": -124.04042053222656,
|
|
"logps/rejected": -167.5652618408203,
|
|
"loss": 0.5787,
|
|
"margin_dpo/margin_mean": 30.85987663269043,
|
|
"margin_dpo/margin_std": 21.3883113861084,
|
|
"step": 382
|
|
},
|
|
{
|
|
"epoch": 0.5624082232011748,
|
|
"grad_norm": 32.93791580200195,
|
|
"learning_rate": 2.4101862075518037e-07,
|
|
"logits/chosen": -2.270036220550537,
|
|
"logits/rejected": -1.7036347389221191,
|
|
"logps/chosen": -109.92510986328125,
|
|
"logps/ref_chosen": -84.62395477294922,
|
|
"logps/ref_rejected": -71.94377136230469,
|
|
"logps/rejected": -113.90255737304688,
|
|
"loss": 0.7806,
|
|
"margin_dpo/margin_mean": 16.65763282775879,
|
|
"margin_dpo/margin_std": 20.160343170166016,
|
|
"step": 383
|
|
},
|
|
{
|
|
"epoch": 0.5638766519823789,
|
|
"grad_norm": 23.051111221313477,
|
|
"learning_rate": 2.397362428170992e-07,
|
|
"logits/chosen": -1.8162555694580078,
|
|
"logits/rejected": -1.813509225845337,
|
|
"logps/chosen": -84.0343017578125,
|
|
"logps/ref_chosen": -58.861610412597656,
|
|
"logps/ref_rejected": -74.94581604003906,
|
|
"logps/rejected": -122.06885528564453,
|
|
"loss": 0.5823,
|
|
"margin_dpo/margin_mean": 21.950340270996094,
|
|
"margin_dpo/margin_std": 19.539579391479492,
|
|
"step": 384
|
|
},
|
|
{
|
|
"epoch": 0.5653450807635829,
|
|
"grad_norm": 28.523611068725586,
|
|
"learning_rate": 2.3845413533856514e-07,
|
|
"logits/chosen": -2.339524745941162,
|
|
"logits/rejected": -1.4595880508422852,
|
|
"logps/chosen": -108.14667510986328,
|
|
"logps/ref_chosen": -88.04960632324219,
|
|
"logps/ref_rejected": -73.19291687011719,
|
|
"logps/rejected": -112.58067321777344,
|
|
"loss": 0.6257,
|
|
"margin_dpo/margin_mean": 19.29067611694336,
|
|
"margin_dpo/margin_std": 16.32854461669922,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.566813509544787,
|
|
"grad_norm": 24.61203956604004,
|
|
"learning_rate": 2.3717233210430254e-07,
|
|
"logits/chosen": -1.5339938402175903,
|
|
"logits/rejected": -1.359063982963562,
|
|
"logps/chosen": -109.09315490722656,
|
|
"logps/ref_chosen": -83.63232421875,
|
|
"logps/ref_rejected": -115.93226623535156,
|
|
"logps/rejected": -156.15509033203125,
|
|
"loss": 0.626,
|
|
"margin_dpo/margin_mean": 14.761992454528809,
|
|
"margin_dpo/margin_std": 16.876083374023438,
|
|
"step": 386
|
|
},
|
|
{
|
|
"epoch": 0.5682819383259912,
|
|
"grad_norm": 28.160860061645508,
|
|
"learning_rate": 2.3589086689101889e-07,
|
|
"logits/chosen": -1.8218247890472412,
|
|
"logits/rejected": -1.1248939037322998,
|
|
"logps/chosen": -119.5574722290039,
|
|
"logps/ref_chosen": -97.42512512207031,
|
|
"logps/ref_rejected": -67.96300506591797,
|
|
"logps/rejected": -111.73419189453125,
|
|
"loss": 0.686,
|
|
"margin_dpo/margin_mean": 21.638851165771484,
|
|
"margin_dpo/margin_std": 18.822141647338867,
|
|
"step": 387
|
|
},
|
|
{
|
|
"epoch": 0.5697503671071953,
|
|
"grad_norm": 21.124223709106445,
|
|
"learning_rate": 2.3460977346651428e-07,
|
|
"logits/chosen": -1.9199204444885254,
|
|
"logits/rejected": -1.564023494720459,
|
|
"logps/chosen": -100.3346176147461,
|
|
"logps/ref_chosen": -76.29823303222656,
|
|
"logps/ref_rejected": -101.94246673583984,
|
|
"logps/rejected": -154.15386962890625,
|
|
"loss": 0.3795,
|
|
"margin_dpo/margin_mean": 28.17501449584961,
|
|
"margin_dpo/margin_std": 13.74698257446289,
|
|
"step": 388
|
|
},
|
|
{
|
|
"epoch": 0.5712187958883994,
|
|
"grad_norm": 36.40032196044922,
|
|
"learning_rate": 2.3332908558879177e-07,
|
|
"logits/chosen": -1.2836616039276123,
|
|
"logits/rejected": -1.0078399181365967,
|
|
"logps/chosen": -122.41304016113281,
|
|
"logps/ref_chosen": -97.35070037841797,
|
|
"logps/ref_rejected": -91.23686981201172,
|
|
"logps/rejected": -133.83741760253906,
|
|
"loss": 0.7691,
|
|
"margin_dpo/margin_mean": 17.538209915161133,
|
|
"margin_dpo/margin_std": 22.933223724365234,
|
|
"step": 389
|
|
},
|
|
{
|
|
"epoch": 0.5726872246696035,
|
|
"grad_norm": 29.538150787353516,
|
|
"learning_rate": 2.320488370051681e-07,
|
|
"logits/chosen": -1.8550262451171875,
|
|
"logits/rejected": -1.7845234870910645,
|
|
"logps/chosen": -84.25496673583984,
|
|
"logps/ref_chosen": -58.03634262084961,
|
|
"logps/ref_rejected": -74.02693176269531,
|
|
"logps/rejected": -116.31646728515625,
|
|
"loss": 0.7758,
|
|
"margin_dpo/margin_mean": 16.070911407470703,
|
|
"margin_dpo/margin_std": 14.410566329956055,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.5741556534508077,
|
|
"grad_norm": 27.119056701660156,
|
|
"learning_rate": 2.3076906145138405e-07,
|
|
"logits/chosen": -1.7980409860610962,
|
|
"logits/rejected": -1.492868185043335,
|
|
"logps/chosen": -101.29808044433594,
|
|
"logps/ref_chosen": -74.79901885986328,
|
|
"logps/ref_rejected": -100.62217712402344,
|
|
"logps/rejected": -142.77825927734375,
|
|
"loss": 0.5887,
|
|
"margin_dpo/margin_mean": 15.65703010559082,
|
|
"margin_dpo/margin_std": 12.83513069152832,
|
|
"step": 391
|
|
},
|
|
{
|
|
"epoch": 0.5756240822320118,
|
|
"grad_norm": 26.66034507751465,
|
|
"learning_rate": 2.294897926507156e-07,
|
|
"logits/chosen": -1.8430507183074951,
|
|
"logits/rejected": -1.2197606563568115,
|
|
"logps/chosen": -137.8718719482422,
|
|
"logps/ref_chosen": -112.5604019165039,
|
|
"logps/ref_rejected": -74.96153259277344,
|
|
"logps/rejected": -115.8902816772461,
|
|
"loss": 0.6069,
|
|
"margin_dpo/margin_mean": 15.61727237701416,
|
|
"margin_dpo/margin_std": 16.81137466430664,
|
|
"step": 392
|
|
},
|
|
{
|
|
"epoch": 0.5770925110132159,
|
|
"grad_norm": 24.514549255371094,
|
|
"learning_rate": 2.2821106431308543e-07,
|
|
"logits/chosen": -1.7378129959106445,
|
|
"logits/rejected": -1.5551893711090088,
|
|
"logps/chosen": -100.59256744384766,
|
|
"logps/ref_chosen": -81.56492614746094,
|
|
"logps/ref_rejected": -89.94920349121094,
|
|
"logps/rejected": -131.44895935058594,
|
|
"loss": 0.561,
|
|
"margin_dpo/margin_mean": 22.472103118896484,
|
|
"margin_dpo/margin_std": 14.629976272583008,
|
|
"step": 393
|
|
},
|
|
{
|
|
"epoch": 0.57856093979442,
|
|
"grad_norm": 21.149150848388672,
|
|
"learning_rate": 2.2693291013417452e-07,
|
|
"logits/chosen": -1.8245320320129395,
|
|
"logits/rejected": -1.1274913549423218,
|
|
"logps/chosen": -100.30079650878906,
|
|
"logps/ref_chosen": -80.80152130126953,
|
|
"logps/ref_rejected": -57.329986572265625,
|
|
"logps/rejected": -98.2430648803711,
|
|
"loss": 0.4967,
|
|
"margin_dpo/margin_mean": 21.413806915283203,
|
|
"margin_dpo/margin_std": 18.011598587036133,
|
|
"step": 394
|
|
},
|
|
{
|
|
"epoch": 0.580029368575624,
|
|
"grad_norm": 29.29093360900879,
|
|
"learning_rate": 2.2565536379453404e-07,
|
|
"logits/chosen": -2.202932357788086,
|
|
"logits/rejected": -1.7157886028289795,
|
|
"logps/chosen": -108.34182739257812,
|
|
"logps/ref_chosen": -84.28303527832031,
|
|
"logps/ref_rejected": -68.78179931640625,
|
|
"logps/rejected": -105.08010864257812,
|
|
"loss": 0.7926,
|
|
"margin_dpo/margin_mean": 12.239519119262695,
|
|
"margin_dpo/margin_std": 17.91888427734375,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.5814977973568282,
|
|
"grad_norm": 34.08306121826172,
|
|
"learning_rate": 2.2437845895869825e-07,
|
|
"logits/chosen": -1.750239372253418,
|
|
"logits/rejected": -1.5954954624176025,
|
|
"logps/chosen": -82.07247924804688,
|
|
"logps/ref_chosen": -55.12290954589844,
|
|
"logps/ref_rejected": -74.67718505859375,
|
|
"logps/rejected": -116.58084106445312,
|
|
"loss": 0.7967,
|
|
"margin_dpo/margin_mean": 14.95408821105957,
|
|
"margin_dpo/margin_std": 16.51461410522461,
|
|
"step": 396
|
|
},
|
|
{
|
|
"epoch": 0.5829662261380323,
|
|
"grad_norm": 25.127824783325195,
|
|
"learning_rate": 2.2310222927429716e-07,
|
|
"logits/chosen": -1.2238702774047852,
|
|
"logits/rejected": -1.2863905429840088,
|
|
"logps/chosen": -114.83526611328125,
|
|
"logps/ref_chosen": -91.61041259765625,
|
|
"logps/ref_rejected": -130.83035278320312,
|
|
"logps/rejected": -175.09243774414062,
|
|
"loss": 0.6264,
|
|
"margin_dpo/margin_mean": 21.037246704101562,
|
|
"margin_dpo/margin_std": 25.966951370239258,
|
|
"step": 397
|
|
},
|
|
{
|
|
"epoch": 0.5844346549192364,
|
|
"grad_norm": 27.55780792236328,
|
|
"learning_rate": 2.2182670837116972e-07,
|
|
"logits/chosen": -1.5437828302383423,
|
|
"logits/rejected": -1.1038641929626465,
|
|
"logps/chosen": -128.46905517578125,
|
|
"logps/ref_chosen": -100.79484558105469,
|
|
"logps/ref_rejected": -102.5047378540039,
|
|
"logps/rejected": -147.94284057617188,
|
|
"loss": 0.5462,
|
|
"margin_dpo/margin_mean": 17.76388168334961,
|
|
"margin_dpo/margin_std": 17.782602310180664,
|
|
"step": 398
|
|
},
|
|
{
|
|
"epoch": 0.5859030837004405,
|
|
"grad_norm": 16.550373077392578,
|
|
"learning_rate": 2.2055192986047804e-07,
|
|
"logits/chosen": -2.9651870727539062,
|
|
"logits/rejected": -1.4718399047851562,
|
|
"logps/chosen": -150.08575439453125,
|
|
"logps/ref_chosen": -131.33981323242188,
|
|
"logps/ref_rejected": -76.11972045898438,
|
|
"logps/rejected": -118.81426239013672,
|
|
"loss": 0.3647,
|
|
"margin_dpo/margin_mean": 23.948593139648438,
|
|
"margin_dpo/margin_std": 12.468704223632812,
|
|
"step": 399
|
|
},
|
|
{
|
|
"epoch": 0.5873715124816447,
|
|
"grad_norm": 28.322050094604492,
|
|
"learning_rate": 2.192779273338215e-07,
|
|
"logits/chosen": -2.360074043273926,
|
|
"logits/rejected": -1.4844969511032104,
|
|
"logps/chosen": -140.05178833007812,
|
|
"logps/ref_chosen": -118.27375793457031,
|
|
"logps/ref_rejected": -115.4412841796875,
|
|
"logps/rejected": -159.77346801757812,
|
|
"loss": 0.4743,
|
|
"margin_dpo/margin_mean": 22.554155349731445,
|
|
"margin_dpo/margin_std": 18.595291137695312,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5873715124816447,
|
|
"eval_logits/chosen": -1.2117310762405396,
|
|
"eval_logits/rejected": -0.8992021083831787,
|
|
"eval_logps/chosen": -128.40940856933594,
|
|
"eval_logps/ref_chosen": -101.88616943359375,
|
|
"eval_logps/ref_rejected": -92.68607330322266,
|
|
"eval_logps/rejected": -134.59133911132812,
|
|
"eval_loss": 0.42923954129219055,
|
|
"eval_margin_dpo/margin_mean": 15.382046699523926,
|
|
"eval_margin_dpo/margin_std": 16.76236343383789,
|
|
"eval_runtime": 44.0108,
|
|
"eval_samples_per_second": 53.146,
|
|
"eval_steps_per_second": 1.681,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5888399412628488,
|
|
"grad_norm": 27.084718704223633,
|
|
"learning_rate": 2.1800473436235136e-07,
|
|
"logits/chosen": -0.82004714012146,
|
|
"logits/rejected": -0.5782370567321777,
|
|
"logps/chosen": -119.38215637207031,
|
|
"logps/ref_chosen": -90.59347534179688,
|
|
"logps/ref_rejected": -64.001708984375,
|
|
"logps/rejected": -99.23313903808594,
|
|
"loss": 0.8125,
|
|
"margin_dpo/margin_mean": 6.442744255065918,
|
|
"margin_dpo/margin_std": 16.57547378540039,
|
|
"step": 401
|
|
},
|
|
{
|
|
"epoch": 0.5903083700440529,
|
|
"grad_norm": 20.646493911743164,
|
|
"learning_rate": 2.1673238449588665e-07,
|
|
"logits/chosen": -1.7929552793502808,
|
|
"logits/rejected": -1.4944618940353394,
|
|
"logps/chosen": -114.04674530029297,
|
|
"logps/ref_chosen": -92.98797607421875,
|
|
"logps/ref_rejected": -87.38853454589844,
|
|
"logps/rejected": -132.45294189453125,
|
|
"loss": 0.3894,
|
|
"margin_dpo/margin_mean": 24.005657196044922,
|
|
"margin_dpo/margin_std": 14.80632209777832,
|
|
"step": 402
|
|
},
|
|
{
|
|
"epoch": 0.591776798825257,
|
|
"grad_norm": 20.255006790161133,
|
|
"learning_rate": 2.154609112620295e-07,
|
|
"logits/chosen": -2.186984062194824,
|
|
"logits/rejected": -1.762284278869629,
|
|
"logps/chosen": -111.00959777832031,
|
|
"logps/ref_chosen": -91.2931900024414,
|
|
"logps/ref_rejected": -62.71670913696289,
|
|
"logps/rejected": -102.13783264160156,
|
|
"loss": 0.5945,
|
|
"margin_dpo/margin_mean": 19.7047061920166,
|
|
"margin_dpo/margin_std": 19.972158432006836,
|
|
"step": 403
|
|
},
|
|
{
|
|
"epoch": 0.593245227606461,
|
|
"grad_norm": 21.293611526489258,
|
|
"learning_rate": 2.1419034816528218e-07,
|
|
"logits/chosen": -2.028939723968506,
|
|
"logits/rejected": -1.676574945449829,
|
|
"logps/chosen": -100.58467864990234,
|
|
"logps/ref_chosen": -77.146728515625,
|
|
"logps/ref_rejected": -100.78892517089844,
|
|
"logps/rejected": -154.99609375,
|
|
"loss": 0.6096,
|
|
"margin_dpo/margin_mean": 30.76922035217285,
|
|
"margin_dpo/margin_std": 20.257919311523438,
|
|
"step": 404
|
|
},
|
|
{
|
|
"epoch": 0.5947136563876652,
|
|
"grad_norm": 26.865093231201172,
|
|
"learning_rate": 2.129207286861638e-07,
|
|
"logits/chosen": -0.8960381746292114,
|
|
"logits/rejected": -0.6281898021697998,
|
|
"logps/chosen": -140.56105041503906,
|
|
"logps/ref_chosen": -113.60545349121094,
|
|
"logps/ref_rejected": -113.76882934570312,
|
|
"logps/rejected": -163.43182373046875,
|
|
"loss": 0.6095,
|
|
"margin_dpo/margin_mean": 22.707393646240234,
|
|
"margin_dpo/margin_std": 19.417251586914062,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.5961820851688693,
|
|
"grad_norm": 26.39828109741211,
|
|
"learning_rate": 2.1165208628032861e-07,
|
|
"logits/chosen": -2.1540369987487793,
|
|
"logits/rejected": -1.7181463241577148,
|
|
"logps/chosen": -102.27427673339844,
|
|
"logps/ref_chosen": -82.02105712890625,
|
|
"logps/ref_rejected": -90.03308868408203,
|
|
"logps/rejected": -134.26731872558594,
|
|
"loss": 0.6461,
|
|
"margin_dpo/margin_mean": 23.98102378845215,
|
|
"margin_dpo/margin_std": 19.551021575927734,
|
|
"step": 406
|
|
},
|
|
{
|
|
"epoch": 0.5976505139500734,
|
|
"grad_norm": 25.158649444580078,
|
|
"learning_rate": 2.1038445437768375e-07,
|
|
"logits/chosen": -2.754941463470459,
|
|
"logits/rejected": -2.005859851837158,
|
|
"logps/chosen": -102.0631103515625,
|
|
"logps/ref_chosen": -71.41169738769531,
|
|
"logps/ref_rejected": -65.43003845214844,
|
|
"logps/rejected": -109.44853210449219,
|
|
"loss": 0.6719,
|
|
"margin_dpo/margin_mean": 13.367082595825195,
|
|
"margin_dpo/margin_std": 21.264638900756836,
|
|
"step": 407
|
|
},
|
|
{
|
|
"epoch": 0.5991189427312775,
|
|
"grad_norm": 34.24391555786133,
|
|
"learning_rate": 2.0911786638150872e-07,
|
|
"logits/chosen": -0.9452221989631653,
|
|
"logits/rejected": -0.40952903032302856,
|
|
"logps/chosen": -129.79104614257812,
|
|
"logps/ref_chosen": -103.82225036621094,
|
|
"logps/ref_rejected": -66.49540710449219,
|
|
"logps/rejected": -112.33432006835938,
|
|
"loss": 0.7704,
|
|
"margin_dpo/margin_mean": 19.870115280151367,
|
|
"margin_dpo/margin_std": 19.41891860961914,
|
|
"step": 408
|
|
},
|
|
{
|
|
"epoch": 0.6005873715124816,
|
|
"grad_norm": 36.82187271118164,
|
|
"learning_rate": 2.0785235566757517e-07,
|
|
"logits/chosen": -1.7844617366790771,
|
|
"logits/rejected": -1.0885341167449951,
|
|
"logps/chosen": -163.90234375,
|
|
"logps/ref_chosen": -143.14186096191406,
|
|
"logps/ref_rejected": -86.51887512207031,
|
|
"logps/rejected": -126.71371459960938,
|
|
"loss": 0.7749,
|
|
"margin_dpo/margin_mean": 19.43436050415039,
|
|
"margin_dpo/margin_std": 19.317798614501953,
|
|
"step": 409
|
|
},
|
|
{
|
|
"epoch": 0.6020558002936858,
|
|
"grad_norm": 26.417638778686523,
|
|
"learning_rate": 2.065879555832674e-07,
|
|
"logits/chosen": -2.059418201446533,
|
|
"logits/rejected": -1.9821703433990479,
|
|
"logps/chosen": -86.10826110839844,
|
|
"logps/ref_chosen": -60.73199462890625,
|
|
"logps/ref_rejected": -80.75624084472656,
|
|
"logps/rejected": -125.80564880371094,
|
|
"loss": 0.6685,
|
|
"margin_dpo/margin_mean": 19.673145294189453,
|
|
"margin_dpo/margin_std": 14.39936351776123,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.6035242290748899,
|
|
"grad_norm": 26.731914520263672,
|
|
"learning_rate": 2.0532469944670343e-07,
|
|
"logits/chosen": -2.0580475330352783,
|
|
"logits/rejected": -2.118593692779541,
|
|
"logps/chosen": -95.44523620605469,
|
|
"logps/ref_chosen": -66.200927734375,
|
|
"logps/ref_rejected": -75.45953369140625,
|
|
"logps/rejected": -118.89471435546875,
|
|
"loss": 0.6113,
|
|
"margin_dpo/margin_mean": 14.190881729125977,
|
|
"margin_dpo/margin_std": 13.96728515625,
|
|
"step": 411
|
|
},
|
|
{
|
|
"epoch": 0.604992657856094,
|
|
"grad_norm": 24.100954055786133,
|
|
"learning_rate": 2.0406262054585738e-07,
|
|
"logits/chosen": -1.6652562618255615,
|
|
"logits/rejected": -1.6168675422668457,
|
|
"logps/chosen": -96.10821533203125,
|
|
"logps/ref_chosen": -70.4729232788086,
|
|
"logps/ref_rejected": -116.28516387939453,
|
|
"logps/rejected": -161.6231689453125,
|
|
"loss": 0.6431,
|
|
"margin_dpo/margin_mean": 19.70269203186035,
|
|
"margin_dpo/margin_std": 17.422361373901367,
|
|
"step": 412
|
|
},
|
|
{
|
|
"epoch": 0.6064610866372981,
|
|
"grad_norm": 32.71993637084961,
|
|
"learning_rate": 2.0280175213768205e-07,
|
|
"logits/chosen": -0.5341243743896484,
|
|
"logits/rejected": -0.1439867466688156,
|
|
"logps/chosen": -121.38672637939453,
|
|
"logps/ref_chosen": -91.80178833007812,
|
|
"logps/ref_rejected": -78.62126159667969,
|
|
"logps/rejected": -119.86428833007812,
|
|
"loss": 0.7514,
|
|
"margin_dpo/margin_mean": 11.658080101013184,
|
|
"margin_dpo/margin_std": 13.869819641113281,
|
|
"step": 413
|
|
},
|
|
{
|
|
"epoch": 0.6079295154185022,
|
|
"grad_norm": 32.95093536376953,
|
|
"learning_rate": 2.0154212744723247e-07,
|
|
"logits/chosen": -2.0908069610595703,
|
|
"logits/rejected": -1.547807216644287,
|
|
"logps/chosen": -115.67071533203125,
|
|
"logps/ref_chosen": -93.31285095214844,
|
|
"logps/ref_rejected": -87.45198822021484,
|
|
"logps/rejected": -127.25440216064453,
|
|
"loss": 0.7371,
|
|
"margin_dpo/margin_mean": 17.44455337524414,
|
|
"margin_dpo/margin_std": 21.200258255004883,
|
|
"step": 414
|
|
},
|
|
{
|
|
"epoch": 0.6093979441997063,
|
|
"grad_norm": 38.77595520019531,
|
|
"learning_rate": 2.002837796667909e-07,
|
|
"logits/chosen": -1.3557873964309692,
|
|
"logits/rejected": -0.9005389213562012,
|
|
"logps/chosen": -131.7755126953125,
|
|
"logps/ref_chosen": -108.298095703125,
|
|
"logps/ref_rejected": -97.05425262451172,
|
|
"logps/rejected": -146.5677490234375,
|
|
"loss": 0.7499,
|
|
"margin_dpo/margin_mean": 26.036104202270508,
|
|
"margin_dpo/margin_std": 21.959575653076172,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.6108663729809104,
|
|
"grad_norm": 20.369680404663086,
|
|
"learning_rate": 1.990267419549914e-07,
|
|
"logits/chosen": -2.151233196258545,
|
|
"logits/rejected": -1.5413405895233154,
|
|
"logps/chosen": -108.19269561767578,
|
|
"logps/ref_chosen": -84.23023986816406,
|
|
"logps/ref_rejected": -105.33232879638672,
|
|
"logps/rejected": -153.75759887695312,
|
|
"loss": 0.4686,
|
|
"margin_dpo/margin_mean": 24.462818145751953,
|
|
"margin_dpo/margin_std": 19.401506423950195,
|
|
"step": 416
|
|
},
|
|
{
|
|
"epoch": 0.6123348017621145,
|
|
"grad_norm": 24.230409622192383,
|
|
"learning_rate": 1.9777104743594686e-07,
|
|
"logits/chosen": -2.02632474899292,
|
|
"logits/rejected": -0.5686318278312683,
|
|
"logps/chosen": -109.26607513427734,
|
|
"logps/ref_chosen": -89.4459457397461,
|
|
"logps/ref_rejected": -59.425933837890625,
|
|
"logps/rejected": -103.1401596069336,
|
|
"loss": 0.5942,
|
|
"margin_dpo/margin_mean": 23.894100189208984,
|
|
"margin_dpo/margin_std": 23.507366180419922,
|
|
"step": 417
|
|
},
|
|
{
|
|
"epoch": 0.6138032305433186,
|
|
"grad_norm": 26.247756958007812,
|
|
"learning_rate": 1.965167291983757e-07,
|
|
"logits/chosen": -2.066215753555298,
|
|
"logits/rejected": -1.878091812133789,
|
|
"logps/chosen": -109.95904541015625,
|
|
"logps/ref_chosen": -87.28543090820312,
|
|
"logps/ref_rejected": -105.37240600585938,
|
|
"logps/rejected": -155.7923126220703,
|
|
"loss": 0.5106,
|
|
"margin_dpo/margin_mean": 27.746294021606445,
|
|
"margin_dpo/margin_std": 15.890663146972656,
|
|
"step": 418
|
|
},
|
|
{
|
|
"epoch": 0.6152716593245228,
|
|
"grad_norm": 29.40278434753418,
|
|
"learning_rate": 1.9526382029472988e-07,
|
|
"logits/chosen": -1.2009162902832031,
|
|
"logits/rejected": -0.9768263101577759,
|
|
"logps/chosen": -118.16517639160156,
|
|
"logps/ref_chosen": -94.09585571289062,
|
|
"logps/ref_rejected": -115.02862548828125,
|
|
"logps/rejected": -162.2279510498047,
|
|
"loss": 0.7314,
|
|
"margin_dpo/margin_mean": 23.130002975463867,
|
|
"margin_dpo/margin_std": 20.91366195678711,
|
|
"step": 419
|
|
},
|
|
{
|
|
"epoch": 0.6167400881057269,
|
|
"grad_norm": 38.40915298461914,
|
|
"learning_rate": 1.9401235374032425e-07,
|
|
"logits/chosen": -1.9284502267837524,
|
|
"logits/rejected": -1.554548978805542,
|
|
"logps/chosen": -123.48500061035156,
|
|
"logps/ref_chosen": -101.53406524658203,
|
|
"logps/ref_rejected": -94.7691650390625,
|
|
"logps/rejected": -136.91636657714844,
|
|
"loss": 0.598,
|
|
"margin_dpo/margin_mean": 20.196256637573242,
|
|
"margin_dpo/margin_std": 21.781005859375,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.618208516886931,
|
|
"grad_norm": 26.852741241455078,
|
|
"learning_rate": 1.9276236251246653e-07,
|
|
"logits/chosen": -1.4120337963104248,
|
|
"logits/rejected": -1.2915436029434204,
|
|
"logps/chosen": -87.1471939086914,
|
|
"logps/ref_chosen": -57.393409729003906,
|
|
"logps/ref_rejected": -88.12896728515625,
|
|
"logps/rejected": -140.4273681640625,
|
|
"loss": 0.6178,
|
|
"margin_dpo/margin_mean": 22.544641494750977,
|
|
"margin_dpo/margin_std": 18.13541030883789,
|
|
"step": 421
|
|
},
|
|
{
|
|
"epoch": 0.6196769456681351,
|
|
"grad_norm": 28.518985748291016,
|
|
"learning_rate": 1.9151387954958792e-07,
|
|
"logits/chosen": -2.0918030738830566,
|
|
"logits/rejected": -1.6457126140594482,
|
|
"logps/chosen": -158.14785766601562,
|
|
"logps/ref_chosen": -137.25265502929688,
|
|
"logps/ref_rejected": -128.72036743164062,
|
|
"logps/rejected": -174.84373474121094,
|
|
"loss": 0.6448,
|
|
"margin_dpo/margin_mean": 25.22815704345703,
|
|
"margin_dpo/margin_std": 25.092853546142578,
|
|
"step": 422
|
|
},
|
|
{
|
|
"epoch": 0.6211453744493393,
|
|
"grad_norm": 27.90230941772461,
|
|
"learning_rate": 1.902669377503756e-07,
|
|
"logits/chosen": -1.126874566078186,
|
|
"logits/rejected": -1.2383806705474854,
|
|
"logps/chosen": -95.43914794921875,
|
|
"logps/ref_chosen": -61.778358459472656,
|
|
"logps/ref_rejected": -78.25019836425781,
|
|
"logps/rejected": -125.11500549316406,
|
|
"loss": 0.6857,
|
|
"margin_dpo/margin_mean": 13.20401382446289,
|
|
"margin_dpo/margin_std": 14.358932495117188,
|
|
"step": 423
|
|
},
|
|
{
|
|
"epoch": 0.6226138032305433,
|
|
"grad_norm": 31.08055877685547,
|
|
"learning_rate": 1.890215699729057e-07,
|
|
"logits/chosen": -1.6574307680130005,
|
|
"logits/rejected": -0.9652201533317566,
|
|
"logps/chosen": -116.06146240234375,
|
|
"logps/ref_chosen": -92.41514587402344,
|
|
"logps/ref_rejected": -58.8270263671875,
|
|
"logps/rejected": -106.29594421386719,
|
|
"loss": 0.7084,
|
|
"margin_dpo/margin_mean": 23.822601318359375,
|
|
"margin_dpo/margin_std": 23.501611709594727,
|
|
"step": 424
|
|
},
|
|
{
|
|
"epoch": 0.6240822320117474,
|
|
"grad_norm": 31.131589889526367,
|
|
"learning_rate": 1.8777780903377732e-07,
|
|
"logits/chosen": -2.1004793643951416,
|
|
"logits/rejected": -1.985741138458252,
|
|
"logps/chosen": -107.98200988769531,
|
|
"logps/ref_chosen": -82.38971710205078,
|
|
"logps/ref_rejected": -75.65036010742188,
|
|
"logps/rejected": -109.9925308227539,
|
|
"loss": 0.8679,
|
|
"margin_dpo/margin_mean": 8.749892234802246,
|
|
"margin_dpo/margin_std": 17.027074813842773,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.6255506607929515,
|
|
"grad_norm": 31.041427612304688,
|
|
"learning_rate": 1.8653568770724803e-07,
|
|
"logits/chosen": -1.810539960861206,
|
|
"logits/rejected": -1.1514090299606323,
|
|
"logps/chosen": -121.36643981933594,
|
|
"logps/ref_chosen": -100.81852722167969,
|
|
"logps/ref_rejected": -86.2867431640625,
|
|
"logps/rejected": -130.52114868164062,
|
|
"loss": 0.4407,
|
|
"margin_dpo/margin_mean": 23.68649673461914,
|
|
"margin_dpo/margin_std": 17.5459041595459,
|
|
"step": 426
|
|
},
|
|
{
|
|
"epoch": 0.6270190895741556,
|
|
"grad_norm": 20.86272430419922,
|
|
"learning_rate": 1.8529523872436977e-07,
|
|
"logits/chosen": -1.7860926389694214,
|
|
"logits/rejected": -1.2939798831939697,
|
|
"logps/chosen": -113.21662902832031,
|
|
"logps/ref_chosen": -90.550537109375,
|
|
"logps/ref_rejected": -102.77699279785156,
|
|
"logps/rejected": -146.71148681640625,
|
|
"loss": 0.5377,
|
|
"margin_dpo/margin_mean": 21.26839828491211,
|
|
"margin_dpo/margin_std": 21.174129486083984,
|
|
"step": 427
|
|
},
|
|
{
|
|
"epoch": 0.6284875183553598,
|
|
"grad_norm": 39.244056701660156,
|
|
"learning_rate": 1.8405649477212697e-07,
|
|
"logits/chosen": -0.9955465793609619,
|
|
"logits/rejected": -1.0620194673538208,
|
|
"logps/chosen": -125.4493408203125,
|
|
"logps/ref_chosen": -97.28589630126953,
|
|
"logps/ref_rejected": -120.8280029296875,
|
|
"logps/rejected": -171.86065673828125,
|
|
"loss": 0.8885,
|
|
"margin_dpo/margin_mean": 22.869205474853516,
|
|
"margin_dpo/margin_std": 22.71468734741211,
|
|
"step": 428
|
|
},
|
|
{
|
|
"epoch": 0.6299559471365639,
|
|
"grad_norm": 31.844663619995117,
|
|
"learning_rate": 1.828194884925749e-07,
|
|
"logits/chosen": -1.9276843070983887,
|
|
"logits/rejected": -1.3610649108886719,
|
|
"logps/chosen": -149.5,
|
|
"logps/ref_chosen": -119.6482162475586,
|
|
"logps/ref_rejected": -108.09115600585938,
|
|
"logps/rejected": -155.66888427734375,
|
|
"loss": 0.6668,
|
|
"margin_dpo/margin_mean": 17.7259521484375,
|
|
"margin_dpo/margin_std": 19.956687927246094,
|
|
"step": 429
|
|
},
|
|
{
|
|
"epoch": 0.631424375917768,
|
|
"grad_norm": 25.210390090942383,
|
|
"learning_rate": 1.8158425248197928e-07,
|
|
"logits/chosen": -0.9649932384490967,
|
|
"logits/rejected": -1.147021770477295,
|
|
"logps/chosen": -125.53485107421875,
|
|
"logps/ref_chosen": -97.25688934326172,
|
|
"logps/ref_rejected": -125.52217102050781,
|
|
"logps/rejected": -172.47962951660156,
|
|
"loss": 0.6066,
|
|
"margin_dpo/margin_mean": 18.67949104309082,
|
|
"margin_dpo/margin_std": 18.119897842407227,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.6328928046989721,
|
|
"grad_norm": 33.201725006103516,
|
|
"learning_rate": 1.8035081928995788e-07,
|
|
"logits/chosen": -2.278841972351074,
|
|
"logits/rejected": -2.0075526237487793,
|
|
"logps/chosen": -81.04603576660156,
|
|
"logps/ref_chosen": -57.92179870605469,
|
|
"logps/ref_rejected": -86.5896987915039,
|
|
"logps/rejected": -129.18716430664062,
|
|
"loss": 0.773,
|
|
"margin_dpo/margin_mean": 19.473228454589844,
|
|
"margin_dpo/margin_std": 14.711877822875977,
|
|
"step": 431
|
|
},
|
|
{
|
|
"epoch": 0.6343612334801763,
|
|
"grad_norm": 22.698999404907227,
|
|
"learning_rate": 1.791192214186223e-07,
|
|
"logits/chosen": -1.4666633605957031,
|
|
"logits/rejected": -0.8918869495391846,
|
|
"logps/chosen": -106.17467498779297,
|
|
"logps/ref_chosen": -83.56768798828125,
|
|
"logps/ref_rejected": -81.35081481933594,
|
|
"logps/rejected": -129.19357299804688,
|
|
"loss": 0.5115,
|
|
"margin_dpo/margin_mean": 25.235767364501953,
|
|
"margin_dpo/margin_std": 20.750383377075195,
|
|
"step": 432
|
|
},
|
|
{
|
|
"epoch": 0.6358296622613803,
|
|
"grad_norm": 32.29205322265625,
|
|
"learning_rate": 1.7788949132172193e-07,
|
|
"logits/chosen": -1.1999212503433228,
|
|
"logits/rejected": -0.5681695938110352,
|
|
"logps/chosen": -131.34445190429688,
|
|
"logps/ref_chosen": -101.5228042602539,
|
|
"logps/ref_rejected": -108.22576904296875,
|
|
"logps/rejected": -160.79852294921875,
|
|
"loss": 0.836,
|
|
"margin_dpo/margin_mean": 22.7510929107666,
|
|
"margin_dpo/margin_std": 26.510454177856445,
|
|
"step": 433
|
|
},
|
|
{
|
|
"epoch": 0.6372980910425844,
|
|
"grad_norm": 34.036136627197266,
|
|
"learning_rate": 1.7666166140378853e-07,
|
|
"logits/chosen": -2.054288387298584,
|
|
"logits/rejected": -1.399308204650879,
|
|
"logps/chosen": -147.2008056640625,
|
|
"logps/ref_chosen": -124.12574768066406,
|
|
"logps/ref_rejected": -111.97697448730469,
|
|
"logps/rejected": -154.53948974609375,
|
|
"loss": 0.7302,
|
|
"margin_dpo/margin_mean": 19.487464904785156,
|
|
"margin_dpo/margin_std": 28.65212631225586,
|
|
"step": 434
|
|
},
|
|
{
|
|
"epoch": 0.6387665198237885,
|
|
"grad_norm": 24.667842864990234,
|
|
"learning_rate": 1.7543576401928218e-07,
|
|
"logits/chosen": -1.8135147094726562,
|
|
"logits/rejected": -1.378133773803711,
|
|
"logps/chosen": -97.1856460571289,
|
|
"logps/ref_chosen": -75.44041442871094,
|
|
"logps/ref_rejected": -105.867431640625,
|
|
"logps/rejected": -155.25845336914062,
|
|
"loss": 0.5765,
|
|
"margin_dpo/margin_mean": 27.645782470703125,
|
|
"margin_dpo/margin_std": 22.643753051757812,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.6402349486049926,
|
|
"grad_norm": 21.6951961517334,
|
|
"learning_rate": 1.742118314717391e-07,
|
|
"logits/chosen": -1.3429805040359497,
|
|
"logits/rejected": -0.3639574348926544,
|
|
"logps/chosen": -118.84077453613281,
|
|
"logps/ref_chosen": -98.83950805664062,
|
|
"logps/ref_rejected": -65.53834533691406,
|
|
"logps/rejected": -109.84593200683594,
|
|
"loss": 0.4742,
|
|
"margin_dpo/margin_mean": 24.306320190429688,
|
|
"margin_dpo/margin_std": 22.910158157348633,
|
|
"step": 436
|
|
},
|
|
{
|
|
"epoch": 0.6417033773861968,
|
|
"grad_norm": 21.48826789855957,
|
|
"learning_rate": 1.7298989601292036e-07,
|
|
"logits/chosen": -2.178161144256592,
|
|
"logits/rejected": -1.9608675241470337,
|
|
"logps/chosen": -110.85467529296875,
|
|
"logps/ref_chosen": -86.16108703613281,
|
|
"logps/ref_rejected": -85.81136322021484,
|
|
"logps/rejected": -124.95901489257812,
|
|
"loss": 0.6716,
|
|
"margin_dpo/margin_mean": 14.454059600830078,
|
|
"margin_dpo/margin_std": 16.443344116210938,
|
|
"step": 437
|
|
},
|
|
{
|
|
"epoch": 0.6431718061674009,
|
|
"grad_norm": 27.280611038208008,
|
|
"learning_rate": 1.7176998984196144e-07,
|
|
"logits/chosen": -1.978929042816162,
|
|
"logits/rejected": -1.4313712120056152,
|
|
"logps/chosen": -97.4400634765625,
|
|
"logps/ref_chosen": -72.21536254882812,
|
|
"logps/ref_rejected": -86.81999206542969,
|
|
"logps/rejected": -137.0142364501953,
|
|
"loss": 0.5761,
|
|
"margin_dpo/margin_mean": 24.969541549682617,
|
|
"margin_dpo/margin_std": 16.327075958251953,
|
|
"step": 438
|
|
},
|
|
{
|
|
"epoch": 0.644640234948605,
|
|
"grad_norm": 29.87968635559082,
|
|
"learning_rate": 1.7055214510452458e-07,
|
|
"logits/chosen": -1.223572015762329,
|
|
"logits/rejected": -0.7343321442604065,
|
|
"logps/chosen": -119.00277709960938,
|
|
"logps/ref_chosen": -93.64883422851562,
|
|
"logps/ref_rejected": -70.13905334472656,
|
|
"logps/rejected": -112.55131530761719,
|
|
"loss": 0.7894,
|
|
"margin_dpo/margin_mean": 17.058311462402344,
|
|
"margin_dpo/margin_std": 18.533573150634766,
|
|
"step": 439
|
|
},
|
|
{
|
|
"epoch": 0.6461086637298091,
|
|
"grad_norm": 29.423545837402344,
|
|
"learning_rate": 1.6933639389195134e-07,
|
|
"logits/chosen": -1.738755464553833,
|
|
"logits/rejected": -0.7158520817756653,
|
|
"logps/chosen": -166.27590942382812,
|
|
"logps/ref_chosen": -143.11007690429688,
|
|
"logps/ref_rejected": -86.04011535644531,
|
|
"logps/rejected": -133.2398223876953,
|
|
"loss": 0.7767,
|
|
"margin_dpo/margin_mean": 24.033851623535156,
|
|
"margin_dpo/margin_std": 21.511178970336914,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.6475770925110133,
|
|
"grad_norm": 20.80038070678711,
|
|
"learning_rate": 1.681227682404166e-07,
|
|
"logits/chosen": -1.494542121887207,
|
|
"logits/rejected": -0.6456449031829834,
|
|
"logps/chosen": -132.18185424804688,
|
|
"logps/ref_chosen": -109.37970733642578,
|
|
"logps/ref_rejected": -72.19073486328125,
|
|
"logps/rejected": -119.72990417480469,
|
|
"loss": 0.4617,
|
|
"margin_dpo/margin_mean": 24.737010955810547,
|
|
"margin_dpo/margin_std": 11.764477729797363,
|
|
"step": 441
|
|
},
|
|
{
|
|
"epoch": 0.6490455212922174,
|
|
"grad_norm": 19.458789825439453,
|
|
"learning_rate": 1.669113001300851e-07,
|
|
"logits/chosen": -0.5713354349136353,
|
|
"logits/rejected": -0.13276606798171997,
|
|
"logps/chosen": -124.80497741699219,
|
|
"logps/ref_chosen": -98.47917175292969,
|
|
"logps/ref_rejected": -89.41722106933594,
|
|
"logps/rejected": -142.98641967773438,
|
|
"loss": 0.4504,
|
|
"margin_dpo/margin_mean": 27.243377685546875,
|
|
"margin_dpo/margin_std": 16.030248641967773,
|
|
"step": 442
|
|
},
|
|
{
|
|
"epoch": 0.6505139500734214,
|
|
"grad_norm": 31.2533016204834,
|
|
"learning_rate": 1.6570202148426815e-07,
|
|
"logits/chosen": -1.5800436735153198,
|
|
"logits/rejected": -1.1696522235870361,
|
|
"logps/chosen": -128.31686401367188,
|
|
"logps/ref_chosen": -98.59529876708984,
|
|
"logps/ref_rejected": -74.47461700439453,
|
|
"logps/rejected": -118.65472412109375,
|
|
"loss": 0.6083,
|
|
"margin_dpo/margin_mean": 14.458542823791504,
|
|
"margin_dpo/margin_std": 14.307601928710938,
|
|
"step": 443
|
|
},
|
|
{
|
|
"epoch": 0.6519823788546255,
|
|
"grad_norm": 28.26342010498047,
|
|
"learning_rate": 1.6449496416858282e-07,
|
|
"logits/chosen": -2.4062390327453613,
|
|
"logits/rejected": -1.701258897781372,
|
|
"logps/chosen": -142.62533569335938,
|
|
"logps/ref_chosen": -111.86331939697266,
|
|
"logps/ref_rejected": -96.1914291381836,
|
|
"logps/rejected": -144.64889526367188,
|
|
"loss": 0.6585,
|
|
"margin_dpo/margin_mean": 17.695472717285156,
|
|
"margin_dpo/margin_std": 21.756973266601562,
|
|
"step": 444
|
|
},
|
|
{
|
|
"epoch": 0.6534508076358296,
|
|
"grad_norm": 23.41632843017578,
|
|
"learning_rate": 1.6329015999011182e-07,
|
|
"logits/chosen": -1.8755643367767334,
|
|
"logits/rejected": -1.7402284145355225,
|
|
"logps/chosen": -117.87601470947266,
|
|
"logps/ref_chosen": -89.98236083984375,
|
|
"logps/ref_rejected": -78.82273864746094,
|
|
"logps/rejected": -122.12147521972656,
|
|
"loss": 0.5487,
|
|
"margin_dpo/margin_mean": 15.405086517333984,
|
|
"margin_dpo/margin_std": 11.568341255187988,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.6549192364170338,
|
|
"grad_norm": 30.221832275390625,
|
|
"learning_rate": 1.6208764069656578e-07,
|
|
"logits/chosen": -0.7040866613388062,
|
|
"logits/rejected": -0.5425094366073608,
|
|
"logps/chosen": -107.57206726074219,
|
|
"logps/ref_chosen": -83.11263275146484,
|
|
"logps/ref_rejected": -128.77877807617188,
|
|
"logps/rejected": -175.65806579589844,
|
|
"loss": 0.6084,
|
|
"margin_dpo/margin_mean": 22.419845581054688,
|
|
"margin_dpo/margin_std": 18.770353317260742,
|
|
"step": 446
|
|
},
|
|
{
|
|
"epoch": 0.6563876651982379,
|
|
"grad_norm": 26.28083038330078,
|
|
"learning_rate": 1.608874379754465e-07,
|
|
"logits/chosen": -1.621050238609314,
|
|
"logits/rejected": -1.6599102020263672,
|
|
"logps/chosen": -103.27529907226562,
|
|
"logps/ref_chosen": -78.08226013183594,
|
|
"logps/ref_rejected": -104.58940887451172,
|
|
"logps/rejected": -145.8384552001953,
|
|
"loss": 0.6788,
|
|
"margin_dpo/margin_mean": 16.056018829345703,
|
|
"margin_dpo/margin_std": 15.311338424682617,
|
|
"step": 447
|
|
},
|
|
{
|
|
"epoch": 0.657856093979442,
|
|
"grad_norm": 31.752517700195312,
|
|
"learning_rate": 1.5968958345321177e-07,
|
|
"logits/chosen": -1.1484473943710327,
|
|
"logits/rejected": -1.2155144214630127,
|
|
"logps/chosen": -130.1195068359375,
|
|
"logps/ref_chosen": -102.85002136230469,
|
|
"logps/ref_rejected": -134.32081604003906,
|
|
"logps/rejected": -171.00125122070312,
|
|
"loss": 0.6527,
|
|
"margin_dpo/margin_mean": 9.410964965820312,
|
|
"margin_dpo/margin_std": 19.52904510498047,
|
|
"step": 448
|
|
},
|
|
{
|
|
"epoch": 0.6593245227606461,
|
|
"grad_norm": 26.76458740234375,
|
|
"learning_rate": 1.584941086944423e-07,
|
|
"logits/chosen": -1.1806657314300537,
|
|
"logits/rejected": -0.7023051381111145,
|
|
"logps/chosen": -114.85504913330078,
|
|
"logps/ref_chosen": -87.11476135253906,
|
|
"logps/ref_rejected": -80.93266296386719,
|
|
"logps/rejected": -131.17945861816406,
|
|
"loss": 0.6548,
|
|
"margin_dpo/margin_mean": 22.50650978088379,
|
|
"margin_dpo/margin_std": 17.623516082763672,
|
|
"step": 449
|
|
},
|
|
{
|
|
"epoch": 0.6607929515418502,
|
|
"grad_norm": 24.460163116455078,
|
|
"learning_rate": 1.573010452010098e-07,
|
|
"logits/chosen": -2.825883626937866,
|
|
"logits/rejected": -2.340266227722168,
|
|
"logps/chosen": -136.053466796875,
|
|
"logps/ref_chosen": -114.85942077636719,
|
|
"logps/ref_rejected": -149.81314086914062,
|
|
"logps/rejected": -204.9149627685547,
|
|
"loss": 0.4745,
|
|
"margin_dpo/margin_mean": 33.907779693603516,
|
|
"margin_dpo/margin_std": 16.30735206604004,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.6622613803230544,
|
|
"grad_norm": 28.61817741394043,
|
|
"learning_rate": 1.5611042441124687e-07,
|
|
"logits/chosen": -1.5507322549819946,
|
|
"logits/rejected": -1.089903473854065,
|
|
"logps/chosen": -136.8935546875,
|
|
"logps/ref_chosen": -114.61380767822266,
|
|
"logps/ref_rejected": -105.7976303100586,
|
|
"logps/rejected": -156.86073303222656,
|
|
"loss": 0.7374,
|
|
"margin_dpo/margin_mean": 28.783344268798828,
|
|
"margin_dpo/margin_std": 21.23330307006836,
|
|
"step": 451
|
|
},
|
|
{
|
|
"epoch": 0.6637298091042585,
|
|
"grad_norm": 22.830219268798828,
|
|
"learning_rate": 1.549222776991186e-07,
|
|
"logits/chosen": -2.039238929748535,
|
|
"logits/rejected": -1.8188178539276123,
|
|
"logps/chosen": -104.45685577392578,
|
|
"logps/ref_chosen": -82.06895446777344,
|
|
"logps/ref_rejected": -100.78804016113281,
|
|
"logps/rejected": -140.10707092285156,
|
|
"loss": 0.5899,
|
|
"margin_dpo/margin_mean": 16.93113899230957,
|
|
"margin_dpo/margin_std": 14.689157485961914,
|
|
"step": 452
|
|
},
|
|
{
|
|
"epoch": 0.6651982378854625,
|
|
"grad_norm": 18.594497680664062,
|
|
"learning_rate": 1.5373663637339584e-07,
|
|
"logits/chosen": -1.2692456245422363,
|
|
"logits/rejected": -1.0498113632202148,
|
|
"logps/chosen": -106.26356506347656,
|
|
"logps/ref_chosen": -78.82481384277344,
|
|
"logps/ref_rejected": -72.43688201904297,
|
|
"logps/rejected": -120.48109436035156,
|
|
"loss": 0.4381,
|
|
"margin_dpo/margin_mean": 20.605459213256836,
|
|
"margin_dpo/margin_std": 11.411565780639648,
|
|
"step": 453
|
|
},
|
|
{
|
|
"epoch": 0.6666666666666666,
|
|
"grad_norm": 34.69410705566406,
|
|
"learning_rate": 1.5255353167683017e-07,
|
|
"logits/chosen": -1.726050615310669,
|
|
"logits/rejected": -1.4121358394622803,
|
|
"logps/chosen": -133.55905151367188,
|
|
"logps/ref_chosen": -101.30783081054688,
|
|
"logps/ref_rejected": -115.9158935546875,
|
|
"logps/rejected": -164.92752075195312,
|
|
"loss": 0.7431,
|
|
"margin_dpo/margin_mean": 16.760404586791992,
|
|
"margin_dpo/margin_std": 17.026487350463867,
|
|
"step": 454
|
|
},
|
|
{
|
|
"epoch": 0.6681350954478708,
|
|
"grad_norm": 27.073896408081055,
|
|
"learning_rate": 1.5137299478533064e-07,
|
|
"logits/chosen": -2.262284994125366,
|
|
"logits/rejected": -2.4119272232055664,
|
|
"logps/chosen": -107.57990264892578,
|
|
"logps/ref_chosen": -80.74185943603516,
|
|
"logps/ref_rejected": -165.87608337402344,
|
|
"logps/rejected": -218.08432006835938,
|
|
"loss": 0.5704,
|
|
"margin_dpo/margin_mean": 25.370182037353516,
|
|
"margin_dpo/margin_std": 24.86716651916504,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.6696035242290749,
|
|
"grad_norm": 24.588420867919922,
|
|
"learning_rate": 1.5019505680714232e-07,
|
|
"logits/chosen": -1.48960542678833,
|
|
"logits/rejected": -1.4934964179992676,
|
|
"logps/chosen": -104.88959503173828,
|
|
"logps/ref_chosen": -83.01458740234375,
|
|
"logps/ref_rejected": -123.61626434326172,
|
|
"logps/rejected": -166.04608154296875,
|
|
"loss": 0.5251,
|
|
"margin_dpo/margin_mean": 20.554821014404297,
|
|
"margin_dpo/margin_std": 16.57241439819336,
|
|
"step": 456
|
|
},
|
|
{
|
|
"epoch": 0.671071953010279,
|
|
"grad_norm": 19.552265167236328,
|
|
"learning_rate": 1.4901974878202627e-07,
|
|
"logits/chosen": -0.9713010787963867,
|
|
"logits/rejected": -0.8292329907417297,
|
|
"logps/chosen": -110.44456481933594,
|
|
"logps/ref_chosen": -89.19579315185547,
|
|
"logps/ref_rejected": -100.71463012695312,
|
|
"logps/rejected": -149.5691680908203,
|
|
"loss": 0.4447,
|
|
"margin_dpo/margin_mean": 27.605770111083984,
|
|
"margin_dpo/margin_std": 16.504793167114258,
|
|
"step": 457
|
|
},
|
|
{
|
|
"epoch": 0.6725403817914831,
|
|
"grad_norm": 33.34504318237305,
|
|
"learning_rate": 1.4784710168044212e-07,
|
|
"logits/chosen": -1.010892629623413,
|
|
"logits/rejected": -0.6925811767578125,
|
|
"logps/chosen": -123.5889663696289,
|
|
"logps/ref_chosen": -93.04997253417969,
|
|
"logps/ref_rejected": -89.65494537353516,
|
|
"logps/rejected": -135.5701904296875,
|
|
"loss": 0.703,
|
|
"margin_dpo/margin_mean": 15.376252174377441,
|
|
"margin_dpo/margin_std": 23.105487823486328,
|
|
"step": 458
|
|
},
|
|
{
|
|
"epoch": 0.6740088105726872,
|
|
"grad_norm": 25.338581085205078,
|
|
"learning_rate": 1.466771464027316e-07,
|
|
"logits/chosen": -2.277261734008789,
|
|
"logits/rejected": -1.876206636428833,
|
|
"logps/chosen": -102.30770874023438,
|
|
"logps/ref_chosen": -77.70928192138672,
|
|
"logps/ref_rejected": -94.44648742675781,
|
|
"logps/rejected": -137.26004028320312,
|
|
"loss": 0.5821,
|
|
"margin_dpo/margin_mean": 18.215126037597656,
|
|
"margin_dpo/margin_std": 16.228425979614258,
|
|
"step": 459
|
|
},
|
|
{
|
|
"epoch": 0.6754772393538914,
|
|
"grad_norm": 31.598941802978516,
|
|
"learning_rate": 1.4550991377830423e-07,
|
|
"logits/chosen": -1.5439469814300537,
|
|
"logits/rejected": -1.373673439025879,
|
|
"logps/chosen": -120.14390563964844,
|
|
"logps/ref_chosen": -86.8564453125,
|
|
"logps/ref_rejected": -115.38356018066406,
|
|
"logps/rejected": -166.2212371826172,
|
|
"loss": 0.6881,
|
|
"margin_dpo/margin_mean": 17.550220489501953,
|
|
"margin_dpo/margin_std": 15.671613693237305,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6769456681350955,
|
|
"grad_norm": 29.087486267089844,
|
|
"learning_rate": 1.4434543456482518e-07,
|
|
"logits/chosen": -1.0304532051086426,
|
|
"logits/rejected": -0.7353922128677368,
|
|
"logps/chosen": -142.9497833251953,
|
|
"logps/ref_chosen": -110.2314224243164,
|
|
"logps/ref_rejected": -112.15139770507812,
|
|
"logps/rejected": -160.65652465820312,
|
|
"loss": 0.7707,
|
|
"margin_dpo/margin_mean": 15.786758422851562,
|
|
"margin_dpo/margin_std": 16.66317367553711,
|
|
"step": 461
|
|
},
|
|
{
|
|
"epoch": 0.6784140969162996,
|
|
"grad_norm": 25.08791160583496,
|
|
"learning_rate": 1.4318373944740484e-07,
|
|
"logits/chosen": -2.122267723083496,
|
|
"logits/rejected": -1.8464157581329346,
|
|
"logps/chosen": -106.20613098144531,
|
|
"logps/ref_chosen": -79.80224609375,
|
|
"logps/ref_rejected": -77.2978515625,
|
|
"logps/rejected": -121.96664428710938,
|
|
"loss": 0.5766,
|
|
"margin_dpo/margin_mean": 18.264904022216797,
|
|
"margin_dpo/margin_std": 16.625919342041016,
|
|
"step": 462
|
|
},
|
|
{
|
|
"epoch": 0.6798825256975036,
|
|
"grad_norm": 25.841707229614258,
|
|
"learning_rate": 1.4202485903778976e-07,
|
|
"logits/chosen": -2.0914337635040283,
|
|
"logits/rejected": -1.7170106172561646,
|
|
"logps/chosen": -93.4617919921875,
|
|
"logps/ref_chosen": -71.40604400634766,
|
|
"logps/ref_rejected": -76.95889282226562,
|
|
"logps/rejected": -121.64329528808594,
|
|
"loss": 0.5968,
|
|
"margin_dpo/margin_mean": 22.628652572631836,
|
|
"margin_dpo/margin_std": 19.861661911010742,
|
|
"step": 463
|
|
},
|
|
{
|
|
"epoch": 0.6813509544787077,
|
|
"grad_norm": 29.462730407714844,
|
|
"learning_rate": 1.4086882387355658e-07,
|
|
"logits/chosen": -1.1162123680114746,
|
|
"logits/rejected": -1.4878854751586914,
|
|
"logps/chosen": -125.587158203125,
|
|
"logps/ref_chosen": -92.85479736328125,
|
|
"logps/ref_rejected": -121.56739807128906,
|
|
"logps/rejected": -170.02841186523438,
|
|
"loss": 0.6442,
|
|
"margin_dpo/margin_mean": 15.728658676147461,
|
|
"margin_dpo/margin_std": 12.379182815551758,
|
|
"step": 464
|
|
},
|
|
{
|
|
"epoch": 0.6828193832599119,
|
|
"grad_norm": 24.474634170532227,
|
|
"learning_rate": 1.3971566441730714e-07,
|
|
"logits/chosen": -1.3897593021392822,
|
|
"logits/rejected": -1.0794075727462769,
|
|
"logps/chosen": -112.57616424560547,
|
|
"logps/ref_chosen": -86.05903625488281,
|
|
"logps/ref_rejected": -94.044677734375,
|
|
"logps/rejected": -143.59713745117188,
|
|
"loss": 0.4936,
|
|
"margin_dpo/margin_mean": 23.03533172607422,
|
|
"margin_dpo/margin_std": 21.698204040527344,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.684287812041116,
|
|
"grad_norm": 25.171062469482422,
|
|
"learning_rate": 1.3856541105586545e-07,
|
|
"logits/chosen": -1.600342035293579,
|
|
"logits/rejected": -0.7873063087463379,
|
|
"logps/chosen": -118.66007232666016,
|
|
"logps/ref_chosen": -93.65107727050781,
|
|
"logps/ref_rejected": -74.44425964355469,
|
|
"logps/rejected": -123.89339447021484,
|
|
"loss": 0.5221,
|
|
"margin_dpo/margin_mean": 24.440139770507812,
|
|
"margin_dpo/margin_std": 19.990158081054688,
|
|
"step": 466
|
|
},
|
|
{
|
|
"epoch": 0.6857562408223201,
|
|
"grad_norm": 24.633567810058594,
|
|
"learning_rate": 1.3741809409947729e-07,
|
|
"logits/chosen": -1.2185485363006592,
|
|
"logits/rejected": -0.803144097328186,
|
|
"logps/chosen": -127.40275573730469,
|
|
"logps/ref_chosen": -100.60771179199219,
|
|
"logps/ref_rejected": -107.02332305908203,
|
|
"logps/rejected": -163.00355529785156,
|
|
"loss": 0.5664,
|
|
"margin_dpo/margin_mean": 29.18516731262207,
|
|
"margin_dpo/margin_std": 19.719329833984375,
|
|
"step": 467
|
|
},
|
|
{
|
|
"epoch": 0.6872246696035242,
|
|
"grad_norm": 31.386714935302734,
|
|
"learning_rate": 1.362737437810114e-07,
|
|
"logits/chosen": -1.2606921195983887,
|
|
"logits/rejected": -1.089573621749878,
|
|
"logps/chosen": -118.3271255493164,
|
|
"logps/ref_chosen": -96.5201187133789,
|
|
"logps/ref_rejected": -121.30470275878906,
|
|
"logps/rejected": -163.67080688476562,
|
|
"loss": 0.7409,
|
|
"margin_dpo/margin_mean": 20.559078216552734,
|
|
"margin_dpo/margin_std": 20.951251983642578,
|
|
"step": 468
|
|
},
|
|
{
|
|
"epoch": 0.6886930983847284,
|
|
"grad_norm": 27.293046951293945,
|
|
"learning_rate": 1.351323902551631e-07,
|
|
"logits/chosen": -1.0601452589035034,
|
|
"logits/rejected": -1.0791672468185425,
|
|
"logps/chosen": -88.2209701538086,
|
|
"logps/ref_chosen": -57.2293586730957,
|
|
"logps/ref_rejected": -86.61882019042969,
|
|
"logps/rejected": -139.1120147705078,
|
|
"loss": 0.5459,
|
|
"margin_dpo/margin_mean": 21.50157928466797,
|
|
"margin_dpo/margin_std": 19.085735321044922,
|
|
"step": 469
|
|
},
|
|
{
|
|
"epoch": 0.6901615271659325,
|
|
"grad_norm": 22.850967407226562,
|
|
"learning_rate": 1.339940635976592e-07,
|
|
"logits/chosen": -0.9972758889198303,
|
|
"logits/rejected": -0.3706229329109192,
|
|
"logps/chosen": -112.50442504882812,
|
|
"logps/ref_chosen": -87.85397338867188,
|
|
"logps/ref_rejected": -73.04238891601562,
|
|
"logps/rejected": -117.85956573486328,
|
|
"loss": 0.5101,
|
|
"margin_dpo/margin_mean": 20.166732788085938,
|
|
"margin_dpo/margin_std": 13.451013565063477,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.6916299559471366,
|
|
"grad_norm": 20.220306396484375,
|
|
"learning_rate": 1.3285879380446563e-07,
|
|
"logits/chosen": -1.0001336336135864,
|
|
"logits/rejected": -1.0569403171539307,
|
|
"logps/chosen": -130.043701171875,
|
|
"logps/ref_chosen": -98.69927978515625,
|
|
"logps/ref_rejected": -111.39106750488281,
|
|
"logps/rejected": -165.02394104003906,
|
|
"loss": 0.4371,
|
|
"margin_dpo/margin_mean": 22.2884521484375,
|
|
"margin_dpo/margin_std": 12.328178405761719,
|
|
"step": 471
|
|
},
|
|
{
|
|
"epoch": 0.6930983847283406,
|
|
"grad_norm": 27.954729080200195,
|
|
"learning_rate": 1.317266107909975e-07,
|
|
"logits/chosen": -1.4778022766113281,
|
|
"logits/rejected": -0.5768337249755859,
|
|
"logps/chosen": -180.3785400390625,
|
|
"logps/ref_chosen": -154.4603271484375,
|
|
"logps/ref_rejected": -115.84030151367188,
|
|
"logps/rejected": -163.5217742919922,
|
|
"loss": 0.4734,
|
|
"margin_dpo/margin_mean": 21.763275146484375,
|
|
"margin_dpo/margin_std": 16.120174407958984,
|
|
"step": 472
|
|
},
|
|
{
|
|
"epoch": 0.6945668135095447,
|
|
"grad_norm": 27.106035232543945,
|
|
"learning_rate": 1.3059754439133002e-07,
|
|
"logits/chosen": -1.175186276435852,
|
|
"logits/rejected": -0.891939640045166,
|
|
"logps/chosen": -156.43284606933594,
|
|
"logps/ref_chosen": -124.94532775878906,
|
|
"logps/ref_rejected": -98.58340454101562,
|
|
"logps/rejected": -144.3042449951172,
|
|
"loss": 0.6772,
|
|
"margin_dpo/margin_mean": 14.2333345413208,
|
|
"margin_dpo/margin_std": 18.283645629882812,
|
|
"step": 473
|
|
},
|
|
{
|
|
"epoch": 0.6960352422907489,
|
|
"grad_norm": 41.271942138671875,
|
|
"learning_rate": 1.2947162435741277e-07,
|
|
"logits/chosen": -1.574142575263977,
|
|
"logits/rejected": -1.827897071838379,
|
|
"logps/chosen": -98.8539810180664,
|
|
"logps/ref_chosen": -68.98202514648438,
|
|
"logps/ref_rejected": -131.7036590576172,
|
|
"logps/rejected": -183.00289916992188,
|
|
"loss": 0.9112,
|
|
"margin_dpo/margin_mean": 21.427288055419922,
|
|
"margin_dpo/margin_std": 19.77497673034668,
|
|
"step": 474
|
|
},
|
|
{
|
|
"epoch": 0.697503671071953,
|
|
"grad_norm": 27.248205184936523,
|
|
"learning_rate": 1.2834888035828596e-07,
|
|
"logits/chosen": -2.1355481147766113,
|
|
"logits/rejected": -2.075871229171753,
|
|
"logps/chosen": -99.45306396484375,
|
|
"logps/ref_chosen": -73.41651916503906,
|
|
"logps/ref_rejected": -94.41173553466797,
|
|
"logps/rejected": -138.05735778808594,
|
|
"loss": 0.6614,
|
|
"margin_dpo/margin_mean": 17.609081268310547,
|
|
"margin_dpo/margin_std": 17.966053009033203,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.6989720998531571,
|
|
"grad_norm": 22.6929874420166,
|
|
"learning_rate": 1.2722934197929802e-07,
|
|
"logits/chosen": -2.7777254581451416,
|
|
"logits/rejected": -2.2750704288482666,
|
|
"logps/chosen": -102.75193786621094,
|
|
"logps/ref_chosen": -74.20452117919922,
|
|
"logps/ref_rejected": -92.60858917236328,
|
|
"logps/rejected": -143.93138122558594,
|
|
"loss": 0.666,
|
|
"margin_dpo/margin_mean": 22.775375366210938,
|
|
"margin_dpo/margin_std": 21.31452178955078,
|
|
"step": 476
|
|
},
|
|
{
|
|
"epoch": 0.7004405286343612,
|
|
"grad_norm": 24.573259353637695,
|
|
"learning_rate": 1.2611303872132631e-07,
|
|
"logits/chosen": -3.4070348739624023,
|
|
"logits/rejected": -2.3290576934814453,
|
|
"logps/chosen": -130.92428588867188,
|
|
"logps/ref_chosen": -102.4937515258789,
|
|
"logps/ref_rejected": -68.5474853515625,
|
|
"logps/rejected": -120.82994079589844,
|
|
"loss": 0.5533,
|
|
"margin_dpo/margin_mean": 23.8519287109375,
|
|
"margin_dpo/margin_std": 17.174068450927734,
|
|
"step": 477
|
|
},
|
|
{
|
|
"epoch": 0.7019089574155654,
|
|
"grad_norm": 22.8625545501709,
|
|
"learning_rate": 1.2500000000000005e-07,
|
|
"logits/chosen": -1.4574041366577148,
|
|
"logits/rejected": -1.3147239685058594,
|
|
"logps/chosen": -93.35462951660156,
|
|
"logps/ref_chosen": -64.2506103515625,
|
|
"logps/ref_rejected": -69.66351318359375,
|
|
"logps/rejected": -111.02110290527344,
|
|
"loss": 0.5916,
|
|
"margin_dpo/margin_mean": 12.25357437133789,
|
|
"margin_dpo/margin_std": 14.410320281982422,
|
|
"step": 478
|
|
},
|
|
{
|
|
"epoch": 0.7033773861967695,
|
|
"grad_norm": 27.097768783569336,
|
|
"learning_rate": 1.2389025514492456e-07,
|
|
"logits/chosen": -1.1350903511047363,
|
|
"logits/rejected": -1.2377521991729736,
|
|
"logps/chosen": -101.08732604980469,
|
|
"logps/ref_chosen": -73.15229034423828,
|
|
"logps/ref_rejected": -95.76731872558594,
|
|
"logps/rejected": -144.12460327148438,
|
|
"loss": 0.6646,
|
|
"margin_dpo/margin_mean": 20.42224884033203,
|
|
"margin_dpo/margin_std": 16.232749938964844,
|
|
"step": 479
|
|
},
|
|
{
|
|
"epoch": 0.7048458149779736,
|
|
"grad_norm": 27.788637161254883,
|
|
"learning_rate": 1.227838333989088e-07,
|
|
"logits/chosen": -0.924386739730835,
|
|
"logits/rejected": -0.4081732928752899,
|
|
"logps/chosen": -117.9144515991211,
|
|
"logps/ref_chosen": -90.6583251953125,
|
|
"logps/ref_rejected": -78.41166687011719,
|
|
"logps/rejected": -129.16677856445312,
|
|
"loss": 0.5901,
|
|
"margin_dpo/margin_mean": 23.498985290527344,
|
|
"margin_dpo/margin_std": 15.09835433959961,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.7063142437591777,
|
|
"grad_norm": 25.31952476501465,
|
|
"learning_rate": 1.2168076391719489e-07,
|
|
"logits/chosen": -1.2236384153366089,
|
|
"logits/rejected": -0.7734112739562988,
|
|
"logps/chosen": -112.26165771484375,
|
|
"logps/ref_chosen": -77.6314468383789,
|
|
"logps/ref_rejected": -74.5798568725586,
|
|
"logps/rejected": -123.01365661621094,
|
|
"loss": 0.595,
|
|
"margin_dpo/margin_mean": 13.803601264953613,
|
|
"margin_dpo/margin_std": 14.735580444335938,
|
|
"step": 481
|
|
},
|
|
{
|
|
"epoch": 0.7077826725403817,
|
|
"grad_norm": 30.939720153808594,
|
|
"learning_rate": 1.2058107576668938e-07,
|
|
"logits/chosen": -1.545617938041687,
|
|
"logits/rejected": -1.4013819694519043,
|
|
"logps/chosen": -132.59231567382812,
|
|
"logps/ref_chosen": -99.90892791748047,
|
|
"logps/ref_rejected": -100.73772430419922,
|
|
"logps/rejected": -150.3515625,
|
|
"loss": 0.705,
|
|
"margin_dpo/margin_mean": 16.930465698242188,
|
|
"margin_dpo/margin_std": 15.111373901367188,
|
|
"step": 482
|
|
},
|
|
{
|
|
"epoch": 0.7092511013215859,
|
|
"grad_norm": 23.95891571044922,
|
|
"learning_rate": 1.194847979251979e-07,
|
|
"logits/chosen": -2.434985637664795,
|
|
"logits/rejected": -1.8117592334747314,
|
|
"logps/chosen": -138.04351806640625,
|
|
"logps/ref_chosen": -114.92469787597656,
|
|
"logps/ref_rejected": -106.71426391601562,
|
|
"logps/rejected": -157.34286499023438,
|
|
"loss": 0.523,
|
|
"margin_dpo/margin_mean": 27.50977325439453,
|
|
"margin_dpo/margin_std": 21.83698272705078,
|
|
"step": 483
|
|
},
|
|
{
|
|
"epoch": 0.71071953010279,
|
|
"grad_norm": 20.260234832763672,
|
|
"learning_rate": 1.1839195928066101e-07,
|
|
"logits/chosen": -2.028228759765625,
|
|
"logits/rejected": -1.2856285572052002,
|
|
"logps/chosen": -112.14205932617188,
|
|
"logps/ref_chosen": -80.64376831054688,
|
|
"logps/ref_rejected": -62.502586364746094,
|
|
"logps/rejected": -114.17668914794922,
|
|
"loss": 0.5374,
|
|
"margin_dpo/margin_mean": 20.175811767578125,
|
|
"margin_dpo/margin_std": 17.458547592163086,
|
|
"step": 484
|
|
},
|
|
{
|
|
"epoch": 0.7121879588839941,
|
|
"grad_norm": 21.97205924987793,
|
|
"learning_rate": 1.1730258863039347e-07,
|
|
"logits/chosen": -1.9410022497177124,
|
|
"logits/rejected": -1.6192748546600342,
|
|
"logps/chosen": -116.61924743652344,
|
|
"logps/ref_chosen": -92.76929473876953,
|
|
"logps/ref_rejected": -111.85530090332031,
|
|
"logps/rejected": -165.03895568847656,
|
|
"loss": 0.3976,
|
|
"margin_dpo/margin_mean": 29.333690643310547,
|
|
"margin_dpo/margin_std": 20.77867889404297,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.7136563876651982,
|
|
"grad_norm": 28.425851821899414,
|
|
"learning_rate": 1.1621671468032493e-07,
|
|
"logits/chosen": -1.306410312652588,
|
|
"logits/rejected": -0.296735554933548,
|
|
"logps/chosen": -123.26519012451172,
|
|
"logps/ref_chosen": -100.41597747802734,
|
|
"logps/ref_rejected": -100.8256607055664,
|
|
"logps/rejected": -155.62200927734375,
|
|
"loss": 0.5486,
|
|
"margin_dpo/margin_mean": 31.947128295898438,
|
|
"margin_dpo/margin_std": 18.083139419555664,
|
|
"step": 486
|
|
},
|
|
{
|
|
"epoch": 0.7151248164464024,
|
|
"grad_norm": 23.94493293762207,
|
|
"learning_rate": 1.1513436604424378e-07,
|
|
"logits/chosen": -2.2987453937530518,
|
|
"logits/rejected": -1.9822089672088623,
|
|
"logps/chosen": -131.52816772460938,
|
|
"logps/ref_chosen": -100.6910400390625,
|
|
"logps/ref_rejected": -107.24468994140625,
|
|
"logps/rejected": -166.06118774414062,
|
|
"loss": 0.4507,
|
|
"margin_dpo/margin_mean": 27.979373931884766,
|
|
"margin_dpo/margin_std": 18.914302825927734,
|
|
"step": 487
|
|
},
|
|
{
|
|
"epoch": 0.7165932452276065,
|
|
"grad_norm": 27.051164627075195,
|
|
"learning_rate": 1.1405557124304335e-07,
|
|
"logits/chosen": -1.088942527770996,
|
|
"logits/rejected": -0.8271230459213257,
|
|
"logps/chosen": -131.36257934570312,
|
|
"logps/ref_chosen": -107.36306762695312,
|
|
"logps/ref_rejected": -105.19340515136719,
|
|
"logps/rejected": -148.592529296875,
|
|
"loss": 0.5527,
|
|
"margin_dpo/margin_mean": 19.399627685546875,
|
|
"margin_dpo/margin_std": 15.338823318481445,
|
|
"step": 488
|
|
},
|
|
{
|
|
"epoch": 0.7180616740088106,
|
|
"grad_norm": 26.43874168395996,
|
|
"learning_rate": 1.1298035870396985e-07,
|
|
"logits/chosen": -0.9547609090805054,
|
|
"logits/rejected": -0.39513295888900757,
|
|
"logps/chosen": -119.86599731445312,
|
|
"logps/ref_chosen": -95.271240234375,
|
|
"logps/ref_rejected": -81.49215698242188,
|
|
"logps/rejected": -122.27718353271484,
|
|
"loss": 0.6173,
|
|
"margin_dpo/margin_mean": 16.190275192260742,
|
|
"margin_dpo/margin_std": 13.990297317504883,
|
|
"step": 489
|
|
},
|
|
{
|
|
"epoch": 0.7195301027900147,
|
|
"grad_norm": 29.264699935913086,
|
|
"learning_rate": 1.1190875675987355e-07,
|
|
"logits/chosen": -2.5108864307403564,
|
|
"logits/rejected": -2.8467113971710205,
|
|
"logps/chosen": -112.51838684082031,
|
|
"logps/ref_chosen": -85.32609558105469,
|
|
"logps/ref_rejected": -148.7736053466797,
|
|
"logps/rejected": -191.67208862304688,
|
|
"loss": 0.7967,
|
|
"margin_dpo/margin_mean": 15.706191062927246,
|
|
"margin_dpo/margin_std": 23.679283142089844,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.7209985315712188,
|
|
"grad_norm": 28.753334045410156,
|
|
"learning_rate": 1.1084079364846241e-07,
|
|
"logits/chosen": -1.7324990034103394,
|
|
"logits/rejected": -1.075792670249939,
|
|
"logps/chosen": -135.39317321777344,
|
|
"logps/ref_chosen": -105.19956970214844,
|
|
"logps/ref_rejected": -90.82363891601562,
|
|
"logps/rejected": -144.2461700439453,
|
|
"loss": 0.7306,
|
|
"margin_dpo/margin_mean": 23.228939056396484,
|
|
"margin_dpo/margin_std": 21.352033615112305,
|
|
"step": 491
|
|
},
|
|
{
|
|
"epoch": 0.7224669603524229,
|
|
"grad_norm": 26.195093154907227,
|
|
"learning_rate": 1.097764975115576e-07,
|
|
"logits/chosen": -2.059189558029175,
|
|
"logits/rejected": -1.671616792678833,
|
|
"logps/chosen": -91.94832611083984,
|
|
"logps/ref_chosen": -66.54839324951172,
|
|
"logps/ref_rejected": -69.85902404785156,
|
|
"logps/rejected": -113.68507385253906,
|
|
"loss": 0.7929,
|
|
"margin_dpo/margin_mean": 18.426122665405273,
|
|
"margin_dpo/margin_std": 19.221683502197266,
|
|
"step": 492
|
|
},
|
|
{
|
|
"epoch": 0.723935389133627,
|
|
"grad_norm": 27.575870513916016,
|
|
"learning_rate": 1.0871589639435203e-07,
|
|
"logits/chosen": -1.1041479110717773,
|
|
"logits/rejected": -0.9681916236877441,
|
|
"logps/chosen": -162.7125244140625,
|
|
"logps/ref_chosen": -138.72125244140625,
|
|
"logps/ref_rejected": -133.82675170898438,
|
|
"logps/rejected": -184.400390625,
|
|
"loss": 0.5993,
|
|
"margin_dpo/margin_mean": 26.582361221313477,
|
|
"margin_dpo/margin_std": 20.35706329345703,
|
|
"step": 493
|
|
},
|
|
{
|
|
"epoch": 0.7254038179148311,
|
|
"grad_norm": 23.7596492767334,
|
|
"learning_rate": 1.0765901824467166e-07,
|
|
"logits/chosen": -0.6587156057357788,
|
|
"logits/rejected": -0.5145885348320007,
|
|
"logps/chosen": -97.23701477050781,
|
|
"logps/ref_chosen": -66.82516479492188,
|
|
"logps/ref_rejected": -73.68384552001953,
|
|
"logps/rejected": -135.43707275390625,
|
|
"loss": 0.5601,
|
|
"margin_dpo/margin_mean": 31.34137535095215,
|
|
"margin_dpo/margin_std": 21.264366149902344,
|
|
"step": 494
|
|
},
|
|
{
|
|
"epoch": 0.7268722466960352,
|
|
"grad_norm": 34.367488861083984,
|
|
"learning_rate": 1.0660589091223854e-07,
|
|
"logits/chosen": -1.8472561836242676,
|
|
"logits/rejected": -1.2327227592468262,
|
|
"logps/chosen": -108.4244613647461,
|
|
"logps/ref_chosen": -87.1620864868164,
|
|
"logps/ref_rejected": -68.8896255493164,
|
|
"logps/rejected": -108.88243103027344,
|
|
"loss": 0.7539,
|
|
"margin_dpo/margin_mean": 18.730424880981445,
|
|
"margin_dpo/margin_std": 13.805540084838867,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.7283406754772394,
|
|
"grad_norm": 30.489227294921875,
|
|
"learning_rate": 1.0555654214793722e-07,
|
|
"logits/chosen": -2.2407476902008057,
|
|
"logits/rejected": -2.1656553745269775,
|
|
"logps/chosen": -122.41502380371094,
|
|
"logps/ref_chosen": -90.9640884399414,
|
|
"logps/ref_rejected": -107.32502746582031,
|
|
"logps/rejected": -157.75169372558594,
|
|
"loss": 0.6622,
|
|
"margin_dpo/margin_mean": 18.975727081298828,
|
|
"margin_dpo/margin_std": 19.945220947265625,
|
|
"step": 496
|
|
},
|
|
{
|
|
"epoch": 0.7298091042584435,
|
|
"grad_norm": 24.2734432220459,
|
|
"learning_rate": 1.0451099960308374e-07,
|
|
"logits/chosen": -1.2152756452560425,
|
|
"logits/rejected": -0.7649951577186584,
|
|
"logps/chosen": -114.13996124267578,
|
|
"logps/ref_chosen": -89.48576354980469,
|
|
"logps/ref_rejected": -77.33207702636719,
|
|
"logps/rejected": -121.43716430664062,
|
|
"loss": 0.6999,
|
|
"margin_dpo/margin_mean": 19.450889587402344,
|
|
"margin_dpo/margin_std": 14.824586868286133,
|
|
"step": 497
|
|
},
|
|
{
|
|
"epoch": 0.7312775330396476,
|
|
"grad_norm": 31.715137481689453,
|
|
"learning_rate": 1.0346929082869641e-07,
|
|
"logits/chosen": -1.5480034351348877,
|
|
"logits/rejected": -0.859655499458313,
|
|
"logps/chosen": -113.42682647705078,
|
|
"logps/ref_chosen": -86.19139099121094,
|
|
"logps/ref_rejected": -58.00111389160156,
|
|
"logps/rejected": -101.58597564697266,
|
|
"loss": 0.5858,
|
|
"margin_dpo/margin_mean": 16.34941864013672,
|
|
"margin_dpo/margin_std": 10.946194648742676,
|
|
"step": 498
|
|
},
|
|
{
|
|
"epoch": 0.7327459618208517,
|
|
"grad_norm": 26.94870376586914,
|
|
"learning_rate": 1.0243144327477013e-07,
|
|
"logits/chosen": -1.2497620582580566,
|
|
"logits/rejected": -1.404412031173706,
|
|
"logps/chosen": -121.62670135498047,
|
|
"logps/ref_chosen": -93.28053283691406,
|
|
"logps/ref_rejected": -132.08175659179688,
|
|
"logps/rejected": -175.87213134765625,
|
|
"loss": 0.6569,
|
|
"margin_dpo/margin_mean": 15.444196701049805,
|
|
"margin_dpo/margin_std": 16.970027923583984,
|
|
"step": 499
|
|
},
|
|
{
|
|
"epoch": 0.7342143906020558,
|
|
"grad_norm": 30.46588897705078,
|
|
"learning_rate": 1.0139748428955333e-07,
|
|
"logits/chosen": -2.575468063354492,
|
|
"logits/rejected": -2.286226272583008,
|
|
"logps/chosen": -104.04843139648438,
|
|
"logps/ref_chosen": -77.39090728759766,
|
|
"logps/ref_rejected": -98.08084106445312,
|
|
"logps/rejected": -146.70986938476562,
|
|
"loss": 0.7107,
|
|
"margin_dpo/margin_mean": 21.971506118774414,
|
|
"margin_dpo/margin_std": 22.098201751708984,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.7342143906020558,
|
|
"eval_logits/chosen": -1.337835669517517,
|
|
"eval_logits/rejected": -1.0359337329864502,
|
|
"eval_logps/chosen": -131.5918426513672,
|
|
"eval_logps/ref_chosen": -101.88616943359375,
|
|
"eval_logps/ref_rejected": -92.68607330322266,
|
|
"eval_logps/rejected": -138.25230407714844,
|
|
"eval_loss": 0.4213423430919647,
|
|
"eval_margin_dpo/margin_mean": 15.860573768615723,
|
|
"eval_margin_dpo/margin_std": 17.095046997070312,
|
|
"eval_runtime": 44.044,
|
|
"eval_samples_per_second": 53.106,
|
|
"eval_steps_per_second": 1.68,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.73568281938326,
|
|
"grad_norm": 32.96603775024414,
|
|
"learning_rate": 1.0036744111882672e-07,
|
|
"logits/chosen": -1.9527108669281006,
|
|
"logits/rejected": -1.451664924621582,
|
|
"logps/chosen": -104.76618957519531,
|
|
"logps/ref_chosen": -76.12673950195312,
|
|
"logps/ref_rejected": -76.95055389404297,
|
|
"logps/rejected": -124.26972961425781,
|
|
"loss": 0.7031,
|
|
"margin_dpo/margin_mean": 18.67972755432129,
|
|
"margin_dpo/margin_std": 21.08388900756836,
|
|
"step": 501
|
|
},
|
|
{
|
|
"epoch": 0.737151248164464,
|
|
"grad_norm": 26.178499221801758,
|
|
"learning_rate": 9.934134090518592e-08,
|
|
"logits/chosen": -1.0793780088424683,
|
|
"logits/rejected": -0.21455411612987518,
|
|
"logps/chosen": -137.14662170410156,
|
|
"logps/ref_chosen": -105.97865295410156,
|
|
"logps/ref_rejected": -78.07791137695312,
|
|
"logps/rejected": -124.19863891601562,
|
|
"loss": 0.6598,
|
|
"margin_dpo/margin_mean": 14.95276165008545,
|
|
"margin_dpo/margin_std": 16.38261604309082,
|
|
"step": 502
|
|
},
|
|
{
|
|
"epoch": 0.7386196769456681,
|
|
"grad_norm": 29.94610595703125,
|
|
"learning_rate": 9.831921068732571e-08,
|
|
"logits/chosen": -0.928479015827179,
|
|
"logits/rejected": -0.2922493815422058,
|
|
"logps/chosen": -97.82994079589844,
|
|
"logps/ref_chosen": -74.26416015625,
|
|
"logps/ref_rejected": -62.19700241088867,
|
|
"logps/rejected": -107.19987487792969,
|
|
"loss": 0.6637,
|
|
"margin_dpo/margin_mean": 21.43708038330078,
|
|
"margin_dpo/margin_std": 14.342729568481445,
|
|
"step": 503
|
|
},
|
|
{
|
|
"epoch": 0.7400881057268722,
|
|
"grad_norm": 25.398269653320312,
|
|
"learning_rate": 9.730107739932805e-08,
|
|
"logits/chosen": -1.9864063262939453,
|
|
"logits/rejected": -1.3488131761550903,
|
|
"logps/chosen": -123.76087951660156,
|
|
"logps/ref_chosen": -100.04080200195312,
|
|
"logps/ref_rejected": -91.90785217285156,
|
|
"logps/rejected": -140.589111328125,
|
|
"loss": 0.4438,
|
|
"margin_dpo/margin_mean": 24.96117401123047,
|
|
"margin_dpo/margin_std": 15.161877632141113,
|
|
"step": 504
|
|
},
|
|
{
|
|
"epoch": 0.7415565345080763,
|
|
"grad_norm": 36.0130500793457,
|
|
"learning_rate": 9.628696786995188e-08,
|
|
"logits/chosen": -2.0708212852478027,
|
|
"logits/rejected": -1.4549051523208618,
|
|
"logps/chosen": -144.2436065673828,
|
|
"logps/ref_chosen": -120.70248413085938,
|
|
"logps/ref_rejected": -106.37281799316406,
|
|
"logps/rejected": -159.46884155273438,
|
|
"loss": 0.5544,
|
|
"margin_dpo/margin_mean": 29.554908752441406,
|
|
"margin_dpo/margin_std": 23.2249698638916,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.7430249632892805,
|
|
"grad_norm": 22.152841567993164,
|
|
"learning_rate": 9.527690882192635e-08,
|
|
"logits/chosen": -2.4234611988067627,
|
|
"logits/rejected": -1.7693153619766235,
|
|
"logps/chosen": -129.5956268310547,
|
|
"logps/ref_chosen": -99.54842376708984,
|
|
"logps/ref_rejected": -76.66189575195312,
|
|
"logps/rejected": -122.17888641357422,
|
|
"loss": 0.5739,
|
|
"margin_dpo/margin_mean": 15.469791412353516,
|
|
"margin_dpo/margin_std": 15.011087417602539,
|
|
"step": 506
|
|
},
|
|
{
|
|
"epoch": 0.7444933920704846,
|
|
"grad_norm": 30.579790115356445,
|
|
"learning_rate": 9.427092687124691e-08,
|
|
"logits/chosen": -1.8249220848083496,
|
|
"logits/rejected": -1.0810041427612305,
|
|
"logps/chosen": -135.46640014648438,
|
|
"logps/ref_chosen": -110.98868560791016,
|
|
"logps/ref_rejected": -118.09579467773438,
|
|
"logps/rejected": -167.599365234375,
|
|
"loss": 0.6956,
|
|
"margin_dpo/margin_mean": 25.025875091552734,
|
|
"margin_dpo/margin_std": 18.596511840820312,
|
|
"step": 507
|
|
},
|
|
{
|
|
"epoch": 0.7459618208516887,
|
|
"grad_norm": 32.01652908325195,
|
|
"learning_rate": 9.326904852647344e-08,
|
|
"logits/chosen": -1.6956753730773926,
|
|
"logits/rejected": -1.3768240213394165,
|
|
"logps/chosen": -140.34487915039062,
|
|
"logps/ref_chosen": -115.07064819335938,
|
|
"logps/ref_rejected": -159.63973999023438,
|
|
"logps/rejected": -210.63319396972656,
|
|
"loss": 0.7563,
|
|
"margin_dpo/margin_mean": 25.71923065185547,
|
|
"margin_dpo/margin_std": 28.898908615112305,
|
|
"step": 508
|
|
},
|
|
{
|
|
"epoch": 0.7474302496328928,
|
|
"grad_norm": 38.91352081298828,
|
|
"learning_rate": 9.227130018803195e-08,
|
|
"logits/chosen": -1.2427645921707153,
|
|
"logits/rejected": -0.8354566693305969,
|
|
"logps/chosen": -126.22138977050781,
|
|
"logps/ref_chosen": -101.36344909667969,
|
|
"logps/ref_rejected": -108.30143737792969,
|
|
"logps/rejected": -149.70640563964844,
|
|
"loss": 0.8266,
|
|
"margin_dpo/margin_mean": 16.547025680541992,
|
|
"margin_dpo/margin_std": 21.260597229003906,
|
|
"step": 509
|
|
},
|
|
{
|
|
"epoch": 0.748898678414097,
|
|
"grad_norm": 28.172101974487305,
|
|
"learning_rate": 9.127770814751932e-08,
|
|
"logits/chosen": -1.961017370223999,
|
|
"logits/rejected": -1.5561909675598145,
|
|
"logps/chosen": -110.28907775878906,
|
|
"logps/ref_chosen": -74.63158416748047,
|
|
"logps/ref_rejected": -76.96902465820312,
|
|
"logps/rejected": -142.86691284179688,
|
|
"loss": 0.7292,
|
|
"margin_dpo/margin_mean": 30.24038314819336,
|
|
"margin_dpo/margin_std": 28.988506317138672,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.750367107195301,
|
|
"grad_norm": 29.637191772460938,
|
|
"learning_rate": 9.028829858700973e-08,
|
|
"logits/chosen": -2.0202572345733643,
|
|
"logits/rejected": -1.8201736211776733,
|
|
"logps/chosen": -93.00399780273438,
|
|
"logps/ref_chosen": -64.2794189453125,
|
|
"logps/ref_rejected": -98.6783218383789,
|
|
"logps/rejected": -149.43972778320312,
|
|
"loss": 0.7293,
|
|
"margin_dpo/margin_mean": 22.036819458007812,
|
|
"margin_dpo/margin_std": 23.274316787719727,
|
|
"step": 511
|
|
},
|
|
{
|
|
"epoch": 0.7518355359765051,
|
|
"grad_norm": 34.43431854248047,
|
|
"learning_rate": 8.930309757836516e-08,
|
|
"logits/chosen": -2.452205181121826,
|
|
"logits/rejected": -1.9652907848358154,
|
|
"logps/chosen": -110.59320831298828,
|
|
"logps/ref_chosen": -82.41152954101562,
|
|
"logps/ref_rejected": -79.29417419433594,
|
|
"logps/rejected": -126.40013122558594,
|
|
"loss": 0.7216,
|
|
"margin_dpo/margin_mean": 18.924272537231445,
|
|
"margin_dpo/margin_std": 21.845081329345703,
|
|
"step": 512
|
|
},
|
|
{
|
|
"epoch": 0.7533039647577092,
|
|
"grad_norm": 24.879047393798828,
|
|
"learning_rate": 8.832213108254863e-08,
|
|
"logits/chosen": -2.379291534423828,
|
|
"logits/rejected": -2.0403084754943848,
|
|
"logps/chosen": -95.06507110595703,
|
|
"logps/ref_chosen": -68.55144500732422,
|
|
"logps/ref_rejected": -81.16883850097656,
|
|
"logps/rejected": -128.31956481933594,
|
|
"loss": 0.6583,
|
|
"margin_dpo/margin_mean": 20.637096405029297,
|
|
"margin_dpo/margin_std": 14.324064254760742,
|
|
"step": 513
|
|
},
|
|
{
|
|
"epoch": 0.7547723935389133,
|
|
"grad_norm": 26.90195083618164,
|
|
"learning_rate": 8.734542494893954e-08,
|
|
"logits/chosen": -1.9824368953704834,
|
|
"logits/rejected": -1.517698049545288,
|
|
"logps/chosen": -151.19822692871094,
|
|
"logps/ref_chosen": -128.8970947265625,
|
|
"logps/ref_rejected": -122.94134521484375,
|
|
"logps/rejected": -165.733154296875,
|
|
"loss": 0.6437,
|
|
"margin_dpo/margin_mean": 20.49066162109375,
|
|
"margin_dpo/margin_std": 21.157379150390625,
|
|
"step": 514
|
|
},
|
|
{
|
|
"epoch": 0.7562408223201175,
|
|
"grad_norm": 30.2961368560791,
|
|
"learning_rate": 8.637300491465272e-08,
|
|
"logits/chosen": -1.1363587379455566,
|
|
"logits/rejected": -0.9900952577590942,
|
|
"logps/chosen": -116.65281677246094,
|
|
"logps/ref_chosen": -83.93804931640625,
|
|
"logps/ref_rejected": -116.58180236816406,
|
|
"logps/rejected": -160.9093780517578,
|
|
"loss": 0.7953,
|
|
"margin_dpo/margin_mean": 11.612812042236328,
|
|
"margin_dpo/margin_std": 18.925113677978516,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.7577092511013216,
|
|
"grad_norm": 27.436569213867188,
|
|
"learning_rate": 8.540489660386064e-08,
|
|
"logits/chosen": -2.0557169914245605,
|
|
"logits/rejected": -1.6514482498168945,
|
|
"logps/chosen": -132.43113708496094,
|
|
"logps/ref_chosen": -106.92062377929688,
|
|
"logps/ref_rejected": -110.65416717529297,
|
|
"logps/rejected": -160.17745971679688,
|
|
"loss": 0.6418,
|
|
"margin_dpo/margin_mean": 24.01278305053711,
|
|
"margin_dpo/margin_std": 17.438676834106445,
|
|
"step": 516
|
|
},
|
|
{
|
|
"epoch": 0.7591776798825257,
|
|
"grad_norm": 28.32579231262207,
|
|
"learning_rate": 8.444112552711752e-08,
|
|
"logits/chosen": -1.3749854564666748,
|
|
"logits/rejected": -1.1191842555999756,
|
|
"logps/chosen": -110.68536376953125,
|
|
"logps/ref_chosen": -86.00096130371094,
|
|
"logps/ref_rejected": -86.94703674316406,
|
|
"logps/rejected": -137.059326171875,
|
|
"loss": 0.539,
|
|
"margin_dpo/margin_mean": 25.427875518798828,
|
|
"margin_dpo/margin_std": 11.098093032836914,
|
|
"step": 517
|
|
},
|
|
{
|
|
"epoch": 0.7606461086637298,
|
|
"grad_norm": 28.546537399291992,
|
|
"learning_rate": 8.348171708068747e-08,
|
|
"logits/chosen": -1.2067550420761108,
|
|
"logits/rejected": -1.171380639076233,
|
|
"logps/chosen": -104.00096893310547,
|
|
"logps/ref_chosen": -70.59335327148438,
|
|
"logps/ref_rejected": -95.62405395507812,
|
|
"logps/rejected": -142.74261474609375,
|
|
"loss": 0.7406,
|
|
"margin_dpo/margin_mean": 13.710941314697266,
|
|
"margin_dpo/margin_std": 13.447914123535156,
|
|
"step": 518
|
|
},
|
|
{
|
|
"epoch": 0.762114537444934,
|
|
"grad_norm": 19.51987648010254,
|
|
"learning_rate": 8.25266965458755e-08,
|
|
"logits/chosen": -1.9919052124023438,
|
|
"logits/rejected": -1.7074830532073975,
|
|
"logps/chosen": -124.48606872558594,
|
|
"logps/ref_chosen": -96.66695404052734,
|
|
"logps/ref_rejected": -117.94882202148438,
|
|
"logps/rejected": -167.92465209960938,
|
|
"loss": 0.4639,
|
|
"margin_dpo/margin_mean": 22.15669822692871,
|
|
"margin_dpo/margin_std": 15.507266998291016,
|
|
"step": 519
|
|
},
|
|
{
|
|
"epoch": 0.7635829662261381,
|
|
"grad_norm": 21.792402267456055,
|
|
"learning_rate": 8.15760890883607e-08,
|
|
"logits/chosen": -1.5175939798355103,
|
|
"logits/rejected": -0.9286727905273438,
|
|
"logps/chosen": -145.54800415039062,
|
|
"logps/ref_chosen": -116.01941680908203,
|
|
"logps/ref_rejected": -127.23921203613281,
|
|
"logps/rejected": -172.96063232421875,
|
|
"loss": 0.4543,
|
|
"margin_dpo/margin_mean": 16.192813873291016,
|
|
"margin_dpo/margin_std": 14.938676834106445,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.7650513950073421,
|
|
"grad_norm": 23.45681381225586,
|
|
"learning_rate": 8.062991975753378e-08,
|
|
"logits/chosen": -1.2491178512573242,
|
|
"logits/rejected": -1.037899374961853,
|
|
"logps/chosen": -99.95437622070312,
|
|
"logps/ref_chosen": -71.46085357666016,
|
|
"logps/ref_rejected": -79.8829345703125,
|
|
"logps/rejected": -126.59786987304688,
|
|
"loss": 0.5567,
|
|
"margin_dpo/margin_mean": 18.221412658691406,
|
|
"margin_dpo/margin_std": 10.629344940185547,
|
|
"step": 521
|
|
},
|
|
{
|
|
"epoch": 0.7665198237885462,
|
|
"grad_norm": 24.551103591918945,
|
|
"learning_rate": 7.968821348583643e-08,
|
|
"logits/chosen": -0.8299498558044434,
|
|
"logits/rejected": -1.041572093963623,
|
|
"logps/chosen": -69.39967346191406,
|
|
"logps/ref_chosen": -42.4521369934082,
|
|
"logps/ref_rejected": -76.93772888183594,
|
|
"logps/rejected": -126.54615020751953,
|
|
"loss": 0.6518,
|
|
"margin_dpo/margin_mean": 22.660873413085938,
|
|
"margin_dpo/margin_std": 21.646053314208984,
|
|
"step": 522
|
|
},
|
|
{
|
|
"epoch": 0.7679882525697503,
|
|
"grad_norm": 34.3400993347168,
|
|
"learning_rate": 7.875099508810484e-08,
|
|
"logits/chosen": -2.052365779876709,
|
|
"logits/rejected": -0.4013219475746155,
|
|
"logps/chosen": -174.81800842285156,
|
|
"logps/ref_chosen": -148.17559814453125,
|
|
"logps/ref_rejected": -72.78897857666016,
|
|
"logps/rejected": -118.07229614257812,
|
|
"loss": 0.7682,
|
|
"margin_dpo/margin_mean": 18.640913009643555,
|
|
"margin_dpo/margin_std": 19.8717041015625,
|
|
"step": 523
|
|
},
|
|
{
|
|
"epoch": 0.7694566813509545,
|
|
"grad_norm": 24.858745574951172,
|
|
"learning_rate": 7.781828926091535e-08,
|
|
"logits/chosen": -1.7648762464523315,
|
|
"logits/rejected": -1.5137319564819336,
|
|
"logps/chosen": -116.93294525146484,
|
|
"logps/ref_chosen": -93.44391632080078,
|
|
"logps/ref_rejected": -96.07662200927734,
|
|
"logps/rejected": -138.82461547851562,
|
|
"loss": 0.5851,
|
|
"margin_dpo/margin_mean": 19.25895881652832,
|
|
"margin_dpo/margin_std": 13.380806922912598,
|
|
"step": 524
|
|
},
|
|
{
|
|
"epoch": 0.7709251101321586,
|
|
"grad_norm": 31.337230682373047,
|
|
"learning_rate": 7.689012058193384e-08,
|
|
"logits/chosen": -1.9353046417236328,
|
|
"logits/rejected": -1.7126715183258057,
|
|
"logps/chosen": -112.10871124267578,
|
|
"logps/ref_chosen": -83.10411834716797,
|
|
"logps/ref_rejected": -88.51509094238281,
|
|
"logps/rejected": -135.60104370117188,
|
|
"loss": 0.6649,
|
|
"margin_dpo/margin_mean": 18.081356048583984,
|
|
"margin_dpo/margin_std": 19.914634704589844,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.7723935389133627,
|
|
"grad_norm": 25.90157127380371,
|
|
"learning_rate": 7.596651350926836e-08,
|
|
"logits/chosen": -0.9229705929756165,
|
|
"logits/rejected": -0.26788192987442017,
|
|
"logps/chosen": -125.15325164794922,
|
|
"logps/ref_chosen": -94.63446807861328,
|
|
"logps/ref_rejected": -70.77076721191406,
|
|
"logps/rejected": -120.48922729492188,
|
|
"loss": 0.5904,
|
|
"margin_dpo/margin_mean": 19.199670791625977,
|
|
"margin_dpo/margin_std": 16.344276428222656,
|
|
"step": 526
|
|
},
|
|
{
|
|
"epoch": 0.7738619676945668,
|
|
"grad_norm": 22.789775848388672,
|
|
"learning_rate": 7.504749238082414e-08,
|
|
"logits/chosen": -1.9874346256256104,
|
|
"logits/rejected": -1.2692365646362305,
|
|
"logps/chosen": -125.09730529785156,
|
|
"logps/ref_chosen": -100.09386444091797,
|
|
"logps/ref_rejected": -97.19957733154297,
|
|
"logps/rejected": -143.50888061523438,
|
|
"loss": 0.539,
|
|
"margin_dpo/margin_mean": 21.305864334106445,
|
|
"margin_dpo/margin_std": 20.422344207763672,
|
|
"step": 527
|
|
},
|
|
{
|
|
"epoch": 0.775330396475771,
|
|
"grad_norm": 21.659528732299805,
|
|
"learning_rate": 7.413308141366254e-08,
|
|
"logits/chosen": -2.231781005859375,
|
|
"logits/rejected": -1.9260224103927612,
|
|
"logps/chosen": -133.10507202148438,
|
|
"logps/ref_chosen": -110.18800354003906,
|
|
"logps/ref_rejected": -102.22782135009766,
|
|
"logps/rejected": -145.65003967285156,
|
|
"loss": 0.5995,
|
|
"margin_dpo/margin_mean": 20.505146026611328,
|
|
"margin_dpo/margin_std": 19.203224182128906,
|
|
"step": 528
|
|
},
|
|
{
|
|
"epoch": 0.7767988252569751,
|
|
"grad_norm": 29.118371963500977,
|
|
"learning_rate": 7.322330470336313e-08,
|
|
"logits/chosen": -1.4017257690429688,
|
|
"logits/rejected": -1.16348135471344,
|
|
"logps/chosen": -104.53858184814453,
|
|
"logps/ref_chosen": -77.38128662109375,
|
|
"logps/ref_rejected": -95.52617645263672,
|
|
"logps/rejected": -144.15533447265625,
|
|
"loss": 0.6501,
|
|
"margin_dpo/margin_mean": 21.471858978271484,
|
|
"margin_dpo/margin_std": 19.392187118530273,
|
|
"step": 529
|
|
},
|
|
{
|
|
"epoch": 0.7782672540381792,
|
|
"grad_norm": 20.232019424438477,
|
|
"learning_rate": 7.231818622338822e-08,
|
|
"logits/chosen": -0.6151648759841919,
|
|
"logits/rejected": -0.207261323928833,
|
|
"logps/chosen": -104.87347412109375,
|
|
"logps/ref_chosen": -77.66940307617188,
|
|
"logps/ref_rejected": -78.3644027709961,
|
|
"logps/rejected": -124.8089599609375,
|
|
"loss": 0.4763,
|
|
"margin_dpo/margin_mean": 19.240493774414062,
|
|
"margin_dpo/margin_std": 13.067811012268066,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.7797356828193832,
|
|
"grad_norm": 20.91158103942871,
|
|
"learning_rate": 7.141774982445147e-08,
|
|
"logits/chosen": -2.1337180137634277,
|
|
"logits/rejected": -1.653696060180664,
|
|
"logps/chosen": -94.38259887695312,
|
|
"logps/ref_chosen": -67.79977416992188,
|
|
"logps/ref_rejected": -74.13839721679688,
|
|
"logps/rejected": -123.50054931640625,
|
|
"loss": 0.5318,
|
|
"margin_dpo/margin_mean": 22.77933120727539,
|
|
"margin_dpo/margin_std": 19.406822204589844,
|
|
"step": 531
|
|
},
|
|
{
|
|
"epoch": 0.7812041116005873,
|
|
"grad_norm": 27.96685218811035,
|
|
"learning_rate": 7.052201923388953e-08,
|
|
"logits/chosen": -0.6504892706871033,
|
|
"logits/rejected": -0.17150549590587616,
|
|
"logps/chosen": -131.53196716308594,
|
|
"logps/ref_chosen": -103.95175170898438,
|
|
"logps/ref_rejected": -105.47799682617188,
|
|
"logps/rejected": -164.3619842529297,
|
|
"loss": 0.41,
|
|
"margin_dpo/margin_mean": 31.30377197265625,
|
|
"margin_dpo/margin_std": 14.452159881591797,
|
|
"step": 532
|
|
},
|
|
{
|
|
"epoch": 0.7826725403817915,
|
|
"grad_norm": 27.39715003967285,
|
|
"learning_rate": 6.963101805503646e-08,
|
|
"logits/chosen": -2.3065249919891357,
|
|
"logits/rejected": -2.3915162086486816,
|
|
"logps/chosen": -111.36788177490234,
|
|
"logps/ref_chosen": -85.58866119384766,
|
|
"logps/ref_rejected": -123.13199615478516,
|
|
"logps/rejected": -163.6744384765625,
|
|
"loss": 0.7967,
|
|
"margin_dpo/margin_mean": 14.76324462890625,
|
|
"margin_dpo/margin_std": 17.567630767822266,
|
|
"step": 533
|
|
},
|
|
{
|
|
"epoch": 0.7841409691629956,
|
|
"grad_norm": 25.041467666625977,
|
|
"learning_rate": 6.874476976660184e-08,
|
|
"logits/chosen": -1.307136058807373,
|
|
"logits/rejected": -0.8812864422798157,
|
|
"logps/chosen": -117.07095336914062,
|
|
"logps/ref_chosen": -91.5281753540039,
|
|
"logps/ref_rejected": -75.92799377441406,
|
|
"logps/rejected": -126.13896942138672,
|
|
"loss": 0.564,
|
|
"margin_dpo/margin_mean": 24.668193817138672,
|
|
"margin_dpo/margin_std": 17.151203155517578,
|
|
"step": 534
|
|
},
|
|
{
|
|
"epoch": 0.7856093979441997,
|
|
"grad_norm": 23.80198860168457,
|
|
"learning_rate": 6.786329772205246e-08,
|
|
"logits/chosen": -0.8859176635742188,
|
|
"logits/rejected": -0.7408356666564941,
|
|
"logps/chosen": -118.31693267822266,
|
|
"logps/ref_chosen": -93.79539489746094,
|
|
"logps/ref_rejected": -119.77159118652344,
|
|
"logps/rejected": -169.48883056640625,
|
|
"loss": 0.6317,
|
|
"margin_dpo/margin_mean": 25.195722579956055,
|
|
"margin_dpo/margin_std": 20.150222778320312,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.7870778267254038,
|
|
"grad_norm": 24.914581298828125,
|
|
"learning_rate": 6.698662514899638e-08,
|
|
"logits/chosen": -2.2322561740875244,
|
|
"logits/rejected": -1.6124215126037598,
|
|
"logps/chosen": -92.461181640625,
|
|
"logps/ref_chosen": -69.53123474121094,
|
|
"logps/ref_rejected": -68.41876220703125,
|
|
"logps/rejected": -118.23365783691406,
|
|
"loss": 0.5392,
|
|
"margin_dpo/margin_mean": 26.884937286376953,
|
|
"margin_dpo/margin_std": 16.595741271972656,
|
|
"step": 536
|
|
},
|
|
{
|
|
"epoch": 0.788546255506608,
|
|
"grad_norm": 16.780048370361328,
|
|
"learning_rate": 6.611477514857114e-08,
|
|
"logits/chosen": -1.0513997077941895,
|
|
"logits/rejected": -0.77760249376297,
|
|
"logps/chosen": -117.74217224121094,
|
|
"logps/ref_chosen": -95.1636962890625,
|
|
"logps/ref_rejected": -94.2923812866211,
|
|
"logps/rejected": -135.7506866455078,
|
|
"loss": 0.413,
|
|
"margin_dpo/margin_mean": 18.879819869995117,
|
|
"margin_dpo/margin_std": 17.38238525390625,
|
|
"step": 537
|
|
},
|
|
{
|
|
"epoch": 0.7900146842878121,
|
|
"grad_norm": 23.98692512512207,
|
|
"learning_rate": 6.524777069483525e-08,
|
|
"logits/chosen": -1.9336012601852417,
|
|
"logits/rejected": -1.4487353563308716,
|
|
"logps/chosen": -128.1285858154297,
|
|
"logps/ref_chosen": -102.02078247070312,
|
|
"logps/ref_rejected": -89.95391082763672,
|
|
"logps/rejected": -137.56033325195312,
|
|
"loss": 0.5387,
|
|
"margin_dpo/margin_mean": 21.498611450195312,
|
|
"margin_dpo/margin_std": 10.918216705322266,
|
|
"step": 538
|
|
},
|
|
{
|
|
"epoch": 0.7914831130690162,
|
|
"grad_norm": 27.972061157226562,
|
|
"learning_rate": 6.438563463416221e-08,
|
|
"logits/chosen": -1.7122726440429688,
|
|
"logits/rejected": -1.3533068895339966,
|
|
"logps/chosen": -139.71206665039062,
|
|
"logps/ref_chosen": -114.733154296875,
|
|
"logps/ref_rejected": -83.13861083984375,
|
|
"logps/rejected": -125.41854095458984,
|
|
"loss": 0.6208,
|
|
"margin_dpo/margin_mean": 17.301036834716797,
|
|
"margin_dpo/margin_std": 15.256547927856445,
|
|
"step": 539
|
|
},
|
|
{
|
|
"epoch": 0.7929515418502202,
|
|
"grad_norm": 29.838821411132812,
|
|
"learning_rate": 6.352838968463919e-08,
|
|
"logits/chosen": -1.6790308952331543,
|
|
"logits/rejected": -1.3257447481155396,
|
|
"logps/chosen": -132.5393524169922,
|
|
"logps/ref_chosen": -106.70039367675781,
|
|
"logps/ref_rejected": -107.09190368652344,
|
|
"logps/rejected": -153.77479553222656,
|
|
"loss": 0.6429,
|
|
"margin_dpo/margin_mean": 20.843917846679688,
|
|
"margin_dpo/margin_std": 24.12029457092285,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7944199706314243,
|
|
"grad_norm": 24.643421173095703,
|
|
"learning_rate": 6.267605843546767e-08,
|
|
"logits/chosen": -1.0789179801940918,
|
|
"logits/rejected": -0.5022500157356262,
|
|
"logps/chosen": -167.45968627929688,
|
|
"logps/ref_chosen": -135.84811401367188,
|
|
"logps/ref_rejected": -121.25428009033203,
|
|
"logps/rejected": -176.2462615966797,
|
|
"loss": 0.5465,
|
|
"margin_dpo/margin_mean": 23.38041114807129,
|
|
"margin_dpo/margin_std": 16.1546630859375,
|
|
"step": 541
|
|
},
|
|
{
|
|
"epoch": 0.7958883994126285,
|
|
"grad_norm": 24.163631439208984,
|
|
"learning_rate": 6.182866334636888e-08,
|
|
"logits/chosen": -2.3853206634521484,
|
|
"logits/rejected": -2.185912609100342,
|
|
"logps/chosen": -92.60365295410156,
|
|
"logps/ref_chosen": -68.84876251220703,
|
|
"logps/ref_rejected": -100.97695922851562,
|
|
"logps/rejected": -155.03704833984375,
|
|
"loss": 0.5527,
|
|
"margin_dpo/margin_mean": 30.305194854736328,
|
|
"margin_dpo/margin_std": 17.951576232910156,
|
|
"step": 542
|
|
},
|
|
{
|
|
"epoch": 0.7973568281938326,
|
|
"grad_norm": 32.94911575317383,
|
|
"learning_rate": 6.098622674699147e-08,
|
|
"logits/chosen": -0.9092826843261719,
|
|
"logits/rejected": -0.7881707549095154,
|
|
"logps/chosen": -130.04563903808594,
|
|
"logps/ref_chosen": -100.33512115478516,
|
|
"logps/ref_rejected": -99.94261932373047,
|
|
"logps/rejected": -140.88246154785156,
|
|
"loss": 0.8397,
|
|
"margin_dpo/margin_mean": 11.22932243347168,
|
|
"margin_dpo/margin_std": 23.324386596679688,
|
|
"step": 543
|
|
},
|
|
{
|
|
"epoch": 0.7988252569750367,
|
|
"grad_norm": 25.188100814819336,
|
|
"learning_rate": 6.01487708363232e-08,
|
|
"logits/chosen": -1.485346794128418,
|
|
"logits/rejected": -1.2233922481536865,
|
|
"logps/chosen": -130.50936889648438,
|
|
"logps/ref_chosen": -103.06536102294922,
|
|
"logps/ref_rejected": -118.76666259765625,
|
|
"logps/rejected": -167.39077758789062,
|
|
"loss": 0.5924,
|
|
"margin_dpo/margin_mean": 21.180089950561523,
|
|
"margin_dpo/margin_std": 21.94754409790039,
|
|
"step": 544
|
|
},
|
|
{
|
|
"epoch": 0.8002936857562408,
|
|
"grad_norm": 28.579204559326172,
|
|
"learning_rate": 5.9316317682106294e-08,
|
|
"logits/chosen": -1.371840000152588,
|
|
"logits/rejected": -1.1238985061645508,
|
|
"logps/chosen": -113.33309173583984,
|
|
"logps/ref_chosen": -84.86571502685547,
|
|
"logps/ref_rejected": -108.79608154296875,
|
|
"logps/rejected": -151.92977905273438,
|
|
"loss": 0.6574,
|
|
"margin_dpo/margin_mean": 14.666326522827148,
|
|
"margin_dpo/margin_std": 19.2750244140625,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.801762114537445,
|
|
"grad_norm": 28.237146377563477,
|
|
"learning_rate": 5.848888922025552e-08,
|
|
"logits/chosen": -0.9829261898994446,
|
|
"logits/rejected": -0.5799949169158936,
|
|
"logps/chosen": -95.30905151367188,
|
|
"logps/ref_chosen": -70.455078125,
|
|
"logps/ref_rejected": -63.61554718017578,
|
|
"logps/rejected": -110.25160217285156,
|
|
"loss": 0.679,
|
|
"margin_dpo/margin_mean": 21.78207778930664,
|
|
"margin_dpo/margin_std": 12.5681734085083,
|
|
"step": 546
|
|
},
|
|
{
|
|
"epoch": 0.8032305433186491,
|
|
"grad_norm": 20.89057731628418,
|
|
"learning_rate": 5.7666507254280265e-08,
|
|
"logits/chosen": -1.4058809280395508,
|
|
"logits/rejected": -1.014687418937683,
|
|
"logps/chosen": -140.72103881835938,
|
|
"logps/ref_chosen": -107.83323669433594,
|
|
"logps/ref_rejected": -115.82113647460938,
|
|
"logps/rejected": -170.63926696777344,
|
|
"loss": 0.485,
|
|
"margin_dpo/margin_mean": 21.930328369140625,
|
|
"margin_dpo/margin_std": 25.41767120361328,
|
|
"step": 547
|
|
},
|
|
{
|
|
"epoch": 0.8046989720998532,
|
|
"grad_norm": 34.20707321166992,
|
|
"learning_rate": 5.684919345471029e-08,
|
|
"logits/chosen": -2.0937061309814453,
|
|
"logits/rejected": -2.137777090072632,
|
|
"logps/chosen": -98.56684112548828,
|
|
"logps/ref_chosen": -69.65579223632812,
|
|
"logps/ref_rejected": -83.24872589111328,
|
|
"logps/rejected": -128.52069091796875,
|
|
"loss": 0.8659,
|
|
"margin_dpo/margin_mean": 16.36090850830078,
|
|
"margin_dpo/margin_std": 16.906295776367188,
|
|
"step": 548
|
|
},
|
|
{
|
|
"epoch": 0.8061674008810573,
|
|
"grad_norm": 23.242633819580078,
|
|
"learning_rate": 5.603696935852426e-08,
|
|
"logits/chosen": -1.707764983177185,
|
|
"logits/rejected": -0.5650928020477295,
|
|
"logps/chosen": -126.3386001586914,
|
|
"logps/ref_chosen": -105.56031036376953,
|
|
"logps/ref_rejected": -84.10400390625,
|
|
"logps/rejected": -132.23867797851562,
|
|
"loss": 0.5675,
|
|
"margin_dpo/margin_mean": 27.356380462646484,
|
|
"margin_dpo/margin_std": 18.783891677856445,
|
|
"step": 549
|
|
},
|
|
{
|
|
"epoch": 0.8076358296622613,
|
|
"grad_norm": 25.554492950439453,
|
|
"learning_rate": 5.5229856368582376e-08,
|
|
"logits/chosen": 0.1068572849035263,
|
|
"logits/rejected": 0.1513178050518036,
|
|
"logps/chosen": -106.12084197998047,
|
|
"logps/ref_chosen": -76.27791595458984,
|
|
"logps/ref_rejected": -100.49739074707031,
|
|
"logps/rejected": -153.0285186767578,
|
|
"loss": 0.4972,
|
|
"margin_dpo/margin_mean": 22.688209533691406,
|
|
"margin_dpo/margin_std": 14.742086410522461,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.8091042584434655,
|
|
"grad_norm": 22.723278045654297,
|
|
"learning_rate": 5.4427875753062734e-08,
|
|
"logits/chosen": -1.7522798776626587,
|
|
"logits/rejected": -1.0569827556610107,
|
|
"logps/chosen": -127.6473159790039,
|
|
"logps/ref_chosen": -102.71890258789062,
|
|
"logps/ref_rejected": -97.09249877929688,
|
|
"logps/rejected": -145.43980407714844,
|
|
"loss": 0.423,
|
|
"margin_dpo/margin_mean": 23.418901443481445,
|
|
"margin_dpo/margin_std": 14.990714073181152,
|
|
"step": 551
|
|
},
|
|
{
|
|
"epoch": 0.8105726872246696,
|
|
"grad_norm": 24.465240478515625,
|
|
"learning_rate": 5.363104864490034e-08,
|
|
"logits/chosen": -1.6065510511398315,
|
|
"logits/rejected": -1.269682765007019,
|
|
"logps/chosen": -128.99668884277344,
|
|
"logps/ref_chosen": -102.84449005126953,
|
|
"logps/ref_rejected": -108.57992553710938,
|
|
"logps/rejected": -163.40673828125,
|
|
"loss": 0.5215,
|
|
"margin_dpo/margin_mean": 28.674596786499023,
|
|
"margin_dpo/margin_std": 17.37641143798828,
|
|
"step": 552
|
|
},
|
|
{
|
|
"epoch": 0.8120411160058737,
|
|
"grad_norm": 26.768144607543945,
|
|
"learning_rate": 5.2839396041230415e-08,
|
|
"logits/chosen": -0.9789568185806274,
|
|
"logits/rejected": -1.1432013511657715,
|
|
"logps/chosen": -97.16468811035156,
|
|
"logps/ref_chosen": -69.79757690429688,
|
|
"logps/ref_rejected": -102.5894775390625,
|
|
"logps/rejected": -149.51959228515625,
|
|
"loss": 0.5763,
|
|
"margin_dpo/margin_mean": 19.5629940032959,
|
|
"margin_dpo/margin_std": 16.6281795501709,
|
|
"step": 553
|
|
},
|
|
{
|
|
"epoch": 0.8135095447870778,
|
|
"grad_norm": 28.801767349243164,
|
|
"learning_rate": 5.205293880283551e-08,
|
|
"logits/chosen": -1.2124251127243042,
|
|
"logits/rejected": -0.50566166639328,
|
|
"logps/chosen": -139.305419921875,
|
|
"logps/ref_chosen": -111.99177551269531,
|
|
"logps/ref_rejected": -89.77027893066406,
|
|
"logps/rejected": -138.11862182617188,
|
|
"loss": 0.5706,
|
|
"margin_dpo/margin_mean": 21.034685134887695,
|
|
"margin_dpo/margin_std": 11.804601669311523,
|
|
"step": 554
|
|
},
|
|
{
|
|
"epoch": 0.8149779735682819,
|
|
"grad_norm": 28.282394409179688,
|
|
"learning_rate": 5.127169765359515e-08,
|
|
"logits/chosen": -1.374567985534668,
|
|
"logits/rejected": -1.305861473083496,
|
|
"logps/chosen": -106.81584167480469,
|
|
"logps/ref_chosen": -75.38294982910156,
|
|
"logps/ref_rejected": -92.58414459228516,
|
|
"logps/rejected": -148.74456787109375,
|
|
"loss": 0.6231,
|
|
"margin_dpo/margin_mean": 24.7275390625,
|
|
"margin_dpo/margin_std": 17.48480796813965,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.8164464023494861,
|
|
"grad_norm": 28.61037826538086,
|
|
"learning_rate": 5.049569317994012e-08,
|
|
"logits/chosen": -1.6289178133010864,
|
|
"logits/rejected": -0.9854074716567993,
|
|
"logps/chosen": -105.55892944335938,
|
|
"logps/ref_chosen": -79.32157897949219,
|
|
"logps/ref_rejected": -82.64404296875,
|
|
"logps/rejected": -131.9959716796875,
|
|
"loss": 0.553,
|
|
"margin_dpo/margin_mean": 23.114587783813477,
|
|
"margin_dpo/margin_std": 16.753026962280273,
|
|
"step": 556
|
|
},
|
|
{
|
|
"epoch": 0.8179148311306902,
|
|
"grad_norm": 26.416162490844727,
|
|
"learning_rate": 4.9724945830310144e-08,
|
|
"logits/chosen": -1.8984003067016602,
|
|
"logits/rejected": -1.9594836235046387,
|
|
"logps/chosen": -124.50114440917969,
|
|
"logps/ref_chosen": -98.14947509765625,
|
|
"logps/ref_rejected": -123.18280029296875,
|
|
"logps/rejected": -170.90692138671875,
|
|
"loss": 0.6707,
|
|
"margin_dpo/margin_mean": 21.37244415283203,
|
|
"margin_dpo/margin_std": 20.823997497558594,
|
|
"step": 557
|
|
},
|
|
{
|
|
"epoch": 0.8193832599118943,
|
|
"grad_norm": 21.28540802001953,
|
|
"learning_rate": 4.8959475914614554e-08,
|
|
"logits/chosen": -2.3782596588134766,
|
|
"logits/rejected": -1.9224114418029785,
|
|
"logps/chosen": -137.03941345214844,
|
|
"logps/ref_chosen": -108.68679809570312,
|
|
"logps/ref_rejected": -86.3449935913086,
|
|
"logps/rejected": -133.84918212890625,
|
|
"loss": 0.4902,
|
|
"margin_dpo/margin_mean": 19.151586532592773,
|
|
"margin_dpo/margin_std": 13.445259094238281,
|
|
"step": 558
|
|
},
|
|
{
|
|
"epoch": 0.8208516886930984,
|
|
"grad_norm": 25.445751190185547,
|
|
"learning_rate": 4.8199303603697614e-08,
|
|
"logits/chosen": -2.1156458854675293,
|
|
"logits/rejected": -1.5914777517318726,
|
|
"logps/chosen": -136.66253662109375,
|
|
"logps/ref_chosen": -112.66477966308594,
|
|
"logps/ref_rejected": -94.24748992919922,
|
|
"logps/rejected": -147.06002807617188,
|
|
"loss": 0.596,
|
|
"margin_dpo/margin_mean": 28.814769744873047,
|
|
"margin_dpo/margin_std": 15.901989936828613,
|
|
"step": 559
|
|
},
|
|
{
|
|
"epoch": 0.8223201174743024,
|
|
"grad_norm": 25.374601364135742,
|
|
"learning_rate": 4.7444448928806615e-08,
|
|
"logits/chosen": -0.29492858052253723,
|
|
"logits/rejected": -0.17774538695812225,
|
|
"logps/chosen": -104.58026123046875,
|
|
"logps/ref_chosen": -75.38302612304688,
|
|
"logps/ref_rejected": -108.84716033935547,
|
|
"logps/rejected": -159.85110473632812,
|
|
"loss": 0.6773,
|
|
"margin_dpo/margin_mean": 21.806716918945312,
|
|
"margin_dpo/margin_std": 23.361881256103516,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.8237885462555066,
|
|
"grad_norm": 24.0804500579834,
|
|
"learning_rate": 4.669493178106432e-08,
|
|
"logits/chosen": -1.1828765869140625,
|
|
"logits/rejected": -0.6570239067077637,
|
|
"logps/chosen": -121.85696411132812,
|
|
"logps/ref_chosen": -90.67036437988281,
|
|
"logps/ref_rejected": -68.84110260009766,
|
|
"logps/rejected": -123.22027587890625,
|
|
"loss": 0.635,
|
|
"margin_dpo/margin_mean": 23.192569732666016,
|
|
"margin_dpo/margin_std": 19.936168670654297,
|
|
"step": 561
|
|
},
|
|
{
|
|
"epoch": 0.8252569750367107,
|
|
"grad_norm": 28.623416900634766,
|
|
"learning_rate": 4.5950771910944596e-08,
|
|
"logits/chosen": -1.246302843093872,
|
|
"logits/rejected": -1.3074339628219604,
|
|
"logps/chosen": -96.68048858642578,
|
|
"logps/ref_chosen": -70.02510070800781,
|
|
"logps/ref_rejected": -93.74087524414062,
|
|
"logps/rejected": -141.97848510742188,
|
|
"loss": 0.7298,
|
|
"margin_dpo/margin_mean": 21.58222770690918,
|
|
"margin_dpo/margin_std": 18.971715927124023,
|
|
"step": 562
|
|
},
|
|
{
|
|
"epoch": 0.8267254038179148,
|
|
"grad_norm": 37.866973876953125,
|
|
"learning_rate": 4.521198892775202e-08,
|
|
"logits/chosen": -1.2940800189971924,
|
|
"logits/rejected": -0.7970089912414551,
|
|
"logps/chosen": -156.23606872558594,
|
|
"logps/ref_chosen": -124.02999877929688,
|
|
"logps/ref_rejected": -96.42265319824219,
|
|
"logps/rejected": -141.616455078125,
|
|
"loss": 0.7532,
|
|
"margin_dpo/margin_mean": 12.987737655639648,
|
|
"margin_dpo/margin_std": 17.41985511779785,
|
|
"step": 563
|
|
},
|
|
{
|
|
"epoch": 0.8281938325991189,
|
|
"grad_norm": 26.319257736206055,
|
|
"learning_rate": 4.447860229910544e-08,
|
|
"logits/chosen": -1.7149813175201416,
|
|
"logits/rejected": -1.1024806499481201,
|
|
"logps/chosen": -169.05923461914062,
|
|
"logps/ref_chosen": -141.51185607910156,
|
|
"logps/ref_rejected": -112.57762145996094,
|
|
"logps/rejected": -155.19082641601562,
|
|
"loss": 0.628,
|
|
"margin_dpo/margin_mean": 15.065828323364258,
|
|
"margin_dpo/margin_std": 12.99314022064209,
|
|
"step": 564
|
|
},
|
|
{
|
|
"epoch": 0.8296622613803231,
|
|
"grad_norm": 32.7898063659668,
|
|
"learning_rate": 4.375063135042445e-08,
|
|
"logits/chosen": -1.7462910413742065,
|
|
"logits/rejected": -0.5241377353668213,
|
|
"logps/chosen": -139.33755493164062,
|
|
"logps/ref_chosen": -113.30201721191406,
|
|
"logps/ref_rejected": -92.07376098632812,
|
|
"logps/rejected": -143.7384490966797,
|
|
"loss": 0.5127,
|
|
"margin_dpo/margin_mean": 25.629154205322266,
|
|
"margin_dpo/margin_std": 19.308208465576172,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.8311306901615272,
|
|
"grad_norm": 19.79732322692871,
|
|
"learning_rate": 4.3028095264420525e-08,
|
|
"logits/chosen": -1.873335361480713,
|
|
"logits/rejected": -1.0767878293991089,
|
|
"logps/chosen": -168.4169921875,
|
|
"logps/ref_chosen": -147.71287536621094,
|
|
"logps/ref_rejected": -123.13082885742188,
|
|
"logps/rejected": -171.9524383544922,
|
|
"loss": 0.5636,
|
|
"margin_dpo/margin_mean": 28.117481231689453,
|
|
"margin_dpo/margin_std": 14.197080612182617,
|
|
"step": 566
|
|
},
|
|
{
|
|
"epoch": 0.8325991189427313,
|
|
"grad_norm": 28.859512329101562,
|
|
"learning_rate": 4.231101308059165e-08,
|
|
"logits/chosen": -1.336463451385498,
|
|
"logits/rejected": -1.3242238759994507,
|
|
"logps/chosen": -129.6639404296875,
|
|
"logps/ref_chosen": -103.08148193359375,
|
|
"logps/ref_rejected": -135.55751037597656,
|
|
"logps/rejected": -185.67556762695312,
|
|
"loss": 0.6424,
|
|
"margin_dpo/margin_mean": 23.53559112548828,
|
|
"margin_dpo/margin_std": 18.061981201171875,
|
|
"step": 567
|
|
},
|
|
{
|
|
"epoch": 0.8340675477239354,
|
|
"grad_norm": 22.684192657470703,
|
|
"learning_rate": 4.1599403694720145e-08,
|
|
"logits/chosen": -2.0500826835632324,
|
|
"logits/rejected": -1.7743902206420898,
|
|
"logps/chosen": -101.66519165039062,
|
|
"logps/ref_chosen": -79.48320007324219,
|
|
"logps/ref_rejected": -91.81827545166016,
|
|
"logps/rejected": -135.9856414794922,
|
|
"loss": 0.5057,
|
|
"margin_dpo/margin_mean": 21.98537254333496,
|
|
"margin_dpo/margin_std": 21.847187042236328,
|
|
"step": 568
|
|
},
|
|
{
|
|
"epoch": 0.8355359765051396,
|
|
"grad_norm": 33.8665771484375,
|
|
"learning_rate": 4.089328585837512e-08,
|
|
"logits/chosen": -1.0648646354675293,
|
|
"logits/rejected": -0.8371211886405945,
|
|
"logps/chosen": -112.8988265991211,
|
|
"logps/ref_chosen": -84.46160888671875,
|
|
"logps/ref_rejected": -88.43289947509766,
|
|
"logps/rejected": -136.52499389648438,
|
|
"loss": 0.8075,
|
|
"margin_dpo/margin_mean": 19.654869079589844,
|
|
"margin_dpo/margin_std": 22.281082153320312,
|
|
"step": 569
|
|
},
|
|
{
|
|
"epoch": 0.8370044052863436,
|
|
"grad_norm": 21.87103843688965,
|
|
"learning_rate": 4.019267817841834e-08,
|
|
"logits/chosen": -2.1644656658172607,
|
|
"logits/rejected": -1.2819344997406006,
|
|
"logps/chosen": -136.2928924560547,
|
|
"logps/ref_chosen": -110.28079986572266,
|
|
"logps/ref_rejected": -89.88323211669922,
|
|
"logps/rejected": -141.99786376953125,
|
|
"loss": 0.5036,
|
|
"margin_dpo/margin_mean": 26.1025390625,
|
|
"margin_dpo/margin_std": 18.09876251220703,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.8384728340675477,
|
|
"grad_norm": 24.671363830566406,
|
|
"learning_rate": 3.9497599116513705e-08,
|
|
"logits/chosen": -1.0694646835327148,
|
|
"logits/rejected": -1.1418089866638184,
|
|
"logps/chosen": -89.88897705078125,
|
|
"logps/ref_chosen": -63.78746032714844,
|
|
"logps/ref_rejected": -87.41693115234375,
|
|
"logps/rejected": -134.61260986328125,
|
|
"loss": 0.516,
|
|
"margin_dpo/margin_mean": 21.094158172607422,
|
|
"margin_dpo/margin_std": 17.385196685791016,
|
|
"step": 571
|
|
},
|
|
{
|
|
"epoch": 0.8399412628487518,
|
|
"grad_norm": 24.654348373413086,
|
|
"learning_rate": 3.880806698864086e-08,
|
|
"logits/chosen": -2.017566204071045,
|
|
"logits/rejected": -2.3396568298339844,
|
|
"logps/chosen": -114.6158676147461,
|
|
"logps/ref_chosen": -81.1136245727539,
|
|
"logps/ref_rejected": -104.74822998046875,
|
|
"logps/rejected": -151.9698486328125,
|
|
"loss": 0.6028,
|
|
"margin_dpo/margin_mean": 13.719372749328613,
|
|
"margin_dpo/margin_std": 16.3624267578125,
|
|
"step": 572
|
|
},
|
|
{
|
|
"epoch": 0.8414096916299559,
|
|
"grad_norm": 24.63699722290039,
|
|
"learning_rate": 3.812409996461275e-08,
|
|
"logits/chosen": -1.3356173038482666,
|
|
"logits/rejected": -1.1175293922424316,
|
|
"logps/chosen": -92.43038177490234,
|
|
"logps/ref_chosen": -62.825836181640625,
|
|
"logps/ref_rejected": -88.45487213134766,
|
|
"logps/rejected": -143.63778686523438,
|
|
"loss": 0.6495,
|
|
"margin_dpo/margin_mean": 25.578380584716797,
|
|
"margin_dpo/margin_std": 18.0982723236084,
|
|
"step": 573
|
|
},
|
|
{
|
|
"epoch": 0.8428781204111601,
|
|
"grad_norm": 25.474008560180664,
|
|
"learning_rate": 3.74457160675965e-08,
|
|
"logits/chosen": -1.7099212408065796,
|
|
"logits/rejected": -1.0368297100067139,
|
|
"logps/chosen": -99.78987884521484,
|
|
"logps/ref_chosen": -77.86640930175781,
|
|
"logps/ref_rejected": -100.60933685302734,
|
|
"logps/rejected": -152.71780395507812,
|
|
"loss": 0.5408,
|
|
"margin_dpo/margin_mean": 30.184980392456055,
|
|
"margin_dpo/margin_std": 17.348926544189453,
|
|
"step": 574
|
|
},
|
|
{
|
|
"epoch": 0.8443465491923642,
|
|
"grad_norm": 36.82394790649414,
|
|
"learning_rate": 3.677293317363864e-08,
|
|
"logits/chosen": -1.2702062129974365,
|
|
"logits/rejected": -0.623336911201477,
|
|
"logps/chosen": -137.8572998046875,
|
|
"logps/ref_chosen": -113.77069091796875,
|
|
"logps/ref_rejected": -102.4068603515625,
|
|
"logps/rejected": -152.94381713867188,
|
|
"loss": 0.6186,
|
|
"margin_dpo/margin_mean": 26.45035743713379,
|
|
"margin_dpo/margin_std": 12.797685623168945,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.8458149779735683,
|
|
"grad_norm": 27.242877960205078,
|
|
"learning_rate": 3.6105769011194224e-08,
|
|
"logits/chosen": -1.1757344007492065,
|
|
"logits/rejected": -1.024355411529541,
|
|
"logps/chosen": -103.07350158691406,
|
|
"logps/ref_chosen": -73.22354125976562,
|
|
"logps/ref_rejected": -77.03753662109375,
|
|
"logps/rejected": -130.0419921875,
|
|
"loss": 0.728,
|
|
"margin_dpo/margin_mean": 23.154489517211914,
|
|
"margin_dpo/margin_std": 16.945371627807617,
|
|
"step": 576
|
|
},
|
|
{
|
|
"epoch": 0.8472834067547724,
|
|
"grad_norm": 26.261314392089844,
|
|
"learning_rate": 3.5444241160659304e-08,
|
|
"logits/chosen": -2.1024394035339355,
|
|
"logits/rejected": -1.363139033317566,
|
|
"logps/chosen": -132.17788696289062,
|
|
"logps/ref_chosen": -111.53047943115234,
|
|
"logps/ref_rejected": -100.61012268066406,
|
|
"logps/rejected": -150.16122436523438,
|
|
"loss": 0.57,
|
|
"margin_dpo/margin_mean": 28.903701782226562,
|
|
"margin_dpo/margin_std": 21.926589965820312,
|
|
"step": 577
|
|
},
|
|
{
|
|
"epoch": 0.8487518355359766,
|
|
"grad_norm": 28.919174194335938,
|
|
"learning_rate": 3.478836705390808e-08,
|
|
"logits/chosen": -0.9342671036720276,
|
|
"logits/rejected": -1.0065442323684692,
|
|
"logps/chosen": -97.9559326171875,
|
|
"logps/ref_chosen": -69.15601348876953,
|
|
"logps/ref_rejected": -101.06340789794922,
|
|
"logps/rejected": -147.69058227539062,
|
|
"loss": 0.6897,
|
|
"margin_dpo/margin_mean": 17.827260971069336,
|
|
"margin_dpo/margin_std": 14.1982421875,
|
|
"step": 578
|
|
},
|
|
{
|
|
"epoch": 0.8502202643171806,
|
|
"grad_norm": 23.156034469604492,
|
|
"learning_rate": 3.41381639738331e-08,
|
|
"logits/chosen": -2.2614612579345703,
|
|
"logits/rejected": -1.3790278434753418,
|
|
"logps/chosen": -122.92021179199219,
|
|
"logps/ref_chosen": -94.19979858398438,
|
|
"logps/ref_rejected": -71.94280242919922,
|
|
"logps/rejected": -119.78824615478516,
|
|
"loss": 0.5835,
|
|
"margin_dpo/margin_mean": 19.12502670288086,
|
|
"margin_dpo/margin_std": 16.983705520629883,
|
|
"step": 579
|
|
},
|
|
{
|
|
"epoch": 0.8516886930983847,
|
|
"grad_norm": 22.19964027404785,
|
|
"learning_rate": 3.349364905389032e-08,
|
|
"logits/chosen": -0.7070454359054565,
|
|
"logits/rejected": -0.4206308126449585,
|
|
"logps/chosen": -143.45135498046875,
|
|
"logps/ref_chosen": -115.61140441894531,
|
|
"logps/ref_rejected": -82.69953155517578,
|
|
"logps/rejected": -134.01918029785156,
|
|
"loss": 0.5858,
|
|
"margin_dpo/margin_mean": 23.47968864440918,
|
|
"margin_dpo/margin_std": 18.69431495666504,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.8531571218795888,
|
|
"grad_norm": 27.800729751586914,
|
|
"learning_rate": 3.285483927764726e-08,
|
|
"logits/chosen": -1.2207493782043457,
|
|
"logits/rejected": -1.4801889657974243,
|
|
"logps/chosen": -103.3560562133789,
|
|
"logps/ref_chosen": -75.1635971069336,
|
|
"logps/ref_rejected": -101.49029541015625,
|
|
"logps/rejected": -147.3246612548828,
|
|
"loss": 0.6392,
|
|
"margin_dpo/margin_mean": 17.64190101623535,
|
|
"margin_dpo/margin_std": 13.56999397277832,
|
|
"step": 581
|
|
},
|
|
{
|
|
"epoch": 0.8546255506607929,
|
|
"grad_norm": 26.177412033081055,
|
|
"learning_rate": 3.222175147833556e-08,
|
|
"logits/chosen": -1.5780534744262695,
|
|
"logits/rejected": -1.2352222204208374,
|
|
"logps/chosen": -126.73898315429688,
|
|
"logps/ref_chosen": -107.35800170898438,
|
|
"logps/ref_rejected": -118.97886657714844,
|
|
"logps/rejected": -165.69467163085938,
|
|
"loss": 0.6833,
|
|
"margin_dpo/margin_mean": 27.334815979003906,
|
|
"margin_dpo/margin_std": 18.021024703979492,
|
|
"step": 582
|
|
},
|
|
{
|
|
"epoch": 0.856093979441997,
|
|
"grad_norm": 33.17780685424805,
|
|
"learning_rate": 3.159440233840763e-08,
|
|
"logits/chosen": -1.4593960046768188,
|
|
"logits/rejected": -1.1161738634109497,
|
|
"logps/chosen": -130.53958129882812,
|
|
"logps/ref_chosen": -96.88568115234375,
|
|
"logps/ref_rejected": -84.66061401367188,
|
|
"logps/rejected": -130.51904296875,
|
|
"loss": 0.6948,
|
|
"margin_dpo/margin_mean": 12.204545974731445,
|
|
"margin_dpo/margin_std": 18.54879379272461,
|
|
"step": 583
|
|
},
|
|
{
|
|
"epoch": 0.8575624082232012,
|
|
"grad_norm": 31.337125778198242,
|
|
"learning_rate": 3.0972808389096635e-08,
|
|
"logits/chosen": -1.4248698949813843,
|
|
"logits/rejected": -0.8519065380096436,
|
|
"logps/chosen": -122.12081146240234,
|
|
"logps/ref_chosen": -99.00904083251953,
|
|
"logps/ref_rejected": -111.17665100097656,
|
|
"logps/rejected": -155.17088317871094,
|
|
"loss": 0.6102,
|
|
"margin_dpo/margin_mean": 20.88245964050293,
|
|
"margin_dpo/margin_std": 11.362781524658203,
|
|
"step": 584
|
|
},
|
|
{
|
|
"epoch": 0.8590308370044053,
|
|
"grad_norm": 22.652122497558594,
|
|
"learning_rate": 3.035698600998121e-08,
|
|
"logits/chosen": -1.6795928478240967,
|
|
"logits/rejected": -1.4183259010314941,
|
|
"logps/chosen": -96.15231323242188,
|
|
"logps/ref_chosen": -67.79874420166016,
|
|
"logps/ref_rejected": -71.48472595214844,
|
|
"logps/rejected": -123.78256225585938,
|
|
"loss": 0.5842,
|
|
"margin_dpo/margin_mean": 23.94426727294922,
|
|
"margin_dpo/margin_std": 15.578774452209473,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.8604992657856094,
|
|
"grad_norm": 25.997224807739258,
|
|
"learning_rate": 2.974695142855388e-08,
|
|
"logits/chosen": -1.0662152767181396,
|
|
"logits/rejected": -1.0517430305480957,
|
|
"logps/chosen": -92.78762817382812,
|
|
"logps/ref_chosen": -58.51750183105469,
|
|
"logps/ref_rejected": -81.29431915283203,
|
|
"logps/rejected": -133.53366088867188,
|
|
"loss": 0.6058,
|
|
"margin_dpo/margin_mean": 17.969219207763672,
|
|
"margin_dpo/margin_std": 19.24755859375,
|
|
"step": 586
|
|
},
|
|
{
|
|
"epoch": 0.8619676945668135,
|
|
"grad_norm": 24.352365493774414,
|
|
"learning_rate": 2.9142720719793122e-08,
|
|
"logits/chosen": -2.5131826400756836,
|
|
"logits/rejected": -2.127216100692749,
|
|
"logps/chosen": -91.87644958496094,
|
|
"logps/ref_chosen": -64.46495056152344,
|
|
"logps/ref_rejected": -68.1041259765625,
|
|
"logps/rejected": -117.914306640625,
|
|
"loss": 0.7339,
|
|
"margin_dpo/margin_mean": 22.398693084716797,
|
|
"margin_dpo/margin_std": 17.78519058227539,
|
|
"step": 587
|
|
},
|
|
{
|
|
"epoch": 0.8634361233480177,
|
|
"grad_norm": 39.026004791259766,
|
|
"learning_rate": 2.8544309805740018e-08,
|
|
"logits/chosen": -1.4373981952667236,
|
|
"logits/rejected": -1.0331122875213623,
|
|
"logps/chosen": -105.52696228027344,
|
|
"logps/ref_chosen": -77.16580200195312,
|
|
"logps/ref_rejected": -97.94784545898438,
|
|
"logps/rejected": -149.39785766601562,
|
|
"loss": 0.7691,
|
|
"margin_dpo/margin_mean": 23.088871002197266,
|
|
"margin_dpo/margin_std": 21.23520278930664,
|
|
"step": 588
|
|
},
|
|
{
|
|
"epoch": 0.8649045521292217,
|
|
"grad_norm": 25.116107940673828,
|
|
"learning_rate": 2.7951734455078786e-08,
|
|
"logits/chosen": -1.8460958003997803,
|
|
"logits/rejected": -1.2316406965255737,
|
|
"logps/chosen": -132.01768493652344,
|
|
"logps/ref_chosen": -109.15735626220703,
|
|
"logps/ref_rejected": -103.8968505859375,
|
|
"logps/rejected": -153.23219299316406,
|
|
"loss": 0.4628,
|
|
"margin_dpo/margin_mean": 26.475025177001953,
|
|
"margin_dpo/margin_std": 19.995899200439453,
|
|
"step": 589
|
|
},
|
|
{
|
|
"epoch": 0.8663729809104258,
|
|
"grad_norm": 20.96693229675293,
|
|
"learning_rate": 2.736501028272095e-08,
|
|
"logits/chosen": -1.6605994701385498,
|
|
"logits/rejected": -1.5838966369628906,
|
|
"logps/chosen": -100.31023406982422,
|
|
"logps/ref_chosen": -73.52894592285156,
|
|
"logps/ref_rejected": -102.6637954711914,
|
|
"logps/rejected": -153.43533325195312,
|
|
"loss": 0.4637,
|
|
"margin_dpo/margin_mean": 23.990245819091797,
|
|
"margin_dpo/margin_std": 23.85308837890625,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.8678414096916299,
|
|
"grad_norm": 24.580368041992188,
|
|
"learning_rate": 2.678415274939408e-08,
|
|
"logits/chosen": -0.8293085694313049,
|
|
"logits/rejected": -0.7269066572189331,
|
|
"logps/chosen": -104.53363037109375,
|
|
"logps/ref_chosen": -81.24513244628906,
|
|
"logps/ref_rejected": -104.93965148925781,
|
|
"logps/rejected": -156.94107055664062,
|
|
"loss": 0.4883,
|
|
"margin_dpo/margin_mean": 28.712928771972656,
|
|
"margin_dpo/margin_std": 19.021089553833008,
|
|
"step": 591
|
|
},
|
|
{
|
|
"epoch": 0.869309838472834,
|
|
"grad_norm": 26.056900024414062,
|
|
"learning_rate": 2.6209177161234442e-08,
|
|
"logits/chosen": -1.4577271938323975,
|
|
"logits/rejected": -1.3645278215408325,
|
|
"logps/chosen": -106.28244018554688,
|
|
"logps/ref_chosen": -71.50765991210938,
|
|
"logps/ref_rejected": -85.3493423461914,
|
|
"logps/rejected": -132.59182739257812,
|
|
"loss": 0.6852,
|
|
"margin_dpo/margin_mean": 12.467700958251953,
|
|
"margin_dpo/margin_std": 20.66848373413086,
|
|
"step": 592
|
|
},
|
|
{
|
|
"epoch": 0.8707782672540382,
|
|
"grad_norm": 27.79155731201172,
|
|
"learning_rate": 2.564009866938349e-08,
|
|
"logits/chosen": -1.6856143474578857,
|
|
"logits/rejected": -1.3747767210006714,
|
|
"logps/chosen": -99.52073669433594,
|
|
"logps/ref_chosen": -70.61467742919922,
|
|
"logps/ref_rejected": -89.37353515625,
|
|
"logps/rejected": -134.16650390625,
|
|
"loss": 0.7436,
|
|
"margin_dpo/margin_mean": 15.886906623840332,
|
|
"margin_dpo/margin_std": 14.40092945098877,
|
|
"step": 593
|
|
},
|
|
{
|
|
"epoch": 0.8722466960352423,
|
|
"grad_norm": 22.358192443847656,
|
|
"learning_rate": 2.5076932269588708e-08,
|
|
"logits/chosen": -1.6691186428070068,
|
|
"logits/rejected": -1.2223987579345703,
|
|
"logps/chosen": -120.62618255615234,
|
|
"logps/ref_chosen": -91.4361801147461,
|
|
"logps/ref_rejected": -74.34607696533203,
|
|
"logps/rejected": -124.12226104736328,
|
|
"loss": 0.5016,
|
|
"margin_dpo/margin_mean": 20.586183547973633,
|
|
"margin_dpo/margin_std": 17.020692825317383,
|
|
"step": 594
|
|
},
|
|
{
|
|
"epoch": 0.8737151248164464,
|
|
"grad_norm": 27.895854949951172,
|
|
"learning_rate": 2.451969280180849e-08,
|
|
"logits/chosen": -2.383293628692627,
|
|
"logits/rejected": -2.2450737953186035,
|
|
"logps/chosen": -78.96890258789062,
|
|
"logps/ref_chosen": -50.59502410888672,
|
|
"logps/ref_rejected": -65.34422302246094,
|
|
"logps/rejected": -113.44318389892578,
|
|
"loss": 0.7549,
|
|
"margin_dpo/margin_mean": 19.725078582763672,
|
|
"margin_dpo/margin_std": 20.610492706298828,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.8751835535976505,
|
|
"grad_norm": 27.49303436279297,
|
|
"learning_rate": 2.396839494982103e-08,
|
|
"logits/chosen": -0.7340579032897949,
|
|
"logits/rejected": -0.5988126993179321,
|
|
"logps/chosen": -124.57491302490234,
|
|
"logps/ref_chosen": -95.0877456665039,
|
|
"logps/ref_rejected": -117.5943374633789,
|
|
"logps/rejected": -172.03057861328125,
|
|
"loss": 0.6443,
|
|
"margin_dpo/margin_mean": 24.949066162109375,
|
|
"margin_dpo/margin_std": 21.884986877441406,
|
|
"step": 596
|
|
},
|
|
{
|
|
"epoch": 0.8766519823788547,
|
|
"grad_norm": 25.584749221801758,
|
|
"learning_rate": 2.3423053240837514e-08,
|
|
"logits/chosen": -0.8481107354164124,
|
|
"logits/rejected": -0.7759636640548706,
|
|
"logps/chosen": -116.6168212890625,
|
|
"logps/ref_chosen": -87.56103515625,
|
|
"logps/ref_rejected": -105.85301971435547,
|
|
"logps/rejected": -152.90264892578125,
|
|
"loss": 0.5819,
|
|
"margin_dpo/margin_mean": 17.993831634521484,
|
|
"margin_dpo/margin_std": 20.48471450805664,
|
|
"step": 597
|
|
},
|
|
{
|
|
"epoch": 0.8781204111600588,
|
|
"grad_norm": 29.62306785583496,
|
|
"learning_rate": 2.2883682045119062e-08,
|
|
"logits/chosen": -3.3226518630981445,
|
|
"logits/rejected": -2.930318832397461,
|
|
"logps/chosen": -161.9443359375,
|
|
"logps/ref_chosen": -132.37234497070312,
|
|
"logps/ref_rejected": -126.33038330078125,
|
|
"logps/rejected": -172.46737670898438,
|
|
"loss": 0.7032,
|
|
"margin_dpo/margin_mean": 16.564979553222656,
|
|
"margin_dpo/margin_std": 17.32196044921875,
|
|
"step": 598
|
|
},
|
|
{
|
|
"epoch": 0.8795888399412628,
|
|
"grad_norm": 25.984834671020508,
|
|
"learning_rate": 2.2350295575598367e-08,
|
|
"logits/chosen": -1.3305424451828003,
|
|
"logits/rejected": -1.1590290069580078,
|
|
"logps/chosen": -109.2089614868164,
|
|
"logps/ref_chosen": -80.37190246582031,
|
|
"logps/ref_rejected": -98.19518280029297,
|
|
"logps/rejected": -146.1300048828125,
|
|
"loss": 0.5828,
|
|
"margin_dpo/margin_mean": 19.097774505615234,
|
|
"margin_dpo/margin_std": 16.6318302154541,
|
|
"step": 599
|
|
},
|
|
{
|
|
"epoch": 0.8810572687224669,
|
|
"grad_norm": 20.932893753051758,
|
|
"learning_rate": 2.1822907887504932e-08,
|
|
"logits/chosen": -1.6403688192367554,
|
|
"logits/rejected": -1.5137927532196045,
|
|
"logps/chosen": -102.33708190917969,
|
|
"logps/ref_chosen": -73.66311645507812,
|
|
"logps/ref_rejected": -88.36824035644531,
|
|
"logps/rejected": -137.10244750976562,
|
|
"loss": 0.5423,
|
|
"margin_dpo/margin_mean": 20.060239791870117,
|
|
"margin_dpo/margin_std": 17.122276306152344,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8810572687224669,
|
|
"eval_logits/chosen": -1.4537636041641235,
|
|
"eval_logits/rejected": -1.1605932712554932,
|
|
"eval_logps/chosen": -132.64610290527344,
|
|
"eval_logps/ref_chosen": -101.88616943359375,
|
|
"eval_logps/ref_rejected": -92.68607330322266,
|
|
"eval_logps/rejected": -139.3175048828125,
|
|
"eval_loss": 0.41954532265663147,
|
|
"eval_margin_dpo/margin_mean": 15.871517181396484,
|
|
"eval_margin_dpo/margin_std": 17.077098846435547,
|
|
"eval_runtime": 44.0423,
|
|
"eval_samples_per_second": 53.108,
|
|
"eval_steps_per_second": 1.68,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.882525697503671,
|
|
"grad_norm": 26.41422462463379,
|
|
"learning_rate": 2.1301532877994742e-08,
|
|
"logits/chosen": -1.7634140253067017,
|
|
"logits/rejected": -1.8990434408187866,
|
|
"logps/chosen": -104.30206298828125,
|
|
"logps/ref_chosen": -75.11897277832031,
|
|
"logps/ref_rejected": -110.72828674316406,
|
|
"logps/rejected": -170.05686950683594,
|
|
"loss": 0.5885,
|
|
"margin_dpo/margin_mean": 30.145484924316406,
|
|
"margin_dpo/margin_std": 24.454891204833984,
|
|
"step": 601
|
|
},
|
|
{
|
|
"epoch": 0.8839941262848752,
|
|
"grad_norm": 24.838815689086914,
|
|
"learning_rate": 2.0786184285784298e-08,
|
|
"logits/chosen": -1.9122300148010254,
|
|
"logits/rejected": -1.8413580656051636,
|
|
"logps/chosen": -94.12992858886719,
|
|
"logps/ref_chosen": -65.56060791015625,
|
|
"logps/ref_rejected": -70.72991943359375,
|
|
"logps/rejected": -113.13572692871094,
|
|
"loss": 0.5053,
|
|
"margin_dpo/margin_mean": 13.836491584777832,
|
|
"margin_dpo/margin_std": 8.537615776062012,
|
|
"step": 602
|
|
},
|
|
{
|
|
"epoch": 0.8854625550660793,
|
|
"grad_norm": 21.262638092041016,
|
|
"learning_rate": 2.0276875690788204e-08,
|
|
"logits/chosen": -1.533982753753662,
|
|
"logits/rejected": -0.9280251264572144,
|
|
"logps/chosen": -142.13864135742188,
|
|
"logps/ref_chosen": -122.12416076660156,
|
|
"logps/ref_rejected": -129.0844268798828,
|
|
"logps/rejected": -173.14385986328125,
|
|
"loss": 0.4995,
|
|
"margin_dpo/margin_mean": 24.044979095458984,
|
|
"margin_dpo/margin_std": 16.012496948242188,
|
|
"step": 603
|
|
},
|
|
{
|
|
"epoch": 0.8869309838472834,
|
|
"grad_norm": 23.620716094970703,
|
|
"learning_rate": 1.977362051376158e-08,
|
|
"logits/chosen": -2.0624282360076904,
|
|
"logits/rejected": -1.6026830673217773,
|
|
"logps/chosen": -123.72084045410156,
|
|
"logps/ref_chosen": -95.43553161621094,
|
|
"logps/ref_rejected": -88.09769439697266,
|
|
"logps/rejected": -132.3470458984375,
|
|
"loss": 0.5316,
|
|
"margin_dpo/margin_mean": 15.964040756225586,
|
|
"margin_dpo/margin_std": 17.116283416748047,
|
|
"step": 604
|
|
},
|
|
{
|
|
"epoch": 0.8883994126284875,
|
|
"grad_norm": 23.886274337768555,
|
|
"learning_rate": 1.9276432015946446e-08,
|
|
"logits/chosen": -0.8116771578788757,
|
|
"logits/rejected": 0.05416472256183624,
|
|
"logps/chosen": -124.10987854003906,
|
|
"logps/ref_chosen": -98.30979919433594,
|
|
"logps/ref_rejected": -61.28790283203125,
|
|
"logps/rejected": -113.97074127197266,
|
|
"loss": 0.6004,
|
|
"margin_dpo/margin_mean": 26.88275146484375,
|
|
"margin_dpo/margin_std": 12.025161743164062,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.8898678414096917,
|
|
"grad_norm": 19.38317108154297,
|
|
"learning_rate": 1.8785323298722093e-08,
|
|
"logits/chosen": -1.7569808959960938,
|
|
"logits/rejected": -1.498138427734375,
|
|
"logps/chosen": -119.4458999633789,
|
|
"logps/ref_chosen": -91.20582580566406,
|
|
"logps/ref_rejected": -121.39325714111328,
|
|
"logps/rejected": -173.896484375,
|
|
"loss": 0.4465,
|
|
"margin_dpo/margin_mean": 24.26314926147461,
|
|
"margin_dpo/margin_std": 23.452638626098633,
|
|
"step": 606
|
|
},
|
|
{
|
|
"epoch": 0.8913362701908958,
|
|
"grad_norm": 31.742523193359375,
|
|
"learning_rate": 1.8300307303259904e-08,
|
|
"logits/chosen": -1.2356622219085693,
|
|
"logits/rejected": -0.7431513071060181,
|
|
"logps/chosen": -117.7818603515625,
|
|
"logps/ref_chosen": -93.03514099121094,
|
|
"logps/ref_rejected": -91.56881713867188,
|
|
"logps/rejected": -143.45266723632812,
|
|
"loss": 0.7087,
|
|
"margin_dpo/margin_mean": 27.137123107910156,
|
|
"margin_dpo/margin_std": 22.96157455444336,
|
|
"step": 607
|
|
},
|
|
{
|
|
"epoch": 0.8928046989720999,
|
|
"grad_norm": 26.023786544799805,
|
|
"learning_rate": 1.7821396810182437e-08,
|
|
"logits/chosen": -1.4785089492797852,
|
|
"logits/rejected": -0.9715176820755005,
|
|
"logps/chosen": -93.41421508789062,
|
|
"logps/ref_chosen": -67.08824157714844,
|
|
"logps/ref_rejected": -64.46246337890625,
|
|
"logps/rejected": -107.28245544433594,
|
|
"loss": 0.6131,
|
|
"margin_dpo/margin_mean": 16.4940128326416,
|
|
"margin_dpo/margin_std": 14.074356079101562,
|
|
"step": 608
|
|
},
|
|
{
|
|
"epoch": 0.8942731277533039,
|
|
"grad_norm": 22.36457061767578,
|
|
"learning_rate": 1.7348604439226617e-08,
|
|
"logits/chosen": -1.7442258596420288,
|
|
"logits/rejected": -1.2550201416015625,
|
|
"logps/chosen": -112.64973449707031,
|
|
"logps/ref_chosen": -87.85757446289062,
|
|
"logps/ref_rejected": -84.98696899414062,
|
|
"logps/rejected": -131.94284057617188,
|
|
"loss": 0.5371,
|
|
"margin_dpo/margin_mean": 22.16372299194336,
|
|
"margin_dpo/margin_std": 17.413898468017578,
|
|
"step": 609
|
|
},
|
|
{
|
|
"epoch": 0.895741556534508,
|
|
"grad_norm": 35.290225982666016,
|
|
"learning_rate": 1.6881942648911074e-08,
|
|
"logits/chosen": -0.8668674230575562,
|
|
"logits/rejected": -0.7067489624023438,
|
|
"logps/chosen": -124.01420593261719,
|
|
"logps/ref_chosen": -96.38543701171875,
|
|
"logps/ref_rejected": -118.17731475830078,
|
|
"logps/rejected": -175.32977294921875,
|
|
"loss": 0.5567,
|
|
"margin_dpo/margin_mean": 29.52370834350586,
|
|
"margin_dpo/margin_std": 20.365798950195312,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.8972099853157122,
|
|
"grad_norm": 24.000062942504883,
|
|
"learning_rate": 1.6421423736208e-08,
|
|
"logits/chosen": -1.1134871244430542,
|
|
"logits/rejected": -1.3368947505950928,
|
|
"logps/chosen": -87.62871551513672,
|
|
"logps/ref_chosen": -52.06251525878906,
|
|
"logps/ref_rejected": -91.46289825439453,
|
|
"logps/rejected": -145.35049438476562,
|
|
"loss": 0.6434,
|
|
"margin_dpo/margin_mean": 18.321395874023438,
|
|
"margin_dpo/margin_std": 21.715795516967773,
|
|
"step": 611
|
|
},
|
|
{
|
|
"epoch": 0.8986784140969163,
|
|
"grad_norm": 28.72694206237793,
|
|
"learning_rate": 1.5967059836219042e-08,
|
|
"logits/chosen": -2.4680771827697754,
|
|
"logits/rejected": -1.2184339761734009,
|
|
"logps/chosen": -129.73422241210938,
|
|
"logps/ref_chosen": -111.08282470703125,
|
|
"logps/ref_rejected": -92.18126678466797,
|
|
"logps/rejected": -140.9920654296875,
|
|
"loss": 0.6102,
|
|
"margin_dpo/margin_mean": 30.159427642822266,
|
|
"margin_dpo/margin_std": 28.102706909179688,
|
|
"step": 612
|
|
},
|
|
{
|
|
"epoch": 0.9001468428781204,
|
|
"grad_norm": 29.352991104125977,
|
|
"learning_rate": 1.551886292185553e-08,
|
|
"logits/chosen": -0.9738931655883789,
|
|
"logits/rejected": -0.4800354242324829,
|
|
"logps/chosen": -123.05023193359375,
|
|
"logps/ref_chosen": -98.03831481933594,
|
|
"logps/ref_rejected": -96.86331176757812,
|
|
"logps/rejected": -153.18783569335938,
|
|
"loss": 0.4778,
|
|
"margin_dpo/margin_mean": 31.312606811523438,
|
|
"margin_dpo/margin_std": 19.18683624267578,
|
|
"step": 613
|
|
},
|
|
{
|
|
"epoch": 0.9016152716593245,
|
|
"grad_norm": 31.17698860168457,
|
|
"learning_rate": 1.507684480352292e-08,
|
|
"logits/chosen": -1.1176828145980835,
|
|
"logits/rejected": -1.1149662733078003,
|
|
"logps/chosen": -106.69200134277344,
|
|
"logps/ref_chosen": -77.41725158691406,
|
|
"logps/ref_rejected": -141.9147186279297,
|
|
"logps/rejected": -185.89691162109375,
|
|
"loss": 0.557,
|
|
"margin_dpo/margin_mean": 14.707442283630371,
|
|
"margin_dpo/margin_std": 17.06017303466797,
|
|
"step": 614
|
|
},
|
|
{
|
|
"epoch": 0.9030837004405287,
|
|
"grad_norm": 23.951841354370117,
|
|
"learning_rate": 1.4641017128809801e-08,
|
|
"logits/chosen": -2.5860671997070312,
|
|
"logits/rejected": -1.4683854579925537,
|
|
"logps/chosen": -186.31454467773438,
|
|
"logps/ref_chosen": -159.53976440429688,
|
|
"logps/ref_rejected": -85.81703186035156,
|
|
"logps/rejected": -133.49378967285156,
|
|
"loss": 0.6647,
|
|
"margin_dpo/margin_mean": 20.90196418762207,
|
|
"margin_dpo/margin_std": 15.42950439453125,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.9045521292217328,
|
|
"grad_norm": 28.593801498413086,
|
|
"learning_rate": 1.4211391382180637e-08,
|
|
"logits/chosen": -1.352514624595642,
|
|
"logits/rejected": -0.6652741432189941,
|
|
"logps/chosen": -113.18240356445312,
|
|
"logps/ref_chosen": -86.95388793945312,
|
|
"logps/ref_rejected": -72.23883819580078,
|
|
"logps/rejected": -125.2229995727539,
|
|
"loss": 0.655,
|
|
"margin_dpo/margin_mean": 26.75563621520996,
|
|
"margin_dpo/margin_std": 24.228565216064453,
|
|
"step": 616
|
|
},
|
|
{
|
|
"epoch": 0.9060205580029369,
|
|
"grad_norm": 32.52434158325195,
|
|
"learning_rate": 1.378797888467345e-08,
|
|
"logits/chosen": -1.0046977996826172,
|
|
"logits/rejected": -0.2724980413913727,
|
|
"logps/chosen": -133.91485595703125,
|
|
"logps/ref_chosen": -105.39964294433594,
|
|
"logps/ref_rejected": -54.019500732421875,
|
|
"logps/rejected": -101.05567932128906,
|
|
"loss": 0.7685,
|
|
"margin_dpo/margin_mean": 18.52096939086914,
|
|
"margin_dpo/margin_std": 16.058685302734375,
|
|
"step": 617
|
|
},
|
|
{
|
|
"epoch": 0.9074889867841409,
|
|
"grad_norm": 33.953426361083984,
|
|
"learning_rate": 1.3370790793601371e-08,
|
|
"logits/chosen": -2.503714084625244,
|
|
"logits/rejected": -2.1493351459503174,
|
|
"logps/chosen": -137.4913330078125,
|
|
"logps/ref_chosen": -111.18899536132812,
|
|
"logps/ref_rejected": -86.93389892578125,
|
|
"logps/rejected": -132.3260498046875,
|
|
"loss": 0.7211,
|
|
"margin_dpo/margin_mean": 19.089811325073242,
|
|
"margin_dpo/margin_std": 17.1854248046875,
|
|
"step": 618
|
|
},
|
|
{
|
|
"epoch": 0.908957415565345,
|
|
"grad_norm": 26.156822204589844,
|
|
"learning_rate": 1.2959838102258535e-08,
|
|
"logits/chosen": -1.3825751543045044,
|
|
"logits/rejected": -0.729312002658844,
|
|
"logps/chosen": -142.80148315429688,
|
|
"logps/ref_chosen": -122.91197204589844,
|
|
"logps/ref_rejected": -108.35417175292969,
|
|
"logps/rejected": -154.7870330810547,
|
|
"loss": 0.6267,
|
|
"margin_dpo/margin_mean": 26.543346405029297,
|
|
"margin_dpo/margin_std": 19.007911682128906,
|
|
"step": 619
|
|
},
|
|
{
|
|
"epoch": 0.9104258443465492,
|
|
"grad_norm": 31.405254364013672,
|
|
"learning_rate": 1.2555131639630567e-08,
|
|
"logits/chosen": -1.6282209157943726,
|
|
"logits/rejected": -1.5203661918640137,
|
|
"logps/chosen": -98.9208984375,
|
|
"logps/ref_chosen": -66.8671875,
|
|
"logps/ref_rejected": -82.39781188964844,
|
|
"logps/rejected": -130.6840362548828,
|
|
"loss": 0.7628,
|
|
"margin_dpo/margin_mean": 16.232501983642578,
|
|
"margin_dpo/margin_std": 18.031766891479492,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.9118942731277533,
|
|
"grad_norm": 27.288387298583984,
|
|
"learning_rate": 1.2156682070109086e-08,
|
|
"logits/chosen": -1.5042550563812256,
|
|
"logits/rejected": -1.1285921335220337,
|
|
"logps/chosen": -122.62663269042969,
|
|
"logps/ref_chosen": -100.42076110839844,
|
|
"logps/ref_rejected": -85.03102111816406,
|
|
"logps/rejected": -124.19369506835938,
|
|
"loss": 0.697,
|
|
"margin_dpo/margin_mean": 16.956809997558594,
|
|
"margin_dpo/margin_std": 20.150236129760742,
|
|
"step": 621
|
|
},
|
|
{
|
|
"epoch": 0.9133627019089574,
|
|
"grad_norm": 22.930315017700195,
|
|
"learning_rate": 1.1764499893210878e-08,
|
|
"logits/chosen": -0.6095631122589111,
|
|
"logits/rejected": 0.05947205424308777,
|
|
"logps/chosen": -109.48207092285156,
|
|
"logps/ref_chosen": -86.09562683105469,
|
|
"logps/ref_rejected": -58.54158020019531,
|
|
"logps/rejected": -106.65364074707031,
|
|
"loss": 0.4756,
|
|
"margin_dpo/margin_mean": 24.725618362426758,
|
|
"margin_dpo/margin_std": 19.953989028930664,
|
|
"step": 622
|
|
},
|
|
{
|
|
"epoch": 0.9148311306901615,
|
|
"grad_norm": 34.40840148925781,
|
|
"learning_rate": 1.1378595443300998e-08,
|
|
"logits/chosen": -2.1641011238098145,
|
|
"logits/rejected": -2.284421443939209,
|
|
"logps/chosen": -109.72994232177734,
|
|
"logps/ref_chosen": -80.37020874023438,
|
|
"logps/ref_rejected": -114.00752258300781,
|
|
"logps/rejected": -163.298583984375,
|
|
"loss": 0.8517,
|
|
"margin_dpo/margin_mean": 19.931346893310547,
|
|
"margin_dpo/margin_std": 23.866024017333984,
|
|
"step": 623
|
|
},
|
|
{
|
|
"epoch": 0.9162995594713657,
|
|
"grad_norm": 30.061717987060547,
|
|
"learning_rate": 1.0998978889320582e-08,
|
|
"logits/chosen": -1.5450116395950317,
|
|
"logits/rejected": -0.4716716408729553,
|
|
"logps/chosen": -135.16061401367188,
|
|
"logps/ref_chosen": -112.427734375,
|
|
"logps/ref_rejected": -96.3587646484375,
|
|
"logps/rejected": -146.82891845703125,
|
|
"loss": 0.5314,
|
|
"margin_dpo/margin_mean": 27.737272262573242,
|
|
"margin_dpo/margin_std": 21.234542846679688,
|
|
"step": 624
|
|
},
|
|
{
|
|
"epoch": 0.9177679882525698,
|
|
"grad_norm": 24.109561920166016,
|
|
"learning_rate": 1.0625660234518913e-08,
|
|
"logits/chosen": -2.4023971557617188,
|
|
"logits/rejected": -2.229701280593872,
|
|
"logps/chosen": -109.12333679199219,
|
|
"logps/ref_chosen": -75.3917465209961,
|
|
"logps/ref_rejected": -107.67977905273438,
|
|
"logps/rejected": -162.85675048828125,
|
|
"loss": 0.553,
|
|
"margin_dpo/margin_mean": 21.44537353515625,
|
|
"margin_dpo/margin_std": 15.984648704528809,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.9192364170337739,
|
|
"grad_norm": 40.272911071777344,
|
|
"learning_rate": 1.0258649316189721e-08,
|
|
"logits/chosen": -2.329925537109375,
|
|
"logits/rejected": -2.2528748512268066,
|
|
"logps/chosen": -132.60232543945312,
|
|
"logps/ref_chosen": -96.54901123046875,
|
|
"logps/ref_rejected": -101.44801330566406,
|
|
"logps/rejected": -151.58079528808594,
|
|
"loss": 0.8693,
|
|
"margin_dpo/margin_mean": 14.079477310180664,
|
|
"margin_dpo/margin_std": 25.268157958984375,
|
|
"step": 626
|
|
},
|
|
{
|
|
"epoch": 0.920704845814978,
|
|
"grad_norm": 26.63361358642578,
|
|
"learning_rate": 9.897955805412e-09,
|
|
"logits/chosen": -1.4621191024780273,
|
|
"logits/rejected": -1.2315270900726318,
|
|
"logps/chosen": -119.01112365722656,
|
|
"logps/ref_chosen": -88.80218505859375,
|
|
"logps/ref_rejected": -96.42103576660156,
|
|
"logps/rejected": -146.07315063476562,
|
|
"loss": 0.6398,
|
|
"margin_dpo/margin_mean": 19.443174362182617,
|
|
"margin_dpo/margin_std": 14.579084396362305,
|
|
"step": 627
|
|
},
|
|
{
|
|
"epoch": 0.922173274596182,
|
|
"grad_norm": 24.76960563659668,
|
|
"learning_rate": 9.543589206795238e-09,
|
|
"logits/chosen": -1.3107500076293945,
|
|
"logits/rejected": -0.9299607276916504,
|
|
"logps/chosen": -126.72996520996094,
|
|
"logps/ref_chosen": -99.37443542480469,
|
|
"logps/ref_rejected": -100.62442779541016,
|
|
"logps/rejected": -147.0572052001953,
|
|
"loss": 0.5028,
|
|
"margin_dpo/margin_mean": 19.077255249023438,
|
|
"margin_dpo/margin_std": 12.725533485412598,
|
|
"step": 628
|
|
},
|
|
{
|
|
"epoch": 0.9236417033773862,
|
|
"grad_norm": 32.12858581542969,
|
|
"learning_rate": 9.19555885822887e-09,
|
|
"logits/chosen": -1.940063238143921,
|
|
"logits/rejected": -1.3807988166809082,
|
|
"logps/chosen": -121.68482208251953,
|
|
"logps/ref_chosen": -98.16767120361328,
|
|
"logps/ref_rejected": -84.53733825683594,
|
|
"logps/rejected": -129.65162658691406,
|
|
"loss": 0.7023,
|
|
"margin_dpo/margin_mean": 21.597137451171875,
|
|
"margin_dpo/margin_std": 15.778484344482422,
|
|
"step": 629
|
|
},
|
|
{
|
|
"epoch": 0.9251101321585903,
|
|
"grad_norm": 29.0245361328125,
|
|
"learning_rate": 8.85387393063622e-09,
|
|
"logits/chosen": -2.2557272911071777,
|
|
"logits/rejected": -0.403414249420166,
|
|
"logps/chosen": -127.28799438476562,
|
|
"logps/ref_chosen": -102.16020202636719,
|
|
"logps/ref_rejected": -76.82844543457031,
|
|
"logps/rejected": -128.5855712890625,
|
|
"loss": 0.6629,
|
|
"margin_dpo/margin_mean": 26.629322052001953,
|
|
"margin_dpo/margin_std": 19.489566802978516,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.9265785609397944,
|
|
"grad_norm": 19.227121353149414,
|
|
"learning_rate": 8.518543427732949e-09,
|
|
"logits/chosen": -2.5818333625793457,
|
|
"logits/rejected": -1.2536900043487549,
|
|
"logps/chosen": -133.84732055664062,
|
|
"logps/ref_chosen": -114.78280639648438,
|
|
"logps/ref_rejected": -76.31077575683594,
|
|
"logps/rejected": -121.6485824584961,
|
|
"loss": 0.4943,
|
|
"margin_dpo/margin_mean": 26.273296356201172,
|
|
"margin_dpo/margin_std": 15.153189659118652,
|
|
"step": 631
|
|
},
|
|
{
|
|
"epoch": 0.9280469897209985,
|
|
"grad_norm": 23.025691986083984,
|
|
"learning_rate": 8.189576185789637e-09,
|
|
"logits/chosen": -1.9373183250427246,
|
|
"logits/rejected": -1.6466472148895264,
|
|
"logps/chosen": -107.76996612548828,
|
|
"logps/ref_chosen": -80.00565338134766,
|
|
"logps/ref_rejected": -94.28547668457031,
|
|
"logps/rejected": -147.23556518554688,
|
|
"loss": 0.5382,
|
|
"margin_dpo/margin_mean": 25.185754776000977,
|
|
"margin_dpo/margin_std": 19.113544464111328,
|
|
"step": 632
|
|
},
|
|
{
|
|
"epoch": 0.9295154185022027,
|
|
"grad_norm": 31.93626594543457,
|
|
"learning_rate": 7.866980873399015e-09,
|
|
"logits/chosen": -3.0079336166381836,
|
|
"logits/rejected": -2.5928359031677246,
|
|
"logps/chosen": -107.93728637695312,
|
|
"logps/ref_chosen": -79.70408630371094,
|
|
"logps/ref_rejected": -85.23828125,
|
|
"logps/rejected": -133.3818359375,
|
|
"loss": 0.7673,
|
|
"margin_dpo/margin_mean": 19.91036605834961,
|
|
"margin_dpo/margin_std": 21.933469772338867,
|
|
"step": 633
|
|
},
|
|
{
|
|
"epoch": 0.9309838472834068,
|
|
"grad_norm": 24.426607131958008,
|
|
"learning_rate": 7.550765991247654e-09,
|
|
"logits/chosen": -1.3841078281402588,
|
|
"logits/rejected": -1.0601425170898438,
|
|
"logps/chosen": -95.75471496582031,
|
|
"logps/ref_chosen": -65.44131469726562,
|
|
"logps/ref_rejected": -96.48623657226562,
|
|
"logps/rejected": -150.92276000976562,
|
|
"loss": 0.5543,
|
|
"margin_dpo/margin_mean": 24.123126983642578,
|
|
"margin_dpo/margin_std": 15.052120208740234,
|
|
"step": 634
|
|
},
|
|
{
|
|
"epoch": 0.9324522760646109,
|
|
"grad_norm": 26.585651397705078,
|
|
"learning_rate": 7.240939871891699e-09,
|
|
"logits/chosen": -1.7034859657287598,
|
|
"logits/rejected": -1.0522689819335938,
|
|
"logps/chosen": -126.51262664794922,
|
|
"logps/ref_chosen": -99.6771240234375,
|
|
"logps/ref_rejected": -78.4532470703125,
|
|
"logps/rejected": -126.68396759033203,
|
|
"loss": 0.6623,
|
|
"margin_dpo/margin_mean": 21.39522933959961,
|
|
"margin_dpo/margin_std": 16.801891326904297,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.933920704845815,
|
|
"grad_norm": 28.644737243652344,
|
|
"learning_rate": 6.937510679537628e-09,
|
|
"logits/chosen": -0.9782381057739258,
|
|
"logits/rejected": -0.4344422221183777,
|
|
"logps/chosen": -130.71478271484375,
|
|
"logps/ref_chosen": -102.57931518554688,
|
|
"logps/ref_rejected": -88.06050109863281,
|
|
"logps/rejected": -131.12960815429688,
|
|
"loss": 0.6777,
|
|
"margin_dpo/margin_mean": 14.933639526367188,
|
|
"margin_dpo/margin_std": 14.684162139892578,
|
|
"step": 636
|
|
},
|
|
{
|
|
"epoch": 0.9353891336270191,
|
|
"grad_norm": 29.544261932373047,
|
|
"learning_rate": 6.640486409826785e-09,
|
|
"logits/chosen": -1.4342460632324219,
|
|
"logits/rejected": -1.0915610790252686,
|
|
"logps/chosen": -98.80889892578125,
|
|
"logps/ref_chosen": -73.93675994873047,
|
|
"logps/ref_rejected": -76.05081176757812,
|
|
"logps/rejected": -121.78729248046875,
|
|
"loss": 0.7009,
|
|
"margin_dpo/margin_mean": 20.864341735839844,
|
|
"margin_dpo/margin_std": 27.72216033935547,
|
|
"step": 637
|
|
},
|
|
{
|
|
"epoch": 0.9368575624082232,
|
|
"grad_norm": 15.684412956237793,
|
|
"learning_rate": 6.349874889624962e-09,
|
|
"logits/chosen": -2.4961729049682617,
|
|
"logits/rejected": -0.7357807159423828,
|
|
"logps/chosen": -162.9862518310547,
|
|
"logps/ref_chosen": -146.49359130859375,
|
|
"logps/ref_rejected": -72.0327377319336,
|
|
"logps/rejected": -122.63823699951172,
|
|
"loss": 0.3867,
|
|
"margin_dpo/margin_mean": 34.11281967163086,
|
|
"margin_dpo/margin_std": 22.033653259277344,
|
|
"step": 638
|
|
},
|
|
{
|
|
"epoch": 0.9383259911894273,
|
|
"grad_norm": 41.89670181274414,
|
|
"learning_rate": 6.065683776815933e-09,
|
|
"logits/chosen": -2.0428102016448975,
|
|
"logits/rejected": -0.9309650659561157,
|
|
"logps/chosen": -126.6545181274414,
|
|
"logps/ref_chosen": -99.16008758544922,
|
|
"logps/ref_rejected": -78.15130615234375,
|
|
"logps/rejected": -126.32666015625,
|
|
"loss": 0.8664,
|
|
"margin_dpo/margin_mean": 20.680925369262695,
|
|
"margin_dpo/margin_std": 17.583145141601562,
|
|
"step": 639
|
|
},
|
|
{
|
|
"epoch": 0.9397944199706314,
|
|
"grad_norm": 26.69725227355957,
|
|
"learning_rate": 5.7879205600998296e-09,
|
|
"logits/chosen": -0.8534713387489319,
|
|
"logits/rejected": -0.35765373706817627,
|
|
"logps/chosen": -100.805419921875,
|
|
"logps/ref_chosen": -70.03657531738281,
|
|
"logps/ref_rejected": -72.30453491210938,
|
|
"logps/rejected": -126.19176483154297,
|
|
"loss": 0.5627,
|
|
"margin_dpo/margin_mean": 23.118385314941406,
|
|
"margin_dpo/margin_std": 11.315099716186523,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.9412628487518355,
|
|
"grad_norm": 24.453815460205078,
|
|
"learning_rate": 5.516592558795746e-09,
|
|
"logits/chosen": -1.6536837816238403,
|
|
"logits/rejected": -1.5010559558868408,
|
|
"logps/chosen": -113.69627380371094,
|
|
"logps/ref_chosen": -86.68313598632812,
|
|
"logps/ref_rejected": -109.86448669433594,
|
|
"logps/rejected": -158.95619201660156,
|
|
"loss": 0.5443,
|
|
"margin_dpo/margin_mean": 22.078567504882812,
|
|
"margin_dpo/margin_std": 24.4644775390625,
|
|
"step": 641
|
|
},
|
|
{
|
|
"epoch": 0.9427312775330396,
|
|
"grad_norm": 27.494384765625,
|
|
"learning_rate": 5.251706922648868e-09,
|
|
"logits/chosen": -0.8695433139801025,
|
|
"logits/rejected": -0.6438785791397095,
|
|
"logps/chosen": -112.55367279052734,
|
|
"logps/ref_chosen": -81.20620727539062,
|
|
"logps/ref_rejected": -126.77632141113281,
|
|
"logps/rejected": -176.6457061767578,
|
|
"loss": 0.6432,
|
|
"margin_dpo/margin_mean": 18.521900177001953,
|
|
"margin_dpo/margin_std": 18.387256622314453,
|
|
"step": 642
|
|
},
|
|
{
|
|
"epoch": 0.9441997063142438,
|
|
"grad_norm": 21.128366470336914,
|
|
"learning_rate": 4.993270631642038e-09,
|
|
"logits/chosen": -1.7625564336776733,
|
|
"logits/rejected": -1.2028493881225586,
|
|
"logps/chosen": -116.45796203613281,
|
|
"logps/ref_chosen": -89.84759521484375,
|
|
"logps/ref_rejected": -74.28402709960938,
|
|
"logps/rejected": -121.64019012451172,
|
|
"loss": 0.5499,
|
|
"margin_dpo/margin_mean": 20.74580955505371,
|
|
"margin_dpo/margin_std": 12.552116394042969,
|
|
"step": 643
|
|
},
|
|
{
|
|
"epoch": 0.9456681350954479,
|
|
"grad_norm": 28.339998245239258,
|
|
"learning_rate": 4.741290495811873e-09,
|
|
"logits/chosen": -1.7750461101531982,
|
|
"logits/rejected": -1.5512135028839111,
|
|
"logps/chosen": -106.68426513671875,
|
|
"logps/ref_chosen": -76.54374694824219,
|
|
"logps/ref_rejected": -97.76353454589844,
|
|
"logps/rejected": -146.94818115234375,
|
|
"loss": 0.7342,
|
|
"margin_dpo/margin_mean": 19.04413414001465,
|
|
"margin_dpo/margin_std": 20.908920288085938,
|
|
"step": 644
|
|
},
|
|
{
|
|
"epoch": 0.947136563876652,
|
|
"grad_norm": 32.95466232299805,
|
|
"learning_rate": 4.495773155069299e-09,
|
|
"logits/chosen": -1.6883985996246338,
|
|
"logits/rejected": -0.7081927061080933,
|
|
"logps/chosen": -137.83843994140625,
|
|
"logps/ref_chosen": -108.44778442382812,
|
|
"logps/ref_rejected": -65.57743072509766,
|
|
"logps/rejected": -107.06454467773438,
|
|
"loss": 0.8305,
|
|
"margin_dpo/margin_mean": 12.096460342407227,
|
|
"margin_dpo/margin_std": 17.232433319091797,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.9486049926578561,
|
|
"grad_norm": 23.519771575927734,
|
|
"learning_rate": 4.256725079024553e-09,
|
|
"logits/chosen": -1.1280121803283691,
|
|
"logits/rejected": -0.7711001038551331,
|
|
"logps/chosen": -124.54080963134766,
|
|
"logps/ref_chosen": -99.14352416992188,
|
|
"logps/ref_rejected": -115.10133361816406,
|
|
"logps/rejected": -167.08096313476562,
|
|
"loss": 0.4552,
|
|
"margin_dpo/margin_mean": 26.582338333129883,
|
|
"margin_dpo/margin_std": 12.872812271118164,
|
|
"step": 646
|
|
},
|
|
{
|
|
"epoch": 0.9500734214390602,
|
|
"grad_norm": 32.113014221191406,
|
|
"learning_rate": 4.024152566816791e-09,
|
|
"logits/chosen": -0.8513507843017578,
|
|
"logits/rejected": -0.7660804390907288,
|
|
"logps/chosen": -114.12860107421875,
|
|
"logps/ref_chosen": -82.64013671875,
|
|
"logps/ref_rejected": -98.2857666015625,
|
|
"logps/rejected": -149.02532958984375,
|
|
"loss": 0.7834,
|
|
"margin_dpo/margin_mean": 19.25110626220703,
|
|
"margin_dpo/margin_std": 20.14596176147461,
|
|
"step": 647
|
|
},
|
|
{
|
|
"epoch": 0.9515418502202643,
|
|
"grad_norm": 26.356727600097656,
|
|
"learning_rate": 3.798061746947995e-09,
|
|
"logits/chosen": -1.376887559890747,
|
|
"logits/rejected": -1.3346166610717773,
|
|
"logps/chosen": -109.87057495117188,
|
|
"logps/ref_chosen": -77.24684143066406,
|
|
"logps/ref_rejected": -99.16560363769531,
|
|
"logps/rejected": -153.75718688964844,
|
|
"loss": 0.5122,
|
|
"margin_dpo/margin_mean": 21.967853546142578,
|
|
"margin_dpo/margin_std": 16.269351959228516,
|
|
"step": 648
|
|
},
|
|
{
|
|
"epoch": 0.9530102790014684,
|
|
"grad_norm": 24.207809448242188,
|
|
"learning_rate": 3.5784585771215235e-09,
|
|
"logits/chosen": -1.8359942436218262,
|
|
"logits/rejected": -1.2063957452774048,
|
|
"logps/chosen": -131.0638427734375,
|
|
"logps/ref_chosen": -105.46896362304688,
|
|
"logps/ref_rejected": -77.46502685546875,
|
|
"logps/rejected": -116.54486846923828,
|
|
"loss": 0.609,
|
|
"margin_dpo/margin_mean": 13.484968185424805,
|
|
"margin_dpo/margin_std": 18.346370697021484,
|
|
"step": 649
|
|
},
|
|
{
|
|
"epoch": 0.9544787077826725,
|
|
"grad_norm": 37.710941314697266,
|
|
"learning_rate": 3.3653488440851253e-09,
|
|
"logits/chosen": -1.2717688083648682,
|
|
"logits/rejected": -1.2444267272949219,
|
|
"logps/chosen": -109.78580474853516,
|
|
"logps/ref_chosen": -83.3130111694336,
|
|
"logps/ref_rejected": -103.46926879882812,
|
|
"logps/rejected": -155.64500427246094,
|
|
"loss": 0.6071,
|
|
"margin_dpo/margin_mean": 25.702922821044922,
|
|
"margin_dpo/margin_std": 18.235645294189453,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.9559471365638766,
|
|
"grad_norm": 20.15422248840332,
|
|
"learning_rate": 3.158738163478475e-09,
|
|
"logits/chosen": -2.506378412246704,
|
|
"logits/rejected": -1.909285306930542,
|
|
"logps/chosen": -125.57213592529297,
|
|
"logps/ref_chosen": -109.26248168945312,
|
|
"logps/ref_rejected": -135.50436401367188,
|
|
"logps/rejected": -186.22874450683594,
|
|
"loss": 0.4133,
|
|
"margin_dpo/margin_mean": 34.41473388671875,
|
|
"margin_dpo/margin_std": 15.260412216186523,
|
|
"step": 651
|
|
},
|
|
{
|
|
"epoch": 0.9574155653450808,
|
|
"grad_norm": 23.561294555664062,
|
|
"learning_rate": 2.9586319796851555e-09,
|
|
"logits/chosen": -1.4572404623031616,
|
|
"logits/rejected": -1.3910325765609741,
|
|
"logps/chosen": -103.1104736328125,
|
|
"logps/ref_chosen": -77.7144775390625,
|
|
"logps/ref_rejected": -108.62936401367188,
|
|
"logps/rejected": -159.1099853515625,
|
|
"loss": 0.5575,
|
|
"margin_dpo/margin_mean": 25.084617614746094,
|
|
"margin_dpo/margin_std": 15.043981552124023,
|
|
"step": 652
|
|
},
|
|
{
|
|
"epoch": 0.9588839941262849,
|
|
"grad_norm": 28.391294479370117,
|
|
"learning_rate": 2.7650355656892166e-09,
|
|
"logits/chosen": -2.1806952953338623,
|
|
"logits/rejected": -2.2581558227539062,
|
|
"logps/chosen": -119.31652069091797,
|
|
"logps/ref_chosen": -93.11857604980469,
|
|
"logps/ref_rejected": -131.34259033203125,
|
|
"logps/rejected": -177.6236114501953,
|
|
"loss": 0.5885,
|
|
"margin_dpo/margin_mean": 20.083072662353516,
|
|
"margin_dpo/margin_std": 16.388137817382812,
|
|
"step": 653
|
|
},
|
|
{
|
|
"epoch": 0.960352422907489,
|
|
"grad_norm": 31.213104248046875,
|
|
"learning_rate": 2.577954022936174e-09,
|
|
"logits/chosen": -1.9482306241989136,
|
|
"logits/rejected": -1.3727693557739258,
|
|
"logps/chosen": -109.99600219726562,
|
|
"logps/ref_chosen": -78.7916488647461,
|
|
"logps/ref_rejected": -91.75021362304688,
|
|
"logps/rejected": -140.29420471191406,
|
|
"loss": 0.7289,
|
|
"margin_dpo/margin_mean": 17.339649200439453,
|
|
"margin_dpo/margin_std": 15.09589672088623,
|
|
"step": 654
|
|
},
|
|
{
|
|
"epoch": 0.9618208516886931,
|
|
"grad_norm": 27.248577117919922,
|
|
"learning_rate": 2.397392281198729e-09,
|
|
"logits/chosen": -1.7276451587677002,
|
|
"logits/rejected": -1.683257818222046,
|
|
"logps/chosen": -106.54402923583984,
|
|
"logps/ref_chosen": -74.46823120117188,
|
|
"logps/ref_rejected": -75.98558807373047,
|
|
"logps/rejected": -121.86129760742188,
|
|
"loss": 0.72,
|
|
"margin_dpo/margin_mean": 13.799907684326172,
|
|
"margin_dpo/margin_std": 12.526430130004883,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.9632892804698973,
|
|
"grad_norm": 22.945934295654297,
|
|
"learning_rate": 2.223355098446622e-09,
|
|
"logits/chosen": -1.748338222503662,
|
|
"logits/rejected": -1.9085040092468262,
|
|
"logps/chosen": -111.42201232910156,
|
|
"logps/ref_chosen": -80.48881530761719,
|
|
"logps/ref_rejected": -135.5433349609375,
|
|
"logps/rejected": -194.20220947265625,
|
|
"loss": 0.4748,
|
|
"margin_dpo/margin_mean": 27.725685119628906,
|
|
"margin_dpo/margin_std": 16.44278335571289,
|
|
"step": 656
|
|
},
|
|
{
|
|
"epoch": 0.9647577092511013,
|
|
"grad_norm": 25.630002975463867,
|
|
"learning_rate": 2.055847060721566e-09,
|
|
"logits/chosen": -1.4626106023788452,
|
|
"logits/rejected": -1.0344831943511963,
|
|
"logps/chosen": -116.6616439819336,
|
|
"logps/ref_chosen": -91.72633361816406,
|
|
"logps/ref_rejected": -99.731689453125,
|
|
"logps/rejected": -147.25933837890625,
|
|
"loss": 0.6366,
|
|
"margin_dpo/margin_mean": 22.592342376708984,
|
|
"margin_dpo/margin_std": 27.992549896240234,
|
|
"step": 657
|
|
},
|
|
{
|
|
"epoch": 0.9662261380323054,
|
|
"grad_norm": 24.88594627380371,
|
|
"learning_rate": 1.8948725820160662e-09,
|
|
"logits/chosen": -1.8801562786102295,
|
|
"logits/rejected": -1.3535478115081787,
|
|
"logps/chosen": -146.45626831054688,
|
|
"logps/ref_chosen": -112.55894470214844,
|
|
"logps/ref_rejected": -104.8104019165039,
|
|
"logps/rejected": -156.04794311523438,
|
|
"loss": 0.5872,
|
|
"margin_dpo/margin_mean": 17.340206146240234,
|
|
"margin_dpo/margin_std": 20.778215408325195,
|
|
"step": 658
|
|
},
|
|
{
|
|
"epoch": 0.9676945668135095,
|
|
"grad_norm": 24.542510986328125,
|
|
"learning_rate": 1.7404359041573723e-09,
|
|
"logits/chosen": -2.352703094482422,
|
|
"logits/rejected": -1.7414582967758179,
|
|
"logps/chosen": -104.23818969726562,
|
|
"logps/ref_chosen": -76.53956604003906,
|
|
"logps/ref_rejected": -71.9808120727539,
|
|
"logps/rejected": -126.01404571533203,
|
|
"loss": 0.532,
|
|
"margin_dpo/margin_mean": 26.33460235595703,
|
|
"margin_dpo/margin_std": 24.5662841796875,
|
|
"step": 659
|
|
},
|
|
{
|
|
"epoch": 0.9691629955947136,
|
|
"grad_norm": 23.398681640625,
|
|
"learning_rate": 1.592541096695571e-09,
|
|
"logits/chosen": -2.1632814407348633,
|
|
"logits/rejected": -1.345157504081726,
|
|
"logps/chosen": -111.40882873535156,
|
|
"logps/ref_chosen": -84.62733459472656,
|
|
"logps/ref_rejected": -69.92381286621094,
|
|
"logps/rejected": -124.7333984375,
|
|
"loss": 0.54,
|
|
"margin_dpo/margin_mean": 28.028095245361328,
|
|
"margin_dpo/margin_std": 18.708595275878906,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.9706314243759178,
|
|
"grad_norm": 29.665576934814453,
|
|
"learning_rate": 1.4511920567963908e-09,
|
|
"logits/chosen": -1.6013872623443604,
|
|
"logits/rejected": -1.1411731243133545,
|
|
"logps/chosen": -123.57489013671875,
|
|
"logps/ref_chosen": -92.84153747558594,
|
|
"logps/ref_rejected": -114.14533996582031,
|
|
"logps/rejected": -167.12649536132812,
|
|
"loss": 0.5883,
|
|
"margin_dpo/margin_mean": 22.2477970123291,
|
|
"margin_dpo/margin_std": 23.344955444335938,
|
|
"step": 661
|
|
},
|
|
{
|
|
"epoch": 0.9720998531571219,
|
|
"grad_norm": 24.84825897216797,
|
|
"learning_rate": 1.3163925091384532e-09,
|
|
"logits/chosen": -1.088799238204956,
|
|
"logits/rejected": -0.6112481355667114,
|
|
"logps/chosen": -112.46092987060547,
|
|
"logps/ref_chosen": -82.68042755126953,
|
|
"logps/ref_rejected": -87.19029235839844,
|
|
"logps/rejected": -137.53738403320312,
|
|
"loss": 0.4635,
|
|
"margin_dpo/margin_mean": 20.56658935546875,
|
|
"margin_dpo/margin_std": 13.022397994995117,
|
|
"step": 662
|
|
},
|
|
{
|
|
"epoch": 0.973568281938326,
|
|
"grad_norm": 22.896821975708008,
|
|
"learning_rate": 1.1881460058152382e-09,
|
|
"logits/chosen": -1.5960707664489746,
|
|
"logits/rejected": -1.2925134897232056,
|
|
"logps/chosen": -135.431640625,
|
|
"logps/ref_chosen": -106.22300720214844,
|
|
"logps/ref_rejected": -130.58547973632812,
|
|
"logps/rejected": -179.110595703125,
|
|
"loss": 0.5211,
|
|
"margin_dpo/margin_mean": 19.316484451293945,
|
|
"margin_dpo/margin_std": 15.610393524169922,
|
|
"step": 663
|
|
},
|
|
{
|
|
"epoch": 0.9750367107195301,
|
|
"grad_norm": 27.367639541625977,
|
|
"learning_rate": 1.066455926241383e-09,
|
|
"logits/chosen": -1.6287367343902588,
|
|
"logits/rejected": -1.190144658088684,
|
|
"logps/chosen": -117.2126693725586,
|
|
"logps/ref_chosen": -93.286376953125,
|
|
"logps/ref_rejected": -97.53064727783203,
|
|
"logps/rejected": -151.74496459960938,
|
|
"loss": 0.5275,
|
|
"margin_dpo/margin_mean": 30.288034439086914,
|
|
"margin_dpo/margin_std": 14.453622817993164,
|
|
"step": 664
|
|
},
|
|
{
|
|
"epoch": 0.9765051395007343,
|
|
"grad_norm": 22.599021911621094,
|
|
"learning_rate": 9.513254770636137e-10,
|
|
"logits/chosen": -2.1870265007019043,
|
|
"logits/rejected": -1.971346139907837,
|
|
"logps/chosen": -131.42825317382812,
|
|
"logps/ref_chosen": -105.4872817993164,
|
|
"logps/ref_rejected": -104.466064453125,
|
|
"logps/rejected": -145.77651977539062,
|
|
"loss": 0.564,
|
|
"margin_dpo/margin_mean": 15.369461059570312,
|
|
"margin_dpo/margin_std": 11.922895431518555,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.9779735682819384,
|
|
"grad_norm": 31.737276077270508,
|
|
"learning_rate": 8.427576920763956e-10,
|
|
"logits/chosen": -1.3515913486480713,
|
|
"logits/rejected": -1.0945022106170654,
|
|
"logps/chosen": -116.85716247558594,
|
|
"logps/ref_chosen": -85.33709716796875,
|
|
"logps/ref_rejected": -90.60565185546875,
|
|
"logps/rejected": -136.8424835205078,
|
|
"loss": 0.815,
|
|
"margin_dpo/margin_mean": 14.71677017211914,
|
|
"margin_dpo/margin_std": 17.84575653076172,
|
|
"step": 666
|
|
},
|
|
{
|
|
"epoch": 0.9794419970631424,
|
|
"grad_norm": 26.36996078491211,
|
|
"learning_rate": 7.407554321417764e-10,
|
|
"logits/chosen": -1.5280002355575562,
|
|
"logits/rejected": -0.8938305377960205,
|
|
"logps/chosen": -105.82881927490234,
|
|
"logps/ref_chosen": -80.93008422851562,
|
|
"logps/ref_rejected": -68.24456787109375,
|
|
"logps/rejected": -120.00591278076172,
|
|
"loss": 0.4763,
|
|
"margin_dpo/margin_mean": 26.86260223388672,
|
|
"margin_dpo/margin_std": 18.75882911682129,
|
|
"step": 667
|
|
},
|
|
{
|
|
"epoch": 0.9809104258443465,
|
|
"grad_norm": 23.50409698486328,
|
|
"learning_rate": 6.453213851142225e-10,
|
|
"logits/chosen": -1.4292489290237427,
|
|
"logits/rejected": -1.478631615638733,
|
|
"logps/chosen": -104.58021545410156,
|
|
"logps/ref_chosen": -76.60061645507812,
|
|
"logps/ref_rejected": -118.1635513305664,
|
|
"logps/rejected": -168.6148223876953,
|
|
"loss": 0.4135,
|
|
"margin_dpo/margin_mean": 22.471668243408203,
|
|
"margin_dpo/margin_std": 17.230484008789062,
|
|
"step": 668
|
|
},
|
|
{
|
|
"epoch": 0.9823788546255506,
|
|
"grad_norm": 24.116369247436523,
|
|
"learning_rate": 5.564580657695939e-10,
|
|
"logits/chosen": -1.528371810913086,
|
|
"logits/rejected": -0.9692767858505249,
|
|
"logps/chosen": -120.40367126464844,
|
|
"logps/ref_chosen": -91.93134307861328,
|
|
"logps/ref_rejected": -62.09255599975586,
|
|
"logps/rejected": -104.82398986816406,
|
|
"loss": 0.5988,
|
|
"margin_dpo/margin_mean": 14.25910758972168,
|
|
"margin_dpo/margin_std": 19.90908432006836,
|
|
"step": 669
|
|
},
|
|
{
|
|
"epoch": 0.9838472834067548,
|
|
"grad_norm": 21.280534744262695,
|
|
"learning_rate": 4.741678157389739e-10,
|
|
"logits/chosen": -1.3773136138916016,
|
|
"logits/rejected": -1.061061978340149,
|
|
"logps/chosen": -98.46882629394531,
|
|
"logps/ref_chosen": -71.73137664794922,
|
|
"logps/ref_rejected": -83.80799102783203,
|
|
"logps/rejected": -138.5927276611328,
|
|
"loss": 0.4798,
|
|
"margin_dpo/margin_mean": 28.047285079956055,
|
|
"margin_dpo/margin_std": 18.901243209838867,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.9853157121879589,
|
|
"grad_norm": 22.35253143310547,
|
|
"learning_rate": 3.9845280344705245e-10,
|
|
"logits/chosen": -0.859630823135376,
|
|
"logits/rejected": -0.6215537190437317,
|
|
"logps/chosen": -131.175048828125,
|
|
"logps/ref_chosen": -97.33171844482422,
|
|
"logps/ref_rejected": -89.43772888183594,
|
|
"logps/rejected": -140.61334228515625,
|
|
"loss": 0.5645,
|
|
"margin_dpo/margin_mean": 17.332304000854492,
|
|
"margin_dpo/margin_std": 15.1148042678833,
|
|
"step": 671
|
|
},
|
|
{
|
|
"epoch": 0.986784140969163,
|
|
"grad_norm": 28.22157096862793,
|
|
"learning_rate": 3.293150240547549e-10,
|
|
"logits/chosen": -2.014643907546997,
|
|
"logits/rejected": -1.8692243099212646,
|
|
"logps/chosen": -108.41804504394531,
|
|
"logps/ref_chosen": -77.21369171142578,
|
|
"logps/ref_rejected": -94.98027038574219,
|
|
"logps/rejected": -139.38845825195312,
|
|
"loss": 0.6272,
|
|
"margin_dpo/margin_mean": 13.203832626342773,
|
|
"margin_dpo/margin_std": 15.262359619140625,
|
|
"step": 672
|
|
},
|
|
{
|
|
"epoch": 0.9882525697503671,
|
|
"grad_norm": 22.13953399658203,
|
|
"learning_rate": 2.6675629940689504e-10,
|
|
"logits/chosen": -0.9023149013519287,
|
|
"logits/rejected": -0.7241397500038147,
|
|
"logps/chosen": -91.1622543334961,
|
|
"logps/ref_chosen": -63.30126190185547,
|
|
"logps/ref_rejected": -101.4169921875,
|
|
"logps/rejected": -152.5889892578125,
|
|
"loss": 0.505,
|
|
"margin_dpo/margin_mean": 23.310997009277344,
|
|
"margin_dpo/margin_std": 15.188114166259766,
|
|
"step": 673
|
|
},
|
|
{
|
|
"epoch": 0.9897209985315712,
|
|
"grad_norm": 20.051538467407227,
|
|
"learning_rate": 2.1077827798404725e-10,
|
|
"logits/chosen": -1.5894590616226196,
|
|
"logits/rejected": -1.1531003713607788,
|
|
"logps/chosen": -109.16806030273438,
|
|
"logps/ref_chosen": -80.00570678710938,
|
|
"logps/ref_rejected": -73.41873168945312,
|
|
"logps/rejected": -128.1361083984375,
|
|
"loss": 0.4786,
|
|
"margin_dpo/margin_mean": 25.555042266845703,
|
|
"margin_dpo/margin_std": 18.46479606628418,
|
|
"step": 674
|
|
},
|
|
{
|
|
"epoch": 0.9911894273127754,
|
|
"grad_norm": 177.50648498535156,
|
|
"learning_rate": 1.6138243485910863e-10,
|
|
"logits/chosen": -0.9243202805519104,
|
|
"logits/rejected": -0.7027927041053772,
|
|
"logps/chosen": -87.20614624023438,
|
|
"logps/ref_chosen": -63.118011474609375,
|
|
"logps/ref_rejected": -69.97030639648438,
|
|
"logps/rejected": -111.49307250976562,
|
|
"loss": 0.7257,
|
|
"margin_dpo/margin_mean": 17.43462371826172,
|
|
"margin_dpo/margin_std": 15.88136100769043,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.9926578560939795,
|
|
"grad_norm": 26.805587768554688,
|
|
"learning_rate": 1.1857007165852472e-10,
|
|
"logits/chosen": -1.7543052434921265,
|
|
"logits/rejected": -1.1985046863555908,
|
|
"logps/chosen": -131.3642578125,
|
|
"logps/ref_chosen": -100.23760986328125,
|
|
"logps/ref_rejected": -98.99859619140625,
|
|
"logps/rejected": -151.52993774414062,
|
|
"loss": 0.4528,
|
|
"margin_dpo/margin_mean": 21.404701232910156,
|
|
"margin_dpo/margin_std": 14.885082244873047,
|
|
"step": 676
|
|
},
|
|
{
|
|
"epoch": 0.9941262848751835,
|
|
"grad_norm": 17.16691780090332,
|
|
"learning_rate": 8.23423165278725e-11,
|
|
"logits/chosen": -2.323134660720825,
|
|
"logits/rejected": -2.3097894191741943,
|
|
"logps/chosen": -106.279541015625,
|
|
"logps/ref_chosen": -80.32097625732422,
|
|
"logps/ref_rejected": -89.61705017089844,
|
|
"logps/rejected": -137.46066284179688,
|
|
"loss": 0.3924,
|
|
"margin_dpo/margin_mean": 21.885032653808594,
|
|
"margin_dpo/margin_std": 18.007171630859375,
|
|
"step": 677
|
|
},
|
|
{
|
|
"epoch": 0.9955947136563876,
|
|
"grad_norm": 20.216367721557617,
|
|
"learning_rate": 5.270012410216185e-11,
|
|
"logits/chosen": -1.1461702585220337,
|
|
"logits/rejected": -0.45331066846847534,
|
|
"logps/chosen": -117.91678619384766,
|
|
"logps/ref_chosen": -89.85308837890625,
|
|
"logps/ref_rejected": -62.9010009765625,
|
|
"logps/rejected": -115.28776550292969,
|
|
"loss": 0.4801,
|
|
"margin_dpo/margin_mean": 24.323078155517578,
|
|
"margin_dpo/margin_std": 20.244949340820312,
|
|
"step": 678
|
|
},
|
|
{
|
|
"epoch": 0.9970631424375918,
|
|
"grad_norm": 22.173934936523438,
|
|
"learning_rate": 2.9644275480772416e-11,
|
|
"logits/chosen": -2.05856990814209,
|
|
"logits/rejected": -1.7644459009170532,
|
|
"logps/chosen": -92.57217407226562,
|
|
"logps/ref_chosen": -63.82649230957031,
|
|
"logps/ref_rejected": -77.2712631225586,
|
|
"logps/rejected": -130.9364013671875,
|
|
"loss": 0.4935,
|
|
"margin_dpo/margin_mean": 24.919471740722656,
|
|
"margin_dpo/margin_std": 18.897235870361328,
|
|
"step": 679
|
|
},
|
|
{
|
|
"epoch": 0.9985315712187959,
|
|
"grad_norm": 29.65631103515625,
|
|
"learning_rate": 1.31753782067201e-11,
|
|
"logits/chosen": -1.9342844486236572,
|
|
"logits/rejected": -1.8356274366378784,
|
|
"logps/chosen": -110.40392303466797,
|
|
"logps/ref_chosen": -81.73397064208984,
|
|
"logps/ref_rejected": -114.59278869628906,
|
|
"logps/rejected": -164.33082580566406,
|
|
"loss": 0.5882,
|
|
"margin_dpo/margin_mean": 21.06808090209961,
|
|
"margin_dpo/margin_std": 22.42251205444336,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"grad_norm": 29.214645385742188,
|
|
"learning_rate": 3.2938662507808745e-12,
|
|
"logits/chosen": -1.2867460250854492,
|
|
"logits/rejected": -1.1931159496307373,
|
|
"logps/chosen": -133.166748046875,
|
|
"logps/ref_chosen": -103.17874145507812,
|
|
"logps/ref_rejected": -113.08921813964844,
|
|
"logps/rejected": -161.36431884765625,
|
|
"loss": 0.5985,
|
|
"margin_dpo/margin_mean": 18.287107467651367,
|
|
"margin_dpo/margin_std": 15.459449768066406,
|
|
"step": 681
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 681,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.7553340482816823,
|
|
"train_runtime": 3298.7616,
|
|
"train_samples_per_second": 13.216,
|
|
"train_steps_per_second": 0.206
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 681,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 200,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|