097e33be47
Model: W-61/mistral-7b-base-beta-dpo-hh-helpful-4xh200-batch-64 Source: Original Platform
13790 lines
555 KiB
JSON
13790 lines
555 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 100,
|
|
"global_step": 681,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"beta_dpo/beta": 0.1005711555480957,
|
|
"beta_dpo/beta_margin_grad_mean": -0.49939221143722534,
|
|
"beta_dpo/beta_margin_grad_std": 0.007166189141571522,
|
|
"beta_dpo/beta_margin_mean": 0.002432748908177018,
|
|
"beta_dpo/beta_margin_std": 0.028674542903900146,
|
|
"beta_dpo/beta_used": 0.1005711555480957,
|
|
"beta_dpo/beta_used_raw": 0.1005711555480957,
|
|
"beta_dpo/gap_mean": 0.0036213158164173365,
|
|
"beta_dpo/gap_std": 0.04173510894179344,
|
|
"beta_dpo/loss_margin_mean": 0.024216145277023315,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.0014684287812041115,
|
|
"grad_norm": 150.13343811035156,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -3.1338672637939453,
|
|
"logits/rejected": -3.1045522689819336,
|
|
"loss": 1.385,
|
|
"step": 1
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10121786594390869,
|
|
"beta_dpo/beta_margin_grad_mean": -0.5002189874649048,
|
|
"beta_dpo/beta_margin_grad_std": 0.006193624343723059,
|
|
"beta_dpo/beta_margin_mean": -0.0008768404368311167,
|
|
"beta_dpo/beta_margin_std": 0.02477947063744068,
|
|
"beta_dpo/beta_used": 0.10121786594390869,
|
|
"beta_dpo/beta_used_raw": 0.10121786594390869,
|
|
"beta_dpo/gap_mean": 0.0025136363692581654,
|
|
"beta_dpo/gap_std": 0.08383626490831375,
|
|
"beta_dpo/loss_margin_mean": -0.008541211485862732,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.002936857562408223,
|
|
"grad_norm": 156.69430541992188,
|
|
"learning_rate": 7.246376811594203e-09,
|
|
"logits/chosen": -3.149817943572998,
|
|
"logits/rejected": -3.133805751800537,
|
|
"loss": 1.3838,
|
|
"step": 2
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10055717825889587,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4997645616531372,
|
|
"beta_dpo/beta_margin_grad_std": 0.006223098374903202,
|
|
"beta_dpo/beta_margin_mean": 0.0009417525725439191,
|
|
"beta_dpo/beta_margin_std": 0.024896962568163872,
|
|
"beta_dpo/beta_used": 0.10055717825889587,
|
|
"beta_dpo/beta_used_raw": 0.10055717825889587,
|
|
"beta_dpo/gap_mean": 0.0029192278161644936,
|
|
"beta_dpo/gap_std": 0.113316610455513,
|
|
"beta_dpo/loss_margin_mean": 0.009323984384536743,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.004405286343612335,
|
|
"grad_norm": 169.9915771484375,
|
|
"learning_rate": 1.4492753623188406e-08,
|
|
"logits/chosen": -3.1339285373687744,
|
|
"logits/rejected": -3.0974249839782715,
|
|
"loss": 1.3851,
|
|
"step": 3
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10269482433795929,
|
|
"beta_dpo/beta_margin_grad_mean": -0.49976256489753723,
|
|
"beta_dpo/beta_margin_grad_std": 0.006724101025611162,
|
|
"beta_dpo/beta_margin_mean": 0.0009488743380643427,
|
|
"beta_dpo/beta_margin_std": 0.026902994140982628,
|
|
"beta_dpo/beta_used": 0.10269482433795929,
|
|
"beta_dpo/beta_used_raw": 0.10269482433795929,
|
|
"beta_dpo/gap_mean": 0.0034820283763110638,
|
|
"beta_dpo/gap_std": 0.14156511425971985,
|
|
"beta_dpo/loss_margin_mean": 0.00913846492767334,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.005873715124816446,
|
|
"grad_norm": 153.58120727539062,
|
|
"learning_rate": 2.1739130434782606e-08,
|
|
"logits/chosen": -3.124056816101074,
|
|
"logits/rejected": -3.1173763275146484,
|
|
"loss": 1.3814,
|
|
"step": 4
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.09814765304327011,
|
|
"beta_dpo/beta_margin_grad_mean": -0.5008248090744019,
|
|
"beta_dpo/beta_margin_grad_std": 0.005996009334921837,
|
|
"beta_dpo/beta_margin_mean": -0.00329973129555583,
|
|
"beta_dpo/beta_margin_std": 0.02398756332695484,
|
|
"beta_dpo/beta_used": 0.09814765304327011,
|
|
"beta_dpo/beta_used_raw": 0.09814765304327011,
|
|
"beta_dpo/gap_mean": 0.0016051515704020858,
|
|
"beta_dpo/gap_std": 0.1615678071975708,
|
|
"beta_dpo/loss_margin_mean": -0.03306567668914795,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.007342143906020558,
|
|
"grad_norm": 179.1089324951172,
|
|
"learning_rate": 2.898550724637681e-08,
|
|
"logits/chosen": -3.109281063079834,
|
|
"logits/rejected": -3.089329481124878,
|
|
"loss": 1.3892,
|
|
"step": 5
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.09695610404014587,
|
|
"beta_dpo/beta_margin_grad_mean": -0.500086784362793,
|
|
"beta_dpo/beta_margin_grad_std": 0.006598716601729393,
|
|
"beta_dpo/beta_margin_mean": -0.00034629780566319823,
|
|
"beta_dpo/beta_margin_std": 0.02640008181333542,
|
|
"beta_dpo/beta_used": 0.09695610404014587,
|
|
"beta_dpo/beta_used_raw": 0.09695610404014587,
|
|
"beta_dpo/gap_mean": -0.002422480145469308,
|
|
"beta_dpo/gap_std": 0.18281012773513794,
|
|
"beta_dpo/loss_margin_mean": -0.00355510413646698,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.00881057268722467,
|
|
"grad_norm": 171.80946350097656,
|
|
"learning_rate": 3.6231884057971014e-08,
|
|
"logits/chosen": -3.1225814819335938,
|
|
"logits/rejected": -3.0773398876190186,
|
|
"loss": 1.3915,
|
|
"step": 6
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10045399516820908,
|
|
"beta_dpo/beta_margin_grad_mean": -0.49887970089912415,
|
|
"beta_dpo/beta_margin_grad_std": 0.005884926300495863,
|
|
"beta_dpo/beta_margin_mean": 0.004481919575482607,
|
|
"beta_dpo/beta_margin_std": 0.02354278229176998,
|
|
"beta_dpo/beta_used": 0.10045399516820908,
|
|
"beta_dpo/beta_used_raw": 0.10045399516820908,
|
|
"beta_dpo/gap_mean": 0.004135521594434977,
|
|
"beta_dpo/gap_std": 0.1954803466796875,
|
|
"beta_dpo/loss_margin_mean": 0.04471883177757263,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.010279001468428781,
|
|
"grad_norm": 165.5727081298828,
|
|
"learning_rate": 4.347826086956521e-08,
|
|
"logits/chosen": -3.1420326232910156,
|
|
"logits/rejected": -3.1474711894989014,
|
|
"loss": 1.3852,
|
|
"step": 7
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.09605514258146286,
|
|
"beta_dpo/beta_margin_grad_mean": -0.5012519359588623,
|
|
"beta_dpo/beta_margin_grad_std": 0.006095360033214092,
|
|
"beta_dpo/beta_margin_mean": -0.005007945001125336,
|
|
"beta_dpo/beta_margin_std": 0.02438473515212536,
|
|
"beta_dpo/beta_used": 0.09605514258146286,
|
|
"beta_dpo/beta_used_raw": 0.09605514258146286,
|
|
"beta_dpo/gap_mean": -0.0014673962723463774,
|
|
"beta_dpo/gap_std": 0.20564857125282288,
|
|
"beta_dpo/loss_margin_mean": -0.052922144532203674,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.011747430249632892,
|
|
"grad_norm": 163.31930541992188,
|
|
"learning_rate": 5.0724637681159424e-08,
|
|
"logits/chosen": -3.198502540588379,
|
|
"logits/rejected": -3.1550021171569824,
|
|
"loss": 1.3927,
|
|
"step": 8
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10017342120409012,
|
|
"beta_dpo/beta_margin_grad_mean": -0.5000122785568237,
|
|
"beta_dpo/beta_margin_grad_std": 0.006785591598600149,
|
|
"beta_dpo/beta_margin_mean": -4.894528683507815e-05,
|
|
"beta_dpo/beta_margin_std": 0.02714758738875389,
|
|
"beta_dpo/beta_used": 0.10017342120409012,
|
|
"beta_dpo/beta_used_raw": 0.10017342120409012,
|
|
"beta_dpo/gap_mean": -0.007042712531983852,
|
|
"beta_dpo/gap_std": 0.21777689456939697,
|
|
"beta_dpo/loss_margin_mean": -0.0016328543424606323,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.013215859030837005,
|
|
"grad_norm": 181.66322326660156,
|
|
"learning_rate": 5.797101449275362e-08,
|
|
"logits/chosen": -3.1416659355163574,
|
|
"logits/rejected": -3.1250786781311035,
|
|
"loss": 1.3867,
|
|
"step": 9
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.09914623945951462,
|
|
"beta_dpo/beta_margin_grad_mean": -0.5005456805229187,
|
|
"beta_dpo/beta_margin_grad_std": 0.005261780694127083,
|
|
"beta_dpo/beta_margin_mean": -0.0021829898469150066,
|
|
"beta_dpo/beta_margin_std": 0.021049022674560547,
|
|
"beta_dpo/beta_used": 0.09914623945951462,
|
|
"beta_dpo/beta_used_raw": 0.09914623945951462,
|
|
"beta_dpo/gap_mean": -0.006503046490252018,
|
|
"beta_dpo/gap_std": 0.21944838762283325,
|
|
"beta_dpo/loss_margin_mean": -0.02211102843284607,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.014684287812041116,
|
|
"grad_norm": 168.22305297851562,
|
|
"learning_rate": 6.521739130434782e-08,
|
|
"logits/chosen": -3.10261869430542,
|
|
"logits/rejected": -3.080986499786377,
|
|
"loss": 1.3884,
|
|
"step": 10
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10375332087278366,
|
|
"beta_dpo/beta_margin_grad_mean": -0.49827736616134644,
|
|
"beta_dpo/beta_margin_grad_std": 0.006540252361446619,
|
|
"beta_dpo/beta_margin_mean": 0.00689197052270174,
|
|
"beta_dpo/beta_margin_std": 0.026165250688791275,
|
|
"beta_dpo/beta_used": 0.10375332087278366,
|
|
"beta_dpo/beta_used_raw": 0.10375332087278366,
|
|
"beta_dpo/gap_mean": 0.002864137524738908,
|
|
"beta_dpo/gap_std": 0.22579887509346008,
|
|
"beta_dpo/loss_margin_mean": 0.06643152236938477,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.016152716593245228,
|
|
"grad_norm": 186.03494262695312,
|
|
"learning_rate": 7.246376811594203e-08,
|
|
"logits/chosen": -3.155540943145752,
|
|
"logits/rejected": -3.142815113067627,
|
|
"loss": 1.3796,
|
|
"step": 11
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10144880414009094,
|
|
"beta_dpo/beta_margin_grad_mean": -0.49903959035873413,
|
|
"beta_dpo/beta_margin_grad_std": 0.006939120590686798,
|
|
"beta_dpo/beta_margin_mean": 0.003841817146167159,
|
|
"beta_dpo/beta_margin_std": 0.02776145376265049,
|
|
"beta_dpo/beta_used": 0.10144880414009094,
|
|
"beta_dpo/beta_used_raw": 0.10144880414009094,
|
|
"beta_dpo/gap_mean": 0.01069832406938076,
|
|
"beta_dpo/gap_std": 0.23313455283641815,
|
|
"beta_dpo/loss_margin_mean": 0.037982091307640076,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.01762114537444934,
|
|
"grad_norm": 202.00311279296875,
|
|
"learning_rate": 7.971014492753623e-08,
|
|
"logits/chosen": -3.1653075218200684,
|
|
"logits/rejected": -3.153486967086792,
|
|
"loss": 1.3828,
|
|
"step": 12
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10425792634487152,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4985834062099457,
|
|
"beta_dpo/beta_margin_grad_std": 0.007317332550883293,
|
|
"beta_dpo/beta_margin_mean": 0.005666609387844801,
|
|
"beta_dpo/beta_margin_std": 0.02927694283425808,
|
|
"beta_dpo/beta_used": 0.10425792634487152,
|
|
"beta_dpo/beta_used_raw": 0.10425792634487152,
|
|
"beta_dpo/gap_mean": 0.01822437345981598,
|
|
"beta_dpo/gap_std": 0.24303942918777466,
|
|
"beta_dpo/loss_margin_mean": 0.054477810859680176,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.01908957415565345,
|
|
"grad_norm": 165.83859252929688,
|
|
"learning_rate": 8.695652173913042e-08,
|
|
"logits/chosen": -3.167670726776123,
|
|
"logits/rejected": -3.135024070739746,
|
|
"loss": 1.377,
|
|
"step": 13
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10415012389421463,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4957675039768219,
|
|
"beta_dpo/beta_margin_grad_std": 0.00820655096322298,
|
|
"beta_dpo/beta_margin_mean": 0.016939442604780197,
|
|
"beta_dpo/beta_margin_std": 0.03285452350974083,
|
|
"beta_dpo/beta_used": 0.10415012389421463,
|
|
"beta_dpo/beta_used_raw": 0.10415012389421463,
|
|
"beta_dpo/gap_mean": 0.039918702095746994,
|
|
"beta_dpo/gap_std": 0.2563033699989319,
|
|
"beta_dpo/loss_margin_mean": 0.16253972053527832,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.020558002936857563,
|
|
"grad_norm": 203.8751983642578,
|
|
"learning_rate": 9.420289855072464e-08,
|
|
"logits/chosen": -3.132795572280884,
|
|
"logits/rejected": -3.1252737045288086,
|
|
"loss": 1.3751,
|
|
"step": 14
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10431154817342758,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4954010248184204,
|
|
"beta_dpo/beta_margin_grad_std": 0.006738570984452963,
|
|
"beta_dpo/beta_margin_mean": 0.018400877714157104,
|
|
"beta_dpo/beta_margin_std": 0.026965487748384476,
|
|
"beta_dpo/beta_used": 0.10431154817342758,
|
|
"beta_dpo/beta_used_raw": 0.10431154817342758,
|
|
"beta_dpo/gap_mean": 0.06402108818292618,
|
|
"beta_dpo/gap_std": 0.2579445540904999,
|
|
"beta_dpo/loss_margin_mean": 0.17562128603458405,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.022026431718061675,
|
|
"grad_norm": 158.30233764648438,
|
|
"learning_rate": 1.0144927536231885e-07,
|
|
"logits/chosen": -3.1343517303466797,
|
|
"logits/rejected": -3.124833106994629,
|
|
"loss": 1.3722,
|
|
"step": 15
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10335851460695267,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4952828586101532,
|
|
"beta_dpo/beta_margin_grad_std": 0.007043875753879547,
|
|
"beta_dpo/beta_margin_mean": 0.018874231725931168,
|
|
"beta_dpo/beta_margin_std": 0.028187856078147888,
|
|
"beta_dpo/beta_used": 0.10335851460695267,
|
|
"beta_dpo/beta_used_raw": 0.10335851460695267,
|
|
"beta_dpo/gap_mean": 0.08750247955322266,
|
|
"beta_dpo/gap_std": 0.25985032320022583,
|
|
"beta_dpo/loss_margin_mean": 0.1823047697544098,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.023494860499265784,
|
|
"grad_norm": 162.9868621826172,
|
|
"learning_rate": 1.0869565217391303e-07,
|
|
"logits/chosen": -3.1242918968200684,
|
|
"logits/rejected": -3.090561628341675,
|
|
"loss": 1.3716,
|
|
"step": 16
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10424471646547318,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4921805262565613,
|
|
"beta_dpo/beta_margin_grad_std": 0.009317115880548954,
|
|
"beta_dpo/beta_margin_mean": 0.031297143548727036,
|
|
"beta_dpo/beta_margin_std": 0.037318408489227295,
|
|
"beta_dpo/beta_used": 0.10424471646547318,
|
|
"beta_dpo/beta_used_raw": 0.10424471646547318,
|
|
"beta_dpo/gap_mean": 0.12156766653060913,
|
|
"beta_dpo/gap_std": 0.273958683013916,
|
|
"beta_dpo/loss_margin_mean": 0.2991063594818115,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.024963289280469897,
|
|
"grad_norm": 171.33746337890625,
|
|
"learning_rate": 1.1594202898550725e-07,
|
|
"logits/chosen": -3.1508283615112305,
|
|
"logits/rejected": -3.0991227626800537,
|
|
"loss": 1.3663,
|
|
"step": 17
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10741108655929565,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4899873435497284,
|
|
"beta_dpo/beta_margin_grad_std": 0.009942273609340191,
|
|
"beta_dpo/beta_margin_mean": 0.040072713047266006,
|
|
"beta_dpo/beta_margin_std": 0.039800975471735,
|
|
"beta_dpo/beta_used": 0.10741108655929565,
|
|
"beta_dpo/beta_used_raw": 0.10741108655929565,
|
|
"beta_dpo/gap_mean": 0.1662050485610962,
|
|
"beta_dpo/gap_std": 0.2930639386177063,
|
|
"beta_dpo/loss_margin_mean": 0.37321096658706665,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.02643171806167401,
|
|
"grad_norm": 172.73681640625,
|
|
"learning_rate": 1.2318840579710146e-07,
|
|
"logits/chosen": -3.1059744358062744,
|
|
"logits/rejected": -3.0668582916259766,
|
|
"loss": 1.3556,
|
|
"step": 18
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10548347979784012,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4869752824306488,
|
|
"beta_dpo/beta_margin_grad_std": 0.01236420962959528,
|
|
"beta_dpo/beta_margin_mean": 0.05215656757354736,
|
|
"beta_dpo/beta_margin_std": 0.04959164187312126,
|
|
"beta_dpo/beta_used": 0.10548347979784012,
|
|
"beta_dpo/beta_used_raw": 0.10548347979784012,
|
|
"beta_dpo/gap_mean": 0.21926948428153992,
|
|
"beta_dpo/gap_std": 0.317290723323822,
|
|
"beta_dpo/loss_margin_mean": 0.49080538749694824,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.027900146842878122,
|
|
"grad_norm": 154.02589416503906,
|
|
"learning_rate": 1.3043478260869563e-07,
|
|
"logits/chosen": -3.137395143508911,
|
|
"logits/rejected": -3.095036506652832,
|
|
"loss": 1.3537,
|
|
"step": 19
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10729610919952393,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4847802519798279,
|
|
"beta_dpo/beta_margin_grad_std": 0.01249635312706232,
|
|
"beta_dpo/beta_margin_mean": 0.06094222515821457,
|
|
"beta_dpo/beta_margin_std": 0.05008528009057045,
|
|
"beta_dpo/beta_used": 0.10729610919952393,
|
|
"beta_dpo/beta_used_raw": 0.10729610919952393,
|
|
"beta_dpo/gap_mean": 0.28570470213890076,
|
|
"beta_dpo/gap_std": 0.34619662165641785,
|
|
"beta_dpo/loss_margin_mean": 0.5685427784919739,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.02936857562408223,
|
|
"grad_norm": 151.9236602783203,
|
|
"learning_rate": 1.3768115942028986e-07,
|
|
"logits/chosen": -3.151339530944824,
|
|
"logits/rejected": -3.1141912937164307,
|
|
"loss": 1.3433,
|
|
"step": 20
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1047445610165596,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4856947958469391,
|
|
"beta_dpo/beta_margin_grad_std": 0.015066784806549549,
|
|
"beta_dpo/beta_margin_mean": 0.057291969656944275,
|
|
"beta_dpo/beta_margin_std": 0.0604330375790596,
|
|
"beta_dpo/beta_used": 0.1047445610165596,
|
|
"beta_dpo/beta_used_raw": 0.1047445610165596,
|
|
"beta_dpo/gap_mean": 0.3368791937828064,
|
|
"beta_dpo/gap_std": 0.3872354030609131,
|
|
"beta_dpo/loss_margin_mean": 0.5463607311248779,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.030837004405286344,
|
|
"grad_norm": 139.0661163330078,
|
|
"learning_rate": 1.4492753623188405e-07,
|
|
"logits/chosen": -3.0985090732574463,
|
|
"logits/rejected": -3.0805447101593018,
|
|
"loss": 1.3432,
|
|
"step": 21
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10737676173448563,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4813934862613678,
|
|
"beta_dpo/beta_margin_grad_std": 0.013753235340118408,
|
|
"beta_dpo/beta_margin_mean": 0.074525848031044,
|
|
"beta_dpo/beta_margin_std": 0.055176492780447006,
|
|
"beta_dpo/beta_used": 0.10737676173448563,
|
|
"beta_dpo/beta_used_raw": 0.10737676173448563,
|
|
"beta_dpo/gap_mean": 0.40355396270751953,
|
|
"beta_dpo/gap_std": 0.4153197705745697,
|
|
"beta_dpo/loss_margin_mean": 0.6897796988487244,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.032305433186490456,
|
|
"grad_norm": 172.8441162109375,
|
|
"learning_rate": 1.5217391304347825e-07,
|
|
"logits/chosen": -3.1092402935028076,
|
|
"logits/rejected": -3.09182071685791,
|
|
"loss": 1.3307,
|
|
"step": 22
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10904830694198608,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4761696755886078,
|
|
"beta_dpo/beta_margin_grad_std": 0.021913398057222366,
|
|
"beta_dpo/beta_margin_mean": 0.09563583135604858,
|
|
"beta_dpo/beta_margin_std": 0.08828449249267578,
|
|
"beta_dpo/beta_used": 0.10904830694198608,
|
|
"beta_dpo/beta_used_raw": 0.10904830694198608,
|
|
"beta_dpo/gap_mean": 0.4720858931541443,
|
|
"beta_dpo/gap_std": 0.4659385085105896,
|
|
"beta_dpo/loss_margin_mean": 0.8740031123161316,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.033773861967694566,
|
|
"grad_norm": 156.1903076171875,
|
|
"learning_rate": 1.5942028985507245e-07,
|
|
"logits/chosen": -3.1103243827819824,
|
|
"logits/rejected": -3.088109016418457,
|
|
"loss": 1.3201,
|
|
"step": 23
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.11075541377067566,
|
|
"beta_dpo/beta_margin_grad_mean": -0.471403568983078,
|
|
"beta_dpo/beta_margin_grad_std": 0.021366121247410774,
|
|
"beta_dpo/beta_margin_mean": 0.11474256962537766,
|
|
"beta_dpo/beta_margin_std": 0.08595109730958939,
|
|
"beta_dpo/beta_used": 0.11075541377067566,
|
|
"beta_dpo/beta_used_raw": 0.11075541377067566,
|
|
"beta_dpo/gap_mean": 0.578666090965271,
|
|
"beta_dpo/gap_std": 0.5306879281997681,
|
|
"beta_dpo/loss_margin_mean": 1.0362656116485596,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.03524229074889868,
|
|
"grad_norm": 177.53704833984375,
|
|
"learning_rate": 1.6666666666666665e-07,
|
|
"logits/chosen": -3.085505723953247,
|
|
"logits/rejected": -3.097721815109253,
|
|
"loss": 1.305,
|
|
"step": 24
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10202641040086746,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47696781158447266,
|
|
"beta_dpo/beta_margin_grad_std": 0.02352500520646572,
|
|
"beta_dpo/beta_margin_mean": 0.09244953840970993,
|
|
"beta_dpo/beta_margin_std": 0.09470401704311371,
|
|
"beta_dpo/beta_used": 0.10202641040086746,
|
|
"beta_dpo/beta_used_raw": 0.10202641040086746,
|
|
"beta_dpo/gap_mean": 0.6436024904251099,
|
|
"beta_dpo/gap_std": 0.5977407693862915,
|
|
"beta_dpo/loss_margin_mean": 0.9032971858978271,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.03671071953010279,
|
|
"grad_norm": 133.51351928710938,
|
|
"learning_rate": 1.7391304347826085e-07,
|
|
"logits/chosen": -3.1361374855041504,
|
|
"logits/rejected": -3.1234917640686035,
|
|
"loss": 1.319,
|
|
"step": 25
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.11549623310565948,
|
|
"beta_dpo/beta_margin_grad_mean": -0.453942209482193,
|
|
"beta_dpo/beta_margin_grad_std": 0.04830946400761604,
|
|
"beta_dpo/beta_margin_mean": 0.18718303740024567,
|
|
"beta_dpo/beta_margin_std": 0.20004823803901672,
|
|
"beta_dpo/beta_used": 0.11549623310565948,
|
|
"beta_dpo/beta_used_raw": 0.11549623310565948,
|
|
"beta_dpo/gap_mean": 0.7885938882827759,
|
|
"beta_dpo/gap_std": 0.7514917850494385,
|
|
"beta_dpo/loss_margin_mean": 1.6038861274719238,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.0381791483113069,
|
|
"grad_norm": 163.3068389892578,
|
|
"learning_rate": 1.8115942028985507e-07,
|
|
"logits/chosen": -3.134460687637329,
|
|
"logits/rejected": -3.140258550643921,
|
|
"loss": 1.27,
|
|
"step": 26
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1283809393644333,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4395935833454132,
|
|
"beta_dpo/beta_margin_grad_std": 0.03977083042263985,
|
|
"beta_dpo/beta_margin_mean": 0.24452631175518036,
|
|
"beta_dpo/beta_margin_std": 0.16269095242023468,
|
|
"beta_dpo/beta_used": 0.1283809393644333,
|
|
"beta_dpo/beta_used_raw": 0.1283809393644333,
|
|
"beta_dpo/gap_mean": 1.0029573440551758,
|
|
"beta_dpo/gap_std": 0.8620645999908447,
|
|
"beta_dpo/loss_margin_mean": 1.8610539436340332,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.039647577092511016,
|
|
"grad_norm": 196.32107543945312,
|
|
"learning_rate": 1.8840579710144927e-07,
|
|
"logits/chosen": -3.0956711769104004,
|
|
"logits/rejected": -3.0962371826171875,
|
|
"loss": 1.2053,
|
|
"step": 27
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.12041008472442627,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44316551089286804,
|
|
"beta_dpo/beta_margin_grad_std": 0.03816642612218857,
|
|
"beta_dpo/beta_margin_mean": 0.2298518419265747,
|
|
"beta_dpo/beta_margin_std": 0.15634086728096008,
|
|
"beta_dpo/beta_used": 0.12041008472442627,
|
|
"beta_dpo/beta_used_raw": 0.12041008472442627,
|
|
"beta_dpo/gap_mean": 1.1611372232437134,
|
|
"beta_dpo/gap_std": 0.9274260997772217,
|
|
"beta_dpo/loss_margin_mean": 1.9079362154006958,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.041116005873715125,
|
|
"grad_norm": 173.4256134033203,
|
|
"learning_rate": 1.9565217391304347e-07,
|
|
"logits/chosen": -3.1474990844726562,
|
|
"logits/rejected": -3.1038711071014404,
|
|
"loss": 1.2161,
|
|
"step": 28
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.12488596141338348,
|
|
"beta_dpo/beta_margin_grad_mean": -0.43044498562812805,
|
|
"beta_dpo/beta_margin_grad_std": 0.04702915996313095,
|
|
"beta_dpo/beta_margin_mean": 0.2829776108264923,
|
|
"beta_dpo/beta_margin_std": 0.19504757225513458,
|
|
"beta_dpo/beta_used": 0.12488596141338348,
|
|
"beta_dpo/beta_used_raw": 0.12488596141338348,
|
|
"beta_dpo/gap_mean": 1.347472906112671,
|
|
"beta_dpo/gap_std": 1.0383023023605347,
|
|
"beta_dpo/loss_margin_mean": 2.274510145187378,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.042584434654919234,
|
|
"grad_norm": 177.54214477539062,
|
|
"learning_rate": 2.028985507246377e-07,
|
|
"logits/chosen": -3.128300666809082,
|
|
"logits/rejected": -3.105536937713623,
|
|
"loss": 1.1833,
|
|
"step": 29
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.12252659350633621,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4280206561088562,
|
|
"beta_dpo/beta_margin_grad_std": 0.05462159961462021,
|
|
"beta_dpo/beta_margin_mean": 0.29425692558288574,
|
|
"beta_dpo/beta_margin_std": 0.22929762303829193,
|
|
"beta_dpo/beta_used": 0.12252659350633621,
|
|
"beta_dpo/beta_used_raw": 0.12252659350633621,
|
|
"beta_dpo/gap_mean": 1.5580029487609863,
|
|
"beta_dpo/gap_std": 1.1984624862670898,
|
|
"beta_dpo/loss_margin_mean": 2.384845018386841,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.04405286343612335,
|
|
"grad_norm": 179.45391845703125,
|
|
"learning_rate": 2.1014492753623187e-07,
|
|
"logits/chosen": -3.109475612640381,
|
|
"logits/rejected": -3.0789778232574463,
|
|
"loss": 1.1672,
|
|
"step": 30
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.12765327095985413,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4269483983516693,
|
|
"beta_dpo/beta_margin_grad_std": 0.06185346469283104,
|
|
"beta_dpo/beta_margin_mean": 0.3004482090473175,
|
|
"beta_dpo/beta_margin_std": 0.2656671404838562,
|
|
"beta_dpo/beta_used": 0.12765327095985413,
|
|
"beta_dpo/beta_used_raw": 0.12765327095985413,
|
|
"beta_dpo/gap_mean": 1.7079854011535645,
|
|
"beta_dpo/gap_std": 1.3484312295913696,
|
|
"beta_dpo/loss_margin_mean": 2.3504257202148438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.04552129221732746,
|
|
"grad_norm": 166.8577880859375,
|
|
"learning_rate": 2.1739130434782607e-07,
|
|
"logits/chosen": -3.1314797401428223,
|
|
"logits/rejected": -3.0868959426879883,
|
|
"loss": 1.1353,
|
|
"step": 31
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10552072525024414,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4227868914604187,
|
|
"beta_dpo/beta_margin_grad_std": 0.06014800816774368,
|
|
"beta_dpo/beta_margin_mean": 0.31740450859069824,
|
|
"beta_dpo/beta_margin_std": 0.25505444407463074,
|
|
"beta_dpo/beta_used": 0.10552072525024414,
|
|
"beta_dpo/beta_used_raw": 0.10552072525024414,
|
|
"beta_dpo/gap_mean": 1.9182875156402588,
|
|
"beta_dpo/gap_std": 1.5188119411468506,
|
|
"beta_dpo/loss_margin_mean": 2.994873046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.04698972099853157,
|
|
"grad_norm": 143.5096435546875,
|
|
"learning_rate": 2.2463768115942027e-07,
|
|
"logits/chosen": -3.1662778854370117,
|
|
"logits/rejected": -3.149886131286621,
|
|
"loss": 1.1878,
|
|
"step": 32
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.12314489483833313,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41513192653656006,
|
|
"beta_dpo/beta_margin_grad_std": 0.07031543552875519,
|
|
"beta_dpo/beta_margin_mean": 0.3495492935180664,
|
|
"beta_dpo/beta_margin_std": 0.30640801787376404,
|
|
"beta_dpo/beta_used": 0.12314489483833313,
|
|
"beta_dpo/beta_used_raw": 0.12314489483833313,
|
|
"beta_dpo/gap_mean": 2.081599235534668,
|
|
"beta_dpo/gap_std": 1.7187418937683105,
|
|
"beta_dpo/loss_margin_mean": 2.8416450023651123,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.048458149779735685,
|
|
"grad_norm": 178.19117736816406,
|
|
"learning_rate": 2.318840579710145e-07,
|
|
"logits/chosen": -3.126152276992798,
|
|
"logits/rejected": -3.110962152481079,
|
|
"loss": 1.1123,
|
|
"step": 33
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10388785600662231,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4212900698184967,
|
|
"beta_dpo/beta_margin_grad_std": 0.0669008269906044,
|
|
"beta_dpo/beta_margin_mean": 0.3246135115623474,
|
|
"beta_dpo/beta_margin_std": 0.28224509954452515,
|
|
"beta_dpo/beta_used": 0.10388785600662231,
|
|
"beta_dpo/beta_used_raw": 0.10388785600662231,
|
|
"beta_dpo/gap_mean": 2.2918877601623535,
|
|
"beta_dpo/gap_std": 1.891815185546875,
|
|
"beta_dpo/loss_margin_mean": 3.123857021331787,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.049926578560939794,
|
|
"grad_norm": 142.10772705078125,
|
|
"learning_rate": 2.391304347826087e-07,
|
|
"logits/chosen": -3.1223511695861816,
|
|
"logits/rejected": -3.1378636360168457,
|
|
"loss": 1.1663,
|
|
"step": 34
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1047118604183197,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40423837304115295,
|
|
"beta_dpo/beta_margin_grad_std": 0.07508349418640137,
|
|
"beta_dpo/beta_margin_mean": 0.39983880519866943,
|
|
"beta_dpo/beta_margin_std": 0.32720947265625,
|
|
"beta_dpo/beta_used": 0.1047118604183197,
|
|
"beta_dpo/beta_used_raw": 0.1047118604183197,
|
|
"beta_dpo/gap_mean": 2.542179584503174,
|
|
"beta_dpo/gap_std": 2.094670295715332,
|
|
"beta_dpo/loss_margin_mean": 3.8020293712615967,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.0513950073421439,
|
|
"grad_norm": 136.2096710205078,
|
|
"learning_rate": 2.463768115942029e-07,
|
|
"logits/chosen": -3.1520180702209473,
|
|
"logits/rejected": -3.1538925170898438,
|
|
"loss": 1.1385,
|
|
"step": 35
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.12119206041097641,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3750362694263458,
|
|
"beta_dpo/beta_margin_grad_std": 0.11262792348861694,
|
|
"beta_dpo/beta_margin_mean": 0.5494691729545593,
|
|
"beta_dpo/beta_margin_std": 0.5312042832374573,
|
|
"beta_dpo/beta_used": 0.12119206041097641,
|
|
"beta_dpo/beta_used_raw": 0.12119206041097641,
|
|
"beta_dpo/gap_mean": 2.822805881500244,
|
|
"beta_dpo/gap_std": 2.430403709411621,
|
|
"beta_dpo/loss_margin_mean": 4.381009578704834,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.05286343612334802,
|
|
"grad_norm": 132.07125854492188,
|
|
"learning_rate": 2.536231884057971e-07,
|
|
"logits/chosen": -3.1347692012786865,
|
|
"logits/rejected": -3.133504629135132,
|
|
"loss": 1.062,
|
|
"step": 36
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.12719795107841492,
|
|
"beta_dpo/beta_margin_grad_mean": -0.368235319852829,
|
|
"beta_dpo/beta_margin_grad_std": 0.10667005181312561,
|
|
"beta_dpo/beta_margin_mean": 0.579422652721405,
|
|
"beta_dpo/beta_margin_std": 0.5222266912460327,
|
|
"beta_dpo/beta_used": 0.12719795107841492,
|
|
"beta_dpo/beta_used_raw": 0.12719795107841492,
|
|
"beta_dpo/gap_mean": 3.1904327869415283,
|
|
"beta_dpo/gap_std": 2.7459893226623535,
|
|
"beta_dpo/loss_margin_mean": 4.56009578704834,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.05433186490455213,
|
|
"grad_norm": 143.18426513671875,
|
|
"learning_rate": 2.6086956521739126e-07,
|
|
"logits/chosen": -3.1382253170013428,
|
|
"logits/rejected": -3.1056854724884033,
|
|
"loss": 1.0,
|
|
"step": 37
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.14245614409446716,
|
|
"beta_dpo/beta_margin_grad_mean": -0.31650885939598083,
|
|
"beta_dpo/beta_margin_grad_std": 0.14402246475219727,
|
|
"beta_dpo/beta_margin_mean": 0.8959528803825378,
|
|
"beta_dpo/beta_margin_std": 0.8295892477035522,
|
|
"beta_dpo/beta_used": 0.14245614409446716,
|
|
"beta_dpo/beta_used_raw": 0.14245614409446716,
|
|
"beta_dpo/gap_mean": 3.688507556915283,
|
|
"beta_dpo/gap_std": 3.2539007663726807,
|
|
"beta_dpo/loss_margin_mean": 6.270021438598633,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.055800293685756244,
|
|
"grad_norm": 143.8856201171875,
|
|
"learning_rate": 2.681159420289855e-07,
|
|
"logits/chosen": -3.1051831245422363,
|
|
"logits/rejected": -3.11987042427063,
|
|
"loss": 0.9095,
|
|
"step": 38
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1562386155128479,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28196191787719727,
|
|
"beta_dpo/beta_margin_grad_std": 0.12678173184394836,
|
|
"beta_dpo/beta_margin_mean": 1.0526187419891357,
|
|
"beta_dpo/beta_margin_std": 0.7619715929031372,
|
|
"beta_dpo/beta_used": 0.1562386155128479,
|
|
"beta_dpo/beta_used_raw": 0.1562386155128479,
|
|
"beta_dpo/gap_mean": 4.267126560211182,
|
|
"beta_dpo/gap_std": 3.6263532638549805,
|
|
"beta_dpo/loss_margin_mean": 6.750538349151611,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.05726872246696035,
|
|
"grad_norm": 157.51174926757812,
|
|
"learning_rate": 2.753623188405797e-07,
|
|
"logits/chosen": -3.107410430908203,
|
|
"logits/rejected": -3.0857043266296387,
|
|
"loss": 0.7892,
|
|
"step": 39
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1138225644826889,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3327762484550476,
|
|
"beta_dpo/beta_margin_grad_std": 0.1230630874633789,
|
|
"beta_dpo/beta_margin_mean": 0.7856088876724243,
|
|
"beta_dpo/beta_margin_std": 0.7016869783401489,
|
|
"beta_dpo/beta_used": 0.1138225644826889,
|
|
"beta_dpo/beta_used_raw": 0.1138225644826889,
|
|
"beta_dpo/gap_mean": 4.675539016723633,
|
|
"beta_dpo/gap_std": 3.911595582962036,
|
|
"beta_dpo/loss_margin_mean": 6.719384670257568,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.05873715124816446,
|
|
"grad_norm": 121.4354248046875,
|
|
"learning_rate": 2.8260869565217386e-07,
|
|
"logits/chosen": -3.154458999633789,
|
|
"logits/rejected": -3.135817050933838,
|
|
"loss": 0.942,
|
|
"step": 40
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.17786243557929993,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2580529749393463,
|
|
"beta_dpo/beta_margin_grad_std": 0.18610239028930664,
|
|
"beta_dpo/beta_margin_mean": 1.4926997423171997,
|
|
"beta_dpo/beta_margin_std": 1.711531162261963,
|
|
"beta_dpo/beta_used": 0.17786243557929993,
|
|
"beta_dpo/beta_used_raw": 0.17786243557929993,
|
|
"beta_dpo/gap_mean": 5.223352432250977,
|
|
"beta_dpo/gap_std": 4.3833465576171875,
|
|
"beta_dpo/loss_margin_mean": 8.071098327636719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.06020558002936858,
|
|
"grad_norm": 153.85923767089844,
|
|
"learning_rate": 2.898550724637681e-07,
|
|
"logits/chosen": -3.1468467712402344,
|
|
"logits/rejected": -3.1274843215942383,
|
|
"loss": 0.6769,
|
|
"step": 41
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1443341076374054,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28167596459388733,
|
|
"beta_dpo/beta_margin_grad_std": 0.18971994519233704,
|
|
"beta_dpo/beta_margin_mean": 1.44333815574646,
|
|
"beta_dpo/beta_margin_std": 1.6360913515090942,
|
|
"beta_dpo/beta_used": 0.1443341076374054,
|
|
"beta_dpo/beta_used_raw": 0.1443341076374054,
|
|
"beta_dpo/gap_mean": 5.9276628494262695,
|
|
"beta_dpo/gap_std": 5.334522724151611,
|
|
"beta_dpo/loss_margin_mean": 9.092480659484863,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.06167400881057269,
|
|
"grad_norm": 157.87091064453125,
|
|
"learning_rate": 2.971014492753623e-07,
|
|
"logits/chosen": -3.130484104156494,
|
|
"logits/rejected": -3.144623279571533,
|
|
"loss": 0.8067,
|
|
"step": 42
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.18256396055221558,
|
|
"beta_dpo/beta_margin_grad_mean": -0.20482036471366882,
|
|
"beta_dpo/beta_margin_grad_std": 0.17849688231945038,
|
|
"beta_dpo/beta_margin_mean": 1.8994702100753784,
|
|
"beta_dpo/beta_margin_std": 1.475931167602539,
|
|
"beta_dpo/beta_used": 0.18256396055221558,
|
|
"beta_dpo/beta_used_raw": 0.18256396055221558,
|
|
"beta_dpo/gap_mean": 6.808272361755371,
|
|
"beta_dpo/gap_std": 5.8962321281433105,
|
|
"beta_dpo/loss_margin_mean": 10.406832695007324,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.0631424375917768,
|
|
"grad_norm": 143.8301239013672,
|
|
"learning_rate": 3.043478260869565e-07,
|
|
"logits/chosen": -3.1377265453338623,
|
|
"logits/rejected": -3.1264193058013916,
|
|
"loss": 0.5973,
|
|
"step": 43
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.24507826566696167,
|
|
"beta_dpo/beta_margin_grad_mean": -0.14221227169036865,
|
|
"beta_dpo/beta_margin_grad_std": 0.19724884629249573,
|
|
"beta_dpo/beta_margin_mean": 3.0058138370513916,
|
|
"beta_dpo/beta_margin_std": 2.3838889598846436,
|
|
"beta_dpo/beta_used": 0.24507826566696167,
|
|
"beta_dpo/beta_used_raw": 0.24507826566696167,
|
|
"beta_dpo/gap_mean": 7.831865310668945,
|
|
"beta_dpo/gap_std": 6.489755153656006,
|
|
"beta_dpo/loss_margin_mean": 12.265984535217285,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.06461086637298091,
|
|
"grad_norm": 115.14469146728516,
|
|
"learning_rate": 3.115942028985507e-07,
|
|
"logits/chosen": -3.0733070373535156,
|
|
"logits/rejected": -3.0947670936584473,
|
|
"loss": 0.3312,
|
|
"step": 44
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.038562677800655365,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4185938835144043,
|
|
"beta_dpo/beta_margin_grad_std": 0.0780901312828064,
|
|
"beta_dpo/beta_margin_mean": 0.3413236737251282,
|
|
"beta_dpo/beta_margin_std": 0.34545838832855225,
|
|
"beta_dpo/beta_used": 0.038562677800655365,
|
|
"beta_dpo/beta_used_raw": 0.038562677800655365,
|
|
"beta_dpo/gap_mean": 8.08301067352295,
|
|
"beta_dpo/gap_std": 6.940834045410156,
|
|
"beta_dpo/loss_margin_mean": 8.900261878967285,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.06607929515418502,
|
|
"grad_norm": 59.289710998535156,
|
|
"learning_rate": 3.188405797101449e-07,
|
|
"logits/chosen": -3.0954208374023438,
|
|
"logits/rejected": -3.097357749938965,
|
|
"loss": 1.1443,
|
|
"step": 45
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.11945217847824097,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2516638934612274,
|
|
"beta_dpo/beta_margin_grad_std": 0.16823652386665344,
|
|
"beta_dpo/beta_margin_mean": 1.4459818601608276,
|
|
"beta_dpo/beta_margin_std": 1.3947581052780151,
|
|
"beta_dpo/beta_used": 0.11945217847824097,
|
|
"beta_dpo/beta_used_raw": 0.11945217847824097,
|
|
"beta_dpo/gap_mean": 8.649866104125977,
|
|
"beta_dpo/gap_std": 7.57880973815918,
|
|
"beta_dpo/loss_margin_mean": 11.921540260314941,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.06754772393538913,
|
|
"grad_norm": 121.44014739990234,
|
|
"learning_rate": 3.260869565217391e-07,
|
|
"logits/chosen": -3.073902130126953,
|
|
"logits/rejected": -3.071622371673584,
|
|
"loss": 0.6997,
|
|
"step": 46
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.09873463958501816,
|
|
"beta_dpo/beta_margin_grad_mean": -0.30619868636131287,
|
|
"beta_dpo/beta_margin_grad_std": 0.17535436153411865,
|
|
"beta_dpo/beta_margin_mean": 1.112424612045288,
|
|
"beta_dpo/beta_margin_std": 1.307705283164978,
|
|
"beta_dpo/beta_used": 0.09873463958501816,
|
|
"beta_dpo/beta_used_raw": 0.09873463958501816,
|
|
"beta_dpo/gap_mean": 8.923969268798828,
|
|
"beta_dpo/gap_std": 8.19182014465332,
|
|
"beta_dpo/loss_margin_mean": 10.096163749694824,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.06901615271659324,
|
|
"grad_norm": 100.65027618408203,
|
|
"learning_rate": 3.333333333333333e-07,
|
|
"logits/chosen": -3.120870351791382,
|
|
"logits/rejected": -3.112800121307373,
|
|
"loss": 0.7862,
|
|
"step": 47
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.04364791885018349,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3776269853115082,
|
|
"beta_dpo/beta_margin_grad_std": 0.10796888172626495,
|
|
"beta_dpo/beta_margin_mean": 0.5390008687973022,
|
|
"beta_dpo/beta_margin_std": 0.5236583948135376,
|
|
"beta_dpo/beta_used": 0.04364791885018349,
|
|
"beta_dpo/beta_used_raw": 0.04364791885018349,
|
|
"beta_dpo/gap_mean": 9.664556503295898,
|
|
"beta_dpo/gap_std": 8.99693489074707,
|
|
"beta_dpo/loss_margin_mean": 12.434694290161133,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.07048458149779736,
|
|
"grad_norm": 72.16007232666016,
|
|
"learning_rate": 3.4057971014492755e-07,
|
|
"logits/chosen": -3.109793186187744,
|
|
"logits/rejected": -3.0930979251861572,
|
|
"loss": 1.067,
|
|
"step": 48
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1467229723930359,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2280292958021164,
|
|
"beta_dpo/beta_margin_grad_std": 0.1954279989004135,
|
|
"beta_dpo/beta_margin_mean": 2.1265885829925537,
|
|
"beta_dpo/beta_margin_std": 2.2465431690216064,
|
|
"beta_dpo/beta_used": 0.1467229723930359,
|
|
"beta_dpo/beta_used_raw": 0.1467229723930359,
|
|
"beta_dpo/gap_mean": 10.395795822143555,
|
|
"beta_dpo/gap_std": 9.3504638671875,
|
|
"beta_dpo/loss_margin_mean": 13.992044448852539,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.07195301027900147,
|
|
"grad_norm": 86.9437484741211,
|
|
"learning_rate": 3.478260869565217e-07,
|
|
"logits/chosen": -3.131073474884033,
|
|
"logits/rejected": -3.105538845062256,
|
|
"loss": 0.6067,
|
|
"step": 49
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.18687647581100464,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22798167169094086,
|
|
"beta_dpo/beta_margin_grad_std": 0.24264167249202728,
|
|
"beta_dpo/beta_margin_mean": 2.600717306137085,
|
|
"beta_dpo/beta_margin_std": 3.0297393798828125,
|
|
"beta_dpo/beta_used": 0.18687647581100464,
|
|
"beta_dpo/beta_used_raw": 0.18687647581100464,
|
|
"beta_dpo/gap_mean": 11.018354415893555,
|
|
"beta_dpo/gap_std": 10.152142524719238,
|
|
"beta_dpo/loss_margin_mean": 13.667196273803711,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.07342143906020558,
|
|
"grad_norm": 170.4370574951172,
|
|
"learning_rate": 3.5507246376811595e-07,
|
|
"logits/chosen": -3.116657257080078,
|
|
"logits/rejected": -3.1149511337280273,
|
|
"loss": 0.5317,
|
|
"step": 50
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0934867113828659,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3069649636745453,
|
|
"beta_dpo/beta_margin_grad_std": 0.1902894377708435,
|
|
"beta_dpo/beta_margin_mean": 1.3371574878692627,
|
|
"beta_dpo/beta_margin_std": 1.9382656812667847,
|
|
"beta_dpo/beta_used": 0.0934867113828659,
|
|
"beta_dpo/beta_used_raw": 0.0934867113828659,
|
|
"beta_dpo/gap_mean": 11.582159996032715,
|
|
"beta_dpo/gap_std": 11.08337688446045,
|
|
"beta_dpo/loss_margin_mean": 14.272186279296875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.07488986784140969,
|
|
"grad_norm": 96.85399627685547,
|
|
"learning_rate": 3.6231884057971015e-07,
|
|
"logits/chosen": -3.06636643409729,
|
|
"logits/rejected": -3.0568933486938477,
|
|
"loss": 0.7809,
|
|
"step": 51
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.18484123051166534,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21149781346321106,
|
|
"beta_dpo/beta_margin_grad_std": 0.19529053568840027,
|
|
"beta_dpo/beta_margin_mean": 3.6904523372650146,
|
|
"beta_dpo/beta_margin_std": 4.510212421417236,
|
|
"beta_dpo/beta_used": 0.18484123051166534,
|
|
"beta_dpo/beta_used_raw": 0.18484123051166534,
|
|
"beta_dpo/gap_mean": 12.820097923278809,
|
|
"beta_dpo/gap_std": 11.931066513061523,
|
|
"beta_dpo/loss_margin_mean": 18.8443546295166,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.0763582966226138,
|
|
"grad_norm": 73.05899810791016,
|
|
"learning_rate": 3.695652173913043e-07,
|
|
"logits/chosen": -3.0794599056243896,
|
|
"logits/rejected": -3.0700924396514893,
|
|
"loss": 0.5789,
|
|
"step": 52
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.28637969493865967,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12516933679580688,
|
|
"beta_dpo/beta_margin_grad_std": 0.2069372832775116,
|
|
"beta_dpo/beta_margin_mean": 6.572714328765869,
|
|
"beta_dpo/beta_margin_std": 7.698074817657471,
|
|
"beta_dpo/beta_used": 0.28637969493865967,
|
|
"beta_dpo/beta_used_raw": 0.28637969493865967,
|
|
"beta_dpo/gap_mean": 13.912508010864258,
|
|
"beta_dpo/gap_std": 12.778947830200195,
|
|
"beta_dpo/loss_margin_mean": 20.386211395263672,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.07782672540381791,
|
|
"grad_norm": 187.00868225097656,
|
|
"learning_rate": 3.7681159420289855e-07,
|
|
"logits/chosen": -3.0854148864746094,
|
|
"logits/rejected": -3.0860137939453125,
|
|
"loss": 0.2734,
|
|
"step": 53
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.06887197494506836,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3133368492126465,
|
|
"beta_dpo/beta_margin_grad_std": 0.20501112937927246,
|
|
"beta_dpo/beta_margin_mean": 1.3893598318099976,
|
|
"beta_dpo/beta_margin_std": 1.9802852869033813,
|
|
"beta_dpo/beta_used": 0.06887197494506836,
|
|
"beta_dpo/beta_used_raw": 0.06331352889537811,
|
|
"beta_dpo/gap_mean": 15.25615119934082,
|
|
"beta_dpo/gap_std": 13.717507362365723,
|
|
"beta_dpo/loss_margin_mean": 19.47393226623535,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.07929515418502203,
|
|
"grad_norm": 57.23472213745117,
|
|
"learning_rate": 3.8405797101449274e-07,
|
|
"logits/chosen": -3.093733310699463,
|
|
"logits/rejected": -3.0969040393829346,
|
|
"loss": 0.8662,
|
|
"step": 54
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.06593922525644302,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3532218635082245,
|
|
"beta_dpo/beta_margin_grad_std": 0.2077246904373169,
|
|
"beta_dpo/beta_margin_mean": 1.131407380104065,
|
|
"beta_dpo/beta_margin_std": 1.861047387123108,
|
|
"beta_dpo/beta_used": 0.06593922525644302,
|
|
"beta_dpo/beta_used_raw": 0.06507368385791779,
|
|
"beta_dpo/gap_mean": 16.030441284179688,
|
|
"beta_dpo/gap_std": 15.258533477783203,
|
|
"beta_dpo/loss_margin_mean": 18.9444522857666,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.08076358296622614,
|
|
"grad_norm": 94.53569030761719,
|
|
"learning_rate": 3.9130434782608694e-07,
|
|
"logits/chosen": -3.090271472930908,
|
|
"logits/rejected": -3.0747573375701904,
|
|
"loss": 0.9575,
|
|
"step": 55
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3255024254322052,
|
|
"beta_dpo/beta_margin_grad_mean": -0.16037730872631073,
|
|
"beta_dpo/beta_margin_grad_std": 0.3197742998600006,
|
|
"beta_dpo/beta_margin_mean": 7.480155944824219,
|
|
"beta_dpo/beta_margin_std": 8.26274585723877,
|
|
"beta_dpo/beta_used": 0.3255024254322052,
|
|
"beta_dpo/beta_used_raw": 0.3255024254322052,
|
|
"beta_dpo/gap_mean": 17.08382797241211,
|
|
"beta_dpo/gap_std": 16.365951538085938,
|
|
"beta_dpo/loss_margin_mean": 22.620737075805664,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.08223201174743025,
|
|
"grad_norm": 467.24908447265625,
|
|
"learning_rate": 3.9855072463768114e-07,
|
|
"logits/chosen": -3.066891670227051,
|
|
"logits/rejected": -3.0782933235168457,
|
|
"loss": 0.4909,
|
|
"step": 56
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.13110555708408356,
|
|
"beta_dpo/beta_margin_grad_mean": -0.16527362167835236,
|
|
"beta_dpo/beta_margin_grad_std": 0.23161666095256805,
|
|
"beta_dpo/beta_margin_mean": 3.3455374240875244,
|
|
"beta_dpo/beta_margin_std": 3.1305649280548096,
|
|
"beta_dpo/beta_used": 0.13110555708408356,
|
|
"beta_dpo/beta_used_raw": 0.13110555708408356,
|
|
"beta_dpo/gap_mean": 18.27362823486328,
|
|
"beta_dpo/gap_std": 17.460128784179688,
|
|
"beta_dpo/loss_margin_mean": 25.040273666381836,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.08370044052863436,
|
|
"grad_norm": 155.8965606689453,
|
|
"learning_rate": 4.057971014492754e-07,
|
|
"logits/chosen": -3.085702419281006,
|
|
"logits/rejected": -3.0846567153930664,
|
|
"loss": 0.5031,
|
|
"step": 57
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0819333866238594,
|
|
"beta_dpo/beta_margin_grad_mean": -0.1960979849100113,
|
|
"beta_dpo/beta_margin_grad_std": 0.16590198874473572,
|
|
"beta_dpo/beta_margin_mean": 2.0763485431671143,
|
|
"beta_dpo/beta_margin_std": 1.8784925937652588,
|
|
"beta_dpo/beta_used": 0.0819333866238594,
|
|
"beta_dpo/beta_used_raw": 0.0819333866238594,
|
|
"beta_dpo/gap_mean": 19.728099822998047,
|
|
"beta_dpo/gap_std": 18.534076690673828,
|
|
"beta_dpo/loss_margin_mean": 25.43709373474121,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.08516886930983847,
|
|
"grad_norm": 105.10377502441406,
|
|
"learning_rate": 4.1304347826086954e-07,
|
|
"logits/chosen": -3.0557703971862793,
|
|
"logits/rejected": -3.044924259185791,
|
|
"loss": 0.5304,
|
|
"step": 58
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.19281019270420074,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28002673387527466,
|
|
"beta_dpo/beta_margin_grad_std": 0.255830854177475,
|
|
"beta_dpo/beta_margin_mean": 5.561700344085693,
|
|
"beta_dpo/beta_margin_std": 8.050152778625488,
|
|
"beta_dpo/beta_used": 0.19281019270420074,
|
|
"beta_dpo/beta_used_raw": 0.06114870309829712,
|
|
"beta_dpo/gap_mean": 21.075639724731445,
|
|
"beta_dpo/gap_std": 19.727760314941406,
|
|
"beta_dpo/loss_margin_mean": 25.989961624145508,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.08663729809104258,
|
|
"grad_norm": 139.58779907226562,
|
|
"learning_rate": 4.2028985507246374e-07,
|
|
"logits/chosen": -3.0765440464019775,
|
|
"logits/rejected": -3.0723140239715576,
|
|
"loss": 0.7404,
|
|
"step": 59
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3099629878997803,
|
|
"beta_dpo/beta_margin_grad_mean": -0.16709139943122864,
|
|
"beta_dpo/beta_margin_grad_std": 0.31567007303237915,
|
|
"beta_dpo/beta_margin_mean": 7.2313055992126465,
|
|
"beta_dpo/beta_margin_std": 9.950211524963379,
|
|
"beta_dpo/beta_used": 0.3099629878997803,
|
|
"beta_dpo/beta_used_raw": 0.3099629878997803,
|
|
"beta_dpo/gap_mean": 21.46399688720703,
|
|
"beta_dpo/gap_std": 21.259231567382812,
|
|
"beta_dpo/loss_margin_mean": 21.761423110961914,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.0881057268722467,
|
|
"grad_norm": 332.7749328613281,
|
|
"learning_rate": 4.2753623188405794e-07,
|
|
"logits/chosen": -3.051283836364746,
|
|
"logits/rejected": -3.03759503364563,
|
|
"loss": 0.2553,
|
|
"step": 60
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0911061018705368,
|
|
"beta_dpo/beta_margin_grad_mean": -0.31341421604156494,
|
|
"beta_dpo/beta_margin_grad_std": 0.25546589493751526,
|
|
"beta_dpo/beta_margin_mean": 2.5058257579803467,
|
|
"beta_dpo/beta_margin_std": 4.593942642211914,
|
|
"beta_dpo/beta_used": 0.0911061018705368,
|
|
"beta_dpo/beta_used_raw": 0.0911061018705368,
|
|
"beta_dpo/gap_mean": 22.091663360595703,
|
|
"beta_dpo/gap_std": 22.40526580810547,
|
|
"beta_dpo/loss_margin_mean": 26.832210540771484,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.08957415565345081,
|
|
"grad_norm": 144.1743927001953,
|
|
"learning_rate": 4.3478260869565214e-07,
|
|
"logits/chosen": -3.0594305992126465,
|
|
"logits/rejected": -3.0812244415283203,
|
|
"loss": 0.8416,
|
|
"step": 61
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.055423423647880554,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3313938081264496,
|
|
"beta_dpo/beta_margin_grad_std": 0.21602469682693481,
|
|
"beta_dpo/beta_margin_mean": 1.2362252473831177,
|
|
"beta_dpo/beta_margin_std": 1.8517836332321167,
|
|
"beta_dpo/beta_used": 0.055423423647880554,
|
|
"beta_dpo/beta_used_raw": -0.009761884808540344,
|
|
"beta_dpo/gap_mean": 22.36334228515625,
|
|
"beta_dpo/gap_std": 22.1772403717041,
|
|
"beta_dpo/loss_margin_mean": 22.912548065185547,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.09104258443465492,
|
|
"grad_norm": 53.500511169433594,
|
|
"learning_rate": 4.420289855072464e-07,
|
|
"logits/chosen": -3.0460290908813477,
|
|
"logits/rejected": -3.0380520820617676,
|
|
"loss": 0.8303,
|
|
"step": 62
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.027289319783449173,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39005380868911743,
|
|
"beta_dpo/beta_margin_grad_std": 0.17873267829418182,
|
|
"beta_dpo/beta_margin_mean": 0.7003328204154968,
|
|
"beta_dpo/beta_margin_std": 1.3609882593154907,
|
|
"beta_dpo/beta_used": 0.027289319783449173,
|
|
"beta_dpo/beta_used_raw": 0.017231859266757965,
|
|
"beta_dpo/gap_mean": 23.183303833007812,
|
|
"beta_dpo/gap_std": 23.588115692138672,
|
|
"beta_dpo/loss_margin_mean": 27.170320510864258,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.09251101321585903,
|
|
"grad_norm": 58.18537521362305,
|
|
"learning_rate": 4.4927536231884053e-07,
|
|
"logits/chosen": -3.0674667358398438,
|
|
"logits/rejected": -3.0683016777038574,
|
|
"loss": 1.0376,
|
|
"step": 63
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4925476312637329,
|
|
"beta_dpo/beta_margin_grad_std": 0.00753110833466053,
|
|
"beta_dpo/beta_margin_mean": 0.029822714626789093,
|
|
"beta_dpo/beta_margin_std": 0.030163373798131943,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.11208631843328476,
|
|
"beta_dpo/gap_mean": 24.51315689086914,
|
|
"beta_dpo/gap_std": 25.101673126220703,
|
|
"beta_dpo/loss_margin_mean": 29.822711944580078,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.09397944199706314,
|
|
"grad_norm": 3.092960834503174,
|
|
"learning_rate": 4.5652173913043473e-07,
|
|
"logits/chosen": -3.0635104179382324,
|
|
"logits/rejected": -3.0607059001922607,
|
|
"loss": 1.3655,
|
|
"step": 64
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2129736691713333,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2961460053920746,
|
|
"beta_dpo/beta_margin_grad_std": 0.2763463854789734,
|
|
"beta_dpo/beta_margin_mean": 7.6759843826293945,
|
|
"beta_dpo/beta_margin_std": 12.091070175170898,
|
|
"beta_dpo/beta_used": 0.2129736691713333,
|
|
"beta_dpo/beta_used_raw": 0.18820346891880035,
|
|
"beta_dpo/gap_mean": 25.80198097229004,
|
|
"beta_dpo/gap_std": 26.017024993896484,
|
|
"beta_dpo/loss_margin_mean": 32.39765167236328,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.09544787077826726,
|
|
"grad_norm": 288.397705078125,
|
|
"learning_rate": 4.63768115942029e-07,
|
|
"logits/chosen": -3.04362416267395,
|
|
"logits/rejected": -3.0484161376953125,
|
|
"loss": 0.8591,
|
|
"step": 65
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.02076115272939205,
|
|
"beta_dpo/beta_margin_grad_mean": -0.37283429503440857,
|
|
"beta_dpo/beta_margin_grad_std": 0.1666054129600525,
|
|
"beta_dpo/beta_margin_mean": 0.713555097579956,
|
|
"beta_dpo/beta_margin_std": 1.0996434688568115,
|
|
"beta_dpo/beta_used": 0.02076115272939205,
|
|
"beta_dpo/beta_used_raw": -0.16835710406303406,
|
|
"beta_dpo/gap_mean": 26.47984504699707,
|
|
"beta_dpo/gap_std": 27.09063148498535,
|
|
"beta_dpo/loss_margin_mean": 27.242469787597656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.09691629955947137,
|
|
"grad_norm": 32.42694091796875,
|
|
"learning_rate": 4.7101449275362313e-07,
|
|
"logits/chosen": -3.029677391052246,
|
|
"logits/rejected": -3.0298986434936523,
|
|
"loss": 1.0385,
|
|
"step": 66
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4936298727989197,
|
|
"beta_dpo/beta_margin_grad_std": 0.007120794616639614,
|
|
"beta_dpo/beta_margin_mean": 0.025488976389169693,
|
|
"beta_dpo/beta_margin_std": 0.02850145660340786,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.2273913323879242,
|
|
"beta_dpo/gap_mean": 25.91208267211914,
|
|
"beta_dpo/gap_std": 27.34076499938965,
|
|
"beta_dpo/loss_margin_mean": 25.488975524902344,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.09838472834067548,
|
|
"grad_norm": 3.0101864337921143,
|
|
"learning_rate": 4.782608695652174e-07,
|
|
"logits/chosen": -2.986940860748291,
|
|
"logits/rejected": -2.976597547531128,
|
|
"loss": 1.366,
|
|
"step": 67
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.14753571152687073,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2845630645751953,
|
|
"beta_dpo/beta_margin_grad_std": 0.2652169167995453,
|
|
"beta_dpo/beta_margin_mean": 5.07893180847168,
|
|
"beta_dpo/beta_margin_std": 6.9394354820251465,
|
|
"beta_dpo/beta_used": 0.14753571152687073,
|
|
"beta_dpo/beta_used_raw": 0.058987803757190704,
|
|
"beta_dpo/gap_mean": 26.669593811035156,
|
|
"beta_dpo/gap_std": 27.67668914794922,
|
|
"beta_dpo/loss_margin_mean": 31.38116455078125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.09985315712187959,
|
|
"grad_norm": 293.31982421875,
|
|
"learning_rate": 4.855072463768116e-07,
|
|
"logits/chosen": -3.0039591789245605,
|
|
"logits/rejected": -2.98842191696167,
|
|
"loss": 0.88,
|
|
"step": 68
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2622404098510742,
|
|
"beta_dpo/beta_margin_grad_mean": -0.30106690526008606,
|
|
"beta_dpo/beta_margin_grad_std": 0.28205162286758423,
|
|
"beta_dpo/beta_margin_mean": 10.098217010498047,
|
|
"beta_dpo/beta_margin_std": 17.523113250732422,
|
|
"beta_dpo/beta_used": 0.2622404098510742,
|
|
"beta_dpo/beta_used_raw": 0.10589639842510223,
|
|
"beta_dpo/gap_mean": 28.50058364868164,
|
|
"beta_dpo/gap_std": 28.632122039794922,
|
|
"beta_dpo/loss_margin_mean": 36.14906311035156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1013215859030837,
|
|
"grad_norm": 702.201416015625,
|
|
"learning_rate": 4.927536231884058e-07,
|
|
"logits/chosen": -3.010495662689209,
|
|
"logits/rejected": -3.016476631164551,
|
|
"loss": 0.8454,
|
|
"step": 69
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.05058491602540016,
|
|
"beta_dpo/beta_margin_grad_mean": -0.314628541469574,
|
|
"beta_dpo/beta_margin_grad_std": 0.21072426438331604,
|
|
"beta_dpo/beta_margin_mean": 1.6962952613830566,
|
|
"beta_dpo/beta_margin_std": 2.6675915718078613,
|
|
"beta_dpo/beta_used": 0.05058491602540016,
|
|
"beta_dpo/beta_used_raw": -0.013440538197755814,
|
|
"beta_dpo/gap_mean": 29.656864166259766,
|
|
"beta_dpo/gap_std": 29.526466369628906,
|
|
"beta_dpo/loss_margin_mean": 34.522823333740234,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1027900146842878,
|
|
"grad_norm": 88.88278198242188,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": -2.9925642013549805,
|
|
"logits/rejected": -2.9963603019714355,
|
|
"loss": 0.8527,
|
|
"step": 70
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0875883623957634,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2784726917743683,
|
|
"beta_dpo/beta_margin_grad_std": 0.24897883832454681,
|
|
"beta_dpo/beta_margin_mean": 4.192900657653809,
|
|
"beta_dpo/beta_margin_std": 6.242672443389893,
|
|
"beta_dpo/beta_used": 0.0875883623957634,
|
|
"beta_dpo/beta_used_raw": 0.009463444352149963,
|
|
"beta_dpo/gap_mean": 31.732118606567383,
|
|
"beta_dpo/gap_std": 30.914306640625,
|
|
"beta_dpo/loss_margin_mean": 40.439083099365234,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.10425844346549193,
|
|
"grad_norm": 84.22056579589844,
|
|
"learning_rate": 4.999967061337492e-07,
|
|
"logits/chosen": -2.9915552139282227,
|
|
"logits/rejected": -3.0017127990722656,
|
|
"loss": 0.8371,
|
|
"step": 71
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.08755672723054886,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22227872908115387,
|
|
"beta_dpo/beta_margin_grad_std": 0.20551539957523346,
|
|
"beta_dpo/beta_margin_mean": 3.4542064666748047,
|
|
"beta_dpo/beta_margin_std": 4.351135730743408,
|
|
"beta_dpo/beta_used": 0.08755672723054886,
|
|
"beta_dpo/beta_used_raw": 0.08755672723054886,
|
|
"beta_dpo/gap_mean": 33.012107849121094,
|
|
"beta_dpo/gap_std": 31.48851776123047,
|
|
"beta_dpo/loss_margin_mean": 40.631866455078125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.10572687224669604,
|
|
"grad_norm": 129.3617401123047,
|
|
"learning_rate": 4.999868246217933e-07,
|
|
"logits/chosen": -3.0176591873168945,
|
|
"logits/rejected": -3.012908935546875,
|
|
"loss": 0.6462,
|
|
"step": 72
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.03140610456466675,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3549874424934387,
|
|
"beta_dpo/beta_margin_grad_std": 0.22458156943321228,
|
|
"beta_dpo/beta_margin_mean": 1.2976723909378052,
|
|
"beta_dpo/beta_margin_std": 2.2271358966827393,
|
|
"beta_dpo/beta_used": 0.03140610456466675,
|
|
"beta_dpo/beta_used_raw": 0.02235252410173416,
|
|
"beta_dpo/gap_mean": 34.28472137451172,
|
|
"beta_dpo/gap_std": 33.075035095214844,
|
|
"beta_dpo/loss_margin_mean": 39.549720764160156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.10719530102790015,
|
|
"grad_norm": 72.28749084472656,
|
|
"learning_rate": 4.999703557245192e-07,
|
|
"logits/chosen": -3.002220630645752,
|
|
"logits/rejected": -3.003396987915039,
|
|
"loss": 1.0018,
|
|
"step": 73
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2684296667575836,
|
|
"beta_dpo/beta_margin_grad_mean": -0.11206705868244171,
|
|
"beta_dpo/beta_margin_grad_std": 0.24692973494529724,
|
|
"beta_dpo/beta_margin_mean": 12.898226737976074,
|
|
"beta_dpo/beta_margin_std": 19.086931228637695,
|
|
"beta_dpo/beta_used": 0.2684296667575836,
|
|
"beta_dpo/beta_used_raw": 0.2684296667575836,
|
|
"beta_dpo/gap_mean": 36.407623291015625,
|
|
"beta_dpo/gap_std": 34.9376220703125,
|
|
"beta_dpo/loss_margin_mean": 47.21061325073242,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.10866372980910426,
|
|
"grad_norm": 242.15377807617188,
|
|
"learning_rate": 4.999472998758977e-07,
|
|
"logits/chosen": -2.9633467197418213,
|
|
"logits/rejected": -2.9860920906066895,
|
|
"loss": 0.2967,
|
|
"step": 74
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.051932115107774734,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2812905013561249,
|
|
"beta_dpo/beta_margin_grad_std": 0.22937369346618652,
|
|
"beta_dpo/beta_margin_mean": 2.8386423587799072,
|
|
"beta_dpo/beta_margin_std": 4.261453151702881,
|
|
"beta_dpo/beta_used": 0.051932115107774734,
|
|
"beta_dpo/beta_used_raw": -0.15860339999198914,
|
|
"beta_dpo/gap_mean": 38.04350280761719,
|
|
"beta_dpo/gap_std": 36.71551513671875,
|
|
"beta_dpo/loss_margin_mean": 46.44905471801758,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.11013215859030837,
|
|
"grad_norm": 59.50148391723633,
|
|
"learning_rate": 4.999176576834721e-07,
|
|
"logits/chosen": -2.9508605003356934,
|
|
"logits/rejected": -2.9791858196258545,
|
|
"loss": 0.792,
|
|
"step": 75
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0431789867579937,
|
|
"beta_dpo/beta_margin_grad_mean": -0.32823190093040466,
|
|
"beta_dpo/beta_margin_grad_std": 0.24035446345806122,
|
|
"beta_dpo/beta_margin_mean": 1.565407156944275,
|
|
"beta_dpo/beta_margin_std": 2.5664196014404297,
|
|
"beta_dpo/beta_used": 0.0431789867579937,
|
|
"beta_dpo/beta_used_raw": -0.04499024525284767,
|
|
"beta_dpo/gap_mean": 38.48039245605469,
|
|
"beta_dpo/gap_std": 36.71884536743164,
|
|
"beta_dpo/loss_margin_mean": 35.94696044921875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.11160058737151249,
|
|
"grad_norm": 67.32833099365234,
|
|
"learning_rate": 4.998814299283415e-07,
|
|
"logits/chosen": -2.9746932983398438,
|
|
"logits/rejected": -2.968583822250366,
|
|
"loss": 0.8447,
|
|
"step": 76
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.34446391463279724,
|
|
"beta_dpo/beta_margin_grad_mean": -0.08970285952091217,
|
|
"beta_dpo/beta_margin_grad_std": 0.22384564578533173,
|
|
"beta_dpo/beta_margin_mean": 14.092086791992188,
|
|
"beta_dpo/beta_margin_std": 18.515459060668945,
|
|
"beta_dpo/beta_used": 0.34446391463279724,
|
|
"beta_dpo/beta_used_raw": 0.34446391463279724,
|
|
"beta_dpo/gap_mean": 39.179561614990234,
|
|
"beta_dpo/gap_std": 37.16301727294922,
|
|
"beta_dpo/loss_margin_mean": 42.844810485839844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1130690161527166,
|
|
"grad_norm": 293.4575500488281,
|
|
"learning_rate": 4.998386175651409e-07,
|
|
"logits/chosen": -2.947664499282837,
|
|
"logits/rejected": -2.959230899810791,
|
|
"loss": 0.1321,
|
|
"step": 77
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4911826550960541,
|
|
"beta_dpo/beta_margin_grad_std": 0.008943051099777222,
|
|
"beta_dpo/beta_margin_mean": 0.035288915038108826,
|
|
"beta_dpo/beta_margin_std": 0.035820893943309784,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.23828990757465363,
|
|
"beta_dpo/gap_mean": 38.626731872558594,
|
|
"beta_dpo/gap_std": 37.65977478027344,
|
|
"beta_dpo/loss_margin_mean": 35.28891372680664,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1145374449339207,
|
|
"grad_norm": 3.816556692123413,
|
|
"learning_rate": 4.997892217220159e-07,
|
|
"logits/chosen": -2.9786555767059326,
|
|
"logits/rejected": -2.973895788192749,
|
|
"loss": 1.3538,
|
|
"step": 78
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.48892611265182495,
|
|
"beta_dpo/beta_margin_grad_std": 0.010049103759229183,
|
|
"beta_dpo/beta_margin_mean": 0.044323720037937164,
|
|
"beta_dpo/beta_margin_std": 0.04024270921945572,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.21949611604213715,
|
|
"beta_dpo/gap_mean": 39.299583435058594,
|
|
"beta_dpo/gap_std": 37.97666931152344,
|
|
"beta_dpo/loss_margin_mean": 44.3237190246582,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.11600587371512482,
|
|
"grad_norm": 3.6560094356536865,
|
|
"learning_rate": 4.997332437005931e-07,
|
|
"logits/chosen": -2.9516005516052246,
|
|
"logits/rejected": -2.9511117935180664,
|
|
"loss": 1.3528,
|
|
"step": 79
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4893885552883148,
|
|
"beta_dpo/beta_margin_grad_std": 0.012339146807789803,
|
|
"beta_dpo/beta_margin_mean": 0.04248877242207527,
|
|
"beta_dpo/beta_margin_std": 0.049461763352155685,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.5071743726730347,
|
|
"beta_dpo/gap_mean": 40.222755432128906,
|
|
"beta_dpo/gap_std": 39.75575256347656,
|
|
"beta_dpo/loss_margin_mean": 42.48876953125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.11747430249632893,
|
|
"grad_norm": 3.7161951065063477,
|
|
"learning_rate": 4.996706849759452e-07,
|
|
"logits/chosen": -2.991058826446533,
|
|
"logits/rejected": -2.986891269683838,
|
|
"loss": 1.3566,
|
|
"step": 80
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.00884567853063345,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40446242690086365,
|
|
"beta_dpo/beta_margin_grad_std": 0.1263885498046875,
|
|
"beta_dpo/beta_margin_mean": 0.4591708183288574,
|
|
"beta_dpo/beta_margin_std": 0.710827648639679,
|
|
"beta_dpo/beta_used": 0.00884567853063345,
|
|
"beta_dpo/beta_used_raw": -0.16323187947273254,
|
|
"beta_dpo/gap_mean": 41.52234649658203,
|
|
"beta_dpo/gap_std": 41.37035369873047,
|
|
"beta_dpo/loss_margin_mean": 49.2286491394043,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.11894273127753303,
|
|
"grad_norm": 29.591167449951172,
|
|
"learning_rate": 4.996015471965529e-07,
|
|
"logits/chosen": -2.9471867084503174,
|
|
"logits/rejected": -2.958087921142578,
|
|
"loss": 1.1115,
|
|
"step": 81
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.17800916731357574,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3327975273132324,
|
|
"beta_dpo/beta_margin_grad_std": 0.30031806230545044,
|
|
"beta_dpo/beta_margin_mean": 6.430704116821289,
|
|
"beta_dpo/beta_margin_std": 13.450794219970703,
|
|
"beta_dpo/beta_used": 0.17800916731357574,
|
|
"beta_dpo/beta_used_raw": 0.1494421511888504,
|
|
"beta_dpo/gap_mean": 41.84193420410156,
|
|
"beta_dpo/gap_std": 42.8049201965332,
|
|
"beta_dpo/loss_margin_mean": 45.061553955078125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.12041116005873716,
|
|
"grad_norm": 565.0650634765625,
|
|
"learning_rate": 4.995258321842611e-07,
|
|
"logits/chosen": -2.938267230987549,
|
|
"logits/rejected": -2.9391956329345703,
|
|
"loss": 1.6144,
|
|
"step": 82
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.18749435245990753,
|
|
"beta_dpo/beta_margin_grad_mean": -0.271941602230072,
|
|
"beta_dpo/beta_margin_grad_std": 0.2540948987007141,
|
|
"beta_dpo/beta_margin_mean": 9.913958549499512,
|
|
"beta_dpo/beta_margin_std": 13.994511604309082,
|
|
"beta_dpo/beta_used": 0.18749435245990753,
|
|
"beta_dpo/beta_used_raw": 0.03422471880912781,
|
|
"beta_dpo/gap_mean": 43.76371765136719,
|
|
"beta_dpo/gap_std": 42.843292236328125,
|
|
"beta_dpo/loss_margin_mean": 50.59626770019531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.12187958883994127,
|
|
"grad_norm": 307.31402587890625,
|
|
"learning_rate": 4.994435419342304e-07,
|
|
"logits/chosen": -2.9267525672912598,
|
|
"logits/rejected": -2.9482855796813965,
|
|
"loss": 1.123,
|
|
"step": 83
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.43998458981513977,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26115646958351135,
|
|
"beta_dpo/beta_margin_grad_std": 0.2607704699039459,
|
|
"beta_dpo/beta_margin_mean": 23.472091674804688,
|
|
"beta_dpo/beta_margin_std": 32.818458557128906,
|
|
"beta_dpo/beta_used": 0.43998458981513977,
|
|
"beta_dpo/beta_used_raw": 0.12223160266876221,
|
|
"beta_dpo/gap_mean": 44.48904800415039,
|
|
"beta_dpo/gap_std": 41.942283630371094,
|
|
"beta_dpo/loss_margin_mean": 44.652137756347656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.12334801762114538,
|
|
"grad_norm": 2.4324653148651123,
|
|
"learning_rate": 4.993546786148857e-07,
|
|
"logits/chosen": -2.9470911026000977,
|
|
"logits/rejected": -2.941443920135498,
|
|
"loss": 0.6775,
|
|
"step": 84
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.49015551805496216,
|
|
"beta_dpo/beta_margin_grad_std": 0.009804553352296352,
|
|
"beta_dpo/beta_margin_mean": 0.03939983248710632,
|
|
"beta_dpo/beta_margin_std": 0.03925201669335365,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.16602811217308044,
|
|
"beta_dpo/gap_mean": 43.268497467041016,
|
|
"beta_dpo/gap_std": 41.009002685546875,
|
|
"beta_dpo/loss_margin_mean": 39.39983367919922,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.12481644640234948,
|
|
"grad_norm": 4.367675304412842,
|
|
"learning_rate": 4.992592445678582e-07,
|
|
"logits/chosen": -2.961709976196289,
|
|
"logits/rejected": -2.935671806335449,
|
|
"loss": 1.3481,
|
|
"step": 85
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4886876940727234,
|
|
"beta_dpo/beta_margin_grad_std": 0.01566956751048565,
|
|
"beta_dpo/beta_margin_mean": 0.04532717540860176,
|
|
"beta_dpo/beta_margin_std": 0.06287750601768494,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.7040857076644897,
|
|
"beta_dpo/gap_mean": 43.424720764160156,
|
|
"beta_dpo/gap_std": 43.841766357421875,
|
|
"beta_dpo/loss_margin_mean": 45.32717514038086,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1262848751835536,
|
|
"grad_norm": 3.6461679935455322,
|
|
"learning_rate": 4.991572423079235e-07,
|
|
"logits/chosen": -2.920724391937256,
|
|
"logits/rejected": -2.9186758995056152,
|
|
"loss": 1.3568,
|
|
"step": 86
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.04776458814740181,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23744255304336548,
|
|
"beta_dpo/beta_margin_grad_std": 0.22597795724868774,
|
|
"beta_dpo/beta_margin_mean": 2.473987340927124,
|
|
"beta_dpo/beta_margin_std": 3.041849374771118,
|
|
"beta_dpo/beta_used": 0.04776458814740181,
|
|
"beta_dpo/beta_used_raw": 0.04776458814740181,
|
|
"beta_dpo/gap_mean": 44.674156188964844,
|
|
"beta_dpo/gap_std": 46.15048599243164,
|
|
"beta_dpo/loss_margin_mean": 51.3042106628418,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1277533039647577,
|
|
"grad_norm": 171.111083984375,
|
|
"learning_rate": 4.990486745229364e-07,
|
|
"logits/chosen": -2.9184794425964355,
|
|
"logits/rejected": -2.9294486045837402,
|
|
"loss": 0.7438,
|
|
"step": 87
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2957174479961395,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12127989530563354,
|
|
"beta_dpo/beta_margin_grad_std": 0.29749196767807007,
|
|
"beta_dpo/beta_margin_mean": 18.622495651245117,
|
|
"beta_dpo/beta_margin_std": 17.863021850585938,
|
|
"beta_dpo/beta_used": 0.2957174479961395,
|
|
"beta_dpo/beta_used_raw": 0.2957174479961395,
|
|
"beta_dpo/gap_mean": 47.820945739746094,
|
|
"beta_dpo/gap_std": 48.210693359375,
|
|
"beta_dpo/loss_margin_mean": 62.004920959472656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.12922173274596183,
|
|
"grad_norm": 559.2757568359375,
|
|
"learning_rate": 4.989335440737586e-07,
|
|
"logits/chosen": -2.9339442253112793,
|
|
"logits/rejected": -2.943288564682007,
|
|
"loss": 0.7024,
|
|
"step": 88
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4874421954154968,
|
|
"beta_dpo/beta_margin_grad_std": 0.013273374177515507,
|
|
"beta_dpo/beta_margin_mean": 0.05029499903321266,
|
|
"beta_dpo/beta_margin_std": 0.053264521062374115,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.4985317587852478,
|
|
"beta_dpo/gap_mean": 48.497169494628906,
|
|
"beta_dpo/gap_std": 49.19569396972656,
|
|
"beta_dpo/loss_margin_mean": 50.29499435424805,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.13069016152716592,
|
|
"grad_norm": 4.085449695587158,
|
|
"learning_rate": 4.988118539941847e-07,
|
|
"logits/chosen": -2.910324811935425,
|
|
"logits/rejected": -2.9079842567443848,
|
|
"loss": 1.3484,
|
|
"step": 89
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4830739498138428,
|
|
"beta_dpo/beta_margin_grad_std": 0.019520951434969902,
|
|
"beta_dpo/beta_margin_mean": 0.06789281219244003,
|
|
"beta_dpo/beta_margin_std": 0.07855159044265747,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.08338849246501923,
|
|
"beta_dpo/gap_mean": 50.64426040649414,
|
|
"beta_dpo/gap_std": 53.176513671875,
|
|
"beta_dpo/loss_margin_mean": 67.89280700683594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.13215859030837004,
|
|
"grad_norm": 5.08858585357666,
|
|
"learning_rate": 4.986836074908615e-07,
|
|
"logits/chosen": -2.881624221801758,
|
|
"logits/rejected": -2.917466878890991,
|
|
"loss": 1.3397,
|
|
"step": 90
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.046673670411109924,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28842246532440186,
|
|
"beta_dpo/beta_margin_grad_std": 0.22475126385688782,
|
|
"beta_dpo/beta_margin_mean": 2.8846089839935303,
|
|
"beta_dpo/beta_margin_std": 4.184475898742676,
|
|
"beta_dpo/beta_used": 0.046673670411109924,
|
|
"beta_dpo/beta_used_raw": -0.17408108711242676,
|
|
"beta_dpo/gap_mean": 52.84674835205078,
|
|
"beta_dpo/gap_std": 54.52159118652344,
|
|
"beta_dpo/loss_margin_mean": 56.735435485839844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.13362701908957417,
|
|
"grad_norm": 93.58074951171875,
|
|
"learning_rate": 4.985488079432037e-07,
|
|
"logits/chosen": -2.899129629135132,
|
|
"logits/rejected": -2.9001574516296387,
|
|
"loss": 0.7869,
|
|
"step": 91
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4871750771999359,
|
|
"beta_dpo/beta_margin_grad_std": 0.014737357385456562,
|
|
"beta_dpo/beta_margin_mean": 0.051369354128837585,
|
|
"beta_dpo/beta_margin_std": 0.059110358357429504,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.3833653926849365,
|
|
"beta_dpo/gap_mean": 53.06073760986328,
|
|
"beta_dpo/gap_std": 55.06708526611328,
|
|
"beta_dpo/loss_margin_mean": 51.36935043334961,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.13509544787077826,
|
|
"grad_norm": 4.548574924468994,
|
|
"learning_rate": 4.984074589033043e-07,
|
|
"logits/chosen": -2.8663196563720703,
|
|
"logits/rejected": -2.865586757659912,
|
|
"loss": 1.3421,
|
|
"step": 92
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0362042672932148,
|
|
"beta_dpo/beta_margin_grad_mean": -0.33124664425849915,
|
|
"beta_dpo/beta_margin_grad_std": 0.2518288195133209,
|
|
"beta_dpo/beta_margin_mean": 1.77645742893219,
|
|
"beta_dpo/beta_margin_std": 2.907167911529541,
|
|
"beta_dpo/beta_used": 0.0362042672932148,
|
|
"beta_dpo/beta_used_raw": -0.0553596131503582,
|
|
"beta_dpo/gap_mean": 53.00493240356445,
|
|
"beta_dpo/gap_std": 55.078792572021484,
|
|
"beta_dpo/loss_margin_mean": 55.34242630004883,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.13656387665198239,
|
|
"grad_norm": 138.0321807861328,
|
|
"learning_rate": 4.982595640958425e-07,
|
|
"logits/chosen": -2.8780605792999268,
|
|
"logits/rejected": -2.865324020385742,
|
|
"loss": 0.8654,
|
|
"step": 93
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.09114255011081696,
|
|
"beta_dpo/beta_margin_grad_mean": -0.289016455411911,
|
|
"beta_dpo/beta_margin_grad_std": 0.25749266147613525,
|
|
"beta_dpo/beta_margin_mean": 6.2431559562683105,
|
|
"beta_dpo/beta_margin_std": 10.020861625671387,
|
|
"beta_dpo/beta_used": 0.09114255011081696,
|
|
"beta_dpo/beta_used_raw": -0.06984854489564896,
|
|
"beta_dpo/gap_mean": 55.642555236816406,
|
|
"beta_dpo/gap_std": 57.18549346923828,
|
|
"beta_dpo/loss_margin_mean": 68.45297241210938,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.13803230543318648,
|
|
"grad_norm": 414.5119934082031,
|
|
"learning_rate": 4.98105127417984e-07,
|
|
"logits/chosen": -2.9009084701538086,
|
|
"logits/rejected": -2.900761127471924,
|
|
"loss": 0.8078,
|
|
"step": 94
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4851512312889099,
|
|
"beta_dpo/beta_margin_grad_std": 0.015362189151346684,
|
|
"beta_dpo/beta_margin_mean": 0.059510547667741776,
|
|
"beta_dpo/beta_margin_std": 0.061817847192287445,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.7674299478530884,
|
|
"beta_dpo/gap_mean": 57.458091735839844,
|
|
"beta_dpo/gap_std": 58.81836700439453,
|
|
"beta_dpo/loss_margin_mean": 59.51054763793945,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1395007342143906,
|
|
"grad_norm": 4.450385570526123,
|
|
"learning_rate": 4.979441529392784e-07,
|
|
"logits/chosen": -2.855530261993408,
|
|
"logits/rejected": -2.8571863174438477,
|
|
"loss": 1.344,
|
|
"step": 95
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4837089478969574,
|
|
"beta_dpo/beta_margin_grad_std": 0.01984335109591484,
|
|
"beta_dpo/beta_margin_mean": 0.06533493846654892,
|
|
"beta_dpo/beta_margin_std": 0.07974996417760849,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.7220409512519836,
|
|
"beta_dpo/gap_mean": 58.48717498779297,
|
|
"beta_dpo/gap_std": 60.73028564453125,
|
|
"beta_dpo/loss_margin_mean": 65.3349380493164,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.14096916299559473,
|
|
"grad_norm": 4.7258100509643555,
|
|
"learning_rate": 4.977766449015534e-07,
|
|
"logits/chosen": -2.904493570327759,
|
|
"logits/rejected": -2.9026832580566406,
|
|
"loss": 1.3426,
|
|
"step": 96
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.11098214983940125,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2807328701019287,
|
|
"beta_dpo/beta_margin_grad_std": 0.2636745870113373,
|
|
"beta_dpo/beta_margin_mean": 8.650201797485352,
|
|
"beta_dpo/beta_margin_std": 13.248647689819336,
|
|
"beta_dpo/beta_used": 0.11098214983940125,
|
|
"beta_dpo/beta_used_raw": -0.17043042182922363,
|
|
"beta_dpo/gap_mean": 59.17014694213867,
|
|
"beta_dpo/gap_std": 62.05863952636719,
|
|
"beta_dpo/loss_margin_mean": 67.58939361572266,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.14243759177679882,
|
|
"grad_norm": 179.77455139160156,
|
|
"learning_rate": 4.976026077188012e-07,
|
|
"logits/chosen": -2.905517578125,
|
|
"logits/rejected": -2.897564649581909,
|
|
"loss": 0.9284,
|
|
"step": 97
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.49735867977142334,
|
|
"beta_dpo/beta_margin_grad_mean": -0.13786101341247559,
|
|
"beta_dpo/beta_margin_grad_std": 0.32877954840660095,
|
|
"beta_dpo/beta_margin_mean": 29.118545532226562,
|
|
"beta_dpo/beta_margin_std": 44.36186599731445,
|
|
"beta_dpo/beta_used": 0.49735867977142334,
|
|
"beta_dpo/beta_used_raw": 0.49735867977142334,
|
|
"beta_dpo/gap_mean": 60.218910217285156,
|
|
"beta_dpo/gap_std": 62.831886291503906,
|
|
"beta_dpo/loss_margin_mean": 62.66773223876953,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.14390602055800295,
|
|
"grad_norm": 1460.3056640625,
|
|
"learning_rate": 4.974220459770639e-07,
|
|
"logits/chosen": -2.874201774597168,
|
|
"logits/rejected": -2.884645462036133,
|
|
"loss": 1.6259,
|
|
"step": 98
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.27869686484336853,
|
|
"beta_dpo/beta_margin_grad_mean": -0.07528817653656006,
|
|
"beta_dpo/beta_margin_grad_std": 0.24065373837947845,
|
|
"beta_dpo/beta_margin_mean": 21.852785110473633,
|
|
"beta_dpo/beta_margin_std": 22.130836486816406,
|
|
"beta_dpo/beta_used": 0.27869686484336853,
|
|
"beta_dpo/beta_used_raw": 0.27869686484336853,
|
|
"beta_dpo/gap_mean": 63.37742614746094,
|
|
"beta_dpo/gap_std": 64.15032958984375,
|
|
"beta_dpo/loss_margin_mean": 77.13097381591797,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.14537444933920704,
|
|
"grad_norm": 384.5785217285156,
|
|
"learning_rate": 4.972349644343108e-07,
|
|
"logits/chosen": -2.845553398132324,
|
|
"logits/rejected": -2.8533787727355957,
|
|
"loss": 0.4593,
|
|
"step": 99
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4882340431213379,
|
|
"beta_dpo/beta_margin_grad_std": 0.015319556929171085,
|
|
"beta_dpo/beta_margin_mean": 0.047130510210990906,
|
|
"beta_dpo/beta_margin_std": 0.06143619865179062,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.3977210819721222,
|
|
"beta_dpo/gap_mean": 61.136131286621094,
|
|
"beta_dpo/gap_std": 64.02367401123047,
|
|
"beta_dpo/loss_margin_mean": 47.13050842285156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.14684287812041116,
|
|
"grad_norm": 5.179396629333496,
|
|
"learning_rate": 4.970413680203148e-07,
|
|
"logits/chosen": -2.8722152709960938,
|
|
"logits/rejected": -2.8694002628326416,
|
|
"loss": 1.3346,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.14684287812041116,
|
|
"eval_beta_dpo/beta": 0.021090898662805557,
|
|
"eval_beta_dpo/beta_margin_grad_mean": -0.4726831018924713,
|
|
"eval_beta_dpo/beta_margin_grad_std": 0.04030865803360939,
|
|
"eval_beta_dpo/beta_margin_mean": 1.4684983491897583,
|
|
"eval_beta_dpo/beta_margin_std": 2.0474720001220703,
|
|
"eval_beta_dpo/beta_used": 0.021090898662805557,
|
|
"eval_beta_dpo/beta_used_raw": -1.2172808647155762,
|
|
"eval_beta_dpo/gap_mean": 60.651268005371094,
|
|
"eval_beta_dpo/gap_std": 63.85259246826172,
|
|
"eval_beta_dpo/loss_margin_mean": 38.69658660888672,
|
|
"eval_beta_dpo/mask_keep_frac": 1.0,
|
|
"eval_logits/chosen": -2.9129130840301514,
|
|
"eval_logits/rejected": -2.903272867202759,
|
|
"eval_loss": 0.7824556827545166,
|
|
"eval_runtime": 37.8912,
|
|
"eval_samples_per_second": 61.729,
|
|
"eval_steps_per_second": 1.953,
|
|
"step": 100
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4872121512889862,
|
|
"beta_dpo/beta_margin_grad_std": 0.0187575314193964,
|
|
"beta_dpo/beta_margin_mean": 0.05126000568270683,
|
|
"beta_dpo/beta_margin_std": 0.07528804987668991,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.9154506325721741,
|
|
"beta_dpo/gap_mean": 59.639793395996094,
|
|
"beta_dpo/gap_std": 64.89390563964844,
|
|
"beta_dpo/loss_margin_mean": 51.26000213623047,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.14831130690161526,
|
|
"grad_norm": 4.951231002807617,
|
|
"learning_rate": 4.968412618365215e-07,
|
|
"logits/chosen": -2.889308452606201,
|
|
"logits/rejected": -2.883979320526123,
|
|
"loss": 1.3445,
|
|
"step": 101
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.48730704188346863,
|
|
"beta_dpo/beta_margin_grad_std": 0.01587892696261406,
|
|
"beta_dpo/beta_margin_mean": 0.050841983407735825,
|
|
"beta_dpo/beta_margin_std": 0.06364595144987106,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.11553104221820831,
|
|
"beta_dpo/gap_mean": 57.58677673339844,
|
|
"beta_dpo/gap_std": 65.677001953125,
|
|
"beta_dpo/loss_margin_mean": 50.84197998046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.14977973568281938,
|
|
"grad_norm": 5.152194023132324,
|
|
"learning_rate": 4.966346511559149e-07,
|
|
"logits/chosen": -2.8980913162231445,
|
|
"logits/rejected": -2.883908987045288,
|
|
"loss": 1.3336,
|
|
"step": 102
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9354996681213379,
|
|
"beta_dpo/beta_margin_grad_mean": -0.15053774416446686,
|
|
"beta_dpo/beta_margin_grad_std": 0.3507133424282074,
|
|
"beta_dpo/beta_margin_mean": 78.73807525634766,
|
|
"beta_dpo/beta_margin_std": 91.95064544677734,
|
|
"beta_dpo/beta_used": 0.9354996681213379,
|
|
"beta_dpo/beta_used_raw": 0.9354996681213379,
|
|
"beta_dpo/gap_mean": 61.26586151123047,
|
|
"beta_dpo/gap_std": 69.46250915527344,
|
|
"beta_dpo/loss_margin_mean": 81.90780639648438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1512481644640235,
|
|
"grad_norm": 3375.601318359375,
|
|
"learning_rate": 4.964215414228785e-07,
|
|
"logits/chosen": -2.829054117202759,
|
|
"logits/rejected": -2.808652639389038,
|
|
"loss": 1.8738,
|
|
"step": 103
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.84881192445755,
|
|
"beta_dpo/beta_margin_grad_mean": -0.10420799255371094,
|
|
"beta_dpo/beta_margin_grad_std": 0.2966291308403015,
|
|
"beta_dpo/beta_margin_mean": 72.4658432006836,
|
|
"beta_dpo/beta_margin_std": 76.15467834472656,
|
|
"beta_dpo/beta_used": 0.84881192445755,
|
|
"beta_dpo/beta_used_raw": 0.84881192445755,
|
|
"beta_dpo/gap_mean": 65.47351837158203,
|
|
"beta_dpo/gap_std": 72.57354736328125,
|
|
"beta_dpo/loss_margin_mean": 84.90675354003906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1527165932452276,
|
|
"grad_norm": 844.726806640625,
|
|
"learning_rate": 4.96201938253052e-07,
|
|
"logits/chosen": -2.822493553161621,
|
|
"logits/rejected": -2.817017078399658,
|
|
"loss": 0.5082,
|
|
"step": 104
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.607758104801178,
|
|
"beta_dpo/beta_margin_grad_mean": -0.07786455005407333,
|
|
"beta_dpo/beta_margin_grad_std": 0.25590455532073975,
|
|
"beta_dpo/beta_margin_mean": 51.5904541015625,
|
|
"beta_dpo/beta_margin_std": 54.65573501586914,
|
|
"beta_dpo/beta_used": 0.607758104801178,
|
|
"beta_dpo/beta_used_raw": 0.607758104801178,
|
|
"beta_dpo/gap_mean": 68.90283203125,
|
|
"beta_dpo/gap_std": 75.64898681640625,
|
|
"beta_dpo/loss_margin_mean": 87.5045394897461,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.15418502202643172,
|
|
"grad_norm": 1462.4896240234375,
|
|
"learning_rate": 4.959758474331832e-07,
|
|
"logits/chosen": -2.8367412090301514,
|
|
"logits/rejected": -2.831176280975342,
|
|
"loss": 0.8486,
|
|
"step": 105
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.483272522687912,
|
|
"beta_dpo/beta_margin_grad_std": 0.01717246323823929,
|
|
"beta_dpo/beta_margin_mean": 0.06702879071235657,
|
|
"beta_dpo/beta_margin_std": 0.06890414655208588,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.6237323880195618,
|
|
"beta_dpo/gap_mean": 69.55685424804688,
|
|
"beta_dpo/gap_std": 74.42953491210938,
|
|
"beta_dpo/loss_margin_mean": 67.0287857055664,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.15565345080763582,
|
|
"grad_norm": 5.48669958114624,
|
|
"learning_rate": 4.957432749209755e-07,
|
|
"logits/chosen": -2.825288772583008,
|
|
"logits/rejected": -2.8292627334594727,
|
|
"loss": 1.3305,
|
|
"step": 106
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.19774837791919708,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2614811658859253,
|
|
"beta_dpo/beta_margin_grad_std": 0.25258755683898926,
|
|
"beta_dpo/beta_margin_mean": 17.963912963867188,
|
|
"beta_dpo/beta_margin_std": 28.16929054260254,
|
|
"beta_dpo/beta_used": 0.19774837791919708,
|
|
"beta_dpo/beta_used_raw": 0.027784347534179688,
|
|
"beta_dpo/gap_mean": 72.93892669677734,
|
|
"beta_dpo/gap_std": 76.43939208984375,
|
|
"beta_dpo/loss_margin_mean": 91.99864959716797,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.15712187958883994,
|
|
"grad_norm": 554.6088256835938,
|
|
"learning_rate": 4.955042268449307e-07,
|
|
"logits/chosen": -2.838146448135376,
|
|
"logits/rejected": -2.829848289489746,
|
|
"loss": 0.9258,
|
|
"step": 107
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47412681579589844,
|
|
"beta_dpo/beta_margin_grad_std": 0.02628808096051216,
|
|
"beta_dpo/beta_margin_mean": 0.10398299247026443,
|
|
"beta_dpo/beta_margin_std": 0.10614117234945297,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.3238527774810791,
|
|
"beta_dpo/gap_mean": 78.68049621582031,
|
|
"beta_dpo/gap_std": 82.48477172851562,
|
|
"beta_dpo/loss_margin_mean": 103.98298645019531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.15859030837004406,
|
|
"grad_norm": 5.760761260986328,
|
|
"learning_rate": 4.952587095041881e-07,
|
|
"logits/chosen": -2.799206256866455,
|
|
"logits/rejected": -2.7892160415649414,
|
|
"loss": 1.317,
|
|
"step": 108
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.07913082838058472,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2913077771663666,
|
|
"beta_dpo/beta_margin_grad_std": 0.26053932309150696,
|
|
"beta_dpo/beta_margin_mean": 7.3264055252075195,
|
|
"beta_dpo/beta_margin_std": 12.864703178405762,
|
|
"beta_dpo/beta_used": 0.07913082838058472,
|
|
"beta_dpo/beta_used_raw": 0.018090281635522842,
|
|
"beta_dpo/gap_mean": 80.97541809082031,
|
|
"beta_dpo/gap_std": 86.63990783691406,
|
|
"beta_dpo/loss_margin_mean": 87.2574234008789,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.16005873715124816,
|
|
"grad_norm": 48.93942642211914,
|
|
"learning_rate": 4.95006729368358e-07,
|
|
"logits/chosen": -2.8221731185913086,
|
|
"logits/rejected": -2.8154239654541016,
|
|
"loss": 0.7203,
|
|
"step": 109
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.13006778061389923,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29096195101737976,
|
|
"beta_dpo/beta_margin_grad_std": 0.2567855417728424,
|
|
"beta_dpo/beta_margin_mean": 11.700284957885742,
|
|
"beta_dpo/beta_margin_std": 19.173776626586914,
|
|
"beta_dpo/beta_used": 0.13006778061389923,
|
|
"beta_dpo/beta_used_raw": -0.47315120697021484,
|
|
"beta_dpo/gap_mean": 80.23991394042969,
|
|
"beta_dpo/gap_std": 88.77531433105469,
|
|
"beta_dpo/loss_margin_mean": 69.10839080810547,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.16152716593245228,
|
|
"grad_norm": 311.8143310546875,
|
|
"learning_rate": 4.947482930773511e-07,
|
|
"logits/chosen": -2.800227165222168,
|
|
"logits/rejected": -2.790717601776123,
|
|
"loss": 1.1166,
|
|
"step": 110
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.23672765493392944,
|
|
"beta_dpo/beta_margin_grad_mean": -0.32680338621139526,
|
|
"beta_dpo/beta_margin_grad_std": 0.30344852805137634,
|
|
"beta_dpo/beta_margin_mean": 21.393625259399414,
|
|
"beta_dpo/beta_margin_std": 45.63887023925781,
|
|
"beta_dpo/beta_used": 0.23672765493392944,
|
|
"beta_dpo/beta_used_raw": 0.19301117956638336,
|
|
"beta_dpo/gap_mean": 80.5330810546875,
|
|
"beta_dpo/gap_std": 92.93572998046875,
|
|
"beta_dpo/loss_margin_mean": 91.79338073730469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.16299559471365638,
|
|
"grad_norm": 1797.791015625,
|
|
"learning_rate": 4.944834074412042e-07,
|
|
"logits/chosen": -2.82328724861145,
|
|
"logits/rejected": -2.8216376304626465,
|
|
"loss": 5.3229,
|
|
"step": 111
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3619656264781952,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3380275368690491,
|
|
"beta_dpo/beta_margin_grad_std": 0.3120715320110321,
|
|
"beta_dpo/beta_margin_mean": 25.55278968811035,
|
|
"beta_dpo/beta_margin_std": 48.8296012878418,
|
|
"beta_dpo/beta_used": 0.3619656264781952,
|
|
"beta_dpo/beta_used_raw": -0.08405748009681702,
|
|
"beta_dpo/gap_mean": 78.98956298828125,
|
|
"beta_dpo/gap_std": 91.90986633300781,
|
|
"beta_dpo/loss_margin_mean": 65.13848114013672,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1644640234948605,
|
|
"grad_norm": 1818.7918701171875,
|
|
"learning_rate": 4.942120794399002e-07,
|
|
"logits/chosen": -2.7841193675994873,
|
|
"logits/rejected": -2.7855634689331055,
|
|
"loss": 2.5001,
|
|
"step": 112
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.48207101225852966,
|
|
"beta_dpo/beta_margin_grad_std": 0.01914754882454872,
|
|
"beta_dpo/beta_margin_mean": 0.07188614457845688,
|
|
"beta_dpo/beta_margin_std": 0.07695025205612183,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.5030999779701233,
|
|
"beta_dpo/gap_mean": 77.31889343261719,
|
|
"beta_dpo/gap_std": 88.37618255615234,
|
|
"beta_dpo/loss_margin_mean": 71.88613891601562,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.16593245227606462,
|
|
"grad_norm": 6.126889705657959,
|
|
"learning_rate": 4.939343162231841e-07,
|
|
"logits/chosen": -2.798726797103882,
|
|
"logits/rejected": -2.7898099422454834,
|
|
"loss": 1.3211,
|
|
"step": 113
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4782230257987976,
|
|
"beta_dpo/beta_margin_grad_std": 0.02653990499675274,
|
|
"beta_dpo/beta_margin_mean": 0.08755876123905182,
|
|
"beta_dpo/beta_margin_std": 0.10747722536325455,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.9737315773963928,
|
|
"beta_dpo/gap_mean": 78.52099609375,
|
|
"beta_dpo/gap_std": 91.858642578125,
|
|
"beta_dpo/loss_margin_mean": 87.55876159667969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.16740088105726872,
|
|
"grad_norm": 6.043005466461182,
|
|
"learning_rate": 4.936501251103751e-07,
|
|
"logits/chosen": -2.7921528816223145,
|
|
"logits/rejected": -2.7812323570251465,
|
|
"loss": 1.3277,
|
|
"step": 114
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.09022793918848038,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3093552589416504,
|
|
"beta_dpo/beta_margin_grad_std": 0.27903029322624207,
|
|
"beta_dpo/beta_margin_mean": 9.817399024963379,
|
|
"beta_dpo/beta_margin_std": 18.483304977416992,
|
|
"beta_dpo/beta_used": 0.09022793918848038,
|
|
"beta_dpo/beta_used_raw": -0.45726335048675537,
|
|
"beta_dpo/gap_mean": 80.5860824584961,
|
|
"beta_dpo/gap_std": 93.43727111816406,
|
|
"beta_dpo/loss_margin_mean": 95.46105194091797,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.16886930983847284,
|
|
"grad_norm": 466.8434753417969,
|
|
"learning_rate": 4.933595135901732e-07,
|
|
"logits/chosen": -2.8372421264648438,
|
|
"logits/rejected": -2.8418526649475098,
|
|
"loss": 1.2961,
|
|
"step": 115
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47841930389404297,
|
|
"beta_dpo/beta_margin_grad_std": 0.022723974660038948,
|
|
"beta_dpo/beta_margin_mean": 0.08669696003198624,
|
|
"beta_dpo/beta_margin_std": 0.09222324937582016,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.2080257534980774,
|
|
"beta_dpo/gap_mean": 82.31340026855469,
|
|
"beta_dpo/gap_std": 93.35620880126953,
|
|
"beta_dpo/loss_margin_mean": 86.69695281982422,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.17033773861967694,
|
|
"grad_norm": 6.324546813964844,
|
|
"learning_rate": 4.930624893204624e-07,
|
|
"logits/chosen": -2.785477876663208,
|
|
"logits/rejected": -2.792661190032959,
|
|
"loss": 1.3117,
|
|
"step": 116
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4851381480693817,
|
|
"beta_dpo/beta_margin_grad_std": 0.022359393537044525,
|
|
"beta_dpo/beta_margin_mean": 0.05954824388027191,
|
|
"beta_dpo/beta_margin_std": 0.08978039026260376,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.7292430400848389,
|
|
"beta_dpo/gap_mean": 79.99589538574219,
|
|
"beta_dpo/gap_std": 94.4775619506836,
|
|
"beta_dpo/loss_margin_mean": 59.548240661621094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.17180616740088106,
|
|
"grad_norm": 6.232842445373535,
|
|
"learning_rate": 4.927590601281083e-07,
|
|
"logits/chosen": -2.785600185394287,
|
|
"logits/rejected": -2.775449752807617,
|
|
"loss": 1.322,
|
|
"step": 117
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.38719773292541504,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12059462815523148,
|
|
"beta_dpo/beta_margin_grad_std": 0.2853429317474365,
|
|
"beta_dpo/beta_margin_mean": 31.55919647216797,
|
|
"beta_dpo/beta_margin_std": 51.115875244140625,
|
|
"beta_dpo/beta_used": 0.38719773292541504,
|
|
"beta_dpo/beta_used_raw": 0.38719773292541504,
|
|
"beta_dpo/gap_mean": 79.46942138671875,
|
|
"beta_dpo/gap_std": 92.74491882324219,
|
|
"beta_dpo/loss_margin_mean": 87.44821166992188,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.17327459618208516,
|
|
"grad_norm": 740.7146606445312,
|
|
"learning_rate": 4.924492340087524e-07,
|
|
"logits/chosen": -2.816103935241699,
|
|
"logits/rejected": -2.816561222076416,
|
|
"loss": 1.2551,
|
|
"step": 118
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6606523394584656,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3558006286621094,
|
|
"beta_dpo/beta_margin_grad_std": 0.3170710504055023,
|
|
"beta_dpo/beta_margin_mean": 56.66292190551758,
|
|
"beta_dpo/beta_margin_std": 127.7380599975586,
|
|
"beta_dpo/beta_used": 0.6606523394584656,
|
|
"beta_dpo/beta_used_raw": 0.13558882474899292,
|
|
"beta_dpo/gap_mean": 84.13648986816406,
|
|
"beta_dpo/gap_std": 96.36943817138672,
|
|
"beta_dpo/loss_margin_mean": 101.86002349853516,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.17474302496328928,
|
|
"grad_norm": 4218.83154296875,
|
|
"learning_rate": 4.92133019126601e-07,
|
|
"logits/chosen": -2.8366334438323975,
|
|
"logits/rejected": -2.8262441158294678,
|
|
"loss": 4.2288,
|
|
"step": 119
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.5045459270477295,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12338397651910782,
|
|
"beta_dpo/beta_margin_grad_std": 0.3060937523841858,
|
|
"beta_dpo/beta_margin_mean": 165.48951721191406,
|
|
"beta_dpo/beta_margin_std": 231.87692260742188,
|
|
"beta_dpo/beta_used": 1.5045459270477295,
|
|
"beta_dpo/beta_used_raw": 1.5045459270477295,
|
|
"beta_dpo/gap_mean": 87.93060302734375,
|
|
"beta_dpo/gap_std": 102.44084167480469,
|
|
"beta_dpo/loss_margin_mean": 109.52928924560547,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1762114537444934,
|
|
"grad_norm": 2128.182373046875,
|
|
"learning_rate": 4.918104238142103e-07,
|
|
"logits/chosen": -2.851163864135742,
|
|
"logits/rejected": -2.836759090423584,
|
|
"loss": 1.3594,
|
|
"step": 120
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9340792894363403,
|
|
"beta_dpo/beta_margin_grad_mean": -0.06197686493396759,
|
|
"beta_dpo/beta_margin_grad_std": 0.23998677730560303,
|
|
"beta_dpo/beta_margin_mean": 120.16432189941406,
|
|
"beta_dpo/beta_margin_std": 121.8327865600586,
|
|
"beta_dpo/beta_used": 0.9340792894363403,
|
|
"beta_dpo/beta_used_raw": 0.9340792894363403,
|
|
"beta_dpo/gap_mean": 95.33363342285156,
|
|
"beta_dpo/gap_std": 105.84669494628906,
|
|
"beta_dpo/loss_margin_mean": 130.99896240234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1776798825256975,
|
|
"grad_norm": 790.7160034179688,
|
|
"learning_rate": 4.91481456572267e-07,
|
|
"logits/chosen": -2.795456886291504,
|
|
"logits/rejected": -2.79376220703125,
|
|
"loss": 1.2803,
|
|
"step": 121
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.41401243209838867,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26532647013664246,
|
|
"beta_dpo/beta_margin_grad_std": 0.2661411166191101,
|
|
"beta_dpo/beta_margin_mean": 51.30147171020508,
|
|
"beta_dpo/beta_margin_std": 88.02983856201172,
|
|
"beta_dpo/beta_used": 0.41401243209838867,
|
|
"beta_dpo/beta_used_raw": 0.17554566264152527,
|
|
"beta_dpo/gap_mean": 101.07794952392578,
|
|
"beta_dpo/gap_std": 108.308349609375,
|
|
"beta_dpo/loss_margin_mean": 126.12928009033203,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.17914831130690162,
|
|
"grad_norm": 3183.692138671875,
|
|
"learning_rate": 4.911461260693638e-07,
|
|
"logits/chosen": -2.7693471908569336,
|
|
"logits/rejected": -2.7797422409057617,
|
|
"loss": 5.005,
|
|
"step": 122
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8816313743591309,
|
|
"beta_dpo/beta_margin_grad_mean": -0.30386897921562195,
|
|
"beta_dpo/beta_margin_grad_std": 0.29489797353744507,
|
|
"beta_dpo/beta_margin_mean": 100.96495056152344,
|
|
"beta_dpo/beta_margin_std": 171.0264434814453,
|
|
"beta_dpo/beta_used": 0.8816313743591309,
|
|
"beta_dpo/beta_used_raw": 0.12362289428710938,
|
|
"beta_dpo/gap_mean": 101.99317932128906,
|
|
"beta_dpo/gap_std": 108.65696716308594,
|
|
"beta_dpo/loss_margin_mean": 91.89662170410156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.18061674008810572,
|
|
"grad_norm": 4966.56640625,
|
|
"learning_rate": 4.908044411417711e-07,
|
|
"logits/chosen": -2.8212316036224365,
|
|
"logits/rejected": -2.8233444690704346,
|
|
"loss": 5.5543,
|
|
"step": 123
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2012283205986023,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29456260800361633,
|
|
"beta_dpo/beta_margin_grad_std": 0.2699277400970459,
|
|
"beta_dpo/beta_margin_mean": 25.60476303100586,
|
|
"beta_dpo/beta_margin_std": 42.12246322631836,
|
|
"beta_dpo/beta_used": 0.2012283205986023,
|
|
"beta_dpo/beta_used_raw": 0.014203429222106934,
|
|
"beta_dpo/gap_mean": 104.23336791992188,
|
|
"beta_dpo/gap_std": 109.76822662353516,
|
|
"beta_dpo/loss_margin_mean": 127.76998901367188,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.18208516886930984,
|
|
"grad_norm": 713.97998046875,
|
|
"learning_rate": 4.904564107932048e-07,
|
|
"logits/chosen": -2.8201169967651367,
|
|
"logits/rejected": -2.839149236679077,
|
|
"loss": 1.1241,
|
|
"step": 124
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4756603538990021,
|
|
"beta_dpo/beta_margin_grad_std": 0.025525817647576332,
|
|
"beta_dpo/beta_margin_mean": 0.09777088463306427,
|
|
"beta_dpo/beta_margin_std": 0.10292989015579224,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8198720216751099,
|
|
"beta_dpo/gap_mean": 103.85806274414062,
|
|
"beta_dpo/gap_std": 108.71501159667969,
|
|
"beta_dpo/loss_margin_mean": 97.7708740234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.18355359765051396,
|
|
"grad_norm": 7.420654296875,
|
|
"learning_rate": 4.90102044194588e-07,
|
|
"logits/chosen": -2.781724452972412,
|
|
"logits/rejected": -2.7918858528137207,
|
|
"loss": 1.3016,
|
|
"step": 125
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.04341450706124306,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3020491302013397,
|
|
"beta_dpo/beta_margin_grad_std": 0.25825148820877075,
|
|
"beta_dpo/beta_margin_mean": 5.03449010848999,
|
|
"beta_dpo/beta_margin_std": 8.020267486572266,
|
|
"beta_dpo/beta_used": 0.04341450706124306,
|
|
"beta_dpo/beta_used_raw": -0.4769749641418457,
|
|
"beta_dpo/gap_mean": 104.70053100585938,
|
|
"beta_dpo/gap_std": 109.29353332519531,
|
|
"beta_dpo/loss_margin_mean": 104.23768615722656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.18502202643171806,
|
|
"grad_norm": 176.98025512695312,
|
|
"learning_rate": 4.897413506838102e-07,
|
|
"logits/chosen": -2.812808036804199,
|
|
"logits/rejected": -2.8147010803222656,
|
|
"loss": 0.8898,
|
|
"step": 126
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4809127449989319,
|
|
"beta_dpo/beta_margin_grad_std": 0.018567463383078575,
|
|
"beta_dpo/beta_margin_mean": 0.07651624083518982,
|
|
"beta_dpo/beta_margin_std": 0.0745573341846466,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.2902711629867554,
|
|
"beta_dpo/gap_mean": 100.42000579833984,
|
|
"beta_dpo/gap_std": 105.15016174316406,
|
|
"beta_dpo/loss_margin_mean": 76.5162353515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.18649045521292218,
|
|
"grad_norm": 6.344285488128662,
|
|
"learning_rate": 4.89374339765481e-07,
|
|
"logits/chosen": -2.828252077102661,
|
|
"logits/rejected": -2.825937271118164,
|
|
"loss": 1.3116,
|
|
"step": 127
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47901013493537903,
|
|
"beta_dpo/beta_margin_grad_std": 0.026977377012372017,
|
|
"beta_dpo/beta_margin_mean": 0.08433213084936142,
|
|
"beta_dpo/beta_margin_std": 0.10864510387182236,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.2069345712661743,
|
|
"beta_dpo/gap_mean": 96.0081558227539,
|
|
"beta_dpo/gap_std": 103.26679229736328,
|
|
"beta_dpo/loss_margin_mean": 84.3321304321289,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.18795888399412627,
|
|
"grad_norm": 6.212590217590332,
|
|
"learning_rate": 4.890010211106795e-07,
|
|
"logits/chosen": -2.8211920261383057,
|
|
"logits/rejected": -2.818962574005127,
|
|
"loss": 1.3148,
|
|
"step": 128
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10110783576965332,
|
|
"beta_dpo/beta_margin_grad_mean": -0.33091118931770325,
|
|
"beta_dpo/beta_margin_grad_std": 0.2910465598106384,
|
|
"beta_dpo/beta_margin_mean": 10.52313232421875,
|
|
"beta_dpo/beta_margin_std": 18.802413940429688,
|
|
"beta_dpo/beta_used": 0.10110783576965332,
|
|
"beta_dpo/beta_used_raw": 0.025442659854888916,
|
|
"beta_dpo/gap_mean": 95.70207214355469,
|
|
"beta_dpo/gap_std": 105.74406433105469,
|
|
"beta_dpo/loss_margin_mean": 88.13529205322266,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.1894273127753304,
|
|
"grad_norm": 417.793212890625,
|
|
"learning_rate": 4.88621404556699e-07,
|
|
"logits/chosen": -2.8037643432617188,
|
|
"logits/rejected": -2.787684440612793,
|
|
"loss": 1.4084,
|
|
"step": 129
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.23459400236606598,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2973823845386505,
|
|
"beta_dpo/beta_margin_grad_std": 0.28484469652175903,
|
|
"beta_dpo/beta_margin_mean": 29.821685791015625,
|
|
"beta_dpo/beta_margin_std": 45.19208526611328,
|
|
"beta_dpo/beta_used": 0.23459400236606598,
|
|
"beta_dpo/beta_used_raw": -0.19728092849254608,
|
|
"beta_dpo/gap_mean": 95.61531066894531,
|
|
"beta_dpo/gap_std": 104.48291015625,
|
|
"beta_dpo/loss_margin_mean": 108.31153869628906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.19089574155653452,
|
|
"grad_norm": 1495.203125,
|
|
"learning_rate": 4.882355001067891e-07,
|
|
"logits/chosen": -2.765773057937622,
|
|
"logits/rejected": -2.762284278869629,
|
|
"loss": 2.5681,
|
|
"step": 130
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4765341281890869,
|
|
"beta_dpo/beta_margin_grad_std": 0.02474939450621605,
|
|
"beta_dpo/beta_margin_mean": 0.09421201795339584,
|
|
"beta_dpo/beta_margin_std": 0.09967362135648727,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.4088224470615387,
|
|
"beta_dpo/gap_mean": 96.90589904785156,
|
|
"beta_dpo/gap_std": 104.24870300292969,
|
|
"beta_dpo/loss_margin_mean": 94.2120132446289,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.19236417033773862,
|
|
"grad_norm": 7.157093048095703,
|
|
"learning_rate": 4.878433179298909e-07,
|
|
"logits/chosen": -2.7631754875183105,
|
|
"logits/rejected": -2.775458574295044,
|
|
"loss": 1.301,
|
|
"step": 131
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.017363857477903366,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3200368583202362,
|
|
"beta_dpo/beta_margin_grad_std": 0.22496141493320465,
|
|
"beta_dpo/beta_margin_mean": 1.8344775438308716,
|
|
"beta_dpo/beta_margin_std": 3.168743371963501,
|
|
"beta_dpo/beta_used": 0.017363857477903366,
|
|
"beta_dpo/beta_used_raw": -1.0121409893035889,
|
|
"beta_dpo/gap_mean": 95.9954833984375,
|
|
"beta_dpo/gap_std": 105.45658874511719,
|
|
"beta_dpo/loss_margin_mean": 85.24566650390625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.19383259911894274,
|
|
"grad_norm": 54.672882080078125,
|
|
"learning_rate": 4.874448683603694e-07,
|
|
"logits/chosen": -2.77361798286438,
|
|
"logits/rejected": -2.784986972808838,
|
|
"loss": 0.8296,
|
|
"step": 132
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.44009122252464294,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2839052081108093,
|
|
"beta_dpo/beta_margin_grad_std": 0.28118520975112915,
|
|
"beta_dpo/beta_margin_mean": 48.52168273925781,
|
|
"beta_dpo/beta_margin_std": 70.87135314941406,
|
|
"beta_dpo/beta_used": 0.44009122252464294,
|
|
"beta_dpo/beta_used_raw": 0.10653135180473328,
|
|
"beta_dpo/gap_mean": 96.31338500976562,
|
|
"beta_dpo/gap_std": 104.94859313964844,
|
|
"beta_dpo/loss_margin_mean": 107.33547973632812,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.19530102790014683,
|
|
"grad_norm": 1724.233642578125,
|
|
"learning_rate": 4.870401618977415e-07,
|
|
"logits/chosen": -2.7555744647979736,
|
|
"logits/rejected": -2.765404224395752,
|
|
"loss": 2.2334,
|
|
"step": 133
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.45935988426208496,
|
|
"beta_dpo/beta_margin_grad_mean": -0.306671679019928,
|
|
"beta_dpo/beta_margin_grad_std": 0.29106321930885315,
|
|
"beta_dpo/beta_margin_mean": 41.79468536376953,
|
|
"beta_dpo/beta_margin_std": 67.19827270507812,
|
|
"beta_dpo/beta_used": 0.45935988426208496,
|
|
"beta_dpo/beta_used_raw": 0.19374334812164307,
|
|
"beta_dpo/gap_mean": 95.26974487304688,
|
|
"beta_dpo/gap_std": 103.39216613769531,
|
|
"beta_dpo/loss_margin_mean": 87.6424560546875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.19676945668135096,
|
|
"grad_norm": 1905.6392822265625,
|
|
"learning_rate": 4.866292092063986e-07,
|
|
"logits/chosen": -2.7540433406829834,
|
|
"logits/rejected": -2.7586114406585693,
|
|
"loss": 2.5789,
|
|
"step": 134
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3179980516433716,
|
|
"beta_dpo/beta_margin_grad_mean": -0.0668000727891922,
|
|
"beta_dpo/beta_margin_grad_std": 0.24074798822402954,
|
|
"beta_dpo/beta_margin_mean": 37.95073318481445,
|
|
"beta_dpo/beta_margin_std": 41.624820709228516,
|
|
"beta_dpo/beta_used": 0.3179980516433716,
|
|
"beta_dpo/beta_used_raw": 0.3179980516433716,
|
|
"beta_dpo/gap_mean": 98.79299926757812,
|
|
"beta_dpo/gap_std": 105.39846801757812,
|
|
"beta_dpo/loss_margin_mean": 123.87374114990234,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.19823788546255505,
|
|
"grad_norm": 400.1125183105469,
|
|
"learning_rate": 4.862120211153265e-07,
|
|
"logits/chosen": -2.7113471031188965,
|
|
"logits/rejected": -2.7307372093200684,
|
|
"loss": 0.6672,
|
|
"step": 135
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4736141860485077,
|
|
"beta_dpo/beta_margin_grad_std": 0.03395112603902817,
|
|
"beta_dpo/beta_margin_mean": 0.10649430751800537,
|
|
"beta_dpo/beta_margin_std": 0.13846558332443237,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.6484628915786743,
|
|
"beta_dpo/gap_mean": 102.35041809082031,
|
|
"beta_dpo/gap_std": 113.31024169921875,
|
|
"beta_dpo/loss_margin_mean": 106.49430847167969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.19970631424375918,
|
|
"grad_norm": 6.583731651306152,
|
|
"learning_rate": 4.857886086178193e-07,
|
|
"logits/chosen": -2.7721986770629883,
|
|
"logits/rejected": -2.7757081985473633,
|
|
"loss": 1.3157,
|
|
"step": 136
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8055553436279297,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28008514642715454,
|
|
"beta_dpo/beta_margin_grad_std": 0.2794659733772278,
|
|
"beta_dpo/beta_margin_mean": 109.63809967041016,
|
|
"beta_dpo/beta_margin_std": 171.1824188232422,
|
|
"beta_dpo/beta_used": 0.8055553436279297,
|
|
"beta_dpo/beta_used_raw": -0.391549289226532,
|
|
"beta_dpo/gap_mean": 106.54948425292969,
|
|
"beta_dpo/gap_std": 117.57719421386719,
|
|
"beta_dpo/loss_margin_mean": 136.09088134765625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2011747430249633,
|
|
"grad_norm": 694.3067626953125,
|
|
"learning_rate": 4.853589828711902e-07,
|
|
"logits/chosen": -2.7071657180786133,
|
|
"logits/rejected": -2.72584867477417,
|
|
"loss": 0.9732,
|
|
"step": 137
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.631058394908905,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27331289649009705,
|
|
"beta_dpo/beta_margin_grad_std": 0.26683762669563293,
|
|
"beta_dpo/beta_margin_mean": 88.83894348144531,
|
|
"beta_dpo/beta_margin_std": 135.77273559570312,
|
|
"beta_dpo/beta_used": 0.631058394908905,
|
|
"beta_dpo/beta_used_raw": 0.5807164311408997,
|
|
"beta_dpo/gap_mean": 108.60539245605469,
|
|
"beta_dpo/gap_std": 118.61262512207031,
|
|
"beta_dpo/loss_margin_mean": 118.49124145507812,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2026431718061674,
|
|
"grad_norm": 1751.6961669921875,
|
|
"learning_rate": 4.849231551964771e-07,
|
|
"logits/chosen": -2.6887621879577637,
|
|
"logits/rejected": -2.6839466094970703,
|
|
"loss": 2.7148,
|
|
"step": 138
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47053390741348267,
|
|
"beta_dpo/beta_margin_grad_std": 0.02328990399837494,
|
|
"beta_dpo/beta_margin_mean": 0.1183198019862175,
|
|
"beta_dpo/beta_margin_std": 0.09397386014461517,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.4085111618041992,
|
|
"beta_dpo/gap_mean": 110.96134948730469,
|
|
"beta_dpo/gap_std": 114.99424743652344,
|
|
"beta_dpo/loss_margin_mean": 118.31979370117188,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.20411160058737152,
|
|
"grad_norm": 7.088428020477295,
|
|
"learning_rate": 4.844811370781446e-07,
|
|
"logits/chosen": -2.668765068054199,
|
|
"logits/rejected": -2.664041757583618,
|
|
"loss": 1.2878,
|
|
"step": 139
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4751591980457306,
|
|
"beta_dpo/beta_margin_grad_std": 0.024874050170183182,
|
|
"beta_dpo/beta_margin_mean": 0.09969595074653625,
|
|
"beta_dpo/beta_margin_std": 0.1000821441411972,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.47669804096221924,
|
|
"beta_dpo/gap_mean": 110.13583374023438,
|
|
"beta_dpo/gap_std": 112.09087371826172,
|
|
"beta_dpo/loss_margin_mean": 99.6959457397461,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2055800293685756,
|
|
"grad_norm": 7.575645446777344,
|
|
"learning_rate": 4.840329401637809e-07,
|
|
"logits/chosen": -2.6947057247161865,
|
|
"logits/rejected": -2.6926677227020264,
|
|
"loss": 1.2898,
|
|
"step": 140
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47280803322792053,
|
|
"beta_dpo/beta_margin_grad_std": 0.036303650587797165,
|
|
"beta_dpo/beta_margin_mean": 0.10958551615476608,
|
|
"beta_dpo/beta_margin_std": 0.14771895110607147,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.6926910877227783,
|
|
"beta_dpo/gap_mean": 108.89583587646484,
|
|
"beta_dpo/gap_std": 115.7603759765625,
|
|
"beta_dpo/loss_margin_mean": 109.58551025390625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.20704845814977973,
|
|
"grad_norm": 7.19344425201416,
|
|
"learning_rate": 4.83578576263792e-07,
|
|
"logits/chosen": -2.7385268211364746,
|
|
"logits/rejected": -2.7286226749420166,
|
|
"loss": 1.2944,
|
|
"step": 141
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.18145199120044708,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2992917001247406,
|
|
"beta_dpo/beta_margin_grad_std": 0.2895396947860718,
|
|
"beta_dpo/beta_margin_mean": 23.18352508544922,
|
|
"beta_dpo/beta_margin_std": 34.87459945678711,
|
|
"beta_dpo/beta_used": 0.18145199120044708,
|
|
"beta_dpo/beta_used_raw": -0.9675522446632385,
|
|
"beta_dpo/gap_mean": 111.3647232055664,
|
|
"beta_dpo/gap_std": 120.15746307373047,
|
|
"beta_dpo/loss_margin_mean": 122.43708801269531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.20851688693098386,
|
|
"grad_norm": 1052.642822265625,
|
|
"learning_rate": 4.83118057351089e-07,
|
|
"logits/chosen": -2.668506622314453,
|
|
"logits/rejected": -2.6709957122802734,
|
|
"loss": 1.5849,
|
|
"step": 142
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4766398072242737,
|
|
"beta_dpo/beta_margin_grad_std": 0.030281823128461838,
|
|
"beta_dpo/beta_margin_mean": 0.09403061121702194,
|
|
"beta_dpo/beta_margin_std": 0.1224864274263382,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.4053492546081543,
|
|
"beta_dpo/gap_mean": 110.77619171142578,
|
|
"beta_dpo/gap_std": 120.38482666015625,
|
|
"beta_dpo/loss_margin_mean": 94.03060913085938,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.20998531571218795,
|
|
"grad_norm": 7.156264781951904,
|
|
"learning_rate": 4.826513955607734e-07,
|
|
"logits/chosen": -2.6267662048339844,
|
|
"logits/rejected": -2.6181480884552,
|
|
"loss": 1.3042,
|
|
"step": 143
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47435808181762695,
|
|
"beta_dpo/beta_margin_grad_std": 0.02727697044610977,
|
|
"beta_dpo/beta_margin_mean": 0.10298500955104828,
|
|
"beta_dpo/beta_margin_std": 0.10976456105709076,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.3952561616897583,
|
|
"beta_dpo/gap_mean": 107.01061248779297,
|
|
"beta_dpo/gap_std": 117.98554992675781,
|
|
"beta_dpo/loss_margin_mean": 102.98500061035156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.21145374449339208,
|
|
"grad_norm": 7.684736251831055,
|
|
"learning_rate": 4.821786031898176e-07,
|
|
"logits/chosen": -2.6427910327911377,
|
|
"logits/rejected": -2.642993211746216,
|
|
"loss": 1.2919,
|
|
"step": 144
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3937012255191803,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2757228910923004,
|
|
"beta_dpo/beta_margin_grad_std": 0.26662591099739075,
|
|
"beta_dpo/beta_margin_mean": 40.2667236328125,
|
|
"beta_dpo/beta_margin_std": 75.03095245361328,
|
|
"beta_dpo/beta_used": 0.3937012255191803,
|
|
"beta_dpo/beta_used_raw": 0.18985168635845184,
|
|
"beta_dpo/gap_mean": 107.73323822021484,
|
|
"beta_dpo/gap_std": 117.16415405273438,
|
|
"beta_dpo/loss_margin_mean": 111.22066497802734,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.21292217327459617,
|
|
"grad_norm": 894.4259033203125,
|
|
"learning_rate": 4.816996926967401e-07,
|
|
"logits/chosen": -2.630441665649414,
|
|
"logits/rejected": -2.6299538612365723,
|
|
"loss": 0.6781,
|
|
"step": 145
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.4095514714717865,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12312614917755127,
|
|
"beta_dpo/beta_margin_grad_std": 0.3112068176269531,
|
|
"beta_dpo/beta_margin_mean": 43.80937957763672,
|
|
"beta_dpo/beta_margin_std": 50.46400833129883,
|
|
"beta_dpo/beta_used": 0.4095514714717865,
|
|
"beta_dpo/beta_used_raw": 0.4095514714717865,
|
|
"beta_dpo/gap_mean": 108.25813293457031,
|
|
"beta_dpo/gap_std": 116.30998992919922,
|
|
"beta_dpo/loss_margin_mean": 108.27872467041016,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2143906020558003,
|
|
"grad_norm": 1232.63330078125,
|
|
"learning_rate": 4.812146767012779e-07,
|
|
"logits/chosen": -2.6039223670959473,
|
|
"logits/rejected": -2.601820707321167,
|
|
"loss": 1.0263,
|
|
"step": 146
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47363805770874023,
|
|
"beta_dpo/beta_margin_grad_std": 0.029436958953738213,
|
|
"beta_dpo/beta_margin_mean": 0.1060258075594902,
|
|
"beta_dpo/beta_margin_std": 0.11908337473869324,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.618476152420044,
|
|
"beta_dpo/gap_mean": 107.93118286132812,
|
|
"beta_dpo/gap_std": 118.50152587890625,
|
|
"beta_dpo/loss_margin_mean": 106.02580261230469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.21585903083700442,
|
|
"grad_norm": 7.791094779968262,
|
|
"learning_rate": 4.807235679840536e-07,
|
|
"logits/chosen": -2.580909490585327,
|
|
"logits/rejected": -2.580644369125366,
|
|
"loss": 1.2941,
|
|
"step": 147
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.021544385701417923,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3400629460811615,
|
|
"beta_dpo/beta_margin_grad_std": 0.26523733139038086,
|
|
"beta_dpo/beta_margin_mean": 3.093792676925659,
|
|
"beta_dpo/beta_margin_std": 6.440648078918457,
|
|
"beta_dpo/beta_used": 0.021544385701417923,
|
|
"beta_dpo/beta_used_raw": -0.8347901105880737,
|
|
"beta_dpo/gap_mean": 104.83203125,
|
|
"beta_dpo/gap_std": 118.19056701660156,
|
|
"beta_dpo/loss_margin_mean": 100.39215850830078,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2173274596182085,
|
|
"grad_norm": 217.89871215820312,
|
|
"learning_rate": 4.802263794862384e-07,
|
|
"logits/chosen": -2.570409059524536,
|
|
"logits/rejected": -2.576443672180176,
|
|
"loss": 1.2305,
|
|
"step": 148
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.199530228972435,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3008774518966675,
|
|
"beta_dpo/beta_margin_grad_std": 0.2911630868911743,
|
|
"beta_dpo/beta_margin_mean": 34.24274826049805,
|
|
"beta_dpo/beta_margin_std": 55.22300720214844,
|
|
"beta_dpo/beta_used": 0.199530228972435,
|
|
"beta_dpo/beta_used_raw": -0.7738866806030273,
|
|
"beta_dpo/gap_mean": 112.32363891601562,
|
|
"beta_dpo/gap_std": 125.47573852539062,
|
|
"beta_dpo/loss_margin_mean": 131.21102905273438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.21879588839941264,
|
|
"grad_norm": 2060.334228515625,
|
|
"learning_rate": 4.797231243092118e-07,
|
|
"logits/chosen": -2.5769052505493164,
|
|
"logits/rejected": -2.581183910369873,
|
|
"loss": 3.0704,
|
|
"step": 149
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.46728765964508057,
|
|
"beta_dpo/beta_margin_grad_std": 0.03217744454741478,
|
|
"beta_dpo/beta_margin_mean": 0.1318235695362091,
|
|
"beta_dpo/beta_margin_std": 0.1308116763830185,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.7136021852493286,
|
|
"beta_dpo/gap_mean": 114.93362426757812,
|
|
"beta_dpo/gap_std": 126.79879760742188,
|
|
"beta_dpo/loss_margin_mean": 131.8235626220703,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.22026431718061673,
|
|
"grad_norm": 8.056601524353027,
|
|
"learning_rate": 4.792138157142157e-07,
|
|
"logits/chosen": -2.5865211486816406,
|
|
"logits/rejected": -2.5941879749298096,
|
|
"loss": 1.2894,
|
|
"step": 150
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4679759740829468,
|
|
"beta_dpo/beta_margin_grad_std": 0.03260992467403412,
|
|
"beta_dpo/beta_margin_mean": 0.12910982966423035,
|
|
"beta_dpo/beta_margin_std": 0.13299117982387543,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.161845862865448,
|
|
"beta_dpo/gap_mean": 117.1995849609375,
|
|
"beta_dpo/gap_std": 127.47024536132812,
|
|
"beta_dpo/loss_margin_mean": 129.1098175048828,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.22173274596182085,
|
|
"grad_norm": 8.165633201599121,
|
|
"learning_rate": 4.786984671220053e-07,
|
|
"logits/chosen": -2.597113847732544,
|
|
"logits/rejected": -2.6131339073181152,
|
|
"loss": 1.2784,
|
|
"step": 151
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.35655510425567627,
|
|
"beta_dpo/beta_margin_grad_mean": -0.293893039226532,
|
|
"beta_dpo/beta_margin_grad_std": 0.28947779536247253,
|
|
"beta_dpo/beta_margin_mean": 48.276973724365234,
|
|
"beta_dpo/beta_margin_std": 90.70513916015625,
|
|
"beta_dpo/beta_used": 0.35655510425567627,
|
|
"beta_dpo/beta_used_raw": 0.10587197542190552,
|
|
"beta_dpo/gap_mean": 121.5870590209961,
|
|
"beta_dpo/gap_std": 130.81991577148438,
|
|
"beta_dpo/loss_margin_mean": 142.88314819335938,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.22320117474302498,
|
|
"grad_norm": 2186.73388671875,
|
|
"learning_rate": 4.78177092112495e-07,
|
|
"logits/chosen": -2.5673489570617676,
|
|
"logits/rejected": -2.573914051055908,
|
|
"loss": 1.5166,
|
|
"step": 152
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.46919021010398865,
|
|
"beta_dpo/beta_margin_grad_std": 0.04115996137261391,
|
|
"beta_dpo/beta_margin_mean": 0.12469355762004852,
|
|
"beta_dpo/beta_margin_std": 0.16790282726287842,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.7168277502059937,
|
|
"beta_dpo/gap_mean": 122.23858642578125,
|
|
"beta_dpo/gap_std": 137.7699737548828,
|
|
"beta_dpo/loss_margin_mean": 124.69355773925781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.22466960352422907,
|
|
"grad_norm": 7.622184753417969,
|
|
"learning_rate": 4.776497044244016e-07,
|
|
"logits/chosen": -2.5728824138641357,
|
|
"logits/rejected": -2.5736918449401855,
|
|
"loss": 1.2987,
|
|
"step": 153
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0757020711898804,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28336623311042786,
|
|
"beta_dpo/beta_margin_grad_std": 0.2811123728752136,
|
|
"beta_dpo/beta_margin_mean": 181.07421875,
|
|
"beta_dpo/beta_margin_std": 271.3897705078125,
|
|
"beta_dpo/beta_used": 1.0757020711898804,
|
|
"beta_dpo/beta_used_raw": 0.654621958732605,
|
|
"beta_dpo/gap_mean": 123.79661560058594,
|
|
"beta_dpo/gap_std": 139.25714111328125,
|
|
"beta_dpo/loss_margin_mean": 138.57278442382812,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2261380323054332,
|
|
"grad_norm": 9847.91015625,
|
|
"learning_rate": 4.771163179548808e-07,
|
|
"logits/chosen": -2.5638508796691895,
|
|
"logits/rejected": -2.5663564205169678,
|
|
"loss": 2.0635,
|
|
"step": 154
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47398510575294495,
|
|
"beta_dpo/beta_margin_grad_std": 0.026723386719822884,
|
|
"beta_dpo/beta_margin_mean": 0.10449974238872528,
|
|
"beta_dpo/beta_margin_std": 0.10794886201620102,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.4330192804336548,
|
|
"beta_dpo/gap_mean": 123.12739562988281,
|
|
"beta_dpo/gap_std": 134.49668884277344,
|
|
"beta_dpo/loss_margin_mean": 104.49974060058594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2276064610866373,
|
|
"grad_norm": 8.36424446105957,
|
|
"learning_rate": 4.7657694675916247e-07,
|
|
"logits/chosen": -2.595834255218506,
|
|
"logits/rejected": -2.5887231826782227,
|
|
"loss": 1.2924,
|
|
"step": 155
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47591692209243774,
|
|
"beta_dpo/beta_margin_grad_std": 0.03880538046360016,
|
|
"beta_dpo/beta_margin_mean": 0.09696371853351593,
|
|
"beta_dpo/beta_margin_std": 0.15683069825172424,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.1627048254013062,
|
|
"beta_dpo/gap_mean": 119.81661224365234,
|
|
"beta_dpo/gap_std": 137.55975341796875,
|
|
"beta_dpo/loss_margin_mean": 96.96371459960938,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2290748898678414,
|
|
"grad_norm": 8.904071807861328,
|
|
"learning_rate": 4.7603160505017893e-07,
|
|
"logits/chosen": -2.5694189071655273,
|
|
"logits/rejected": -2.5704991817474365,
|
|
"loss": 1.2922,
|
|
"step": 156
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0197237730026245,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26143965125083923,
|
|
"beta_dpo/beta_margin_grad_std": 0.26461806893348694,
|
|
"beta_dpo/beta_margin_mean": 192.70529174804688,
|
|
"beta_dpo/beta_margin_std": 259.1637268066406,
|
|
"beta_dpo/beta_used": 1.0197237730026245,
|
|
"beta_dpo/beta_used_raw": 0.7222856283187866,
|
|
"beta_dpo/gap_mean": 125.96708679199219,
|
|
"beta_dpo/gap_std": 138.84896850585938,
|
|
"beta_dpo/loss_margin_mean": 174.92384338378906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2305433186490455,
|
|
"grad_norm": 6987.59423828125,
|
|
"learning_rate": 4.7548030719819154e-07,
|
|
"logits/chosen": -2.514146327972412,
|
|
"logits/rejected": -2.5194973945617676,
|
|
"loss": 2.6962,
|
|
"step": 157
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9159993529319763,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2739473581314087,
|
|
"beta_dpo/beta_margin_grad_std": 0.26920560002326965,
|
|
"beta_dpo/beta_margin_mean": 162.5165252685547,
|
|
"beta_dpo/beta_margin_std": 260.4980773925781,
|
|
"beta_dpo/beta_used": 0.9159993529319763,
|
|
"beta_dpo/beta_used_raw": 0.8316487669944763,
|
|
"beta_dpo/gap_mean": 135.50296020507812,
|
|
"beta_dpo/gap_std": 145.57350158691406,
|
|
"beta_dpo/loss_margin_mean": 182.47454833984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.23201174743024963,
|
|
"grad_norm": 5398.83154296875,
|
|
"learning_rate": 4.7492306773041136e-07,
|
|
"logits/chosen": -2.5161776542663574,
|
|
"logits/rejected": -2.5338191986083984,
|
|
"loss": 5.0943,
|
|
"step": 158
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.36463621258735657,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3097352981567383,
|
|
"beta_dpo/beta_margin_grad_std": 0.3000122308731079,
|
|
"beta_dpo/beta_margin_mean": 49.08452224731445,
|
|
"beta_dpo/beta_margin_std": 94.18856811523438,
|
|
"beta_dpo/beta_used": 0.36463621258735657,
|
|
"beta_dpo/beta_used_raw": -0.35086220502853394,
|
|
"beta_dpo/gap_mean": 138.55572509765625,
|
|
"beta_dpo/gap_std": 149.52554321289062,
|
|
"beta_dpo/loss_margin_mean": 143.02760314941406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.23348017621145375,
|
|
"grad_norm": 3679.53076171875,
|
|
"learning_rate": 4.743599013306165e-07,
|
|
"logits/chosen": -2.5350537300109863,
|
|
"logits/rejected": -2.527538299560547,
|
|
"loss": 5.0817,
|
|
"step": 159
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4628070592880249,
|
|
"beta_dpo/beta_margin_grad_std": 0.04738787189126015,
|
|
"beta_dpo/beta_margin_mean": 0.15084336698055267,
|
|
"beta_dpo/beta_margin_std": 0.19365453720092773,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.107983946800232,
|
|
"beta_dpo/gap_mean": 142.3861083984375,
|
|
"beta_dpo/gap_std": 158.1815185546875,
|
|
"beta_dpo/loss_margin_mean": 150.84335327148438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.23494860499265785,
|
|
"grad_norm": 9.592857360839844,
|
|
"learning_rate": 4.737908228387656e-07,
|
|
"logits/chosen": -2.50848388671875,
|
|
"logits/rejected": -2.5071470737457275,
|
|
"loss": 1.2723,
|
|
"step": 160
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5015167593955994,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3111899793148041,
|
|
"beta_dpo/beta_margin_grad_std": 0.29640379548072815,
|
|
"beta_dpo/beta_margin_mean": 87.97203063964844,
|
|
"beta_dpo/beta_margin_std": 136.19284057617188,
|
|
"beta_dpo/beta_used": 0.5015167593955994,
|
|
"beta_dpo/beta_used_raw": -0.03423714637756348,
|
|
"beta_dpo/gap_mean": 142.8735809326172,
|
|
"beta_dpo/gap_std": 155.4887237548828,
|
|
"beta_dpo/loss_margin_mean": 143.24179077148438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.23641703377386197,
|
|
"grad_norm": 4087.4951171875,
|
|
"learning_rate": 4.7321584725060594e-07,
|
|
"logits/chosen": -2.4902877807617188,
|
|
"logits/rejected": -2.498593330383301,
|
|
"loss": 5.4119,
|
|
"step": 161
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.463805228471756,
|
|
"beta_dpo/beta_margin_grad_std": 0.03792537748813629,
|
|
"beta_dpo/beta_margin_mean": 0.1462424099445343,
|
|
"beta_dpo/beta_margin_std": 0.1551448106765747,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.097806692123413,
|
|
"beta_dpo/gap_mean": 143.18414306640625,
|
|
"beta_dpo/gap_std": 155.8494873046875,
|
|
"beta_dpo/loss_margin_mean": 146.24240112304688,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.23788546255506607,
|
|
"grad_norm": 8.91688346862793,
|
|
"learning_rate": 4.7263498971727905e-07,
|
|
"logits/chosen": -2.5275540351867676,
|
|
"logits/rejected": -2.533555030822754,
|
|
"loss": 1.2697,
|
|
"step": 162
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.14172588288784027,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29044896364212036,
|
|
"beta_dpo/beta_margin_grad_std": 0.2808626890182495,
|
|
"beta_dpo/beta_margin_mean": 20.941862106323242,
|
|
"beta_dpo/beta_margin_std": 32.66022872924805,
|
|
"beta_dpo/beta_used": 0.14172588288784027,
|
|
"beta_dpo/beta_used_raw": -0.5184394717216492,
|
|
"beta_dpo/gap_mean": 141.9095916748047,
|
|
"beta_dpo/gap_std": 151.08145141601562,
|
|
"beta_dpo/loss_margin_mean": 143.454345703125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2393538913362702,
|
|
"grad_norm": 561.4199829101562,
|
|
"learning_rate": 4.720482655449212e-07,
|
|
"logits/chosen": -2.4482381343841553,
|
|
"logits/rejected": -2.4447622299194336,
|
|
"loss": 1.4418,
|
|
"step": 163
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.23951520025730133,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2740349769592285,
|
|
"beta_dpo/beta_margin_grad_std": 0.2672988772392273,
|
|
"beta_dpo/beta_margin_mean": 33.166099548339844,
|
|
"beta_dpo/beta_margin_std": 47.26707458496094,
|
|
"beta_dpo/beta_used": 0.23951520025730133,
|
|
"beta_dpo/beta_used_raw": 0.11707229167222977,
|
|
"beta_dpo/gap_mean": 142.1658935546875,
|
|
"beta_dpo/gap_std": 142.63870239257812,
|
|
"beta_dpo/loss_margin_mean": 141.0481414794922,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.24082232011747431,
|
|
"grad_norm": 1184.495849609375,
|
|
"learning_rate": 4.714556901942599e-07,
|
|
"logits/chosen": -2.453787326812744,
|
|
"logits/rejected": -2.461193799972534,
|
|
"loss": 0.6742,
|
|
"step": 164
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4727182984352112,
|
|
"beta_dpo/beta_margin_grad_std": 0.033597320318222046,
|
|
"beta_dpo/beta_margin_mean": 0.10977371782064438,
|
|
"beta_dpo/beta_margin_std": 0.13549566268920898,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.5455293655395508,
|
|
"beta_dpo/gap_mean": 138.08554077148438,
|
|
"beta_dpo/gap_std": 140.043701171875,
|
|
"beta_dpo/loss_margin_mean": 109.77371215820312,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2422907488986784,
|
|
"grad_norm": 9.582477569580078,
|
|
"learning_rate": 4.708572792802069e-07,
|
|
"logits/chosen": -2.466679096221924,
|
|
"logits/rejected": -2.4749114513397217,
|
|
"loss": 1.2815,
|
|
"step": 165
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.46377497911453247,
|
|
"beta_dpo/beta_margin_grad_std": 0.05350736901164055,
|
|
"beta_dpo/beta_margin_mean": 0.14767391979694366,
|
|
"beta_dpo/beta_margin_std": 0.22187907993793488,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.600358784198761,
|
|
"beta_dpo/gap_mean": 137.3470916748047,
|
|
"beta_dpo/gap_std": 151.5578155517578,
|
|
"beta_dpo/loss_margin_mean": 147.6739044189453,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.24375917767988253,
|
|
"grad_norm": 9.58260440826416,
|
|
"learning_rate": 4.702530485714461e-07,
|
|
"logits/chosen": -2.4487009048461914,
|
|
"logits/rejected": -2.4638147354125977,
|
|
"loss": 1.2681,
|
|
"step": 166
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.7830126881599426,
|
|
"beta_dpo/beta_margin_grad_mean": -0.10973574221134186,
|
|
"beta_dpo/beta_margin_grad_std": 0.3119940757751465,
|
|
"beta_dpo/beta_margin_mean": 129.75360107421875,
|
|
"beta_dpo/beta_margin_std": 127.3109359741211,
|
|
"beta_dpo/beta_used": 0.7830126881599426,
|
|
"beta_dpo/beta_used_raw": 0.7830126881599426,
|
|
"beta_dpo/gap_mean": 141.41099548339844,
|
|
"beta_dpo/gap_std": 154.25009155273438,
|
|
"beta_dpo/loss_margin_mean": 165.0268096923828,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.24522760646108663,
|
|
"grad_norm": 6046.77734375,
|
|
"learning_rate": 4.6964301399001877e-07,
|
|
"logits/chosen": -2.447455883026123,
|
|
"logits/rejected": -2.469198703765869,
|
|
"loss": 7.7956,
|
|
"step": 167
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4695003628730774,
|
|
"beta_dpo/beta_margin_grad_std": 0.03412729129195213,
|
|
"beta_dpo/beta_margin_mean": 0.12282736599445343,
|
|
"beta_dpo/beta_margin_std": 0.13787633180618286,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.081223487854004,
|
|
"beta_dpo/gap_mean": 140.6132354736328,
|
|
"beta_dpo/gap_std": 153.59951782226562,
|
|
"beta_dpo/loss_margin_mean": 122.82736206054688,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.24669603524229075,
|
|
"grad_norm": 9.591252326965332,
|
|
"learning_rate": 4.690271916109034e-07,
|
|
"logits/chosen": -2.4348056316375732,
|
|
"logits/rejected": -2.4319422245025635,
|
|
"loss": 1.2879,
|
|
"step": 168
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2029220461845398,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3294808864593506,
|
|
"beta_dpo/beta_margin_grad_std": 0.3101998269557953,
|
|
"beta_dpo/beta_margin_mean": 32.48569869995117,
|
|
"beta_dpo/beta_margin_std": 66.5421371459961,
|
|
"beta_dpo/beta_used": 0.2029220461845398,
|
|
"beta_dpo/beta_used_raw": -0.917236328125,
|
|
"beta_dpo/gap_mean": 139.76837158203125,
|
|
"beta_dpo/gap_std": 157.51992797851562,
|
|
"beta_dpo/loss_margin_mean": 134.8309326171875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.24816446402349487,
|
|
"grad_norm": 1549.650146484375,
|
|
"learning_rate": 4.6840559766159235e-07,
|
|
"logits/chosen": -2.412907361984253,
|
|
"logits/rejected": -2.4144067764282227,
|
|
"loss": 1.9765,
|
|
"step": 169
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.14525021612644196,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2680424451828003,
|
|
"beta_dpo/beta_margin_grad_std": 0.2648860812187195,
|
|
"beta_dpo/beta_margin_mean": 20.61962890625,
|
|
"beta_dpo/beta_margin_std": 32.76333236694336,
|
|
"beta_dpo/beta_used": 0.14525021612644196,
|
|
"beta_dpo/beta_used_raw": -0.05808502435684204,
|
|
"beta_dpo/gap_mean": 137.88375854492188,
|
|
"beta_dpo/gap_std": 152.96636962890625,
|
|
"beta_dpo/loss_margin_mean": 133.55470275878906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.24963289280469897,
|
|
"grad_norm": 42.14877700805664,
|
|
"learning_rate": 4.6777824852166437e-07,
|
|
"logits/chosen": -2.4181408882141113,
|
|
"logits/rejected": -2.4320292472839355,
|
|
"loss": 0.6382,
|
|
"step": 170
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4662558138370514,
|
|
"beta_dpo/beta_margin_grad_std": 0.0472683347761631,
|
|
"beta_dpo/beta_margin_mean": 0.1371498852968216,
|
|
"beta_dpo/beta_margin_std": 0.19521786272525787,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.053070068359375,
|
|
"beta_dpo/gap_mean": 136.4842987060547,
|
|
"beta_dpo/gap_std": 157.8990478515625,
|
|
"beta_dpo/loss_margin_mean": 137.14987182617188,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2511013215859031,
|
|
"grad_norm": 10.087845802307129,
|
|
"learning_rate": 4.6714516072235273e-07,
|
|
"logits/chosen": -2.4267027378082275,
|
|
"logits/rejected": -2.4207305908203125,
|
|
"loss": 1.2906,
|
|
"step": 171
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4663008153438568,
|
|
"beta_dpo/beta_margin_grad_std": 0.034671008586883545,
|
|
"beta_dpo/beta_margin_mean": 0.1361597627401352,
|
|
"beta_dpo/beta_margin_std": 0.1432521492242813,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8160032629966736,
|
|
"beta_dpo/gap_mean": 136.5276336669922,
|
|
"beta_dpo/gap_std": 157.35430908203125,
|
|
"beta_dpo/loss_margin_mean": 136.15975952148438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2525697503671072,
|
|
"grad_norm": 9.957985877990723,
|
|
"learning_rate": 4.6650635094610966e-07,
|
|
"logits/chosen": -2.444511890411377,
|
|
"logits/rejected": -2.4361934661865234,
|
|
"loss": 1.2706,
|
|
"step": 172
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.47176581621170044,
|
|
"beta_dpo/beta_margin_grad_std": 0.035718757659196854,
|
|
"beta_dpo/beta_margin_mean": 0.11384298652410507,
|
|
"beta_dpo/beta_margin_std": 0.14550843834877014,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.0770390033721924,
|
|
"beta_dpo/gap_mean": 133.58248901367188,
|
|
"beta_dpo/gap_std": 156.02670288085938,
|
|
"beta_dpo/loss_margin_mean": 113.84297943115234,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2540381791483113,
|
|
"grad_norm": 9.779240608215332,
|
|
"learning_rate": 4.6586183602616687e-07,
|
|
"logits/chosen": -2.432872772216797,
|
|
"logits/rejected": -2.435973882675171,
|
|
"loss": 1.2937,
|
|
"step": 173
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3803338408470154,
|
|
"beta_dpo/beta_margin_grad_mean": -0.30775538086891174,
|
|
"beta_dpo/beta_margin_grad_std": 0.29982587695121765,
|
|
"beta_dpo/beta_margin_mean": 62.02728271484375,
|
|
"beta_dpo/beta_margin_std": 124.6036605834961,
|
|
"beta_dpo/beta_used": 0.3803338408470154,
|
|
"beta_dpo/beta_used_raw": 0.21193695068359375,
|
|
"beta_dpo/gap_mean": 137.41583251953125,
|
|
"beta_dpo/gap_std": 162.42489624023438,
|
|
"beta_dpo/loss_margin_mean": 164.56101989746094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2555066079295154,
|
|
"grad_norm": 3307.25830078125,
|
|
"learning_rate": 4.652116329460919e-07,
|
|
"logits/chosen": -2.427743434906006,
|
|
"logits/rejected": -2.437385082244873,
|
|
"loss": 4.3908,
|
|
"step": 174
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.29845502972602844,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29286932945251465,
|
|
"beta_dpo/beta_margin_grad_std": 0.2887791693210602,
|
|
"beta_dpo/beta_margin_mean": 52.850128173828125,
|
|
"beta_dpo/beta_margin_std": 88.22852325439453,
|
|
"beta_dpo/beta_used": 0.29845502972602844,
|
|
"beta_dpo/beta_used_raw": 0.1955851912498474,
|
|
"beta_dpo/gap_mean": 142.67230224609375,
|
|
"beta_dpo/gap_std": 161.5905303955078,
|
|
"beta_dpo/loss_margin_mean": 167.7036590576172,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.25697503671071953,
|
|
"grad_norm": 3898.72021484375,
|
|
"learning_rate": 4.645557588393406e-07,
|
|
"logits/chosen": -2.424152374267578,
|
|
"logits/rejected": -2.4236903190612793,
|
|
"loss": 9.4346,
|
|
"step": 175
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5137513875961304,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2328094094991684,
|
|
"beta_dpo/beta_margin_grad_std": 0.23490336537361145,
|
|
"beta_dpo/beta_margin_mean": 96.55097198486328,
|
|
"beta_dpo/beta_margin_std": 138.6733856201172,
|
|
"beta_dpo/beta_used": 0.5137513875961304,
|
|
"beta_dpo/beta_used_raw": -0.3851640224456787,
|
|
"beta_dpo/gap_mean": 148.3616943359375,
|
|
"beta_dpo/gap_std": 161.01882934570312,
|
|
"beta_dpo/loss_margin_mean": 163.68028259277344,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.25844346549192365,
|
|
"grad_norm": 6.7193403244018555,
|
|
"learning_rate": 4.638942309888058e-07,
|
|
"logits/chosen": -2.455569267272949,
|
|
"logits/rejected": -2.4624271392822266,
|
|
"loss": 0.6383,
|
|
"step": 176
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4615153968334198,
|
|
"beta_dpo/beta_margin_grad_std": 0.045166175812482834,
|
|
"beta_dpo/beta_margin_mean": 0.15591758489608765,
|
|
"beta_dpo/beta_margin_std": 0.18478171527385712,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.3913229703903198,
|
|
"beta_dpo/gap_mean": 148.726806640625,
|
|
"beta_dpo/gap_std": 166.01434326171875,
|
|
"beta_dpo/loss_margin_mean": 155.91757202148438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2599118942731278,
|
|
"grad_norm": 9.99582576751709,
|
|
"learning_rate": 4.6322706682636137e-07,
|
|
"logits/chosen": -2.4966821670532227,
|
|
"logits/rejected": -2.503170967102051,
|
|
"loss": 1.2699,
|
|
"step": 177
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6906166076660156,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24617867171764374,
|
|
"beta_dpo/beta_margin_grad_std": 0.243428036570549,
|
|
"beta_dpo/beta_margin_mean": 125.11310577392578,
|
|
"beta_dpo/beta_margin_std": 196.64479064941406,
|
|
"beta_dpo/beta_used": 0.6906166076660156,
|
|
"beta_dpo/beta_used_raw": 0.5849722027778625,
|
|
"beta_dpo/gap_mean": 156.6367645263672,
|
|
"beta_dpo/gap_std": 169.2756805419922,
|
|
"beta_dpo/loss_margin_mean": 194.61502075195312,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.26138032305433184,
|
|
"grad_norm": 1372.7340087890625,
|
|
"learning_rate": 4.6255428393240354e-07,
|
|
"logits/chosen": -2.419686794281006,
|
|
"logits/rejected": -2.419022560119629,
|
|
"loss": 0.643,
|
|
"step": 178
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.46199724078178406,
|
|
"beta_dpo/beta_margin_grad_std": 0.0452277697622776,
|
|
"beta_dpo/beta_margin_mean": 0.15396226942539215,
|
|
"beta_dpo/beta_margin_std": 0.18460071086883545,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.5662036538124084,
|
|
"beta_dpo/gap_mean": 158.12814331054688,
|
|
"beta_dpo/gap_std": 171.18023681640625,
|
|
"beta_dpo/loss_margin_mean": 153.96226501464844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.26284875183553597,
|
|
"grad_norm": 10.445080757141113,
|
|
"learning_rate": 4.6187590003538724e-07,
|
|
"logits/chosen": -2.497511863708496,
|
|
"logits/rejected": -2.508460283279419,
|
|
"loss": 1.249,
|
|
"step": 179
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10046719759702682,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3064371645450592,
|
|
"beta_dpo/beta_margin_grad_std": 0.29360902309417725,
|
|
"beta_dpo/beta_margin_mean": 18.78900146484375,
|
|
"beta_dpo/beta_margin_std": 31.81175422668457,
|
|
"beta_dpo/beta_used": 0.10046719759702682,
|
|
"beta_dpo/beta_used_raw": -0.13104525208473206,
|
|
"beta_dpo/gap_mean": 157.84716796875,
|
|
"beta_dpo/gap_std": 173.5079345703125,
|
|
"beta_dpo/loss_margin_mean": 168.60824584960938,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2643171806167401,
|
|
"grad_norm": 1487.51171875,
|
|
"learning_rate": 4.611919330113591e-07,
|
|
"logits/chosen": -2.4725236892700195,
|
|
"logits/rejected": -2.4876785278320312,
|
|
"loss": 2.5514,
|
|
"step": 180
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.468180775642395,
|
|
"beta_dpo/beta_margin_grad_std": 0.03856905922293663,
|
|
"beta_dpo/beta_margin_mean": 0.12825387716293335,
|
|
"beta_dpo/beta_margin_std": 0.15621516108512878,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8151113390922546,
|
|
"beta_dpo/gap_mean": 154.2423095703125,
|
|
"beta_dpo/gap_std": 171.04672241210938,
|
|
"beta_dpo/loss_margin_mean": 128.25387573242188,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2657856093979442,
|
|
"grad_norm": 9.46091365814209,
|
|
"learning_rate": 4.605024008834863e-07,
|
|
"logits/chosen": -2.453889846801758,
|
|
"logits/rejected": -2.453495979309082,
|
|
"loss": 1.2548,
|
|
"step": 181
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.074347972869873,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24131838977336884,
|
|
"beta_dpo/beta_margin_grad_std": 0.24388866126537323,
|
|
"beta_dpo/beta_margin_mean": 206.86056518554688,
|
|
"beta_dpo/beta_margin_std": 286.9940490722656,
|
|
"beta_dpo/beta_used": 1.074347972869873,
|
|
"beta_dpo/beta_used_raw": 0.8183754086494446,
|
|
"beta_dpo/gap_mean": 159.13601684570312,
|
|
"beta_dpo/gap_std": 169.01043701171875,
|
|
"beta_dpo/loss_margin_mean": 191.2897491455078,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.26725403817914833,
|
|
"grad_norm": 6.596961975097656,
|
|
"learning_rate": 4.598073218215817e-07,
|
|
"logits/chosen": -2.4499645233154297,
|
|
"logits/rejected": -2.452929973602295,
|
|
"loss": 0.6226,
|
|
"step": 182
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.46204835176467896,
|
|
"beta_dpo/beta_margin_grad_std": 0.037284255027770996,
|
|
"beta_dpo/beta_margin_mean": 0.15333117544651031,
|
|
"beta_dpo/beta_margin_std": 0.152970090508461,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.040143370628357,
|
|
"beta_dpo/gap_mean": 160.8644561767578,
|
|
"beta_dpo/gap_std": 167.21820068359375,
|
|
"beta_dpo/loss_margin_mean": 153.33116149902344,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2687224669603524,
|
|
"grad_norm": 10.506147384643555,
|
|
"learning_rate": 4.5910671414162484e-07,
|
|
"logits/chosen": -2.4273524284362793,
|
|
"logits/rejected": -2.4347071647644043,
|
|
"loss": 1.2515,
|
|
"step": 183
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4735415279865265,
|
|
"beta_dpo/beta_margin_grad_std": 0.03694302588701248,
|
|
"beta_dpo/beta_margin_mean": 0.10642834007740021,
|
|
"beta_dpo/beta_margin_std": 0.14879478514194489,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0169718265533447,
|
|
"beta_dpo/gap_mean": 150.88897705078125,
|
|
"beta_dpo/gap_std": 165.56298828125,
|
|
"beta_dpo/loss_margin_mean": 106.42832946777344,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2701908957415565,
|
|
"grad_norm": 10.852279663085938,
|
|
"learning_rate": 4.5840059630527985e-07,
|
|
"logits/chosen": -2.424748420715332,
|
|
"logits/rejected": -2.420259952545166,
|
|
"loss": 1.2617,
|
|
"step": 184
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4680558145046234,
|
|
"beta_dpo/beta_margin_grad_std": 0.03759468346834183,
|
|
"beta_dpo/beta_margin_mean": 0.12906821072101593,
|
|
"beta_dpo/beta_margin_std": 0.15372078120708466,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.9787791967391968,
|
|
"beta_dpo/gap_mean": 146.270263671875,
|
|
"beta_dpo/gap_std": 162.84954833984375,
|
|
"beta_dpo/loss_margin_mean": 129.06820678710938,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.27165932452276065,
|
|
"grad_norm": 10.52782917022705,
|
|
"learning_rate": 4.5768898691940836e-07,
|
|
"logits/chosen": -2.3389992713928223,
|
|
"logits/rejected": -2.340506076812744,
|
|
"loss": 1.265,
|
|
"step": 185
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.196396142244339,
|
|
"beta_dpo/beta_margin_grad_mean": -0.259600967168808,
|
|
"beta_dpo/beta_margin_grad_std": 0.25825586915016174,
|
|
"beta_dpo/beta_margin_mean": 36.263214111328125,
|
|
"beta_dpo/beta_margin_std": 56.00040817260742,
|
|
"beta_dpo/beta_used": 0.196396142244339,
|
|
"beta_dpo/beta_used_raw": 0.1395997554063797,
|
|
"beta_dpo/gap_mean": 151.24038696289062,
|
|
"beta_dpo/gap_std": 161.67489624023438,
|
|
"beta_dpo/loss_margin_mean": 186.27423095703125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.27312775330396477,
|
|
"grad_norm": 1914.20361328125,
|
|
"learning_rate": 4.5697190473557947e-07,
|
|
"logits/chosen": -2.3745052814483643,
|
|
"logits/rejected": -2.37973690032959,
|
|
"loss": 0.7652,
|
|
"step": 186
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9558632373809814,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28026333451271057,
|
|
"beta_dpo/beta_margin_grad_std": 0.2789249122142792,
|
|
"beta_dpo/beta_margin_mean": 181.9654083251953,
|
|
"beta_dpo/beta_margin_std": 302.32769775390625,
|
|
"beta_dpo/beta_used": 0.9558632373809814,
|
|
"beta_dpo/beta_used_raw": 0.7472833395004272,
|
|
"beta_dpo/gap_mean": 153.18206787109375,
|
|
"beta_dpo/gap_std": 159.89886474609375,
|
|
"beta_dpo/loss_margin_mean": 162.0480194091797,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2745961820851689,
|
|
"grad_norm": 5661.1728515625,
|
|
"learning_rate": 4.5624936864957555e-07,
|
|
"logits/chosen": -2.2961974143981934,
|
|
"logits/rejected": -2.298018217086792,
|
|
"loss": 6.1816,
|
|
"step": 187
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2359977513551712,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2686719000339508,
|
|
"beta_dpo/beta_margin_grad_std": 0.2627968490123749,
|
|
"beta_dpo/beta_margin_mean": 45.696014404296875,
|
|
"beta_dpo/beta_margin_std": 83.678955078125,
|
|
"beta_dpo/beta_used": 0.2359977513551712,
|
|
"beta_dpo/beta_used_raw": 0.019898220896720886,
|
|
"beta_dpo/gap_mean": 157.99917602539062,
|
|
"beta_dpo/gap_std": 162.6558074951172,
|
|
"beta_dpo/loss_margin_mean": 179.31515502929688,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.27606461086637296,
|
|
"grad_norm": 1707.17724609375,
|
|
"learning_rate": 4.5552139770089454e-07,
|
|
"logits/chosen": -2.2876267433166504,
|
|
"logits/rejected": -2.297708034515381,
|
|
"loss": 0.92,
|
|
"step": 188
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.456310510635376,
|
|
"beta_dpo/beta_margin_grad_std": 0.04195929691195488,
|
|
"beta_dpo/beta_margin_mean": 0.17710651457309723,
|
|
"beta_dpo/beta_margin_std": 0.17350885272026062,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.4375971555709839,
|
|
"beta_dpo/gap_mean": 161.61734008789062,
|
|
"beta_dpo/gap_std": 166.82327270507812,
|
|
"beta_dpo/loss_margin_mean": 177.10650634765625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2775330396475771,
|
|
"grad_norm": 12.050735473632812,
|
|
"learning_rate": 4.5478801107224794e-07,
|
|
"logits/chosen": -2.290733814239502,
|
|
"logits/rejected": -2.3053641319274902,
|
|
"loss": 1.2584,
|
|
"step": 189
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6101582646369934,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24652166664600372,
|
|
"beta_dpo/beta_margin_grad_std": 0.2449052780866623,
|
|
"beta_dpo/beta_margin_mean": 159.73216247558594,
|
|
"beta_dpo/beta_margin_std": 244.68267822265625,
|
|
"beta_dpo/beta_used": 0.6101582646369934,
|
|
"beta_dpo/beta_used_raw": -0.6181744337081909,
|
|
"beta_dpo/gap_mean": 165.5086669921875,
|
|
"beta_dpo/gap_std": 169.28228759765625,
|
|
"beta_dpo/loss_margin_mean": 198.18690490722656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2790014684287812,
|
|
"grad_norm": 2421.037841796875,
|
|
"learning_rate": 4.5404922808905543e-07,
|
|
"logits/chosen": -2.3267829418182373,
|
|
"logits/rejected": -2.345957040786743,
|
|
"loss": 0.7114,
|
|
"step": 190
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9563454389572144,
|
|
"beta_dpo/beta_margin_grad_mean": -0.07813985645771027,
|
|
"beta_dpo/beta_margin_grad_std": 0.268363893032074,
|
|
"beta_dpo/beta_margin_mean": 211.70956420898438,
|
|
"beta_dpo/beta_margin_std": 206.75030517578125,
|
|
"beta_dpo/beta_used": 0.9563454389572144,
|
|
"beta_dpo/beta_used_raw": 0.9563454389572144,
|
|
"beta_dpo/gap_mean": 179.21434020996094,
|
|
"beta_dpo/gap_std": 175.45181274414062,
|
|
"beta_dpo/loss_margin_mean": 225.3310089111328,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.28046989720998533,
|
|
"grad_norm": 3832.57177734375,
|
|
"learning_rate": 4.5330506821893565e-07,
|
|
"logits/chosen": -2.335681915283203,
|
|
"logits/rejected": -2.3289167881011963,
|
|
"loss": 6.3921,
|
|
"step": 191
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.45723795890808105,
|
|
"beta_dpo/beta_margin_grad_std": 0.04705416411161423,
|
|
"beta_dpo/beta_margin_mean": 0.17367665469646454,
|
|
"beta_dpo/beta_margin_std": 0.19400545954704285,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.6269910335540771,
|
|
"beta_dpo/gap_mean": 177.93748474121094,
|
|
"beta_dpo/gap_std": 179.75079345703125,
|
|
"beta_dpo/loss_margin_mean": 173.67665100097656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.28193832599118945,
|
|
"grad_norm": 13.0220365524292,
|
|
"learning_rate": 4.5255555107119336e-07,
|
|
"logits/chosen": -2.293377161026001,
|
|
"logits/rejected": -2.295954465866089,
|
|
"loss": 1.246,
|
|
"step": 192
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4627629220485687,
|
|
"beta_dpo/beta_margin_grad_std": 0.0443439707159996,
|
|
"beta_dpo/beta_margin_mean": 0.1510872095823288,
|
|
"beta_dpo/beta_margin_std": 0.18337495625019073,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.660752773284912,
|
|
"beta_dpo/gap_mean": 175.26287841796875,
|
|
"beta_dpo/gap_std": 181.79058837890625,
|
|
"beta_dpo/loss_margin_mean": 151.0872039794922,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2834067547723935,
|
|
"grad_norm": 11.858062744140625,
|
|
"learning_rate": 4.5180069639630236e-07,
|
|
"logits/chosen": -2.3066320419311523,
|
|
"logits/rejected": -2.331085681915283,
|
|
"loss": 1.2649,
|
|
"step": 193
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.46572333574295044,
|
|
"beta_dpo/beta_margin_grad_std": 0.03027997724711895,
|
|
"beta_dpo/beta_margin_mean": 0.1380530595779419,
|
|
"beta_dpo/beta_margin_std": 0.12353204935789108,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.2002902030944824,
|
|
"beta_dpo/gap_mean": 170.08328247070312,
|
|
"beta_dpo/gap_std": 173.58062744140625,
|
|
"beta_dpo/loss_margin_mean": 138.0530548095703,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.28487518355359764,
|
|
"grad_norm": 11.269763946533203,
|
|
"learning_rate": 4.510405240853854e-07,
|
|
"logits/chosen": -2.218247890472412,
|
|
"logits/rejected": -2.219308376312256,
|
|
"loss": 1.261,
|
|
"step": 194
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.7751163244247437,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09672369062900543,
|
|
"beta_dpo/beta_margin_grad_std": 0.2833358645439148,
|
|
"beta_dpo/beta_margin_mean": 158.73081970214844,
|
|
"beta_dpo/beta_margin_std": 144.5747528076172,
|
|
"beta_dpo/beta_used": 0.7751163244247437,
|
|
"beta_dpo/beta_used_raw": 0.7751163244247437,
|
|
"beta_dpo/gap_mean": 173.27928161621094,
|
|
"beta_dpo/gap_std": 173.402587890625,
|
|
"beta_dpo/loss_margin_mean": 206.54397583007812,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.28634361233480177,
|
|
"grad_norm": 6923.83349609375,
|
|
"learning_rate": 4.5027505416968985e-07,
|
|
"logits/chosen": -2.3047163486480713,
|
|
"logits/rejected": -2.3096752166748047,
|
|
"loss": 5.0274,
|
|
"step": 195
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.239418163895607,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2947882413864136,
|
|
"beta_dpo/beta_margin_grad_std": 0.2903721034526825,
|
|
"beta_dpo/beta_margin_mean": 50.04564666748047,
|
|
"beta_dpo/beta_margin_std": 79.44770050048828,
|
|
"beta_dpo/beta_used": 0.239418163895607,
|
|
"beta_dpo/beta_used_raw": -0.7751691341400146,
|
|
"beta_dpo/gap_mean": 176.44851684570312,
|
|
"beta_dpo/gap_std": 173.1087646484375,
|
|
"beta_dpo/loss_margin_mean": 175.94964599609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2878120411160059,
|
|
"grad_norm": 2369.5244140625,
|
|
"learning_rate": 4.495043068200599e-07,
|
|
"logits/chosen": -2.306786060333252,
|
|
"logits/rejected": -2.316213607788086,
|
|
"loss": 3.7479,
|
|
"step": 196
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.461849182844162,
|
|
"beta_dpo/beta_margin_grad_std": 0.033273905515670776,
|
|
"beta_dpo/beta_margin_mean": 0.15370342135429382,
|
|
"beta_dpo/beta_margin_std": 0.13502883911132812,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0447132587432861,
|
|
"beta_dpo/gap_mean": 170.2903594970703,
|
|
"beta_dpo/gap_std": 166.9288787841797,
|
|
"beta_dpo/loss_margin_mean": 153.7034149169922,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.28928046989721,
|
|
"grad_norm": 12.06312084197998,
|
|
"learning_rate": 4.4872830234640493e-07,
|
|
"logits/chosen": -2.2932043075561523,
|
|
"logits/rejected": -2.2925753593444824,
|
|
"loss": 1.2436,
|
|
"step": 197
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.40779924392700195,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2927795648574829,
|
|
"beta_dpo/beta_margin_grad_std": 0.2834474742412567,
|
|
"beta_dpo/beta_margin_mean": 91.81517028808594,
|
|
"beta_dpo/beta_margin_std": 143.10154724121094,
|
|
"beta_dpo/beta_used": 0.40779924392700195,
|
|
"beta_dpo/beta_used_raw": -0.3979314863681793,
|
|
"beta_dpo/gap_mean": 179.01412963867188,
|
|
"beta_dpo/gap_std": 170.71841430664062,
|
|
"beta_dpo/loss_margin_mean": 226.88037109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2907488986784141,
|
|
"grad_norm": 5604.81884765625,
|
|
"learning_rate": 4.479470611971645e-07,
|
|
"logits/chosen": -2.319179058074951,
|
|
"logits/rejected": -2.3143765926361084,
|
|
"loss": 9.2093,
|
|
"step": 198
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4550706148147583,
|
|
"beta_dpo/beta_margin_grad_std": 0.04198264703154564,
|
|
"beta_dpo/beta_margin_mean": 0.18165776133537292,
|
|
"beta_dpo/beta_margin_std": 0.1720964014530182,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.750559687614441,
|
|
"beta_dpo/gap_mean": 181.78192138671875,
|
|
"beta_dpo/gap_std": 174.14816284179688,
|
|
"beta_dpo/loss_margin_mean": 181.6577606201172,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2922173274596182,
|
|
"grad_norm": 11.930063247680664,
|
|
"learning_rate": 4.471606039587695e-07,
|
|
"logits/chosen": -2.3009300231933594,
|
|
"logits/rejected": -2.294772148132324,
|
|
"loss": 1.2447,
|
|
"step": 199
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.46233460307121277,
|
|
"beta_dpo/beta_margin_grad_std": 0.056460920721292496,
|
|
"beta_dpo/beta_margin_mean": 0.15314218401908875,
|
|
"beta_dpo/beta_margin_std": 0.23340220749378204,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.7534046173095703,
|
|
"beta_dpo/gap_mean": 178.69091796875,
|
|
"beta_dpo/gap_std": 183.65151977539062,
|
|
"beta_dpo/loss_margin_mean": 153.14218139648438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2936857562408223,
|
|
"grad_norm": 12.505084037780762,
|
|
"learning_rate": 4.4636895135509966e-07,
|
|
"logits/chosen": -2.274620532989502,
|
|
"logits/rejected": -2.277723789215088,
|
|
"loss": 1.265,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2936857562408223,
|
|
"eval_beta_dpo/beta": 0.041580744087696075,
|
|
"eval_beta_dpo/beta_margin_grad_mean": -0.4593558609485626,
|
|
"eval_beta_dpo/beta_margin_grad_std": 0.06084528937935829,
|
|
"eval_beta_dpo/beta_margin_mean": 8.074563980102539,
|
|
"eval_beta_dpo/beta_margin_std": 10.459110260009766,
|
|
"eval_beta_dpo/beta_used": 0.041580744087696075,
|
|
"eval_beta_dpo/beta_used_raw": -3.920811414718628,
|
|
"eval_beta_dpo/gap_mean": 175.91966247558594,
|
|
"eval_beta_dpo/gap_std": 183.71018981933594,
|
|
"eval_beta_dpo/loss_margin_mean": 108.90614318847656,
|
|
"eval_beta_dpo/mask_keep_frac": 1.0,
|
|
"eval_logits/chosen": -2.311591863632202,
|
|
"eval_logits/rejected": -2.305934429168701,
|
|
"eval_loss": 1.2116435766220093,
|
|
"eval_runtime": 37.7898,
|
|
"eval_samples_per_second": 61.895,
|
|
"eval_steps_per_second": 1.958,
|
|
"step": 200
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.45610305666923523,
|
|
"beta_dpo/beta_margin_grad_std": 0.05195591226220131,
|
|
"beta_dpo/beta_margin_mean": 0.17831824719905853,
|
|
"beta_dpo/beta_margin_std": 0.21344201266765594,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0059877634048462,
|
|
"beta_dpo/gap_mean": 175.1689453125,
|
|
"beta_dpo/gap_std": 188.05361938476562,
|
|
"beta_dpo/loss_margin_mean": 178.3182373046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.29515418502202645,
|
|
"grad_norm": 14.165757179260254,
|
|
"learning_rate": 4.455721242469372e-07,
|
|
"logits/chosen": -2.3472347259521484,
|
|
"logits/rejected": -2.3590919971466064,
|
|
"loss": 1.2403,
|
|
"step": 201
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.16364361345767975,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28894707560539246,
|
|
"beta_dpo/beta_margin_grad_std": 0.28408119082450867,
|
|
"beta_dpo/beta_margin_mean": 30.87427520751953,
|
|
"beta_dpo/beta_margin_std": 56.27296447753906,
|
|
"beta_dpo/beta_used": 0.16364361345767975,
|
|
"beta_dpo/beta_used_raw": -1.1798573732376099,
|
|
"beta_dpo/gap_mean": 177.89190673828125,
|
|
"beta_dpo/gap_std": 191.12405395507812,
|
|
"beta_dpo/loss_margin_mean": 183.90859985351562,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.2966226138032305,
|
|
"grad_norm": 2054.255126953125,
|
|
"learning_rate": 4.4477014363141755e-07,
|
|
"logits/chosen": -2.2713704109191895,
|
|
"logits/rejected": -2.2855608463287354,
|
|
"loss": 2.1346,
|
|
"step": 202
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.19677655398845673,
|
|
"beta_dpo/beta_margin_grad_mean": -0.30733898282051086,
|
|
"beta_dpo/beta_margin_grad_std": 0.29845014214515686,
|
|
"beta_dpo/beta_margin_mean": 41.14481735229492,
|
|
"beta_dpo/beta_margin_std": 74.64393615722656,
|
|
"beta_dpo/beta_used": 0.19677655398845673,
|
|
"beta_dpo/beta_used_raw": -0.3622090816497803,
|
|
"beta_dpo/gap_mean": 180.37454223632812,
|
|
"beta_dpo/gap_std": 190.1531524658203,
|
|
"beta_dpo/loss_margin_mean": 188.07620239257812,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.29809104258443464,
|
|
"grad_norm": 2106.6591796875,
|
|
"learning_rate": 4.439630306414758e-07,
|
|
"logits/chosen": -2.2831432819366455,
|
|
"logits/rejected": -2.294023275375366,
|
|
"loss": 1.9171,
|
|
"step": 203
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.037158019840717316,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2885894477367401,
|
|
"beta_dpo/beta_margin_grad_std": 0.28164830803871155,
|
|
"beta_dpo/beta_margin_mean": 6.227684497833252,
|
|
"beta_dpo/beta_margin_std": 10.331753730773926,
|
|
"beta_dpo/beta_used": 0.037158019840717316,
|
|
"beta_dpo/beta_used_raw": -0.3006288409233093,
|
|
"beta_dpo/gap_mean": 178.19146728515625,
|
|
"beta_dpo/gap_std": 192.77317810058594,
|
|
"beta_dpo/loss_margin_mean": 169.02920532226562,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.29955947136563876,
|
|
"grad_norm": 258.5797119140625,
|
|
"learning_rate": 4.431508065452897e-07,
|
|
"logits/chosen": -2.3215532302856445,
|
|
"logits/rejected": -2.3090624809265137,
|
|
"loss": 0.9138,
|
|
"step": 204
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.7806830406188965,
|
|
"beta_dpo/beta_margin_grad_mean": -0.07821886986494064,
|
|
"beta_dpo/beta_margin_grad_std": 0.26834189891815186,
|
|
"beta_dpo/beta_margin_mean": 394.4228210449219,
|
|
"beta_dpo/beta_margin_std": 418.83489990234375,
|
|
"beta_dpo/beta_used": 1.7806830406188965,
|
|
"beta_dpo/beta_used_raw": 1.7806830406188965,
|
|
"beta_dpo/gap_mean": 183.717529296875,
|
|
"beta_dpo/gap_std": 191.0102081298828,
|
|
"beta_dpo/loss_margin_mean": 210.12306213378906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3010279001468429,
|
|
"grad_norm": 6468.51318359375,
|
|
"learning_rate": 4.4233349274571974e-07,
|
|
"logits/chosen": -2.2471261024475098,
|
|
"logits/rejected": -2.2409067153930664,
|
|
"loss": 7.2147,
|
|
"step": 205
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5732086300849915,
|
|
"beta_dpo/beta_margin_grad_mean": -0.14021912217140198,
|
|
"beta_dpo/beta_margin_grad_std": 0.3377551734447479,
|
|
"beta_dpo/beta_margin_mean": 113.2298583984375,
|
|
"beta_dpo/beta_margin_std": 139.0940704345703,
|
|
"beta_dpo/beta_used": 0.5732086300849915,
|
|
"beta_dpo/beta_used_raw": 0.5732086300849915,
|
|
"beta_dpo/gap_mean": 186.09320068359375,
|
|
"beta_dpo/gap_std": 190.4658966064453,
|
|
"beta_dpo/loss_margin_mean": 205.77224731445312,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.302496328928047,
|
|
"grad_norm": 7734.55419921875,
|
|
"learning_rate": 4.415111107797445e-07,
|
|
"logits/chosen": -2.221261501312256,
|
|
"logits/rejected": -2.21230411529541,
|
|
"loss": 11.641,
|
|
"step": 206
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44699206948280334,
|
|
"beta_dpo/beta_margin_grad_std": 0.04681679978966713,
|
|
"beta_dpo/beta_margin_mean": 0.21536417305469513,
|
|
"beta_dpo/beta_margin_std": 0.19422101974487305,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.9365102052688599,
|
|
"beta_dpo/gap_mean": 191.98245239257812,
|
|
"beta_dpo/gap_std": 192.3529052734375,
|
|
"beta_dpo/loss_margin_mean": 215.36416625976562,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3039647577092511,
|
|
"grad_norm": 13.226594924926758,
|
|
"learning_rate": 4.4068368231789365e-07,
|
|
"logits/chosen": -2.2933871746063232,
|
|
"logits/rejected": -2.283078193664551,
|
|
"loss": 1.2246,
|
|
"step": 207
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8147650957107544,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09077098220586777,
|
|
"beta_dpo/beta_margin_grad_std": 0.28280818462371826,
|
|
"beta_dpo/beta_margin_mean": 174.40185546875,
|
|
"beta_dpo/beta_margin_std": 171.0436248779297,
|
|
"beta_dpo/beta_used": 0.8147650957107544,
|
|
"beta_dpo/beta_used_raw": 0.8147650957107544,
|
|
"beta_dpo/gap_mean": 196.13491821289062,
|
|
"beta_dpo/gap_std": 191.99342346191406,
|
|
"beta_dpo/loss_margin_mean": 211.70095825195312,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3054331864904552,
|
|
"grad_norm": 5424.021484375,
|
|
"learning_rate": 4.398512291636768e-07,
|
|
"logits/chosen": -2.30989408493042,
|
|
"logits/rejected": -2.319075584411621,
|
|
"loss": 5.355,
|
|
"step": 208
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.45670098066329956,
|
|
"beta_dpo/beta_margin_grad_std": 0.041713543236255646,
|
|
"beta_dpo/beta_margin_mean": 0.17479273676872253,
|
|
"beta_dpo/beta_margin_std": 0.16900895535945892,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.6124803423881531,
|
|
"beta_dpo/gap_mean": 193.11166381835938,
|
|
"beta_dpo/gap_std": 188.4140625,
|
|
"beta_dpo/loss_margin_mean": 174.792724609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3069016152716593,
|
|
"grad_norm": 12.918232917785645,
|
|
"learning_rate": 4.3901377325300857e-07,
|
|
"logits/chosen": -2.2634706497192383,
|
|
"logits/rejected": -2.2491159439086914,
|
|
"loss": 1.2183,
|
|
"step": 209
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.45144328474998474,
|
|
"beta_dpo/beta_margin_grad_std": 0.04742031916975975,
|
|
"beta_dpo/beta_margin_mean": 0.19709433615207672,
|
|
"beta_dpo/beta_margin_std": 0.19505690038204193,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.2010389566421509,
|
|
"beta_dpo/gap_mean": 191.64120483398438,
|
|
"beta_dpo/gap_std": 186.86257934570312,
|
|
"beta_dpo/loss_margin_mean": 197.09432983398438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.30837004405286345,
|
|
"grad_norm": 13.08560848236084,
|
|
"learning_rate": 4.381713366536311e-07,
|
|
"logits/chosen": -2.2852721214294434,
|
|
"logits/rejected": -2.291003704071045,
|
|
"loss": 1.229,
|
|
"step": 210
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4518587291240692,
|
|
"beta_dpo/beta_margin_grad_std": 0.046131283044815063,
|
|
"beta_dpo/beta_margin_mean": 0.19535738229751587,
|
|
"beta_dpo/beta_margin_std": 0.1901794821023941,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.092252492904663,
|
|
"beta_dpo/gap_mean": 192.7459716796875,
|
|
"beta_dpo/gap_std": 187.72750854492188,
|
|
"beta_dpo/loss_margin_mean": 195.3573760986328,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.30983847283406757,
|
|
"grad_norm": 14.05884075164795,
|
|
"learning_rate": 4.373239415645323e-07,
|
|
"logits/chosen": -2.2702016830444336,
|
|
"logits/rejected": -2.2552332878112793,
|
|
"loss": 1.2252,
|
|
"step": 211
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8706912398338318,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26792824268341064,
|
|
"beta_dpo/beta_margin_grad_std": 0.27179181575775146,
|
|
"beta_dpo/beta_margin_mean": 268.76666259765625,
|
|
"beta_dpo/beta_margin_std": 430.33172607421875,
|
|
"beta_dpo/beta_used": 0.8706912398338318,
|
|
"beta_dpo/beta_used_raw": 0.801848292350769,
|
|
"beta_dpo/gap_mean": 207.16200256347656,
|
|
"beta_dpo/gap_std": 197.4569549560547,
|
|
"beta_dpo/loss_margin_mean": 271.86383056640625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.31130690161527164,
|
|
"grad_norm": 12608.0732421875,
|
|
"learning_rate": 4.3647161031536086e-07,
|
|
"logits/chosen": -2.2498250007629395,
|
|
"logits/rejected": -2.2473154067993164,
|
|
"loss": 28.9501,
|
|
"step": 212
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3058314323425293,
|
|
"beta_dpo/beta_margin_grad_mean": -0.30717548727989197,
|
|
"beta_dpo/beta_margin_grad_std": 0.29951781034469604,
|
|
"beta_dpo/beta_margin_mean": 63.14212417602539,
|
|
"beta_dpo/beta_margin_std": 126.7613754272461,
|
|
"beta_dpo/beta_used": 0.3058314323425293,
|
|
"beta_dpo/beta_used_raw": -1.3592239618301392,
|
|
"beta_dpo/gap_mean": 206.8724822998047,
|
|
"beta_dpo/gap_std": 205.34677124023438,
|
|
"beta_dpo/loss_margin_mean": 192.0325164794922,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.31277533039647576,
|
|
"grad_norm": 3455.124267578125,
|
|
"learning_rate": 4.3561436536583774e-07,
|
|
"logits/chosen": -2.2697277069091797,
|
|
"logits/rejected": -2.2519259452819824,
|
|
"loss": 2.4062,
|
|
"step": 213
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.16119687259197235,
|
|
"beta_dpo/beta_margin_grad_mean": -0.30010974407196045,
|
|
"beta_dpo/beta_margin_grad_std": 0.28895044326782227,
|
|
"beta_dpo/beta_margin_mean": 32.903358459472656,
|
|
"beta_dpo/beta_margin_std": 53.89445877075195,
|
|
"beta_dpo/beta_used": 0.16119687259197235,
|
|
"beta_dpo/beta_used_raw": -0.6572102904319763,
|
|
"beta_dpo/gap_mean": 200.41761779785156,
|
|
"beta_dpo/gap_std": 208.61569213867188,
|
|
"beta_dpo/loss_margin_mean": 179.22732543945312,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3142437591776799,
|
|
"grad_norm": 883.9422607421875,
|
|
"learning_rate": 4.3475222930516473e-07,
|
|
"logits/chosen": -2.2414190769195557,
|
|
"logits/rejected": -2.2436366081237793,
|
|
"loss": 1.1557,
|
|
"step": 214
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.7592843770980835,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2566761374473572,
|
|
"beta_dpo/beta_margin_grad_std": 0.26009878516197205,
|
|
"beta_dpo/beta_margin_mean": 188.02699279785156,
|
|
"beta_dpo/beta_margin_std": 280.02783203125,
|
|
"beta_dpo/beta_used": 0.7592843770980835,
|
|
"beta_dpo/beta_used_raw": 0.24615520238876343,
|
|
"beta_dpo/gap_mean": 204.9820556640625,
|
|
"beta_dpo/gap_std": 203.79104614257812,
|
|
"beta_dpo/loss_margin_mean": 223.06326293945312,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.315712187958884,
|
|
"grad_norm": 9.2144136428833,
|
|
"learning_rate": 4.3388522485142885e-07,
|
|
"logits/chosen": -2.2476065158843994,
|
|
"logits/rejected": -2.237344741821289,
|
|
"loss": 0.6064,
|
|
"step": 215
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5309363007545471,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27667322754859924,
|
|
"beta_dpo/beta_margin_grad_std": 0.27724382281303406,
|
|
"beta_dpo/beta_margin_mean": 122.65149688720703,
|
|
"beta_dpo/beta_margin_std": 183.43125915527344,
|
|
"beta_dpo/beta_used": 0.5309363007545471,
|
|
"beta_dpo/beta_used_raw": -1.3396328687667847,
|
|
"beta_dpo/gap_mean": 202.26290893554688,
|
|
"beta_dpo/gap_std": 199.85479736328125,
|
|
"beta_dpo/loss_margin_mean": 198.3951873779297,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.31718061674008813,
|
|
"grad_norm": 4384.37841796875,
|
|
"learning_rate": 4.330133748510036e-07,
|
|
"logits/chosen": -2.247248888015747,
|
|
"logits/rejected": -2.2399697303771973,
|
|
"loss": 2.7083,
|
|
"step": 216
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.16512493789196014,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28015637397766113,
|
|
"beta_dpo/beta_margin_grad_std": 0.2829425632953644,
|
|
"beta_dpo/beta_margin_mean": 38.849979400634766,
|
|
"beta_dpo/beta_margin_std": 62.65520477294922,
|
|
"beta_dpo/beta_used": 0.16512493789196014,
|
|
"beta_dpo/beta_used_raw": -0.6432375311851501,
|
|
"beta_dpo/gap_mean": 211.07345581054688,
|
|
"beta_dpo/gap_std": 202.76217651367188,
|
|
"beta_dpo/loss_margin_mean": 249.60948181152344,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3186490455212922,
|
|
"grad_norm": 2745.58251953125,
|
|
"learning_rate": 4.3213670227794757e-07,
|
|
"logits/chosen": -2.2299795150756836,
|
|
"logits/rejected": -2.2205770015716553,
|
|
"loss": 5.3863,
|
|
"step": 217
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4506628215312958,
|
|
"beta_dpo/beta_margin_grad_std": 0.0487155057489872,
|
|
"beta_dpo/beta_margin_mean": 0.20028528571128845,
|
|
"beta_dpo/beta_margin_std": 0.2005169838666916,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.0722289085388184,
|
|
"beta_dpo/gap_mean": 211.20944213867188,
|
|
"beta_dpo/gap_std": 202.60032653808594,
|
|
"beta_dpo/loss_margin_mean": 200.2852783203125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3201174743024963,
|
|
"grad_norm": 13.857083320617676,
|
|
"learning_rate": 4.3125523023339815e-07,
|
|
"logits/chosen": -2.257091999053955,
|
|
"logits/rejected": -2.25131893157959,
|
|
"loss": 1.2226,
|
|
"step": 218
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4540587365627289,
|
|
"beta_dpo/beta_margin_grad_std": 0.060583606362342834,
|
|
"beta_dpo/beta_margin_mean": 0.18906962871551514,
|
|
"beta_dpo/beta_margin_std": 0.25983956456184387,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.0623416900634766,
|
|
"beta_dpo/gap_mean": 204.9937744140625,
|
|
"beta_dpo/gap_std": 208.16064453125,
|
|
"beta_dpo/loss_margin_mean": 189.06961059570312,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.32158590308370044,
|
|
"grad_norm": 13.54708194732666,
|
|
"learning_rate": 4.303689819449636e-07,
|
|
"logits/chosen": -2.2969722747802734,
|
|
"logits/rejected": -2.297567367553711,
|
|
"loss": 1.2297,
|
|
"step": 219
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.08469678461551666,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3169372081756592,
|
|
"beta_dpo/beta_margin_grad_std": 0.3025418817996979,
|
|
"beta_dpo/beta_margin_mean": 16.391508102416992,
|
|
"beta_dpo/beta_margin_std": 27.10245132446289,
|
|
"beta_dpo/beta_used": 0.08469678461551666,
|
|
"beta_dpo/beta_used_raw": -0.6565529108047485,
|
|
"beta_dpo/gap_mean": 204.47842407226562,
|
|
"beta_dpo/gap_std": 211.26084899902344,
|
|
"beta_dpo/loss_margin_mean": 193.17495727539062,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.32305433186490456,
|
|
"grad_norm": 924.0001831054688,
|
|
"learning_rate": 4.2947798076611047e-07,
|
|
"logits/chosen": -2.285961151123047,
|
|
"logits/rejected": -2.2831854820251465,
|
|
"loss": 1.8538,
|
|
"step": 220
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2386034727096558,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09203282743692398,
|
|
"beta_dpo/beta_margin_grad_std": 0.28641635179519653,
|
|
"beta_dpo/beta_margin_mean": 341.2391052246094,
|
|
"beta_dpo/beta_margin_std": 419.7342834472656,
|
|
"beta_dpo/beta_used": 1.2386034727096558,
|
|
"beta_dpo/beta_used_raw": 1.2386034727096558,
|
|
"beta_dpo/gap_mean": 214.00718688964844,
|
|
"beta_dpo/gap_std": 216.42800903320312,
|
|
"beta_dpo/loss_margin_mean": 272.1820373535156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3245227606461087,
|
|
"grad_norm": 8449.5234375,
|
|
"learning_rate": 4.285822501755485e-07,
|
|
"logits/chosen": -2.263484239578247,
|
|
"logits/rejected": -2.2643935680389404,
|
|
"loss": 11.7807,
|
|
"step": 221
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4465690553188324,
|
|
"beta_dpo/beta_margin_grad_std": 0.05451636388897896,
|
|
"beta_dpo/beta_margin_mean": 0.2180844247341156,
|
|
"beta_dpo/beta_margin_std": 0.2271333932876587,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.0969059467315674,
|
|
"beta_dpo/gap_mean": 215.2980499267578,
|
|
"beta_dpo/gap_std": 219.42501831054688,
|
|
"beta_dpo/loss_margin_mean": 218.08441162109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.32599118942731276,
|
|
"grad_norm": 13.580673217773438,
|
|
"learning_rate": 4.276818137766118e-07,
|
|
"logits/chosen": -2.291238784790039,
|
|
"logits/rejected": -2.294947624206543,
|
|
"loss": 1.2209,
|
|
"step": 222
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44978615641593933,
|
|
"beta_dpo/beta_margin_grad_std": 0.05852412432432175,
|
|
"beta_dpo/beta_margin_mean": 0.20669816434383392,
|
|
"beta_dpo/beta_margin_std": 0.2520856559276581,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.296399474143982,
|
|
"beta_dpo/gap_mean": 216.46047973632812,
|
|
"beta_dpo/gap_std": 226.12139892578125,
|
|
"beta_dpo/loss_margin_mean": 206.69815063476562,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3274596182085169,
|
|
"grad_norm": 13.991449356079102,
|
|
"learning_rate": 4.2677669529663686e-07,
|
|
"logits/chosen": -2.237861156463623,
|
|
"logits/rejected": -2.2309041023254395,
|
|
"loss": 1.2099,
|
|
"step": 223
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44978049397468567,
|
|
"beta_dpo/beta_margin_grad_std": 0.05648142471909523,
|
|
"beta_dpo/beta_margin_mean": 0.20587411522865295,
|
|
"beta_dpo/beta_margin_std": 0.24117736518383026,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.2211096286773682,
|
|
"beta_dpo/gap_mean": 214.611083984375,
|
|
"beta_dpo/gap_std": 228.7374267578125,
|
|
"beta_dpo/loss_margin_mean": 205.87411499023438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.328928046989721,
|
|
"grad_norm": 14.716062545776367,
|
|
"learning_rate": 4.2586691858633747e-07,
|
|
"logits/chosen": -2.2692012786865234,
|
|
"logits/rejected": -2.2605040073394775,
|
|
"loss": 1.2091,
|
|
"step": 224
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.866571843624115,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28175389766693115,
|
|
"beta_dpo/beta_margin_grad_std": 0.284710556268692,
|
|
"beta_dpo/beta_margin_mean": 206.28526306152344,
|
|
"beta_dpo/beta_margin_std": 341.5513610839844,
|
|
"beta_dpo/beta_used": 0.866571843624115,
|
|
"beta_dpo/beta_used_raw": -0.3526184558868408,
|
|
"beta_dpo/gap_mean": 217.59930419921875,
|
|
"beta_dpo/gap_std": 227.94232177734375,
|
|
"beta_dpo/loss_margin_mean": 245.15667724609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3303964757709251,
|
|
"grad_norm": 11778.642578125,
|
|
"learning_rate": 4.249525076191759e-07,
|
|
"logits/chosen": -2.27790904045105,
|
|
"logits/rejected": -2.2668089866638184,
|
|
"loss": 6.3533,
|
|
"step": 225
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.45367875695228577,
|
|
"beta_dpo/beta_margin_grad_std": 0.04829113930463791,
|
|
"beta_dpo/beta_margin_mean": 0.18813456594944,
|
|
"beta_dpo/beta_margin_std": 0.1991308182477951,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.7146835327148438,
|
|
"beta_dpo/gap_mean": 216.79244995117188,
|
|
"beta_dpo/gap_std": 225.35797119140625,
|
|
"beta_dpo/loss_margin_mean": 188.1345672607422,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.33186490455212925,
|
|
"grad_norm": 14.342082977294922,
|
|
"learning_rate": 4.2403348649073167e-07,
|
|
"logits/chosen": -2.3111863136291504,
|
|
"logits/rejected": -2.2828011512756348,
|
|
"loss": 1.2303,
|
|
"step": 226
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3605048954486847,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28487730026245117,
|
|
"beta_dpo/beta_margin_grad_std": 0.2841986417770386,
|
|
"beta_dpo/beta_margin_mean": 109.1982421875,
|
|
"beta_dpo/beta_margin_std": 195.55567932128906,
|
|
"beta_dpo/beta_used": 0.3605048954486847,
|
|
"beta_dpo/beta_used_raw": -0.9117497205734253,
|
|
"beta_dpo/gap_mean": 221.8527069091797,
|
|
"beta_dpo/gap_std": 232.91197204589844,
|
|
"beta_dpo/loss_margin_mean": 260.5398864746094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3333333333333333,
|
|
"grad_norm": 5748.3427734375,
|
|
"learning_rate": 4.2310987941806615e-07,
|
|
"logits/chosen": -2.3186283111572266,
|
|
"logits/rejected": -2.2997145652770996,
|
|
"loss": 6.4073,
|
|
"step": 227
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4516187608242035,
|
|
"beta_dpo/beta_margin_grad_std": 0.057563863694667816,
|
|
"beta_dpo/beta_margin_mean": 0.1965341418981552,
|
|
"beta_dpo/beta_margin_std": 0.23873193562030792,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.252966284751892,
|
|
"beta_dpo/gap_mean": 217.8734130859375,
|
|
"beta_dpo/gap_std": 229.7576141357422,
|
|
"beta_dpo/loss_margin_mean": 196.5341339111328,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.33480176211453744,
|
|
"grad_norm": 15.525106430053711,
|
|
"learning_rate": 4.2218171073908463e-07,
|
|
"logits/chosen": -2.3476057052612305,
|
|
"logits/rejected": -2.339545726776123,
|
|
"loss": 1.2061,
|
|
"step": 228
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1788349598646164,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29465344548225403,
|
|
"beta_dpo/beta_margin_grad_std": 0.28662359714508057,
|
|
"beta_dpo/beta_margin_mean": 40.58890151977539,
|
|
"beta_dpo/beta_margin_std": 64.144775390625,
|
|
"beta_dpo/beta_used": 0.1788349598646164,
|
|
"beta_dpo/beta_used_raw": -0.0582791268825531,
|
|
"beta_dpo/gap_mean": 218.74749755859375,
|
|
"beta_dpo/gap_std": 236.47198486328125,
|
|
"beta_dpo/loss_margin_mean": 228.95513916015625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.33627019089574156,
|
|
"grad_norm": 2251.75146484375,
|
|
"learning_rate": 4.212490049118951e-07,
|
|
"logits/chosen": -2.3584940433502197,
|
|
"logits/rejected": -2.3436903953552246,
|
|
"loss": 2.3098,
|
|
"step": 229
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.795495331287384,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2751595079898834,
|
|
"beta_dpo/beta_margin_grad_std": 0.2707229256629944,
|
|
"beta_dpo/beta_margin_mean": 202.500732421875,
|
|
"beta_dpo/beta_margin_std": 299.9369201660156,
|
|
"beta_dpo/beta_used": 0.795495331287384,
|
|
"beta_dpo/beta_used_raw": 0.5070621967315674,
|
|
"beta_dpo/gap_mean": 219.34637451171875,
|
|
"beta_dpo/gap_std": 225.2493896484375,
|
|
"beta_dpo/loss_margin_mean": 228.44595336914062,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3377386196769457,
|
|
"grad_norm": 9665.818359375,
|
|
"learning_rate": 4.203117865141635e-07,
|
|
"logits/chosen": -2.2865986824035645,
|
|
"logits/rejected": -2.294416904449463,
|
|
"loss": 7.6845,
|
|
"step": 230
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4476536810398102,
|
|
"beta_dpo/beta_margin_grad_std": 0.04368142783641815,
|
|
"beta_dpo/beta_margin_mean": 0.21227702498435974,
|
|
"beta_dpo/beta_margin_std": 0.18027953803539276,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8700859546661377,
|
|
"beta_dpo/gap_mean": 220.33432006835938,
|
|
"beta_dpo/gap_std": 214.647705078125,
|
|
"beta_dpo/loss_margin_mean": 212.2770233154297,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3392070484581498,
|
|
"grad_norm": 14.92269229888916,
|
|
"learning_rate": 4.1937008024246625e-07,
|
|
"logits/chosen": -2.3519227504730225,
|
|
"logits/rejected": -2.328807830810547,
|
|
"loss": 1.1986,
|
|
"step": 231
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.23005230724811554,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2875669300556183,
|
|
"beta_dpo/beta_margin_grad_std": 0.2871084213256836,
|
|
"beta_dpo/beta_margin_mean": 49.535980224609375,
|
|
"beta_dpo/beta_margin_std": 79.29191589355469,
|
|
"beta_dpo/beta_used": 0.23005230724811554,
|
|
"beta_dpo/beta_used_raw": 0.0414634644985199,
|
|
"beta_dpo/gap_mean": 218.3118133544922,
|
|
"beta_dpo/gap_std": 214.149169921875,
|
|
"beta_dpo/loss_margin_mean": 210.24900817871094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3406754772393539,
|
|
"grad_norm": 2202.3125,
|
|
"learning_rate": 4.1842391091163933e-07,
|
|
"logits/chosen": -2.3467612266540527,
|
|
"logits/rejected": -2.3430676460266113,
|
|
"loss": 2.6911,
|
|
"step": 232
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6968293786048889,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24002498388290405,
|
|
"beta_dpo/beta_margin_grad_std": 0.2530527412891388,
|
|
"beta_dpo/beta_margin_mean": 181.13375854492188,
|
|
"beta_dpo/beta_margin_std": 268.4140930175781,
|
|
"beta_dpo/beta_used": 0.6968293786048889,
|
|
"beta_dpo/beta_used_raw": 0.5653680562973022,
|
|
"beta_dpo/gap_mean": 228.13232421875,
|
|
"beta_dpo/gap_std": 221.7039794921875,
|
|
"beta_dpo/loss_margin_mean": 303.7350158691406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.342143906020558,
|
|
"grad_norm": 6852.927734375,
|
|
"learning_rate": 4.174733034541245e-07,
|
|
"logits/chosen": -2.345778226852417,
|
|
"logits/rejected": -2.349656105041504,
|
|
"loss": 9.6398,
|
|
"step": 233
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4342879354953766,
|
|
"beta_dpo/beta_margin_grad_std": 0.06734412163496017,
|
|
"beta_dpo/beta_margin_mean": 0.2719317376613617,
|
|
"beta_dpo/beta_margin_std": 0.2893328368663788,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.735541820526123,
|
|
"beta_dpo/gap_mean": 242.57504272460938,
|
|
"beta_dpo/gap_std": 239.12721252441406,
|
|
"beta_dpo/loss_margin_mean": 271.9317321777344,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3436123348017621,
|
|
"grad_norm": 14.259328842163086,
|
|
"learning_rate": 4.165182829193126e-07,
|
|
"logits/chosen": -2.3349199295043945,
|
|
"logits/rejected": -2.356750965118408,
|
|
"loss": 1.1938,
|
|
"step": 234
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44918081164360046,
|
|
"beta_dpo/beta_margin_grad_std": 0.06389525532722473,
|
|
"beta_dpo/beta_margin_mean": 0.20874598622322083,
|
|
"beta_dpo/beta_margin_std": 0.26976871490478516,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.224062204360962,
|
|
"beta_dpo/gap_mean": 234.73019409179688,
|
|
"beta_dpo/gap_std": 243.65020751953125,
|
|
"beta_dpo/loss_margin_mean": 208.74598693847656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.34508076358296624,
|
|
"grad_norm": 15.3160400390625,
|
|
"learning_rate": 4.1555887447288255e-07,
|
|
"logits/chosen": -2.3654861450195312,
|
|
"logits/rejected": -2.3537168502807617,
|
|
"loss": 1.2086,
|
|
"step": 235
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.22831876575946808,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29526790976524353,
|
|
"beta_dpo/beta_margin_grad_std": 0.2938448190689087,
|
|
"beta_dpo/beta_margin_mean": 50.824398040771484,
|
|
"beta_dpo/beta_margin_std": 78.6937255859375,
|
|
"beta_dpo/beta_used": 0.22831876575946808,
|
|
"beta_dpo/beta_used_raw": -0.8176271319389343,
|
|
"beta_dpo/gap_mean": 235.43679809570312,
|
|
"beta_dpo/gap_std": 244.1046905517578,
|
|
"beta_dpo/loss_margin_mean": 243.5421142578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3465491923641703,
|
|
"grad_norm": 3292.40771484375,
|
|
"learning_rate": 4.1459510339613946e-07,
|
|
"logits/chosen": -2.356201648712158,
|
|
"logits/rejected": -2.3701233863830566,
|
|
"loss": 4.0937,
|
|
"step": 236
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0147018432617188,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25070029497146606,
|
|
"beta_dpo/beta_margin_grad_std": 0.25762683153152466,
|
|
"beta_dpo/beta_margin_mean": 303.3717956542969,
|
|
"beta_dpo/beta_margin_std": 481.146240234375,
|
|
"beta_dpo/beta_used": 1.0147018432617188,
|
|
"beta_dpo/beta_used_raw": -0.4895740747451782,
|
|
"beta_dpo/gap_mean": 241.46392822265625,
|
|
"beta_dpo/gap_std": 249.50332641601562,
|
|
"beta_dpo/loss_margin_mean": 275.3121032714844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.34801762114537443,
|
|
"grad_norm": 11534.1455078125,
|
|
"learning_rate": 4.136269950853473e-07,
|
|
"logits/chosen": -2.380739212036133,
|
|
"logits/rejected": -2.381621837615967,
|
|
"loss": 14.4849,
|
|
"step": 237
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4481091797351837,
|
|
"beta_dpo/beta_margin_grad_std": 0.05575034022331238,
|
|
"beta_dpo/beta_margin_mean": 0.21202851831912994,
|
|
"beta_dpo/beta_margin_std": 0.23184596002101898,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.4691739082336426,
|
|
"beta_dpo/gap_mean": 241.19456481933594,
|
|
"beta_dpo/gap_std": 248.67156982421875,
|
|
"beta_dpo/loss_margin_mean": 212.02850341796875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.34948604992657856,
|
|
"grad_norm": 14.532193183898926,
|
|
"learning_rate": 4.126545750510605e-07,
|
|
"logits/chosen": -2.375584602355957,
|
|
"logits/rejected": -2.3742737770080566,
|
|
"loss": 1.2202,
|
|
"step": 238
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.912995457649231,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29015278816223145,
|
|
"beta_dpo/beta_margin_grad_std": 0.2879817485809326,
|
|
"beta_dpo/beta_margin_mean": 527.4285278320312,
|
|
"beta_dpo/beta_margin_std": 877.66650390625,
|
|
"beta_dpo/beta_used": 1.912995457649231,
|
|
"beta_dpo/beta_used_raw": -0.6588293313980103,
|
|
"beta_dpo/gap_mean": 238.7420654296875,
|
|
"beta_dpo/gap_std": 245.08407592773438,
|
|
"beta_dpo/loss_margin_mean": 228.3488311767578,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3509544787077827,
|
|
"grad_norm": 22015.724609375,
|
|
"learning_rate": 4.116778689174514e-07,
|
|
"logits/chosen": -2.369143486022949,
|
|
"logits/rejected": -2.358672857284546,
|
|
"loss": 39.3869,
|
|
"step": 239
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4522637724876404,
|
|
"beta_dpo/beta_margin_grad_std": 0.04363469406962395,
|
|
"beta_dpo/beta_margin_mean": 0.19304384291172028,
|
|
"beta_dpo/beta_margin_std": 0.17740407586097717,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.5935862064361572,
|
|
"beta_dpo/gap_mean": 231.2987060546875,
|
|
"beta_dpo/gap_std": 231.66029357910156,
|
|
"beta_dpo/loss_margin_mean": 193.04383850097656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3524229074889868,
|
|
"grad_norm": 15.185770988464355,
|
|
"learning_rate": 4.106969024216348e-07,
|
|
"logits/chosen": -2.439711093902588,
|
|
"logits/rejected": -2.4188616275787354,
|
|
"loss": 1.2157,
|
|
"step": 240
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.7368063926696777,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2956547141075134,
|
|
"beta_dpo/beta_margin_grad_std": 0.2953491806983948,
|
|
"beta_dpo/beta_margin_mean": 150.56866455078125,
|
|
"beta_dpo/beta_margin_std": 260.19439697265625,
|
|
"beta_dpo/beta_used": 0.7368063926696777,
|
|
"beta_dpo/beta_used_raw": -0.016419827938079834,
|
|
"beta_dpo/gap_mean": 226.96133422851562,
|
|
"beta_dpo/gap_std": 228.65875244140625,
|
|
"beta_dpo/loss_margin_mean": 235.113037109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.35389133627019087,
|
|
"grad_norm": 11925.3466796875,
|
|
"learning_rate": 4.097117014129903e-07,
|
|
"logits/chosen": -2.4186618328094482,
|
|
"logits/rejected": -2.3823959827423096,
|
|
"loss": 7.4844,
|
|
"step": 241
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0837945938110352,
|
|
"beta_dpo/beta_margin_grad_mean": -0.10944601148366928,
|
|
"beta_dpo/beta_margin_grad_std": 0.3120846450328827,
|
|
"beta_dpo/beta_margin_mean": 249.89736938476562,
|
|
"beta_dpo/beta_margin_std": 290.0773010253906,
|
|
"beta_dpo/beta_used": 1.0837945938110352,
|
|
"beta_dpo/beta_used_raw": 1.0837945938110352,
|
|
"beta_dpo/gap_mean": 229.24334716796875,
|
|
"beta_dpo/gap_std": 231.86746215820312,
|
|
"beta_dpo/loss_margin_mean": 231.20904541015625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.355359765051395,
|
|
"grad_norm": 14227.9248046875,
|
|
"learning_rate": 4.087222918524807e-07,
|
|
"logits/chosen": -2.3967206478118896,
|
|
"logits/rejected": -2.3583226203918457,
|
|
"loss": 12.149,
|
|
"step": 242
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.29786860942840576,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23357638716697693,
|
|
"beta_dpo/beta_margin_grad_std": 0.2237424999475479,
|
|
"beta_dpo/beta_margin_mean": 74.67481231689453,
|
|
"beta_dpo/beta_margin_std": 106.87940216064453,
|
|
"beta_dpo/beta_used": 0.29786860942840576,
|
|
"beta_dpo/beta_used_raw": 0.10270766913890839,
|
|
"beta_dpo/gap_mean": 229.03512573242188,
|
|
"beta_dpo/gap_std": 227.02081298828125,
|
|
"beta_dpo/loss_margin_mean": 232.3207550048828,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3568281938325991,
|
|
"grad_norm": 473.8674621582031,
|
|
"learning_rate": 4.07728699811968e-07,
|
|
"logits/chosen": -2.393955945968628,
|
|
"logits/rejected": -2.363004684448242,
|
|
"loss": 0.6125,
|
|
"step": 243
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44421160221099854,
|
|
"beta_dpo/beta_margin_grad_std": 0.05300465598702431,
|
|
"beta_dpo/beta_margin_mean": 0.227519690990448,
|
|
"beta_dpo/beta_margin_std": 0.22171121835708618,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.1679985523223877,
|
|
"beta_dpo/gap_mean": 230.13543701171875,
|
|
"beta_dpo/gap_std": 226.2316131591797,
|
|
"beta_dpo/loss_margin_mean": 227.51968383789062,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.35829662261380324,
|
|
"grad_norm": 16.667314529418945,
|
|
"learning_rate": 4.067309514735267e-07,
|
|
"logits/chosen": -2.446463108062744,
|
|
"logits/rejected": -2.438051223754883,
|
|
"loss": 1.1932,
|
|
"step": 244
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44525429606437683,
|
|
"beta_dpo/beta_margin_grad_std": 0.05737599730491638,
|
|
"beta_dpo/beta_margin_mean": 0.2234031707048416,
|
|
"beta_dpo/beta_margin_std": 0.2370459884405136,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.410043239593506,
|
|
"beta_dpo/gap_mean": 229.35089111328125,
|
|
"beta_dpo/gap_std": 226.95535278320312,
|
|
"beta_dpo/loss_margin_mean": 223.40316772460938,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.35976505139500736,
|
|
"grad_norm": 15.008759498596191,
|
|
"learning_rate": 4.057290731287531e-07,
|
|
"logits/chosen": -2.4402174949645996,
|
|
"logits/rejected": -2.3967413902282715,
|
|
"loss": 1.2316,
|
|
"step": 245
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44665589928627014,
|
|
"beta_dpo/beta_margin_grad_std": 0.059832848608493805,
|
|
"beta_dpo/beta_margin_mean": 0.21825571358203888,
|
|
"beta_dpo/beta_margin_std": 0.2485092133283615,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.323253631591797,
|
|
"beta_dpo/gap_mean": 227.4248046875,
|
|
"beta_dpo/gap_std": 229.33834838867188,
|
|
"beta_dpo/loss_margin_mean": 218.25570678710938,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.36123348017621143,
|
|
"grad_norm": 14.934514999389648,
|
|
"learning_rate": 4.047230911780736e-07,
|
|
"logits/chosen": -2.4563374519348145,
|
|
"logits/rejected": -2.428318500518799,
|
|
"loss": 1.2305,
|
|
"step": 246
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.007803923450410366,
|
|
"beta_dpo/beta_margin_grad_mean": -0.32257363200187683,
|
|
"beta_dpo/beta_margin_grad_std": 0.2511254549026489,
|
|
"beta_dpo/beta_margin_mean": 2.3317739963531494,
|
|
"beta_dpo/beta_margin_std": 4.475132465362549,
|
|
"beta_dpo/beta_used": 0.007803923450410366,
|
|
"beta_dpo/beta_used_raw": -0.14297455549240112,
|
|
"beta_dpo/gap_mean": 234.86566162109375,
|
|
"beta_dpo/gap_std": 243.15460205078125,
|
|
"beta_dpo/loss_margin_mean": 278.5243225097656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.36270190895741555,
|
|
"grad_norm": 101.28236389160156,
|
|
"learning_rate": 4.0371303213004814e-07,
|
|
"logits/chosen": -2.4293994903564453,
|
|
"logits/rejected": -2.430069923400879,
|
|
"loss": 0.886,
|
|
"step": 247
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.43503713607788086,
|
|
"beta_dpo/beta_margin_grad_std": 0.05003669857978821,
|
|
"beta_dpo/beta_margin_mean": 0.2657236158847809,
|
|
"beta_dpo/beta_margin_std": 0.2158864438533783,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8810434341430664,
|
|
"beta_dpo/gap_mean": 241.89364624023438,
|
|
"beta_dpo/gap_std": 237.35562133789062,
|
|
"beta_dpo/loss_margin_mean": 265.7236022949219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3641703377386197,
|
|
"grad_norm": 14.336139678955078,
|
|
"learning_rate": 4.0269892260067197e-07,
|
|
"logits/chosen": -2.402165174484253,
|
|
"logits/rejected": -2.41436505317688,
|
|
"loss": 1.1786,
|
|
"step": 248
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.45203158259391785,
|
|
"beta_dpo/beta_margin_grad_std": 0.06264989078044891,
|
|
"beta_dpo/beta_margin_mean": 0.19835467636585236,
|
|
"beta_dpo/beta_margin_std": 0.2707987129688263,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.373201847076416,
|
|
"beta_dpo/gap_mean": 237.4274139404297,
|
|
"beta_dpo/gap_std": 237.676513671875,
|
|
"beta_dpo/loss_margin_mean": 198.35467529296875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3656387665198238,
|
|
"grad_norm": 14.760684967041016,
|
|
"learning_rate": 4.0168078931267426e-07,
|
|
"logits/chosen": -2.412367343902588,
|
|
"logits/rejected": -2.3927431106567383,
|
|
"loss": 1.2226,
|
|
"step": 249
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4453158974647522,
|
|
"beta_dpo/beta_margin_grad_std": 0.05328512191772461,
|
|
"beta_dpo/beta_margin_mean": 0.22280624508857727,
|
|
"beta_dpo/beta_margin_std": 0.22093412280082703,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.016047477722168,
|
|
"beta_dpo/gap_mean": 230.994140625,
|
|
"beta_dpo/gap_std": 236.17393493652344,
|
|
"beta_dpo/loss_margin_mean": 222.8062286376953,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3671071953010279,
|
|
"grad_norm": 14.936260223388672,
|
|
"learning_rate": 4.006586590948141e-07,
|
|
"logits/chosen": -2.403975009918213,
|
|
"logits/rejected": -2.3519978523254395,
|
|
"loss": 1.2061,
|
|
"step": 250
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4486519694328308,
|
|
"beta_dpo/beta_margin_grad_std": 0.05090898275375366,
|
|
"beta_dpo/beta_margin_mean": 0.20870457589626312,
|
|
"beta_dpo/beta_margin_std": 0.2092035412788391,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.0009233951568604,
|
|
"beta_dpo/gap_mean": 228.9009552001953,
|
|
"beta_dpo/gap_std": 233.10757446289062,
|
|
"beta_dpo/loss_margin_mean": 208.70457458496094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.368575624082232,
|
|
"grad_norm": 14.669971466064453,
|
|
"learning_rate": 3.9963255888117325e-07,
|
|
"logits/chosen": -2.398078203201294,
|
|
"logits/rejected": -2.362529754638672,
|
|
"loss": 1.2071,
|
|
"step": 251
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4456026554107666,
|
|
"beta_dpo/beta_margin_grad_std": 0.0523262694478035,
|
|
"beta_dpo/beta_margin_mean": 0.22128848731517792,
|
|
"beta_dpo/beta_margin_std": 0.21624049544334412,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0428235530853271,
|
|
"beta_dpo/gap_mean": 224.8851318359375,
|
|
"beta_dpo/gap_std": 229.4764404296875,
|
|
"beta_dpo/loss_margin_mean": 221.28848266601562,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3700440528634361,
|
|
"grad_norm": 16.351465225219727,
|
|
"learning_rate": 3.9860251571044666e-07,
|
|
"logits/chosen": -2.4359705448150635,
|
|
"logits/rejected": -2.3960788249969482,
|
|
"loss": 1.1988,
|
|
"step": 252
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6227975487709045,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29470059275627136,
|
|
"beta_dpo/beta_margin_grad_std": 0.29031190276145935,
|
|
"beta_dpo/beta_margin_mean": 181.15090942382812,
|
|
"beta_dpo/beta_margin_std": 269.0624084472656,
|
|
"beta_dpo/beta_used": 0.6227975487709045,
|
|
"beta_dpo/beta_used_raw": -1.4962077140808105,
|
|
"beta_dpo/gap_mean": 220.82672119140625,
|
|
"beta_dpo/gap_std": 225.36129760742188,
|
|
"beta_dpo/loss_margin_mean": 216.6586456298828,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.37151248164464024,
|
|
"grad_norm": 11820.3720703125,
|
|
"learning_rate": 3.9756855672522986e-07,
|
|
"logits/chosen": -2.4164085388183594,
|
|
"logits/rejected": -2.4211513996124268,
|
|
"loss": 4.0804,
|
|
"step": 253
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.45039060711860657,
|
|
"beta_dpo/beta_margin_grad_std": 0.05692875757813454,
|
|
"beta_dpo/beta_margin_mean": 0.20209303498268127,
|
|
"beta_dpo/beta_margin_std": 0.23470279574394226,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.512616753578186,
|
|
"beta_dpo/gap_mean": 221.57920837402344,
|
|
"beta_dpo/gap_std": 227.94493103027344,
|
|
"beta_dpo/loss_margin_mean": 202.093017578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.37298091042584436,
|
|
"grad_norm": 14.783519744873047,
|
|
"learning_rate": 3.965307091713037e-07,
|
|
"logits/chosen": -2.4098727703094482,
|
|
"logits/rejected": -2.3902459144592285,
|
|
"loss": 1.2072,
|
|
"step": 254
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.7512093782424927,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3025147020816803,
|
|
"beta_dpo/beta_margin_grad_std": 0.2974975109100342,
|
|
"beta_dpo/beta_margin_mean": 497.9840087890625,
|
|
"beta_dpo/beta_margin_std": 947.763427734375,
|
|
"beta_dpo/beta_used": 1.7512093782424927,
|
|
"beta_dpo/beta_used_raw": 1.4928070306777954,
|
|
"beta_dpo/gap_mean": 225.62283325195312,
|
|
"beta_dpo/gap_std": 236.36817932128906,
|
|
"beta_dpo/loss_margin_mean": 245.6376953125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3744493392070485,
|
|
"grad_norm": 11111.490234375,
|
|
"learning_rate": 3.954890003969163e-07,
|
|
"logits/chosen": -2.3819503784179688,
|
|
"logits/rejected": -2.3786773681640625,
|
|
"loss": 0.9409,
|
|
"step": 255
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2602046728134155,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2357739359140396,
|
|
"beta_dpo/beta_margin_grad_std": 0.24245071411132812,
|
|
"beta_dpo/beta_margin_mean": 373.910888671875,
|
|
"beta_dpo/beta_margin_std": 559.2975463867188,
|
|
"beta_dpo/beta_used": 1.2602046728134155,
|
|
"beta_dpo/beta_used_raw": 0.8002493977546692,
|
|
"beta_dpo/gap_mean": 230.1458740234375,
|
|
"beta_dpo/gap_std": 236.79115295410156,
|
|
"beta_dpo/loss_margin_mean": 270.7061767578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.37591776798825255,
|
|
"grad_norm": 9.83940601348877,
|
|
"learning_rate": 3.944434578520628e-07,
|
|
"logits/chosen": -2.378755807876587,
|
|
"logits/rejected": -2.3856723308563232,
|
|
"loss": 0.597,
|
|
"step": 256
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4384312331676483,
|
|
"beta_dpo/beta_margin_grad_std": 0.06231582164764404,
|
|
"beta_dpo/beta_margin_mean": 0.25216352939605713,
|
|
"beta_dpo/beta_margin_std": 0.2593878209590912,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.7316312193870544,
|
|
"beta_dpo/gap_mean": 237.4232177734375,
|
|
"beta_dpo/gap_std": 242.20278930664062,
|
|
"beta_dpo/loss_margin_mean": 252.16351318359375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.37738619676945667,
|
|
"grad_norm": 15.36467456817627,
|
|
"learning_rate": 3.933941090877615e-07,
|
|
"logits/chosen": -2.391475200653076,
|
|
"logits/rejected": -2.376940965652466,
|
|
"loss": 1.1852,
|
|
"step": 257
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.6202783584594727,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12499994784593582,
|
|
"beta_dpo/beta_margin_grad_std": 0.3307187855243683,
|
|
"beta_dpo/beta_margin_mean": 712.9149169921875,
|
|
"beta_dpo/beta_margin_std": 668.4194946289062,
|
|
"beta_dpo/beta_used": 2.6202783584594727,
|
|
"beta_dpo/beta_used_raw": 2.6202783584594727,
|
|
"beta_dpo/gap_mean": 240.86416625976562,
|
|
"beta_dpo/gap_std": 243.09378051757812,
|
|
"beta_dpo/loss_margin_mean": 272.5574035644531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3788546255506608,
|
|
"grad_norm": 16114.0732421875,
|
|
"learning_rate": 3.923409817553284e-07,
|
|
"logits/chosen": -2.3731837272644043,
|
|
"logits/rejected": -2.371438980102539,
|
|
"loss": 16.2011,
|
|
"step": 258
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4404573440551758,
|
|
"beta_dpo/beta_margin_grad_std": 0.05528492480516434,
|
|
"beta_dpo/beta_margin_mean": 0.24297261238098145,
|
|
"beta_dpo/beta_margin_std": 0.22916458547115326,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.3427672386169434,
|
|
"beta_dpo/gap_mean": 244.031982421875,
|
|
"beta_dpo/gap_std": 241.36285400390625,
|
|
"beta_dpo/loss_margin_mean": 242.97259521484375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3803230543318649,
|
|
"grad_norm": 16.28805160522461,
|
|
"learning_rate": 3.9128410360564793e-07,
|
|
"logits/chosen": -2.386549949645996,
|
|
"logits/rejected": -2.3812201023101807,
|
|
"loss": 1.2012,
|
|
"step": 259
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4323839843273163,
|
|
"beta_dpo/beta_margin_grad_std": 0.0644708201289177,
|
|
"beta_dpo/beta_margin_mean": 0.27984052896499634,
|
|
"beta_dpo/beta_margin_std": 0.2821098566055298,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.7506465911865234,
|
|
"beta_dpo/gap_mean": 248.44383239746094,
|
|
"beta_dpo/gap_std": 245.54461669921875,
|
|
"beta_dpo/loss_margin_mean": 279.84051513671875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.38179148311306904,
|
|
"grad_norm": 17.50973892211914,
|
|
"learning_rate": 3.9022350248844246e-07,
|
|
"logits/chosen": -2.427947521209717,
|
|
"logits/rejected": -2.4491896629333496,
|
|
"loss": 1.1866,
|
|
"step": 260
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3739962875843048,
|
|
"beta_dpo/beta_margin_grad_mean": -0.0983000099658966,
|
|
"beta_dpo/beta_margin_grad_std": 0.2856021821498871,
|
|
"beta_dpo/beta_margin_mean": 113.49102020263672,
|
|
"beta_dpo/beta_margin_std": 196.81541442871094,
|
|
"beta_dpo/beta_used": 0.3739962875843048,
|
|
"beta_dpo/beta_used_raw": 0.3739962875843048,
|
|
"beta_dpo/gap_mean": 256.7048034667969,
|
|
"beta_dpo/gap_std": 256.45654296875,
|
|
"beta_dpo/loss_margin_mean": 285.4291076660156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3832599118942731,
|
|
"grad_norm": 6361.6826171875,
|
|
"learning_rate": 3.891592063515376e-07,
|
|
"logits/chosen": -2.3608932495117188,
|
|
"logits/rejected": -2.3539364337921143,
|
|
"loss": 6.9612,
|
|
"step": 261
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44041234254837036,
|
|
"beta_dpo/beta_margin_grad_std": 0.052026648074388504,
|
|
"beta_dpo/beta_margin_mean": 0.24300675094127655,
|
|
"beta_dpo/beta_margin_std": 0.21662083268165588,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.2271695137023926,
|
|
"beta_dpo/gap_mean": 253.71353149414062,
|
|
"beta_dpo/gap_std": 248.6503143310547,
|
|
"beta_dpo/loss_margin_mean": 243.00674438476562,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.38472834067547723,
|
|
"grad_norm": 17.864093780517578,
|
|
"learning_rate": 3.880912432401264e-07,
|
|
"logits/chosen": -2.4036097526550293,
|
|
"logits/rejected": -2.3713953495025635,
|
|
"loss": 1.1904,
|
|
"step": 262
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.3717470169067383,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2676565945148468,
|
|
"beta_dpo/beta_margin_grad_std": 0.27245068550109863,
|
|
"beta_dpo/beta_margin_mean": 446.4638366699219,
|
|
"beta_dpo/beta_margin_std": 755.0382690429688,
|
|
"beta_dpo/beta_used": 1.3717470169067383,
|
|
"beta_dpo/beta_used_raw": -0.1793736219406128,
|
|
"beta_dpo/gap_mean": 260.64508056640625,
|
|
"beta_dpo/gap_std": 252.19851684570312,
|
|
"beta_dpo/loss_margin_mean": 282.16845703125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.38619676945668135,
|
|
"grad_norm": 10648.0771484375,
|
|
"learning_rate": 3.870196412960302e-07,
|
|
"logits/chosen": -2.421666145324707,
|
|
"logits/rejected": -2.383852005004883,
|
|
"loss": 4.4638,
|
|
"step": 263
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.3721956014633179,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2683139741420746,
|
|
"beta_dpo/beta_margin_grad_std": 0.2728201150894165,
|
|
"beta_dpo/beta_margin_mean": 423.73870849609375,
|
|
"beta_dpo/beta_margin_std": 758.1284790039062,
|
|
"beta_dpo/beta_used": 1.3721956014633179,
|
|
"beta_dpo/beta_used_raw": 0.22174859046936035,
|
|
"beta_dpo/gap_mean": 262.8730163574219,
|
|
"beta_dpo/gap_std": 257.6596374511719,
|
|
"beta_dpo/loss_margin_mean": 271.05548095703125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3876651982378855,
|
|
"grad_norm": 5079.34765625,
|
|
"learning_rate": 3.8594442875695665e-07,
|
|
"logits/chosen": -2.4400243759155273,
|
|
"logits/rejected": -2.442418098449707,
|
|
"loss": 3.497,
|
|
"step": 264
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4414041340351105,
|
|
"beta_dpo/beta_margin_grad_std": 0.05608292669057846,
|
|
"beta_dpo/beta_margin_mean": 0.23940885066986084,
|
|
"beta_dpo/beta_margin_std": 0.23345738649368286,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.7446627616882324,
|
|
"beta_dpo/gap_mean": 259.660888671875,
|
|
"beta_dpo/gap_std": 252.59515380859375,
|
|
"beta_dpo/loss_margin_mean": 239.40884399414062,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3891336270190896,
|
|
"grad_norm": 18.96255111694336,
|
|
"learning_rate": 3.848656339557562e-07,
|
|
"logits/chosen": -2.427466869354248,
|
|
"logits/rejected": -2.411646842956543,
|
|
"loss": 1.1797,
|
|
"step": 265
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.16673444211483002,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27364593744277954,
|
|
"beta_dpo/beta_margin_grad_std": 0.27338290214538574,
|
|
"beta_dpo/beta_margin_mean": 39.5683479309082,
|
|
"beta_dpo/beta_margin_std": 59.80198287963867,
|
|
"beta_dpo/beta_used": 0.16673444211483002,
|
|
"beta_dpo/beta_used_raw": -1.1203869581222534,
|
|
"beta_dpo/gap_mean": 247.86456298828125,
|
|
"beta_dpo/gap_std": 249.63990783691406,
|
|
"beta_dpo/loss_margin_mean": 205.1346435546875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.39060205580029367,
|
|
"grad_norm": 1790.510009765625,
|
|
"learning_rate": 3.8378328531967507e-07,
|
|
"logits/chosen": -2.421651840209961,
|
|
"logits/rejected": -2.381192207336426,
|
|
"loss": 2.2698,
|
|
"step": 266
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4446525573730469,
|
|
"beta_dpo/beta_margin_grad_std": 0.06158663332462311,
|
|
"beta_dpo/beta_margin_mean": 0.22752374410629272,
|
|
"beta_dpo/beta_margin_std": 0.2612149715423584,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.89312744140625,
|
|
"beta_dpo/gap_mean": 245.2734375,
|
|
"beta_dpo/gap_std": 250.494384765625,
|
|
"beta_dpo/loss_margin_mean": 227.52374267578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3920704845814978,
|
|
"grad_norm": 17.995609283447266,
|
|
"learning_rate": 3.8269741136960646e-07,
|
|
"logits/chosen": -2.4637513160705566,
|
|
"logits/rejected": -2.419002056121826,
|
|
"loss": 1.2079,
|
|
"step": 267
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0013819551095366478,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4197249114513397,
|
|
"beta_dpo/beta_margin_grad_std": 0.10671903938055038,
|
|
"beta_dpo/beta_margin_mean": 0.35537663102149963,
|
|
"beta_dpo/beta_margin_std": 0.5258194208145142,
|
|
"beta_dpo/beta_used": 0.0013819551095366478,
|
|
"beta_dpo/beta_used_raw": -0.5071589946746826,
|
|
"beta_dpo/gap_mean": 242.955078125,
|
|
"beta_dpo/gap_std": 258.65301513671875,
|
|
"beta_dpo/loss_margin_mean": 249.29928588867188,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3935389133627019,
|
|
"grad_norm": 23.119298934936523,
|
|
"learning_rate": 3.8160804071933894e-07,
|
|
"logits/chosen": -2.447445869445801,
|
|
"logits/rejected": -2.4397740364074707,
|
|
"loss": 1.1212,
|
|
"step": 268
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.2243704795837402,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2187625616788864,
|
|
"beta_dpo/beta_margin_grad_std": 0.41339200735092163,
|
|
"beta_dpo/beta_margin_mean": 724.3359985351562,
|
|
"beta_dpo/beta_margin_std": 1003.6710205078125,
|
|
"beta_dpo/beta_used": 2.2243704795837402,
|
|
"beta_dpo/beta_used_raw": 2.2243704795837402,
|
|
"beta_dpo/gap_mean": 251.21771240234375,
|
|
"beta_dpo/gap_std": 269.38458251953125,
|
|
"beta_dpo/loss_margin_mean": 301.1477966308594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.39500734214390604,
|
|
"grad_norm": 53991.6875,
|
|
"learning_rate": 3.8051520207480204e-07,
|
|
"logits/chosen": -2.4700229167938232,
|
|
"logits/rejected": -2.456906318664551,
|
|
"loss": 69.1583,
|
|
"step": 269
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.0886917114257812,
|
|
"beta_dpo/beta_margin_grad_mean": -0.10937510430812836,
|
|
"beta_dpo/beta_margin_grad_std": 0.3121090829372406,
|
|
"beta_dpo/beta_margin_mean": 528.6803588867188,
|
|
"beta_dpo/beta_margin_std": 832.2965087890625,
|
|
"beta_dpo/beta_used": 2.0886917114257812,
|
|
"beta_dpo/beta_used_raw": 2.0886917114257812,
|
|
"beta_dpo/gap_mean": 256.38360595703125,
|
|
"beta_dpo/gap_std": 277.2386474609375,
|
|
"beta_dpo/loss_margin_mean": 266.50457763671875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.3964757709251101,
|
|
"grad_norm": 28465.341796875,
|
|
"learning_rate": 3.794189242333106e-07,
|
|
"logits/chosen": -2.5167593955993652,
|
|
"logits/rejected": -2.494492530822754,
|
|
"loss": 34.1063,
|
|
"step": 270
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8449544310569763,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26794520020484924,
|
|
"beta_dpo/beta_margin_grad_std": 0.2721104919910431,
|
|
"beta_dpo/beta_margin_mean": 248.46165466308594,
|
|
"beta_dpo/beta_margin_std": 399.88555908203125,
|
|
"beta_dpo/beta_used": 0.8449544310569763,
|
|
"beta_dpo/beta_used_raw": 0.3260602355003357,
|
|
"beta_dpo/gap_mean": 260.4082336425781,
|
|
"beta_dpo/gap_std": 271.07489013671875,
|
|
"beta_dpo/loss_margin_mean": 264.7347412109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.39794419970631423,
|
|
"grad_norm": 8032.013671875,
|
|
"learning_rate": 3.7831923608280514e-07,
|
|
"logits/chosen": -2.4501864910125732,
|
|
"logits/rejected": -2.4311776161193848,
|
|
"loss": 9.2538,
|
|
"step": 271
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.243833541870117,
|
|
"beta_dpo/beta_margin_grad_mean": -0.0751146674156189,
|
|
"beta_dpo/beta_margin_grad_std": 0.25838276743888855,
|
|
"beta_dpo/beta_margin_mean": 774.5305786132812,
|
|
"beta_dpo/beta_margin_std": 909.3794555664062,
|
|
"beta_dpo/beta_used": 2.243833541870117,
|
|
"beta_dpo/beta_used_raw": 2.243833541870117,
|
|
"beta_dpo/gap_mean": 272.1531982421875,
|
|
"beta_dpo/gap_std": 264.27728271484375,
|
|
"beta_dpo/loss_margin_mean": 351.419677734375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.39941262848751835,
|
|
"grad_norm": 15859.11328125,
|
|
"learning_rate": 3.772161666010912e-07,
|
|
"logits/chosen": -2.4539542198181152,
|
|
"logits/rejected": -2.4497289657592773,
|
|
"loss": 6.7279,
|
|
"step": 272
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.32312482595443726,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23232722282409668,
|
|
"beta_dpo/beta_margin_grad_std": 0.2386544793844223,
|
|
"beta_dpo/beta_margin_mean": 110.91842651367188,
|
|
"beta_dpo/beta_margin_std": 163.00204467773438,
|
|
"beta_dpo/beta_used": 0.32312482595443726,
|
|
"beta_dpo/beta_used_raw": -0.6652578115463257,
|
|
"beta_dpo/gap_mean": 282.7765808105469,
|
|
"beta_dpo/gap_std": 264.1724853515625,
|
|
"beta_dpo/loss_margin_mean": 307.410400390625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4008810572687225,
|
|
"grad_norm": 4223.08251953125,
|
|
"learning_rate": 3.761097448550755e-07,
|
|
"logits/chosen": -2.448002815246582,
|
|
"logits/rejected": -2.430415630340576,
|
|
"loss": 5.6684,
|
|
"step": 273
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.436676561832428,
|
|
"beta_dpo/beta_margin_grad_std": 0.050382133573293686,
|
|
"beta_dpo/beta_margin_mean": 0.25775137543678284,
|
|
"beta_dpo/beta_margin_std": 0.2082599699497223,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.682006597518921,
|
|
"beta_dpo/gap_mean": 278.31329345703125,
|
|
"beta_dpo/gap_std": 254.832763671875,
|
|
"beta_dpo/loss_margin_mean": 257.7513427734375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4023494860499266,
|
|
"grad_norm": 16.958667755126953,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": -2.454503297805786,
|
|
"logits/rejected": -2.440640926361084,
|
|
"loss": 1.1892,
|
|
"step": 274
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4363858103752136,
|
|
"beta_dpo/beta_margin_grad_std": 0.06395233422517776,
|
|
"beta_dpo/beta_margin_mean": 0.26140275597572327,
|
|
"beta_dpo/beta_margin_std": 0.2691759765148163,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.7606722116470337,
|
|
"beta_dpo/gap_mean": 273.5849914550781,
|
|
"beta_dpo/gap_std": 255.31951904296875,
|
|
"beta_dpo/loss_margin_mean": 261.4027404785156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.40381791483113066,
|
|
"grad_norm": 15.986796379089355,
|
|
"learning_rate": 3.738869612786737e-07,
|
|
"logits/chosen": -2.4930272102355957,
|
|
"logits/rejected": -2.4809093475341797,
|
|
"loss": 1.1688,
|
|
"step": 275
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.45231759548187256,
|
|
"beta_dpo/beta_margin_grad_std": 0.05670015886425972,
|
|
"beta_dpo/beta_margin_mean": 0.19459043443202972,
|
|
"beta_dpo/beta_margin_std": 0.2346203327178955,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.28926682472229,
|
|
"beta_dpo/gap_mean": 264.0999755859375,
|
|
"beta_dpo/gap_std": 254.43577575683594,
|
|
"beta_dpo/loss_margin_mean": 194.59042358398438,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4052863436123348,
|
|
"grad_norm": 16.828632354736328,
|
|
"learning_rate": 3.7277065802070204e-07,
|
|
"logits/chosen": -2.4924230575561523,
|
|
"logits/rejected": -2.454463005065918,
|
|
"loss": 1.1837,
|
|
"step": 276
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.115616798400879,
|
|
"beta_dpo/beta_margin_grad_mean": -0.10937541723251343,
|
|
"beta_dpo/beta_margin_grad_std": 0.31210899353027344,
|
|
"beta_dpo/beta_margin_mean": 563.19384765625,
|
|
"beta_dpo/beta_margin_std": 618.1759643554688,
|
|
"beta_dpo/beta_used": 2.115616798400879,
|
|
"beta_dpo/beta_used_raw": 2.115616798400879,
|
|
"beta_dpo/gap_mean": 261.0350646972656,
|
|
"beta_dpo/gap_std": 248.45208740234375,
|
|
"beta_dpo/loss_margin_mean": 268.9210510253906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4067547723935389,
|
|
"grad_norm": 6979.076171875,
|
|
"learning_rate": 3.71651119641714e-07,
|
|
"logits/chosen": -2.4732658863067627,
|
|
"logits/rejected": -2.4552745819091797,
|
|
"loss": 4.5947,
|
|
"step": 277
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.25115522742271423,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26586809754371643,
|
|
"beta_dpo/beta_margin_grad_std": 0.27073895931243896,
|
|
"beta_dpo/beta_margin_mean": 77.13081359863281,
|
|
"beta_dpo/beta_margin_std": 126.1852798461914,
|
|
"beta_dpo/beta_used": 0.25115522742271423,
|
|
"beta_dpo/beta_used_raw": -1.3573105335235596,
|
|
"beta_dpo/gap_mean": 265.0778503417969,
|
|
"beta_dpo/gap_std": 248.96527099609375,
|
|
"beta_dpo/loss_margin_mean": 279.1919250488281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.40822320117474303,
|
|
"grad_norm": 2251.078857421875,
|
|
"learning_rate": 3.705283756425872e-07,
|
|
"logits/chosen": -2.524477481842041,
|
|
"logits/rejected": -2.508835792541504,
|
|
"loss": 2.4695,
|
|
"step": 278
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.060165222734212875,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24480833113193512,
|
|
"beta_dpo/beta_margin_grad_std": 0.24604347348213196,
|
|
"beta_dpo/beta_margin_mean": 20.768430709838867,
|
|
"beta_dpo/beta_margin_std": 33.017086029052734,
|
|
"beta_dpo/beta_used": 0.060165222734212875,
|
|
"beta_dpo/beta_used_raw": -1.1045668125152588,
|
|
"beta_dpo/gap_mean": 271.9744873046875,
|
|
"beta_dpo/gap_std": 254.4259490966797,
|
|
"beta_dpo/loss_margin_mean": 303.9395751953125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.40969162995594716,
|
|
"grad_norm": 422.1612243652344,
|
|
"learning_rate": 3.6940245560867e-07,
|
|
"logits/chosen": -2.4369864463806152,
|
|
"logits/rejected": -2.438380718231201,
|
|
"loss": 0.6394,
|
|
"step": 279
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2358628511428833,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12507064640522003,
|
|
"beta_dpo/beta_margin_grad_std": 0.3306926190853119,
|
|
"beta_dpo/beta_margin_mean": 338.0630187988281,
|
|
"beta_dpo/beta_margin_std": 364.5479431152344,
|
|
"beta_dpo/beta_used": 1.2358628511428833,
|
|
"beta_dpo/beta_used_raw": 1.2358628511428833,
|
|
"beta_dpo/gap_mean": 274.9402770996094,
|
|
"beta_dpo/gap_std": 252.64535522460938,
|
|
"beta_dpo/loss_margin_mean": 287.3036804199219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4111600587371512,
|
|
"grad_norm": 10145.9853515625,
|
|
"learning_rate": 3.6827338920900253e-07,
|
|
"logits/chosen": -2.4949872493743896,
|
|
"logits/rejected": -2.48579478263855,
|
|
"loss": 9.5901,
|
|
"step": 280
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4418678879737854,
|
|
"beta_dpo/beta_margin_grad_std": 0.05114522576332092,
|
|
"beta_dpo/beta_margin_mean": 0.23657526075839996,
|
|
"beta_dpo/beta_margin_std": 0.21189118921756744,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.398056745529175,
|
|
"beta_dpo/gap_mean": 269.671630859375,
|
|
"beta_dpo/gap_std": 247.16676330566406,
|
|
"beta_dpo/loss_margin_mean": 236.5752410888672,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.41262848751835535,
|
|
"grad_norm": 14.78842544555664,
|
|
"learning_rate": 3.6714120619553435e-07,
|
|
"logits/chosen": -2.5062947273254395,
|
|
"logits/rejected": -2.4647984504699707,
|
|
"loss": 1.1774,
|
|
"step": 281
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.43719446659088135,
|
|
"beta_dpo/beta_margin_grad_std": 0.06138541176915169,
|
|
"beta_dpo/beta_margin_mean": 0.2587386667728424,
|
|
"beta_dpo/beta_margin_std": 0.26557499170303345,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.9040853977203369,
|
|
"beta_dpo/gap_mean": 265.92706298828125,
|
|
"beta_dpo/gap_std": 249.60317993164062,
|
|
"beta_dpo/loss_margin_mean": 258.7386474609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.41409691629955947,
|
|
"grad_norm": 16.898693084716797,
|
|
"learning_rate": 3.660059364023408e-07,
|
|
"logits/chosen": -2.499563217163086,
|
|
"logits/rejected": -2.4865164756774902,
|
|
"loss": 1.1584,
|
|
"step": 282
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.9231171607971191,
|
|
"beta_dpo/beta_margin_grad_mean": -0.07813180983066559,
|
|
"beta_dpo/beta_margin_grad_std": 0.2683661878108978,
|
|
"beta_dpo/beta_margin_mean": 601.787841796875,
|
|
"beta_dpo/beta_margin_std": 425.41082763671875,
|
|
"beta_dpo/beta_used": 1.9231171607971191,
|
|
"beta_dpo/beta_used_raw": 1.9231171607971191,
|
|
"beta_dpo/gap_mean": 271.40765380859375,
|
|
"beta_dpo/gap_std": 244.66708374023438,
|
|
"beta_dpo/loss_margin_mean": 318.6204528808594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4155653450807636,
|
|
"grad_norm": 14138.8076171875,
|
|
"learning_rate": 3.6486760974483685e-07,
|
|
"logits/chosen": -2.5169782638549805,
|
|
"logits/rejected": -2.4992263317108154,
|
|
"loss": 8.8542,
|
|
"step": 283
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9466410875320435,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25939232110977173,
|
|
"beta_dpo/beta_margin_grad_std": 0.26850318908691406,
|
|
"beta_dpo/beta_margin_mean": 256.5771484375,
|
|
"beta_dpo/beta_margin_std": 401.2411804199219,
|
|
"beta_dpo/beta_used": 0.9466410875320435,
|
|
"beta_dpo/beta_used_raw": 0.6345901489257812,
|
|
"beta_dpo/gap_mean": 276.92742919921875,
|
|
"beta_dpo/gap_std": 244.3406982421875,
|
|
"beta_dpo/loss_margin_mean": 289.7724609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4170337738619677,
|
|
"grad_norm": 7450.4580078125,
|
|
"learning_rate": 3.6372625621898863e-07,
|
|
"logits/chosen": -2.505162239074707,
|
|
"logits/rejected": -2.4776804447174072,
|
|
"loss": 7.3208,
|
|
"step": 284
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4398752450942993,
|
|
"beta_dpo/beta_margin_grad_mean": -0.078125,
|
|
"beta_dpo/beta_margin_grad_std": 0.2683681845664978,
|
|
"beta_dpo/beta_margin_mean": 387.22125244140625,
|
|
"beta_dpo/beta_margin_std": 445.4748229980469,
|
|
"beta_dpo/beta_used": 1.4398752450942993,
|
|
"beta_dpo/beta_used_raw": 1.4398752450942993,
|
|
"beta_dpo/gap_mean": 278.3759765625,
|
|
"beta_dpo/gap_std": 243.76156616210938,
|
|
"beta_dpo/loss_margin_mean": 285.8123779296875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4185022026431718,
|
|
"grad_norm": 20105.54296875,
|
|
"learning_rate": 3.625819059005228e-07,
|
|
"logits/chosen": -2.5132524967193604,
|
|
"logits/rejected": -2.4971303939819336,
|
|
"loss": 20.075,
|
|
"step": 285
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.556891679763794,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12499526143074036,
|
|
"beta_dpo/beta_margin_grad_std": 0.330706387758255,
|
|
"beta_dpo/beta_margin_mean": 447.1572570800781,
|
|
"beta_dpo/beta_margin_std": 412.15057373046875,
|
|
"beta_dpo/beta_used": 1.556891679763794,
|
|
"beta_dpo/beta_used_raw": 1.556891679763794,
|
|
"beta_dpo/gap_mean": 282.3958740234375,
|
|
"beta_dpo/gap_std": 245.08740234375,
|
|
"beta_dpo/loss_margin_mean": 289.4398193359375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4199706314243759,
|
|
"grad_norm": 10238.416015625,
|
|
"learning_rate": 3.614345889441346e-07,
|
|
"logits/chosen": -2.477703094482422,
|
|
"logits/rejected": -2.4490466117858887,
|
|
"loss": 1.207,
|
|
"step": 286
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.450316458940506,
|
|
"beta_dpo/beta_margin_grad_std": 0.058557696640491486,
|
|
"beta_dpo/beta_margin_mean": 0.20236189663410187,
|
|
"beta_dpo/beta_margin_std": 0.24117599427700043,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.1759114265441895,
|
|
"beta_dpo/gap_mean": 271.0549621582031,
|
|
"beta_dpo/gap_std": 246.55422973632812,
|
|
"beta_dpo/loss_margin_mean": 202.36187744140625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.42143906020558003,
|
|
"grad_norm": 15.727797508239746,
|
|
"learning_rate": 3.6028433558269275e-07,
|
|
"logits/chosen": -2.454252004623413,
|
|
"logits/rejected": -2.4215316772460938,
|
|
"loss": 1.1749,
|
|
"step": 287
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.5804258584976196,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22239087522029877,
|
|
"beta_dpo/beta_margin_grad_std": 0.22534386813640594,
|
|
"beta_dpo/beta_margin_mean": 554.5604858398438,
|
|
"beta_dpo/beta_margin_std": 696.531982421875,
|
|
"beta_dpo/beta_used": 1.5804258584976196,
|
|
"beta_dpo/beta_used_raw": 0.8364652991294861,
|
|
"beta_dpo/gap_mean": 272.4178161621094,
|
|
"beta_dpo/gap_std": 239.0502471923828,
|
|
"beta_dpo/loss_margin_mean": 287.3144226074219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.42290748898678415,
|
|
"grad_norm": 12.860248565673828,
|
|
"learning_rate": 3.5913117612644327e-07,
|
|
"logits/chosen": -2.441218137741089,
|
|
"logits/rejected": -2.4160122871398926,
|
|
"loss": 0.5814,
|
|
"step": 288
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.12335401028394699,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2808164358139038,
|
|
"beta_dpo/beta_margin_grad_std": 0.2781689763069153,
|
|
"beta_dpo/beta_margin_mean": 38.710533142089844,
|
|
"beta_dpo/beta_margin_std": 60.30266189575195,
|
|
"beta_dpo/beta_used": 0.12335401028394699,
|
|
"beta_dpo/beta_used_raw": -2.816128730773926,
|
|
"beta_dpo/gap_mean": 267.7777404785156,
|
|
"beta_dpo/gap_std": 242.87863159179688,
|
|
"beta_dpo/loss_margin_mean": 269.4267883300781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4243759177679883,
|
|
"grad_norm": 1399.761962890625,
|
|
"learning_rate": 3.5797514096221024e-07,
|
|
"logits/chosen": -2.432061195373535,
|
|
"logits/rejected": -2.416738986968994,
|
|
"loss": 2.1497,
|
|
"step": 289
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.4147535264492035,
|
|
"beta_dpo/beta_margin_grad_mean": -0.257093608379364,
|
|
"beta_dpo/beta_margin_grad_std": 0.2680981159210205,
|
|
"beta_dpo/beta_margin_mean": 118.29973602294922,
|
|
"beta_dpo/beta_margin_std": 170.95254516601562,
|
|
"beta_dpo/beta_used": 0.4147535264492035,
|
|
"beta_dpo/beta_used_raw": -1.4733829498291016,
|
|
"beta_dpo/gap_mean": 274.8150634765625,
|
|
"beta_dpo/gap_std": 246.7188720703125,
|
|
"beta_dpo/loss_margin_mean": 309.44354248046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.42584434654919234,
|
|
"grad_norm": 4860.09814453125,
|
|
"learning_rate": 3.568162605525952e-07,
|
|
"logits/chosen": -2.382627010345459,
|
|
"logits/rejected": -2.3980727195739746,
|
|
"loss": 0.8021,
|
|
"step": 290
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44023028016090393,
|
|
"beta_dpo/beta_margin_grad_std": 0.05959730222821236,
|
|
"beta_dpo/beta_margin_mean": 0.24392713606357574,
|
|
"beta_dpo/beta_margin_std": 0.24562755227088928,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.8497285842895508,
|
|
"beta_dpo/gap_mean": 271.6788635253906,
|
|
"beta_dpo/gap_std": 250.40158081054688,
|
|
"beta_dpo/loss_margin_mean": 243.9271240234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.42731277533039647,
|
|
"grad_norm": 17.112838745117188,
|
|
"learning_rate": 3.5565456543517485e-07,
|
|
"logits/chosen": -2.4613046646118164,
|
|
"logits/rejected": -2.4365696907043457,
|
|
"loss": 1.1703,
|
|
"step": 291
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.21685589849948883,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3011167347431183,
|
|
"beta_dpo/beta_margin_grad_std": 0.28959137201309204,
|
|
"beta_dpo/beta_margin_mean": 62.53536605834961,
|
|
"beta_dpo/beta_margin_std": 106.66192626953125,
|
|
"beta_dpo/beta_used": 0.21685589849948883,
|
|
"beta_dpo/beta_used_raw": 0.09813689440488815,
|
|
"beta_dpo/gap_mean": 274.6377868652344,
|
|
"beta_dpo/gap_std": 246.9325408935547,
|
|
"beta_dpo/loss_margin_mean": 290.8916015625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4287812041116006,
|
|
"grad_norm": 5888.60498046875,
|
|
"learning_rate": 3.5449008622169583e-07,
|
|
"logits/chosen": -2.3982367515563965,
|
|
"logits/rejected": -2.3866944313049316,
|
|
"loss": 4.748,
|
|
"step": 292
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4417594075202942,
|
|
"beta_dpo/beta_margin_grad_std": 0.06032756716012955,
|
|
"beta_dpo/beta_margin_mean": 0.23812739551067352,
|
|
"beta_dpo/beta_margin_std": 0.24970707297325134,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.6080701351165771,
|
|
"beta_dpo/gap_mean": 269.470703125,
|
|
"beta_dpo/gap_std": 247.3360137939453,
|
|
"beta_dpo/loss_margin_mean": 238.1273956298828,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4302496328928047,
|
|
"grad_norm": 15.048643112182617,
|
|
"learning_rate": 3.5332285359726846e-07,
|
|
"logits/chosen": -2.4533298015594482,
|
|
"logits/rejected": -2.4286036491394043,
|
|
"loss": 1.1695,
|
|
"step": 293
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4407859146595001,
|
|
"beta_dpo/beta_margin_grad_std": 0.053464584052562714,
|
|
"beta_dpo/beta_margin_mean": 0.2411770522594452,
|
|
"beta_dpo/beta_margin_std": 0.22201335430145264,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.280418872833252,
|
|
"beta_dpo/gap_mean": 265.29278564453125,
|
|
"beta_dpo/gap_std": 245.94830322265625,
|
|
"beta_dpo/loss_margin_mean": 241.1770477294922,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.43171806167400884,
|
|
"grad_norm": 17.06812286376953,
|
|
"learning_rate": 3.5215289831955786e-07,
|
|
"logits/chosen": -2.3770015239715576,
|
|
"logits/rejected": -2.384579658508301,
|
|
"loss": 1.1791,
|
|
"step": 294
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.3084752559661865,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2641025185585022,
|
|
"beta_dpo/beta_margin_grad_std": 0.2718868553638458,
|
|
"beta_dpo/beta_margin_mean": 686.6762084960938,
|
|
"beta_dpo/beta_margin_std": 1093.60888671875,
|
|
"beta_dpo/beta_used": 2.3084752559661865,
|
|
"beta_dpo/beta_used_raw": 0.6230998039245605,
|
|
"beta_dpo/gap_mean": 266.80670166015625,
|
|
"beta_dpo/gap_std": 250.25885009765625,
|
|
"beta_dpo/loss_margin_mean": 285.25213623046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4331864904552129,
|
|
"grad_norm": 11.919480323791504,
|
|
"learning_rate": 3.509802512179737e-07,
|
|
"logits/chosen": -2.370579719543457,
|
|
"logits/rejected": -2.3648414611816406,
|
|
"loss": 0.5964,
|
|
"step": 295
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44102582335472107,
|
|
"beta_dpo/beta_margin_grad_std": 0.06035232171416283,
|
|
"beta_dpo/beta_margin_mean": 0.24229387938976288,
|
|
"beta_dpo/beta_margin_std": 0.2558496296405792,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.7434802055358887,
|
|
"beta_dpo/gap_mean": 264.58294677734375,
|
|
"beta_dpo/gap_std": 255.38644409179688,
|
|
"beta_dpo/loss_margin_mean": 242.2938690185547,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.434654919236417,
|
|
"grad_norm": 16.63273811340332,
|
|
"learning_rate": 3.498049431928577e-07,
|
|
"logits/chosen": -2.4266653060913086,
|
|
"logits/rejected": -2.3969693183898926,
|
|
"loss": 1.1781,
|
|
"step": 296
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.43483033776283264,
|
|
"beta_dpo/beta_margin_grad_std": 0.05599094182252884,
|
|
"beta_dpo/beta_margin_mean": 0.266786128282547,
|
|
"beta_dpo/beta_margin_std": 0.2364174723625183,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.5507869720458984,
|
|
"beta_dpo/gap_mean": 262.3899841308594,
|
|
"beta_dpo/gap_std": 251.85415649414062,
|
|
"beta_dpo/loss_margin_mean": 266.7861022949219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.43612334801762115,
|
|
"grad_norm": 17.059518814086914,
|
|
"learning_rate": 3.486270052146694e-07,
|
|
"logits/chosen": -2.384669303894043,
|
|
"logits/rejected": -2.378746509552002,
|
|
"loss": 1.1732,
|
|
"step": 297
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.586780071258545,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2367078959941864,
|
|
"beta_dpo/beta_margin_grad_std": 0.25049635767936707,
|
|
"beta_dpo/beta_margin_mean": 772.3213500976562,
|
|
"beta_dpo/beta_margin_std": 1197.1170654296875,
|
|
"beta_dpo/beta_used": 2.586780071258545,
|
|
"beta_dpo/beta_used_raw": 2.4882137775421143,
|
|
"beta_dpo/gap_mean": 272.2360534667969,
|
|
"beta_dpo/gap_std": 257.9064636230469,
|
|
"beta_dpo/loss_margin_mean": 338.0078430175781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.43759177679882527,
|
|
"grad_norm": 11.470409393310547,
|
|
"learning_rate": 3.474464683231698e-07,
|
|
"logits/chosen": -2.401449680328369,
|
|
"logits/rejected": -2.4212589263916016,
|
|
"loss": 0.571,
|
|
"step": 298
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4342046082019806,
|
|
"beta_dpo/beta_margin_grad_std": 0.055606916546821594,
|
|
"beta_dpo/beta_margin_mean": 0.26877033710479736,
|
|
"beta_dpo/beta_margin_std": 0.2329106479883194,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.441830039024353,
|
|
"beta_dpo/gap_mean": 274.23779296875,
|
|
"beta_dpo/gap_std": 258.99835205078125,
|
|
"beta_dpo/loss_margin_mean": 268.77032470703125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4390602055800294,
|
|
"grad_norm": 15.817339897155762,
|
|
"learning_rate": 3.462633636266041e-07,
|
|
"logits/chosen": -2.361654281616211,
|
|
"logits/rejected": -2.373199224472046,
|
|
"loss": 1.1586,
|
|
"step": 299
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0433894395828247,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2306303083896637,
|
|
"beta_dpo/beta_margin_grad_std": 0.24043621122837067,
|
|
"beta_dpo/beta_margin_mean": 355.8514709472656,
|
|
"beta_dpo/beta_margin_std": 520.4441528320312,
|
|
"beta_dpo/beta_used": 1.0433894395828247,
|
|
"beta_dpo/beta_used_raw": -0.06062328815460205,
|
|
"beta_dpo/gap_mean": 281.2528991699219,
|
|
"beta_dpo/gap_std": 265.603759765625,
|
|
"beta_dpo/loss_margin_mean": 316.542236328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.44052863436123346,
|
|
"grad_norm": 14.615754127502441,
|
|
"learning_rate": 3.4507772230088147e-07,
|
|
"logits/chosen": -2.371137857437134,
|
|
"logits/rejected": -2.378791570663452,
|
|
"loss": 0.5857,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.44052863436123346,
|
|
"eval_beta_dpo/beta": 0.0032366011291742325,
|
|
"eval_beta_dpo/beta_margin_grad_mean": -0.4553416073322296,
|
|
"eval_beta_dpo/beta_margin_grad_std": 0.07153664529323578,
|
|
"eval_beta_dpo/beta_margin_mean": 0.8038776516914368,
|
|
"eval_beta_dpo/beta_margin_std": 1.0105534791946411,
|
|
"eval_beta_dpo/beta_used": 0.0032366011291742325,
|
|
"eval_beta_dpo/beta_used_raw": -7.0407514572143555,
|
|
"eval_beta_dpo/gap_mean": 284.4014892578125,
|
|
"eval_beta_dpo/gap_std": 265.404052734375,
|
|
"eval_beta_dpo/loss_margin_mean": 165.3889923095703,
|
|
"eval_beta_dpo/mask_keep_frac": 1.0,
|
|
"eval_logits/chosen": -2.395129442214966,
|
|
"eval_logits/rejected": -2.3756070137023926,
|
|
"eval_loss": 0.6708105206489563,
|
|
"eval_runtime": 37.7646,
|
|
"eval_samples_per_second": 61.936,
|
|
"eval_steps_per_second": 1.96,
|
|
"step": 300
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.4013200104236603,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24900613725185394,
|
|
"beta_dpo/beta_margin_grad_std": 0.25779998302459717,
|
|
"beta_dpo/beta_margin_mean": 136.3580780029297,
|
|
"beta_dpo/beta_margin_std": 207.45651245117188,
|
|
"beta_dpo/beta_used": 0.4013200104236603,
|
|
"beta_dpo/beta_used_raw": -0.9006747007369995,
|
|
"beta_dpo/gap_mean": 288.820556640625,
|
|
"beta_dpo/gap_std": 268.8150634765625,
|
|
"beta_dpo/loss_margin_mean": 303.63372802734375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4419970631424376,
|
|
"grad_norm": 4913.37890625,
|
|
"learning_rate": 3.4388957558875316e-07,
|
|
"logits/chosen": -2.354623794555664,
|
|
"logits/rejected": -2.343824863433838,
|
|
"loss": 2.6589,
|
|
"step": 301
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2232584953308105,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24484674632549286,
|
|
"beta_dpo/beta_margin_grad_std": 0.2557476758956909,
|
|
"beta_dpo/beta_margin_mean": 401.79315185546875,
|
|
"beta_dpo/beta_margin_std": 578.2942504882812,
|
|
"beta_dpo/beta_used": 1.2232584953308105,
|
|
"beta_dpo/beta_used_raw": 1.0286716222763062,
|
|
"beta_dpo/gap_mean": 292.2723388671875,
|
|
"beta_dpo/gap_std": 271.0513916015625,
|
|
"beta_dpo/loss_margin_mean": 315.1329345703125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4434654919236417,
|
|
"grad_norm": 8826.119140625,
|
|
"learning_rate": 3.426989547989902e-07,
|
|
"logits/chosen": -2.3831279277801514,
|
|
"logits/rejected": -2.3882064819335938,
|
|
"loss": 1.4867,
|
|
"step": 302
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.5807337760925293,
|
|
"beta_dpo/beta_margin_grad_mean": -0.15623971819877625,
|
|
"beta_dpo/beta_margin_grad_std": 0.3630678653717041,
|
|
"beta_dpo/beta_margin_mean": 455.2434387207031,
|
|
"beta_dpo/beta_margin_std": 776.2095336914062,
|
|
"beta_dpo/beta_used": 1.5807337760925293,
|
|
"beta_dpo/beta_used_raw": 1.5807337760925293,
|
|
"beta_dpo/gap_mean": 291.7049560546875,
|
|
"beta_dpo/gap_std": 282.81158447265625,
|
|
"beta_dpo/loss_margin_mean": 283.4997253417969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.44493392070484583,
|
|
"grad_norm": 7618.3427734375,
|
|
"learning_rate": 3.4150589130555773e-07,
|
|
"logits/chosen": -2.403137445449829,
|
|
"logits/rejected": -2.3835113048553467,
|
|
"loss": 9.1621,
|
|
"step": 303
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.647692322731018,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2932804524898529,
|
|
"beta_dpo/beta_margin_grad_std": 0.292863667011261,
|
|
"beta_dpo/beta_margin_mean": 490.46783447265625,
|
|
"beta_dpo/beta_margin_std": 817.7069091796875,
|
|
"beta_dpo/beta_used": 1.647692322731018,
|
|
"beta_dpo/beta_used_raw": 1.154199481010437,
|
|
"beta_dpo/gap_mean": 291.1902160644531,
|
|
"beta_dpo/gap_std": 278.3885498046875,
|
|
"beta_dpo/loss_margin_mean": 290.32611083984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.44640234948604995,
|
|
"grad_norm": 24986.185546875,
|
|
"learning_rate": 3.403104165467883e-07,
|
|
"logits/chosen": -2.42034649848938,
|
|
"logits/rejected": -2.406019687652588,
|
|
"loss": 30.0877,
|
|
"step": 304
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.43377500772476196,
|
|
"beta_dpo/beta_margin_grad_std": 0.06665903329849243,
|
|
"beta_dpo/beta_margin_mean": 0.2723696529865265,
|
|
"beta_dpo/beta_margin_std": 0.27916526794433594,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.7424488067626953,
|
|
"beta_dpo/gap_mean": 286.82373046875,
|
|
"beta_dpo/gap_std": 275.3907775878906,
|
|
"beta_dpo/loss_margin_mean": 272.36962890625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.447870778267254,
|
|
"grad_norm": 19.328815460205078,
|
|
"learning_rate": 3.391125620245535e-07,
|
|
"logits/chosen": -2.408841848373413,
|
|
"logits/rejected": -2.3937630653381348,
|
|
"loss": 1.1722,
|
|
"step": 305
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8187703490257263,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2615983486175537,
|
|
"beta_dpo/beta_margin_grad_std": 0.2689959704875946,
|
|
"beta_dpo/beta_margin_mean": 221.95233154296875,
|
|
"beta_dpo/beta_margin_std": 310.6951599121094,
|
|
"beta_dpo/beta_used": 0.8187703490257263,
|
|
"beta_dpo/beta_used_raw": 0.23448729515075684,
|
|
"beta_dpo/gap_mean": 286.04840087890625,
|
|
"beta_dpo/gap_std": 266.01214599609375,
|
|
"beta_dpo/loss_margin_mean": 279.9851989746094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.44933920704845814,
|
|
"grad_norm": 12.146223068237305,
|
|
"learning_rate": 3.3791235930343417e-07,
|
|
"logits/chosen": -2.376699686050415,
|
|
"logits/rejected": -2.353483200073242,
|
|
"loss": 0.5756,
|
|
"step": 306
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.02242352068424225,
|
|
"beta_dpo/beta_margin_grad_mean": -0.322419136762619,
|
|
"beta_dpo/beta_margin_grad_std": 0.2971925735473633,
|
|
"beta_dpo/beta_margin_mean": 5.65758752822876,
|
|
"beta_dpo/beta_margin_std": 10.40542984008789,
|
|
"beta_dpo/beta_used": 0.02242352068424225,
|
|
"beta_dpo/beta_used_raw": -1.8627017736434937,
|
|
"beta_dpo/gap_mean": 283.6302185058594,
|
|
"beta_dpo/gap_std": 267.28631591796875,
|
|
"beta_dpo/loss_margin_mean": 264.5188293457031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.45080763582966227,
|
|
"grad_norm": 351.361083984375,
|
|
"learning_rate": 3.367098400098881e-07,
|
|
"logits/chosen": -2.3703196048736572,
|
|
"logits/rejected": -2.361326217651367,
|
|
"loss": 1.543,
|
|
"step": 307
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.610724925994873,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3044406473636627,
|
|
"beta_dpo/beta_margin_grad_std": 0.2935749292373657,
|
|
"beta_dpo/beta_margin_mean": 177.69180297851562,
|
|
"beta_dpo/beta_margin_std": 322.2966613769531,
|
|
"beta_dpo/beta_used": 0.610724925994873,
|
|
"beta_dpo/beta_used_raw": 0.18854334950447083,
|
|
"beta_dpo/gap_mean": 280.91876220703125,
|
|
"beta_dpo/gap_std": 270.7321472167969,
|
|
"beta_dpo/loss_margin_mean": 267.55584716796875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4522760646108664,
|
|
"grad_norm": 5888.60400390625,
|
|
"learning_rate": 3.355050358314172e-07,
|
|
"logits/chosen": -2.3571863174438477,
|
|
"logits/rejected": -2.3389384746551514,
|
|
"loss": 11.2154,
|
|
"step": 308
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.43760332465171814,
|
|
"beta_dpo/beta_margin_grad_std": 0.056733082979917526,
|
|
"beta_dpo/beta_margin_mean": 0.254102885723114,
|
|
"beta_dpo/beta_margin_std": 0.23493850231170654,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.7044029235839844,
|
|
"beta_dpo/gap_mean": 273.9566650390625,
|
|
"beta_dpo/gap_std": 264.2417297363281,
|
|
"beta_dpo/loss_margin_mean": 254.1028594970703,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.45374449339207046,
|
|
"grad_norm": 19.618959426879883,
|
|
"learning_rate": 3.3429797851573183e-07,
|
|
"logits/chosen": -2.3406341075897217,
|
|
"logits/rejected": -2.3255674839019775,
|
|
"loss": 1.1658,
|
|
"step": 309
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6274186968803406,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3148829936981201,
|
|
"beta_dpo/beta_margin_grad_std": 0.30497488379478455,
|
|
"beta_dpo/beta_margin_mean": 173.7240447998047,
|
|
"beta_dpo/beta_margin_std": 294.6714172363281,
|
|
"beta_dpo/beta_used": 0.6274186968803406,
|
|
"beta_dpo/beta_used_raw": -0.6384754180908203,
|
|
"beta_dpo/gap_mean": 270.5137939453125,
|
|
"beta_dpo/gap_std": 256.47113037109375,
|
|
"beta_dpo/loss_margin_mean": 253.77883911132812,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4552129221732746,
|
|
"grad_norm": 15305.533203125,
|
|
"learning_rate": 3.3308869986991487e-07,
|
|
"logits/chosen": -2.35772442817688,
|
|
"logits/rejected": -2.3493471145629883,
|
|
"loss": 22.3664,
|
|
"step": 310
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.5589945316314697,
|
|
"beta_dpo/beta_margin_grad_mean": -0.10382533818483353,
|
|
"beta_dpo/beta_margin_grad_std": 0.2990655303001404,
|
|
"beta_dpo/beta_margin_mean": 463.5744323730469,
|
|
"beta_dpo/beta_margin_std": 523.1697387695312,
|
|
"beta_dpo/beta_used": 1.5589945316314697,
|
|
"beta_dpo/beta_used_raw": 1.5589945316314697,
|
|
"beta_dpo/gap_mean": 278.9131164550781,
|
|
"beta_dpo/gap_std": 256.4837341308594,
|
|
"beta_dpo/loss_margin_mean": 317.3361511230469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4566813509544787,
|
|
"grad_norm": 17659.806640625,
|
|
"learning_rate": 3.3187723175958346e-07,
|
|
"logits/chosen": -2.3346574306488037,
|
|
"logits/rejected": -2.313831090927124,
|
|
"loss": 10.112,
|
|
"step": 311
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4359610378742218,
|
|
"beta_dpo/beta_margin_grad_std": 0.06931335479021072,
|
|
"beta_dpo/beta_margin_mean": 0.26454854011535645,
|
|
"beta_dpo/beta_margin_std": 0.29412809014320374,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.6447594165802002,
|
|
"beta_dpo/gap_mean": 275.85595703125,
|
|
"beta_dpo/gap_std": 261.27203369140625,
|
|
"beta_dpo/loss_margin_mean": 264.5485534667969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4581497797356828,
|
|
"grad_norm": 19.33686065673828,
|
|
"learning_rate": 3.306636061080487e-07,
|
|
"logits/chosen": -2.301133155822754,
|
|
"logits/rejected": -2.2737810611724854,
|
|
"loss": 1.1643,
|
|
"step": 312
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4278359115123749,
|
|
"beta_dpo/beta_margin_grad_std": 0.06270962953567505,
|
|
"beta_dpo/beta_margin_mean": 0.29759153723716736,
|
|
"beta_dpo/beta_margin_std": 0.26974210143089294,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.4672694206237793,
|
|
"beta_dpo/gap_mean": 277.2411804199219,
|
|
"beta_dpo/gap_std": 261.7774963378906,
|
|
"beta_dpo/loss_margin_mean": 297.5915222167969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.45961820851688695,
|
|
"grad_norm": 18.111801147460938,
|
|
"learning_rate": 3.2944785489547537e-07,
|
|
"logits/chosen": -2.3542749881744385,
|
|
"logits/rejected": -2.3362579345703125,
|
|
"loss": 1.1587,
|
|
"step": 313
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.42816469073295593,
|
|
"beta_dpo/beta_margin_grad_std": 0.06579755246639252,
|
|
"beta_dpo/beta_margin_mean": 0.29621294140815735,
|
|
"beta_dpo/beta_margin_std": 0.27990517020225525,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.222659111022949,
|
|
"beta_dpo/gap_mean": 284.3181457519531,
|
|
"beta_dpo/gap_std": 268.36395263671875,
|
|
"beta_dpo/loss_margin_mean": 296.2129211425781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.461086637298091,
|
|
"grad_norm": 20.57044792175293,
|
|
"learning_rate": 3.2823001015803857e-07,
|
|
"logits/chosen": -2.3612356185913086,
|
|
"logits/rejected": -2.352372169494629,
|
|
"loss": 1.1678,
|
|
"step": 314
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1255430281162262,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2888107895851135,
|
|
"beta_dpo/beta_margin_grad_std": 0.27396243810653687,
|
|
"beta_dpo/beta_margin_mean": 33.68264389038086,
|
|
"beta_dpo/beta_margin_std": 54.088321685791016,
|
|
"beta_dpo/beta_used": 0.1255430281162262,
|
|
"beta_dpo/beta_used_raw": -1.2729917764663696,
|
|
"beta_dpo/gap_mean": 279.51483154296875,
|
|
"beta_dpo/gap_std": 260.94415283203125,
|
|
"beta_dpo/loss_margin_mean": 245.77093505859375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.46255506607929514,
|
|
"grad_norm": 1714.6759033203125,
|
|
"learning_rate": 3.270101039870797e-07,
|
|
"logits/chosen": -2.284019947052002,
|
|
"logits/rejected": -2.2785418033599854,
|
|
"loss": 2.0985,
|
|
"step": 315
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.617267370223999,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23152686655521393,
|
|
"beta_dpo/beta_margin_grad_std": 0.24057447910308838,
|
|
"beta_dpo/beta_margin_mean": 237.53465270996094,
|
|
"beta_dpo/beta_margin_std": 373.554443359375,
|
|
"beta_dpo/beta_used": 0.617267370223999,
|
|
"beta_dpo/beta_used_raw": -0.2918035387992859,
|
|
"beta_dpo/gap_mean": 287.0285949707031,
|
|
"beta_dpo/gap_std": 267.6070556640625,
|
|
"beta_dpo/loss_margin_mean": 332.6310729980469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.46402349486049926,
|
|
"grad_norm": 13.612666130065918,
|
|
"learning_rate": 3.2578816852826086e-07,
|
|
"logits/chosen": -2.326328754425049,
|
|
"logits/rejected": -2.3280577659606934,
|
|
"loss": 0.5785,
|
|
"step": 316
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3495140075683594,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2776326537132263,
|
|
"beta_dpo/beta_margin_grad_std": 0.2784407138824463,
|
|
"beta_dpo/beta_margin_mean": 142.2916717529297,
|
|
"beta_dpo/beta_margin_std": 227.7239990234375,
|
|
"beta_dpo/beta_used": 0.3495140075683594,
|
|
"beta_dpo/beta_used_raw": -0.26205989718437195,
|
|
"beta_dpo/gap_mean": 290.8403625488281,
|
|
"beta_dpo/gap_std": 272.17449951171875,
|
|
"beta_dpo/loss_margin_mean": 337.1584167480469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4654919236417034,
|
|
"grad_norm": 4855.1845703125,
|
|
"learning_rate": 3.2456423598071783e-07,
|
|
"logits/chosen": -2.3395438194274902,
|
|
"logits/rejected": -2.323230743408203,
|
|
"loss": 5.2066,
|
|
"step": 317
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2519617974758148,
|
|
"beta_dpo/beta_margin_grad_mean": -0.30234721302986145,
|
|
"beta_dpo/beta_margin_grad_std": 0.29342392086982727,
|
|
"beta_dpo/beta_margin_mean": 63.16868591308594,
|
|
"beta_dpo/beta_margin_std": 115.97257995605469,
|
|
"beta_dpo/beta_used": 0.2519617974758148,
|
|
"beta_dpo/beta_used_raw": -0.1758996844291687,
|
|
"beta_dpo/gap_mean": 292.19378662109375,
|
|
"beta_dpo/gap_std": 275.5003662109375,
|
|
"beta_dpo/loss_margin_mean": 260.8631591796875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4669603524229075,
|
|
"grad_norm": 1530.6976318359375,
|
|
"learning_rate": 3.233383385962115e-07,
|
|
"logits/chosen": -2.356491804122925,
|
|
"logits/rejected": -2.3217403888702393,
|
|
"loss": 1.5612,
|
|
"step": 318
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.8876981735229492,
|
|
"beta_dpo/beta_margin_grad_mean": -0.0937500074505806,
|
|
"beta_dpo/beta_margin_grad_std": 0.2914806008338928,
|
|
"beta_dpo/beta_margin_mean": 687.0026245117188,
|
|
"beta_dpo/beta_margin_std": 757.8516845703125,
|
|
"beta_dpo/beta_used": 1.8876981735229492,
|
|
"beta_dpo/beta_used_raw": 1.8876981735229492,
|
|
"beta_dpo/gap_mean": 300.7230529785156,
|
|
"beta_dpo/gap_std": 288.64886474609375,
|
|
"beta_dpo/loss_margin_mean": 364.14324951171875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4684287812041116,
|
|
"grad_norm": 9876.3720703125,
|
|
"learning_rate": 3.2211050867827805e-07,
|
|
"logits/chosen": -2.335461139678955,
|
|
"logits/rejected": -2.3492789268493652,
|
|
"loss": 18.2557,
|
|
"step": 319
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.420613557100296,
|
|
"beta_dpo/beta_margin_grad_std": 0.06336997449398041,
|
|
"beta_dpo/beta_margin_mean": 0.3268977701663971,
|
|
"beta_dpo/beta_margin_std": 0.26918449997901917,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.7549973130226135,
|
|
"beta_dpo/gap_mean": 309.19366455078125,
|
|
"beta_dpo/gap_std": 285.6737060546875,
|
|
"beta_dpo/loss_margin_mean": 326.89776611328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4698972099853157,
|
|
"grad_norm": 18.470361709594727,
|
|
"learning_rate": 3.208807785813777e-07,
|
|
"logits/chosen": -2.343623638153076,
|
|
"logits/rejected": -2.3438785076141357,
|
|
"loss": 1.1248,
|
|
"step": 320
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.1016801595687866,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23686403036117554,
|
|
"beta_dpo/beta_margin_grad_std": 0.24444517493247986,
|
|
"beta_dpo/beta_margin_mean": 430.26971435546875,
|
|
"beta_dpo/beta_margin_std": 625.2273559570312,
|
|
"beta_dpo/beta_used": 1.1016801595687866,
|
|
"beta_dpo/beta_used_raw": -0.355446457862854,
|
|
"beta_dpo/gap_mean": 312.37139892578125,
|
|
"beta_dpo/gap_std": 286.314453125,
|
|
"beta_dpo/loss_margin_mean": 314.2262268066406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4713656387665198,
|
|
"grad_norm": 17.7462158203125,
|
|
"learning_rate": 3.1964918071004217e-07,
|
|
"logits/chosen": -2.3021278381347656,
|
|
"logits/rejected": -2.2860074043273926,
|
|
"loss": 0.5814,
|
|
"step": 321
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.7430135011672974,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2846636474132538,
|
|
"beta_dpo/beta_margin_grad_std": 0.28654375672340393,
|
|
"beta_dpo/beta_margin_mean": 607.9188232421875,
|
|
"beta_dpo/beta_margin_std": 957.2604370117188,
|
|
"beta_dpo/beta_used": 1.7430135011672974,
|
|
"beta_dpo/beta_used_raw": 0.8375918865203857,
|
|
"beta_dpo/gap_mean": 308.1527404785156,
|
|
"beta_dpo/gap_std": 287.0491943359375,
|
|
"beta_dpo/loss_margin_mean": 287.3334045410156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.47283406754772395,
|
|
"grad_norm": 24231.912109375,
|
|
"learning_rate": 3.184157475180207e-07,
|
|
"logits/chosen": -2.332392454147339,
|
|
"logits/rejected": -2.3149595260620117,
|
|
"loss": 23.491,
|
|
"step": 322
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.23395885527133942,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3341367244720459,
|
|
"beta_dpo/beta_margin_grad_std": 0.3162374496459961,
|
|
"beta_dpo/beta_margin_mean": 62.195438385009766,
|
|
"beta_dpo/beta_margin_std": 114.66217041015625,
|
|
"beta_dpo/beta_used": 0.23395885527133942,
|
|
"beta_dpo/beta_used_raw": -3.1040267944335938,
|
|
"beta_dpo/gap_mean": 295.2753601074219,
|
|
"beta_dpo/gap_std": 282.6934814453125,
|
|
"beta_dpo/loss_margin_mean": 235.39443969726562,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.47430249632892807,
|
|
"grad_norm": 3386.258544921875,
|
|
"learning_rate": 3.171805115074251e-07,
|
|
"logits/chosen": -2.3514816761016846,
|
|
"logits/rejected": -2.3433778285980225,
|
|
"loss": 4.9578,
|
|
"step": 323
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3184950053691864,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3133268654346466,
|
|
"beta_dpo/beta_margin_grad_std": 0.3128097951412201,
|
|
"beta_dpo/beta_margin_mean": 79.24275207519531,
|
|
"beta_dpo/beta_margin_std": 142.1294403076172,
|
|
"beta_dpo/beta_used": 0.3184950053691864,
|
|
"beta_dpo/beta_used_raw": 0.007871389389038086,
|
|
"beta_dpo/gap_mean": 291.29779052734375,
|
|
"beta_dpo/gap_std": 282.01904296875,
|
|
"beta_dpo/loss_margin_mean": 317.21417236328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.47577092511013214,
|
|
"grad_norm": 4426.65576171875,
|
|
"learning_rate": 3.1594350522787295e-07,
|
|
"logits/chosen": -2.34005069732666,
|
|
"logits/rejected": -2.3183207511901855,
|
|
"loss": 2.4952,
|
|
"step": 324
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.4672297239303589,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2523164451122284,
|
|
"beta_dpo/beta_margin_grad_std": 0.2602192461490631,
|
|
"beta_dpo/beta_margin_mean": 114.00592803955078,
|
|
"beta_dpo/beta_margin_std": 195.69052124023438,
|
|
"beta_dpo/beta_used": 0.4672297239303589,
|
|
"beta_dpo/beta_used_raw": -2.8321478366851807,
|
|
"beta_dpo/gap_mean": 288.2220458984375,
|
|
"beta_dpo/gap_std": 285.31488037109375,
|
|
"beta_dpo/loss_margin_mean": 241.5183563232422,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.47723935389133626,
|
|
"grad_norm": 3548.100830078125,
|
|
"learning_rate": 3.147047612756302e-07,
|
|
"logits/chosen": -2.3933932781219482,
|
|
"logits/rejected": -2.3574209213256836,
|
|
"loss": 4.5265,
|
|
"step": 325
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.42803052067756653,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24226514995098114,
|
|
"beta_dpo/beta_margin_grad_std": 0.25276702642440796,
|
|
"beta_dpo/beta_margin_mean": 128.09483337402344,
|
|
"beta_dpo/beta_margin_std": 189.2570343017578,
|
|
"beta_dpo/beta_used": 0.42803052067756653,
|
|
"beta_dpo/beta_used_raw": 0.20111998915672302,
|
|
"beta_dpo/gap_mean": 288.9057312011719,
|
|
"beta_dpo/gap_std": 278.6829833984375,
|
|
"beta_dpo/loss_margin_mean": 309.9962158203125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4787077826725404,
|
|
"grad_norm": 12.956615447998047,
|
|
"learning_rate": 3.134643122927519e-07,
|
|
"logits/chosen": -2.4037997722625732,
|
|
"logits/rejected": -2.368350028991699,
|
|
"loss": 0.5699,
|
|
"step": 326
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.713379383087158,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09374991059303284,
|
|
"beta_dpo/beta_margin_grad_std": 0.29148030281066895,
|
|
"beta_dpo/beta_margin_mean": 989.2396850585938,
|
|
"beta_dpo/beta_margin_std": 727.1015625,
|
|
"beta_dpo/beta_used": 2.713379383087158,
|
|
"beta_dpo/beta_used_raw": 2.713379383087158,
|
|
"beta_dpo/gap_mean": 302.56488037109375,
|
|
"beta_dpo/gap_std": 276.96759033203125,
|
|
"beta_dpo/loss_margin_mean": 361.91253662109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4801762114537445,
|
|
"grad_norm": 30234.04296875,
|
|
"learning_rate": 3.1222219096622264e-07,
|
|
"logits/chosen": -2.3508028984069824,
|
|
"logits/rejected": -2.3310558795928955,
|
|
"loss": 8.5624,
|
|
"step": 327
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.522514820098877,
|
|
"beta_dpo/beta_margin_grad_mean": -0.08467871695756912,
|
|
"beta_dpo/beta_margin_grad_std": 0.2693813741207123,
|
|
"beta_dpo/beta_margin_mean": 884.0071411132812,
|
|
"beta_dpo/beta_margin_std": 907.109130859375,
|
|
"beta_dpo/beta_used": 2.522514820098877,
|
|
"beta_dpo/beta_used_raw": 2.522514820098877,
|
|
"beta_dpo/gap_mean": 310.073486328125,
|
|
"beta_dpo/gap_std": 278.1096496582031,
|
|
"beta_dpo/loss_margin_mean": 338.466796875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.48164464023494863,
|
|
"grad_norm": 10285.1142578125,
|
|
"learning_rate": 3.1097843002709427e-07,
|
|
"logits/chosen": -2.3761606216430664,
|
|
"logits/rejected": -2.3547372817993164,
|
|
"loss": 10.5089,
|
|
"step": 328
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.3934481143951416,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2540913224220276,
|
|
"beta_dpo/beta_margin_grad_std": 0.2650849223136902,
|
|
"beta_dpo/beta_margin_mean": 809.6107788085938,
|
|
"beta_dpo/beta_margin_std": 1348.3310546875,
|
|
"beta_dpo/beta_used": 2.3934481143951416,
|
|
"beta_dpo/beta_used_raw": 1.0050034523010254,
|
|
"beta_dpo/gap_mean": 315.6600646972656,
|
|
"beta_dpo/gap_std": 280.899658203125,
|
|
"beta_dpo/loss_margin_mean": 346.1232604980469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4831130690161527,
|
|
"grad_norm": 12.77830696105957,
|
|
"learning_rate": 3.0973306224962437e-07,
|
|
"logits/chosen": -2.3475561141967773,
|
|
"logits/rejected": -2.316133975982666,
|
|
"loss": 0.5727,
|
|
"step": 329
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41881564259529114,
|
|
"beta_dpo/beta_margin_grad_std": 0.0694441944360733,
|
|
"beta_dpo/beta_margin_mean": 0.3351494073867798,
|
|
"beta_dpo/beta_margin_std": 0.2930092513561249,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.9319770336151123,
|
|
"beta_dpo/gap_mean": 319.4997863769531,
|
|
"beta_dpo/gap_std": 284.83905029296875,
|
|
"beta_dpo/loss_margin_mean": 335.1493835449219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4845814977973568,
|
|
"grad_norm": 18.973018646240234,
|
|
"learning_rate": 3.084861204504122e-07,
|
|
"logits/chosen": -2.369523048400879,
|
|
"logits/rejected": -2.3711061477661133,
|
|
"loss": 1.1196,
|
|
"step": 330
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2725904583930969,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23779310286045074,
|
|
"beta_dpo/beta_margin_grad_std": 0.2499072253704071,
|
|
"beta_dpo/beta_margin_mean": 81.00366973876953,
|
|
"beta_dpo/beta_margin_std": 118.57772064208984,
|
|
"beta_dpo/beta_used": 0.2725904583930969,
|
|
"beta_dpo/beta_used_raw": -0.001759946346282959,
|
|
"beta_dpo/gap_mean": 322.59796142578125,
|
|
"beta_dpo/gap_std": 284.0390319824219,
|
|
"beta_dpo/loss_margin_mean": 326.7667541503906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.48604992657856094,
|
|
"grad_norm": 13.570107460021973,
|
|
"learning_rate": 3.072376374875335e-07,
|
|
"logits/chosen": -2.355848789215088,
|
|
"logits/rejected": -2.340954303741455,
|
|
"loss": 0.553,
|
|
"step": 331
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4359738230705261,
|
|
"beta_dpo/beta_margin_grad_std": 0.06110256910324097,
|
|
"beta_dpo/beta_margin_mean": 0.2621835172176361,
|
|
"beta_dpo/beta_margin_std": 0.25429460406303406,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.248039245605469,
|
|
"beta_dpo/gap_mean": 312.61383056640625,
|
|
"beta_dpo/gap_std": 277.851318359375,
|
|
"beta_dpo/loss_margin_mean": 262.1835021972656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.48751835535976507,
|
|
"grad_norm": 19.325729370117188,
|
|
"learning_rate": 3.059876462596758e-07,
|
|
"logits/chosen": -2.3538053035736084,
|
|
"logits/rejected": -2.339722156524658,
|
|
"loss": 1.1728,
|
|
"step": 332
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.793745279312134,
|
|
"beta_dpo/beta_margin_grad_mean": -0.14066281914710999,
|
|
"beta_dpo/beta_margin_grad_std": 0.3476191461086273,
|
|
"beta_dpo/beta_margin_mean": 984.294677734375,
|
|
"beta_dpo/beta_margin_std": 1002.9381713867188,
|
|
"beta_dpo/beta_used": 2.793745279312134,
|
|
"beta_dpo/beta_used_raw": 2.793745279312134,
|
|
"beta_dpo/gap_mean": 315.35302734375,
|
|
"beta_dpo/gap_std": 276.364990234375,
|
|
"beta_dpo/loss_margin_mean": 338.3910217285156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4889867841409692,
|
|
"grad_norm": 13925.4560546875,
|
|
"learning_rate": 3.0473617970527015e-07,
|
|
"logits/chosen": -2.367403268814087,
|
|
"logits/rejected": -2.354949474334717,
|
|
"loss": 17.0446,
|
|
"step": 333
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4284491240978241,
|
|
"beta_dpo/beta_margin_grad_std": 0.0682436004281044,
|
|
"beta_dpo/beta_margin_mean": 0.29454606771469116,
|
|
"beta_dpo/beta_margin_std": 0.28985467553138733,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.959665298461914,
|
|
"beta_dpo/gap_mean": 313.328369140625,
|
|
"beta_dpo/gap_std": 281.157470703125,
|
|
"beta_dpo/loss_margin_mean": 294.5460510253906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.49045521292217326,
|
|
"grad_norm": 18.42644500732422,
|
|
"learning_rate": 3.034832708016243e-07,
|
|
"logits/chosen": -2.3879494667053223,
|
|
"logits/rejected": -2.3688955307006836,
|
|
"loss": 1.1501,
|
|
"step": 334
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4463410973548889,
|
|
"beta_dpo/beta_margin_grad_std": 0.062163643538951874,
|
|
"beta_dpo/beta_margin_mean": 0.22004252672195435,
|
|
"beta_dpo/beta_margin_std": 0.2587723433971405,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.695589065551758,
|
|
"beta_dpo/gap_mean": 296.7751159667969,
|
|
"beta_dpo/gap_std": 274.739501953125,
|
|
"beta_dpo/loss_margin_mean": 220.04251098632812,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4919236417033774,
|
|
"grad_norm": 19.224916458129883,
|
|
"learning_rate": 3.022289525640531e-07,
|
|
"logits/chosen": -2.3774776458740234,
|
|
"logits/rejected": -2.358548164367676,
|
|
"loss": 1.1783,
|
|
"step": 335
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.3821998834609985,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09547650814056396,
|
|
"beta_dpo/beta_margin_grad_std": 0.28482678532600403,
|
|
"beta_dpo/beta_margin_mean": 575.6812133789062,
|
|
"beta_dpo/beta_margin_std": 781.006591796875,
|
|
"beta_dpo/beta_used": 1.3821998834609985,
|
|
"beta_dpo/beta_used_raw": 1.3821998834609985,
|
|
"beta_dpo/gap_mean": 304.68438720703125,
|
|
"beta_dpo/gap_std": 286.78021240234375,
|
|
"beta_dpo/loss_margin_mean": 382.9951171875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4933920704845815,
|
|
"grad_norm": 26202.302734375,
|
|
"learning_rate": 3.009732580450086e-07,
|
|
"logits/chosen": -2.341956853866577,
|
|
"logits/rejected": -2.3473258018493652,
|
|
"loss": 10.1207,
|
|
"step": 336
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8355345726013184,
|
|
"beta_dpo/beta_margin_grad_mean": -0.301108717918396,
|
|
"beta_dpo/beta_margin_grad_std": 0.3031361401081085,
|
|
"beta_dpo/beta_margin_mean": 238.27578735351562,
|
|
"beta_dpo/beta_margin_std": 417.9895324707031,
|
|
"beta_dpo/beta_used": 0.8355345726013184,
|
|
"beta_dpo/beta_used_raw": 0.20989900827407837,
|
|
"beta_dpo/gap_mean": 314.0046081542969,
|
|
"beta_dpo/gap_std": 295.2994079589844,
|
|
"beta_dpo/loss_margin_mean": 319.1441650390625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4948604992657856,
|
|
"grad_norm": 15146.6640625,
|
|
"learning_rate": 2.9971622033320914e-07,
|
|
"logits/chosen": -2.3654699325561523,
|
|
"logits/rejected": -2.3615758419036865,
|
|
"loss": 13.694,
|
|
"step": 337
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.42984750866889954,
|
|
"beta_dpo/beta_margin_grad_std": 0.06152024492621422,
|
|
"beta_dpo/beta_margin_mean": 0.28707996010780334,
|
|
"beta_dpo/beta_margin_std": 0.2556547224521637,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.1681759357452393,
|
|
"beta_dpo/gap_mean": 309.56866455078125,
|
|
"beta_dpo/gap_std": 290.0804138183594,
|
|
"beta_dpo/loss_margin_mean": 287.0799560546875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.49632892804698975,
|
|
"grad_norm": 18.67024803161621,
|
|
"learning_rate": 2.984578725527675e-07,
|
|
"logits/chosen": -2.3676528930664062,
|
|
"logits/rejected": -2.366528034210205,
|
|
"loss": 1.1585,
|
|
"step": 338
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.054119348526001,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2663116455078125,
|
|
"beta_dpo/beta_margin_grad_std": 0.2718449532985687,
|
|
"beta_dpo/beta_margin_mean": 411.3683776855469,
|
|
"beta_dpo/beta_margin_std": 620.0492553710938,
|
|
"beta_dpo/beta_used": 1.054119348526001,
|
|
"beta_dpo/beta_used_raw": -0.3248262405395508,
|
|
"beta_dpo/gap_mean": 306.15435791015625,
|
|
"beta_dpo/gap_std": 285.44110107421875,
|
|
"beta_dpo/loss_margin_mean": 320.0867614746094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.4977973568281938,
|
|
"grad_norm": 24645.341796875,
|
|
"learning_rate": 2.9719824786231796e-07,
|
|
"logits/chosen": -2.3688817024230957,
|
|
"logits/rejected": -2.361849784851074,
|
|
"loss": 6.4518,
|
|
"step": 339
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.684582233428955,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26743197441101074,
|
|
"beta_dpo/beta_margin_grad_std": 0.27371782064437866,
|
|
"beta_dpo/beta_margin_mean": 901.1436767578125,
|
|
"beta_dpo/beta_margin_std": 1441.9869384765625,
|
|
"beta_dpo/beta_used": 2.684582233428955,
|
|
"beta_dpo/beta_used_raw": 2.532560110092163,
|
|
"beta_dpo/gap_mean": 309.4892578125,
|
|
"beta_dpo/gap_std": 288.36248779296875,
|
|
"beta_dpo/loss_margin_mean": 288.3397521972656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.49926578560939794,
|
|
"grad_norm": 13.370800018310547,
|
|
"learning_rate": 2.959373794541426e-07,
|
|
"logits/chosen": -2.3758139610290527,
|
|
"logits/rejected": -2.346534490585327,
|
|
"loss": 0.5592,
|
|
"step": 340
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.065169334411621,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23790918290615082,
|
|
"beta_dpo/beta_margin_grad_std": 0.25097015500068665,
|
|
"beta_dpo/beta_margin_mean": 729.3997192382812,
|
|
"beta_dpo/beta_margin_std": 1149.5667724609375,
|
|
"beta_dpo/beta_used": 2.065169334411621,
|
|
"beta_dpo/beta_used_raw": 1.1916053295135498,
|
|
"beta_dpo/gap_mean": 313.42578125,
|
|
"beta_dpo/gap_std": 293.43829345703125,
|
|
"beta_dpo/loss_margin_mean": 356.7818603515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5007342143906021,
|
|
"grad_norm": 13.331892013549805,
|
|
"learning_rate": 2.946753005532965e-07,
|
|
"logits/chosen": -2.3476555347442627,
|
|
"logits/rejected": -2.346323013305664,
|
|
"loss": 0.5701,
|
|
"step": 341
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4230160415172577,
|
|
"beta_dpo/beta_margin_grad_std": 0.07352254539728165,
|
|
"beta_dpo/beta_margin_mean": 0.3177441358566284,
|
|
"beta_dpo/beta_margin_std": 0.3101942837238312,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.5433149337768555,
|
|
"beta_dpo/gap_mean": 312.7081298828125,
|
|
"beta_dpo/gap_std": 296.92974853515625,
|
|
"beta_dpo/loss_margin_mean": 317.7441101074219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5022026431718062,
|
|
"grad_norm": 20.234586715698242,
|
|
"learning_rate": 2.934120444167326e-07,
|
|
"logits/chosen": -2.400458335876465,
|
|
"logits/rejected": -2.365809917449951,
|
|
"loss": 1.1335,
|
|
"step": 342
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4237779676914215,
|
|
"beta_dpo/beta_margin_grad_std": 0.05391445755958557,
|
|
"beta_dpo/beta_margin_mean": 0.3117174208164215,
|
|
"beta_dpo/beta_margin_std": 0.22616280615329742,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.4838824272155762,
|
|
"beta_dpo/gap_mean": 314.9137268066406,
|
|
"beta_dpo/gap_std": 284.4820556640625,
|
|
"beta_dpo/loss_margin_mean": 311.7174072265625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5036710719530103,
|
|
"grad_norm": 21.022735595703125,
|
|
"learning_rate": 2.9214764433242476e-07,
|
|
"logits/chosen": -2.385115623474121,
|
|
"logits/rejected": -2.39156436920166,
|
|
"loss": 1.1268,
|
|
"step": 343
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2279607057571411,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25877782702445984,
|
|
"beta_dpo/beta_margin_grad_std": 0.2643044590950012,
|
|
"beta_dpo/beta_margin_mean": 545.8978271484375,
|
|
"beta_dpo/beta_margin_std": 796.9879760742188,
|
|
"beta_dpo/beta_used": 1.2279607057571411,
|
|
"beta_dpo/beta_used_raw": -2.5525197982788086,
|
|
"beta_dpo/gap_mean": 309.91912841796875,
|
|
"beta_dpo/gap_std": 284.9899597167969,
|
|
"beta_dpo/loss_margin_mean": 314.7950439453125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5051395007342144,
|
|
"grad_norm": 15404.0849609375,
|
|
"learning_rate": 2.9088213361849126e-07,
|
|
"logits/chosen": -2.3805313110351562,
|
|
"logits/rejected": -2.383361339569092,
|
|
"loss": 3.8253,
|
|
"step": 344
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.3728437423706055,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09375,
|
|
"beta_dpo/beta_margin_grad_std": 0.2914806008338928,
|
|
"beta_dpo/beta_margin_mean": 1225.3790283203125,
|
|
"beta_dpo/beta_margin_std": 1016.9912719726562,
|
|
"beta_dpo/beta_used": 3.3728437423706055,
|
|
"beta_dpo/beta_used_raw": 3.3728437423706055,
|
|
"beta_dpo/gap_mean": 323.52496337890625,
|
|
"beta_dpo/gap_std": 289.0306701660156,
|
|
"beta_dpo/loss_margin_mean": 365.1717529296875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5066079295154186,
|
|
"grad_norm": 20076.240234375,
|
|
"learning_rate": 2.896155456223163e-07,
|
|
"logits/chosen": -2.40063214302063,
|
|
"logits/rejected": -2.3850648403167725,
|
|
"loss": 10.5342,
|
|
"step": 345
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6880494952201843,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25628435611724854,
|
|
"beta_dpo/beta_margin_grad_std": 0.2660585939884186,
|
|
"beta_dpo/beta_margin_mean": 223.1298065185547,
|
|
"beta_dpo/beta_margin_std": 299.98797607421875,
|
|
"beta_dpo/beta_used": 0.6880494952201843,
|
|
"beta_dpo/beta_used_raw": 0.48999911546707153,
|
|
"beta_dpo/gap_mean": 326.8367919921875,
|
|
"beta_dpo/gap_std": 281.955810546875,
|
|
"beta_dpo/loss_margin_mean": 328.8027038574219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5080763582966226,
|
|
"grad_norm": 8900.7021484375,
|
|
"learning_rate": 2.883479137196714e-07,
|
|
"logits/chosen": -2.3676557540893555,
|
|
"logits/rejected": -2.3624587059020996,
|
|
"loss": 6.0288,
|
|
"step": 346
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.44006967544555664,
|
|
"beta_dpo/beta_margin_grad_std": 0.0717720165848732,
|
|
"beta_dpo/beta_margin_mean": 0.24584244191646576,
|
|
"beta_dpo/beta_margin_std": 0.2963031232357025,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.904984474182129,
|
|
"beta_dpo/gap_mean": 316.7066650390625,
|
|
"beta_dpo/gap_std": 282.1290283203125,
|
|
"beta_dpo/loss_margin_mean": 245.84243774414062,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5095447870778267,
|
|
"grad_norm": 21.67890739440918,
|
|
"learning_rate": 2.8707927131383614e-07,
|
|
"logits/chosen": -2.3673133850097656,
|
|
"logits/rejected": -2.3626620769500732,
|
|
"loss": 1.1499,
|
|
"step": 347
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4570677280426025,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23459888994693756,
|
|
"beta_dpo/beta_margin_grad_std": 0.24054688215255737,
|
|
"beta_dpo/beta_margin_mean": 557.3722534179688,
|
|
"beta_dpo/beta_margin_std": 804.013671875,
|
|
"beta_dpo/beta_used": 1.4570677280426025,
|
|
"beta_dpo/beta_used_raw": 1.1090539693832397,
|
|
"beta_dpo/gap_mean": 314.82440185546875,
|
|
"beta_dpo/gap_std": 278.79046630859375,
|
|
"beta_dpo/loss_margin_mean": 317.42047119140625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5110132158590308,
|
|
"grad_norm": 11.807612419128418,
|
|
"learning_rate": 2.858096518347179e-07,
|
|
"logits/chosen": -2.3952393531799316,
|
|
"logits/rejected": -2.3867907524108887,
|
|
"loss": 0.5596,
|
|
"step": 348
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.08341504633426666,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25735506415367126,
|
|
"beta_dpo/beta_margin_grad_std": 0.26621419191360474,
|
|
"beta_dpo/beta_margin_mean": 28.167213439941406,
|
|
"beta_dpo/beta_margin_std": 44.491722106933594,
|
|
"beta_dpo/beta_used": 0.08341504633426666,
|
|
"beta_dpo/beta_used_raw": -0.4417477548122406,
|
|
"beta_dpo/gap_mean": 314.21697998046875,
|
|
"beta_dpo/gap_std": 281.6492004394531,
|
|
"beta_dpo/loss_margin_mean": 326.5581359863281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5124816446402349,
|
|
"grad_norm": 1751.5216064453125,
|
|
"learning_rate": 2.845390887379706e-07,
|
|
"logits/chosen": -2.3977246284484863,
|
|
"logits/rejected": -2.3966641426086426,
|
|
"loss": 2.7493,
|
|
"step": 349
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.8887107372283936,
|
|
"beta_dpo/beta_margin_grad_mean": -0.140625,
|
|
"beta_dpo/beta_margin_grad_std": 0.34763428568840027,
|
|
"beta_dpo/beta_margin_mean": 593.6013793945312,
|
|
"beta_dpo/beta_margin_std": 579.1753540039062,
|
|
"beta_dpo/beta_used": 1.8887107372283936,
|
|
"beta_dpo/beta_used_raw": 1.8887107372283936,
|
|
"beta_dpo/gap_mean": 314.50628662109375,
|
|
"beta_dpo/gap_std": 285.41729736328125,
|
|
"beta_dpo/loss_margin_mean": 308.5635986328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5139500734214391,
|
|
"grad_norm": 25647.646484375,
|
|
"learning_rate": 2.8326761550411346e-07,
|
|
"logits/chosen": -2.409909725189209,
|
|
"logits/rejected": -2.4100773334503174,
|
|
"loss": 28.0702,
|
|
"step": 350
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4200161397457123,
|
|
"beta_dpo/beta_margin_grad_std": 0.07670863717794418,
|
|
"beta_dpo/beta_margin_mean": 0.33481884002685547,
|
|
"beta_dpo/beta_margin_std": 0.3401849567890167,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.0178706645965576,
|
|
"beta_dpo/gap_mean": 318.64599609375,
|
|
"beta_dpo/gap_std": 295.79644775390625,
|
|
"beta_dpo/loss_margin_mean": 334.8188171386719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5154185022026432,
|
|
"grad_norm": 20.02607536315918,
|
|
"learning_rate": 2.819952656376487e-07,
|
|
"logits/chosen": -2.3981099128723145,
|
|
"logits/rejected": -2.4018301963806152,
|
|
"loss": 1.1514,
|
|
"step": 351
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.43255868554115295,
|
|
"beta_dpo/beta_margin_grad_std": 0.06271059066057205,
|
|
"beta_dpo/beta_margin_mean": 0.2771955132484436,
|
|
"beta_dpo/beta_margin_std": 0.26465079188346863,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.255035400390625,
|
|
"beta_dpo/gap_mean": 310.7604675292969,
|
|
"beta_dpo/gap_std": 290.19696044921875,
|
|
"beta_dpo/loss_margin_mean": 277.19549560546875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5168869309838473,
|
|
"grad_norm": 18.892972946166992,
|
|
"learning_rate": 2.8072207266617854e-07,
|
|
"logits/chosen": -2.4018521308898926,
|
|
"logits/rejected": -2.383004903793335,
|
|
"loss": 1.1557,
|
|
"step": 352
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4298509657382965,
|
|
"beta_dpo/beta_margin_grad_std": 0.07284195721149445,
|
|
"beta_dpo/beta_margin_mean": 0.2901245057582855,
|
|
"beta_dpo/beta_margin_std": 0.30724653601646423,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.7248914241790771,
|
|
"beta_dpo/gap_mean": 307.8128356933594,
|
|
"beta_dpo/gap_std": 293.14794921875,
|
|
"beta_dpo/loss_margin_mean": 290.12451171875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5183553597650514,
|
|
"grad_norm": 20.134214401245117,
|
|
"learning_rate": 2.794480701395219e-07,
|
|
"logits/chosen": -2.4077649116516113,
|
|
"logits/rejected": -2.4009876251220703,
|
|
"loss": 1.1411,
|
|
"step": 353
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.42897459864616394,
|
|
"beta_dpo/beta_margin_grad_std": 0.058552030473947525,
|
|
"beta_dpo/beta_margin_mean": 0.29052889347076416,
|
|
"beta_dpo/beta_margin_std": 0.2439156472682953,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.4506525993347168,
|
|
"beta_dpo/gap_mean": 302.3212585449219,
|
|
"beta_dpo/gap_std": 285.1046447753906,
|
|
"beta_dpo/loss_margin_mean": 290.52886962890625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5198237885462555,
|
|
"grad_norm": 23.041349411010742,
|
|
"learning_rate": 2.781732916288303e-07,
|
|
"logits/chosen": -2.4216408729553223,
|
|
"logits/rejected": -2.4001893997192383,
|
|
"loss": 1.138,
|
|
"step": 354
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6849542856216431,
|
|
"beta_dpo/beta_margin_grad_mean": -0.10933418571949005,
|
|
"beta_dpo/beta_margin_grad_std": 0.3119906485080719,
|
|
"beta_dpo/beta_margin_mean": 228.0188446044922,
|
|
"beta_dpo/beta_margin_std": 165.11636352539062,
|
|
"beta_dpo/beta_used": 0.6849542856216431,
|
|
"beta_dpo/beta_used_raw": 0.6849542856216431,
|
|
"beta_dpo/gap_mean": 305.43170166015625,
|
|
"beta_dpo/gap_std": 274.4084777832031,
|
|
"beta_dpo/loss_margin_mean": 328.5459289550781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5212922173274597,
|
|
"grad_norm": 6968.03076171875,
|
|
"learning_rate": 2.7689777072570284e-07,
|
|
"logits/chosen": -2.445467948913574,
|
|
"logits/rejected": -2.4192585945129395,
|
|
"loss": 5.7505,
|
|
"step": 355
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4357198476791382,
|
|
"beta_dpo/beta_margin_grad_std": 0.06848623603582382,
|
|
"beta_dpo/beta_margin_mean": 0.2656390070915222,
|
|
"beta_dpo/beta_margin_std": 0.2923762798309326,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.9701589345932007,
|
|
"beta_dpo/gap_mean": 300.7390441894531,
|
|
"beta_dpo/gap_std": 275.934326171875,
|
|
"beta_dpo/loss_margin_mean": 265.6390075683594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5227606461086637,
|
|
"grad_norm": 20.48623275756836,
|
|
"learning_rate": 2.7562154104130176e-07,
|
|
"logits/chosen": -2.409104824066162,
|
|
"logits/rejected": -2.3870463371276855,
|
|
"loss": 1.1484,
|
|
"step": 356
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.1052085161209106,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25593680143356323,
|
|
"beta_dpo/beta_margin_grad_std": 0.2634008824825287,
|
|
"beta_dpo/beta_margin_mean": 405.9659729003906,
|
|
"beta_dpo/beta_margin_std": 574.3975830078125,
|
|
"beta_dpo/beta_used": 1.1052085161209106,
|
|
"beta_dpo/beta_used_raw": -1.2659064531326294,
|
|
"beta_dpo/gap_mean": 301.724609375,
|
|
"beta_dpo/gap_std": 276.3421325683594,
|
|
"beta_dpo/loss_margin_mean": 287.9976806640625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5242290748898678,
|
|
"grad_norm": 9097.716796875,
|
|
"learning_rate": 2.7434463620546594e-07,
|
|
"logits/chosen": -2.4178619384765625,
|
|
"logits/rejected": -2.4092202186584473,
|
|
"loss": 3.893,
|
|
"step": 357
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4275015890598297,
|
|
"beta_dpo/beta_margin_grad_std": 0.0712355524301529,
|
|
"beta_dpo/beta_margin_mean": 0.29972508549690247,
|
|
"beta_dpo/beta_margin_std": 0.3031264543533325,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.7463812828063965,
|
|
"beta_dpo/gap_mean": 296.3010559082031,
|
|
"beta_dpo/gap_std": 281.1275939941406,
|
|
"beta_dpo/loss_margin_mean": 299.7250671386719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5256975036710719,
|
|
"grad_norm": 19.981340408325195,
|
|
"learning_rate": 2.730670898658255e-07,
|
|
"logits/chosen": -2.4385619163513184,
|
|
"logits/rejected": -2.4157814979553223,
|
|
"loss": 1.1319,
|
|
"step": 358
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.1340972185134888,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3279348611831665,
|
|
"beta_dpo/beta_margin_grad_std": 0.31545180082321167,
|
|
"beta_dpo/beta_margin_mean": 409.3197937011719,
|
|
"beta_dpo/beta_margin_std": 649.3925170898438,
|
|
"beta_dpo/beta_used": 1.1340972185134888,
|
|
"beta_dpo/beta_used_raw": 0.9882941842079163,
|
|
"beta_dpo/gap_mean": 301.54461669921875,
|
|
"beta_dpo/gap_std": 286.58502197265625,
|
|
"beta_dpo/loss_margin_mean": 308.6645812988281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.527165932452276,
|
|
"grad_norm": 32715.87109375,
|
|
"learning_rate": 2.717889356869146e-07,
|
|
"logits/chosen": -2.3811445236206055,
|
|
"logits/rejected": -2.3567399978637695,
|
|
"loss": 38.7465,
|
|
"step": 359
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.06283307075500488,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2699156701564789,
|
|
"beta_dpo/beta_margin_grad_std": 0.27412280440330505,
|
|
"beta_dpo/beta_margin_mean": 18.395671844482422,
|
|
"beta_dpo/beta_margin_std": 25.289981842041016,
|
|
"beta_dpo/beta_used": 0.06283307075500488,
|
|
"beta_dpo/beta_used_raw": -3.388972043991089,
|
|
"beta_dpo/gap_mean": 292.21331787109375,
|
|
"beta_dpo/gap_std": 284.0353088378906,
|
|
"beta_dpo/loss_margin_mean": 261.1766357421875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5286343612334802,
|
|
"grad_norm": 1064.697265625,
|
|
"learning_rate": 2.7051020734928443e-07,
|
|
"logits/chosen": -2.3837945461273193,
|
|
"logits/rejected": -2.376451015472412,
|
|
"loss": 1.0703,
|
|
"step": 360
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4298721253871918,
|
|
"beta_dpo/beta_margin_grad_std": 0.07070347666740417,
|
|
"beta_dpo/beta_margin_mean": 0.28980541229248047,
|
|
"beta_dpo/beta_margin_std": 0.2996097505092621,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.996284544467926,
|
|
"beta_dpo/gap_mean": 288.74700927734375,
|
|
"beta_dpo/gap_std": 280.52886962890625,
|
|
"beta_dpo/loss_margin_mean": 289.8053894042969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5301027900146843,
|
|
"grad_norm": 25.262163162231445,
|
|
"learning_rate": 2.6923093854861593e-07,
|
|
"logits/chosen": -2.4027366638183594,
|
|
"logits/rejected": -2.4041662216186523,
|
|
"loss": 1.1471,
|
|
"step": 361
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.8285510540008545,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25968730449676514,
|
|
"beta_dpo/beta_margin_grad_std": 0.2681870758533478,
|
|
"beta_dpo/beta_margin_mean": 1458.04541015625,
|
|
"beta_dpo/beta_margin_std": 2284.134521484375,
|
|
"beta_dpo/beta_used": 3.8285510540008545,
|
|
"beta_dpo/beta_used_raw": 2.4841794967651367,
|
|
"beta_dpo/gap_mean": 301.5517883300781,
|
|
"beta_dpo/gap_std": 286.40460205078125,
|
|
"beta_dpo/loss_margin_mean": 343.222412109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5315712187958884,
|
|
"grad_norm": 17.703359603881836,
|
|
"learning_rate": 2.679511629948319e-07,
|
|
"logits/chosen": -2.412851333618164,
|
|
"logits/rejected": -2.408217430114746,
|
|
"loss": 0.5786,
|
|
"step": 362
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.8720965385437012,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22648286819458008,
|
|
"beta_dpo/beta_margin_grad_std": 0.2343624234199524,
|
|
"beta_dpo/beta_margin_mean": 804.5293579101562,
|
|
"beta_dpo/beta_margin_std": 1079.2637939453125,
|
|
"beta_dpo/beta_used": 1.8720965385437012,
|
|
"beta_dpo/beta_used_raw": 0.9251274466514587,
|
|
"beta_dpo/gap_mean": 309.974365234375,
|
|
"beta_dpo/gap_std": 281.13372802734375,
|
|
"beta_dpo/loss_margin_mean": 375.29144287109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5330396475770925,
|
|
"grad_norm": 13.820899963378906,
|
|
"learning_rate": 2.6667091441120816e-07,
|
|
"logits/chosen": -2.3960952758789062,
|
|
"logits/rejected": -2.389695167541504,
|
|
"loss": 0.5714,
|
|
"step": 363
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4259724020957947,
|
|
"beta_dpo/beta_margin_grad_std": 0.06279082596302032,
|
|
"beta_dpo/beta_margin_mean": 0.304084450006485,
|
|
"beta_dpo/beta_margin_std": 0.2647766172885895,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.1112940311431885,
|
|
"beta_dpo/gap_mean": 312.33929443359375,
|
|
"beta_dpo/gap_std": 278.56072998046875,
|
|
"beta_dpo/loss_margin_mean": 304.08441162109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5345080763582967,
|
|
"grad_norm": 23.981868743896484,
|
|
"learning_rate": 2.6539022653348575e-07,
|
|
"logits/chosen": -2.423187255859375,
|
|
"logits/rejected": -2.4290435314178467,
|
|
"loss": 1.1401,
|
|
"step": 364
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41367998719215393,
|
|
"beta_dpo/beta_margin_grad_std": 0.06623264402151108,
|
|
"beta_dpo/beta_margin_mean": 0.3594275116920471,
|
|
"beta_dpo/beta_margin_std": 0.30056267976760864,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.2951538562774658,
|
|
"beta_dpo/gap_mean": 319.5771484375,
|
|
"beta_dpo/gap_std": 276.10504150390625,
|
|
"beta_dpo/loss_margin_mean": 359.427490234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5359765051395007,
|
|
"grad_norm": 19.053972244262695,
|
|
"learning_rate": 2.641091331089811e-07,
|
|
"logits/chosen": -2.439345121383667,
|
|
"logits/rejected": -2.4456310272216797,
|
|
"loss": 1.1205,
|
|
"step": 365
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.3038524389266968,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26780223846435547,
|
|
"beta_dpo/beta_margin_grad_std": 0.2728710174560547,
|
|
"beta_dpo/beta_margin_mean": 548.268310546875,
|
|
"beta_dpo/beta_margin_std": 849.1015014648438,
|
|
"beta_dpo/beta_used": 1.3038524389266968,
|
|
"beta_dpo/beta_used_raw": -1.0728014707565308,
|
|
"beta_dpo/gap_mean": 328.21820068359375,
|
|
"beta_dpo/gap_std": 286.6083984375,
|
|
"beta_dpo/loss_margin_mean": 328.7030334472656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5374449339207048,
|
|
"grad_norm": 12437.0556640625,
|
|
"learning_rate": 2.6282766789569736e-07,
|
|
"logits/chosen": -2.40187406539917,
|
|
"logits/rejected": -2.413430690765381,
|
|
"loss": 3.6027,
|
|
"step": 366
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2638746500015259,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2950782775878906,
|
|
"beta_dpo/beta_margin_grad_std": 0.2939762771129608,
|
|
"beta_dpo/beta_margin_mean": 86.77686309814453,
|
|
"beta_dpo/beta_margin_std": 148.9335174560547,
|
|
"beta_dpo/beta_used": 0.2638746500015259,
|
|
"beta_dpo/beta_used_raw": -0.3957434594631195,
|
|
"beta_dpo/gap_mean": 318.469482421875,
|
|
"beta_dpo/gap_std": 279.47314453125,
|
|
"beta_dpo/loss_margin_mean": 298.730712890625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5389133627019089,
|
|
"grad_norm": 4519.6572265625,
|
|
"learning_rate": 2.615458646614349e-07,
|
|
"logits/chosen": -2.404602289199829,
|
|
"logits/rejected": -2.3893771171569824,
|
|
"loss": 3.52,
|
|
"step": 367
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.132652759552002,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2524746060371399,
|
|
"beta_dpo/beta_margin_grad_std": 0.2634090185165405,
|
|
"beta_dpo/beta_margin_mean": 816.8623657226562,
|
|
"beta_dpo/beta_margin_std": 1151.4071044921875,
|
|
"beta_dpo/beta_used": 2.132652759552002,
|
|
"beta_dpo/beta_used_raw": 1.5166137218475342,
|
|
"beta_dpo/gap_mean": 326.68768310546875,
|
|
"beta_dpo/gap_std": 279.3883056640625,
|
|
"beta_dpo/loss_margin_mean": 374.5268859863281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.540381791483113,
|
|
"grad_norm": 4498.89404296875,
|
|
"learning_rate": 2.6026375718290083e-07,
|
|
"logits/chosen": -2.4052700996398926,
|
|
"logits/rejected": -2.4133400917053223,
|
|
"loss": 4.8171,
|
|
"step": 368
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.07468457520008087,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23666079342365265,
|
|
"beta_dpo/beta_margin_grad_std": 0.23870179057121277,
|
|
"beta_dpo/beta_margin_mean": 24.3631534576416,
|
|
"beta_dpo/beta_margin_std": 35.491668701171875,
|
|
"beta_dpo/beta_used": 0.07468457520008087,
|
|
"beta_dpo/beta_used_raw": -0.6055254936218262,
|
|
"beta_dpo/gap_mean": 326.35491943359375,
|
|
"beta_dpo/gap_std": 280.87725830078125,
|
|
"beta_dpo/loss_margin_mean": 311.52703857421875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5418502202643172,
|
|
"grad_norm": 11.01858139038086,
|
|
"learning_rate": 2.589813792448196e-07,
|
|
"logits/chosen": -2.4168224334716797,
|
|
"logits/rejected": -2.4043703079223633,
|
|
"loss": 0.5587,
|
|
"step": 369
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.43343597650527954,
|
|
"beta_dpo/beta_margin_grad_std": 0.06827400624752045,
|
|
"beta_dpo/beta_margin_mean": 0.27436593174934387,
|
|
"beta_dpo/beta_margin_std": 0.2874927818775177,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.793357253074646,
|
|
"beta_dpo/gap_mean": 316.9214782714844,
|
|
"beta_dpo/gap_std": 280.57403564453125,
|
|
"beta_dpo/loss_margin_mean": 274.36590576171875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5433186490455213,
|
|
"grad_norm": 21.518030166625977,
|
|
"learning_rate": 2.5769876463904263e-07,
|
|
"logits/chosen": -2.4265782833099365,
|
|
"logits/rejected": -2.425027847290039,
|
|
"loss": 1.1337,
|
|
"step": 370
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5957542657852173,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25642988085746765,
|
|
"beta_dpo/beta_margin_grad_std": 0.26836514472961426,
|
|
"beta_dpo/beta_margin_mean": 196.8231964111328,
|
|
"beta_dpo/beta_margin_std": 344.92559814453125,
|
|
"beta_dpo/beta_used": 0.5957542657852173,
|
|
"beta_dpo/beta_used_raw": -2.339658498764038,
|
|
"beta_dpo/gap_mean": 319.158203125,
|
|
"beta_dpo/gap_std": 289.86175537109375,
|
|
"beta_dpo/loss_margin_mean": 333.40985107421875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5447870778267254,
|
|
"grad_norm": 8407.552734375,
|
|
"learning_rate": 2.5641594716365744e-07,
|
|
"logits/chosen": -2.393846035003662,
|
|
"logits/rejected": -2.3929755687713623,
|
|
"loss": 21.0427,
|
|
"step": 371
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.440232038497925,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09375,
|
|
"beta_dpo/beta_margin_grad_std": 0.2914806008338928,
|
|
"beta_dpo/beta_margin_mean": 1281.364501953125,
|
|
"beta_dpo/beta_margin_std": 1292.0177001953125,
|
|
"beta_dpo/beta_used": 3.440232038497925,
|
|
"beta_dpo/beta_used_raw": 3.440232038497925,
|
|
"beta_dpo/gap_mean": 320.1593017578125,
|
|
"beta_dpo/gap_std": 292.0980224609375,
|
|
"beta_dpo/loss_margin_mean": 341.0747985839844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5462555066079295,
|
|
"grad_norm": 31112.7265625,
|
|
"learning_rate": 2.551329606220976e-07,
|
|
"logits/chosen": -2.3891334533691406,
|
|
"logits/rejected": -2.360813617706299,
|
|
"loss": 9.5999,
|
|
"step": 372
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41610556840896606,
|
|
"beta_dpo/beta_margin_grad_std": 0.05844424292445183,
|
|
"beta_dpo/beta_margin_mean": 0.34435054659843445,
|
|
"beta_dpo/beta_margin_std": 0.2457403987646103,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.9711276292800903,
|
|
"beta_dpo/gap_mean": 326.219970703125,
|
|
"beta_dpo/gap_std": 283.1405334472656,
|
|
"beta_dpo/loss_margin_mean": 344.35052490234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5477239353891337,
|
|
"grad_norm": 23.21306800842285,
|
|
"learning_rate": 2.538498388222517e-07,
|
|
"logits/chosen": -2.3829376697540283,
|
|
"logits/rejected": -2.3680813312530518,
|
|
"loss": 1.1126,
|
|
"step": 373
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.42734894156455994,
|
|
"beta_dpo/beta_margin_grad_std": 0.06432215124368668,
|
|
"beta_dpo/beta_margin_mean": 0.2995753586292267,
|
|
"beta_dpo/beta_margin_std": 0.2753455340862274,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.3080596923828125,
|
|
"beta_dpo/gap_mean": 326.4972229003906,
|
|
"beta_dpo/gap_std": 282.38189697265625,
|
|
"beta_dpo/loss_margin_mean": 299.5753479003906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5491923641703378,
|
|
"grad_norm": 22.25040054321289,
|
|
"learning_rate": 2.525666155755725e-07,
|
|
"logits/chosen": -2.427107810974121,
|
|
"logits/rejected": -2.4131693840026855,
|
|
"loss": 1.1579,
|
|
"step": 374
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.3931167125701904,
|
|
"beta_dpo/beta_margin_grad_mean": -0.20865072309970856,
|
|
"beta_dpo/beta_margin_grad_std": 0.216493159532547,
|
|
"beta_dpo/beta_margin_mean": 1006.3369140625,
|
|
"beta_dpo/beta_margin_std": 1276.470947265625,
|
|
"beta_dpo/beta_used": 2.3931167125701904,
|
|
"beta_dpo/beta_used_raw": 0.9554816484451294,
|
|
"beta_dpo/gap_mean": 333.14459228515625,
|
|
"beta_dpo/gap_std": 280.5745849609375,
|
|
"beta_dpo/loss_margin_mean": 382.5895690917969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5506607929515418,
|
|
"grad_norm": 13.200187683105469,
|
|
"learning_rate": 2.512833246961859e-07,
|
|
"logits/chosen": -2.409682273864746,
|
|
"logits/rejected": -2.407869815826416,
|
|
"loss": 0.5667,
|
|
"step": 375
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5595067143440247,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2484658807516098,
|
|
"beta_dpo/beta_margin_grad_std": 0.26340359449386597,
|
|
"beta_dpo/beta_margin_mean": 214.9127960205078,
|
|
"beta_dpo/beta_margin_std": 298.3186950683594,
|
|
"beta_dpo/beta_used": 0.5595067143440247,
|
|
"beta_dpo/beta_used_raw": -0.2936529517173767,
|
|
"beta_dpo/gap_mean": 342.8294677734375,
|
|
"beta_dpo/gap_std": 288.7122802734375,
|
|
"beta_dpo/loss_margin_mean": 393.45977783203125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5521292217327459,
|
|
"grad_norm": 8034.6337890625,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": -2.4047350883483887,
|
|
"logits/rejected": -2.384918451309204,
|
|
"loss": 9.2557,
|
|
"step": 376
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4221385717391968,
|
|
"beta_dpo/beta_margin_grad_std": 0.0715164914727211,
|
|
"beta_dpo/beta_margin_mean": 0.32525014877319336,
|
|
"beta_dpo/beta_margin_std": 0.3200809955596924,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -5.223406791687012,
|
|
"beta_dpo/gap_mean": 342.27374267578125,
|
|
"beta_dpo/gap_std": 292.09466552734375,
|
|
"beta_dpo/loss_margin_mean": 325.2501525878906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.55359765051395,
|
|
"grad_norm": 21.198095321655273,
|
|
"learning_rate": 2.487166753038141e-07,
|
|
"logits/chosen": -2.341383695602417,
|
|
"logits/rejected": -2.343954086303711,
|
|
"loss": 1.1565,
|
|
"step": 377
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4200609624385834,
|
|
"beta_dpo/beta_margin_grad_std": 0.06614904850721359,
|
|
"beta_dpo/beta_margin_mean": 0.3278721570968628,
|
|
"beta_dpo/beta_margin_std": 0.2774437367916107,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.6767090559005737,
|
|
"beta_dpo/gap_mean": 338.7351379394531,
|
|
"beta_dpo/gap_std": 294.2265625,
|
|
"beta_dpo/loss_margin_mean": 327.87213134765625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5550660792951542,
|
|
"grad_norm": 20.219329833984375,
|
|
"learning_rate": 2.4743338442442754e-07,
|
|
"logits/chosen": -2.350855827331543,
|
|
"logits/rejected": -2.344240665435791,
|
|
"loss": 1.1113,
|
|
"step": 378
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41115090250968933,
|
|
"beta_dpo/beta_margin_grad_std": 0.08131575584411621,
|
|
"beta_dpo/beta_margin_mean": 0.37218692898750305,
|
|
"beta_dpo/beta_margin_std": 0.3555106520652771,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8637235164642334,
|
|
"beta_dpo/gap_mean": 345.63275146484375,
|
|
"beta_dpo/gap_std": 302.9630126953125,
|
|
"beta_dpo/loss_margin_mean": 372.1868896484375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5565345080763583,
|
|
"grad_norm": 21.625322341918945,
|
|
"learning_rate": 2.461501611777483e-07,
|
|
"logits/chosen": -2.3660264015197754,
|
|
"logits/rejected": -2.391849994659424,
|
|
"loss": 1.096,
|
|
"step": 379
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.558686375617981,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24460071325302124,
|
|
"beta_dpo/beta_margin_grad_std": 0.26073068380355835,
|
|
"beta_dpo/beta_margin_mean": 214.0121307373047,
|
|
"beta_dpo/beta_margin_std": 353.10638427734375,
|
|
"beta_dpo/beta_used": 0.558686375617981,
|
|
"beta_dpo/beta_used_raw": 0.4444287121295929,
|
|
"beta_dpo/gap_mean": 352.9265441894531,
|
|
"beta_dpo/gap_std": 309.6590270996094,
|
|
"beta_dpo/loss_margin_mean": 409.6791076660156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5580029368575624,
|
|
"grad_norm": 13815.2509765625,
|
|
"learning_rate": 2.4486703937790243e-07,
|
|
"logits/chosen": -2.35787034034729,
|
|
"logits/rejected": -2.374066114425659,
|
|
"loss": 2.8621,
|
|
"step": 380
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4248752295970917,
|
|
"beta_dpo/beta_margin_grad_std": 0.07637570053339005,
|
|
"beta_dpo/beta_margin_mean": 0.3129652142524719,
|
|
"beta_dpo/beta_margin_std": 0.3291749358177185,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.8646886348724365,
|
|
"beta_dpo/gap_mean": 352.447998046875,
|
|
"beta_dpo/gap_std": 314.13433837890625,
|
|
"beta_dpo/loss_margin_mean": 312.9652099609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5594713656387665,
|
|
"grad_norm": 20.60698890686035,
|
|
"learning_rate": 2.435840528363426e-07,
|
|
"logits/chosen": -2.350996494293213,
|
|
"logits/rejected": -2.340618133544922,
|
|
"loss": 1.1346,
|
|
"step": 381
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4127730131149292,
|
|
"beta_dpo/beta_margin_grad_std": 0.0697304978966713,
|
|
"beta_dpo/beta_margin_mean": 0.3608226776123047,
|
|
"beta_dpo/beta_margin_std": 0.29635754227638245,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.334915280342102,
|
|
"beta_dpo/gap_mean": 351.69036865234375,
|
|
"beta_dpo/gap_std": 313.22735595703125,
|
|
"beta_dpo/loss_margin_mean": 360.8226623535156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5609397944199707,
|
|
"grad_norm": 20.6890811920166,
|
|
"learning_rate": 2.4230123536095745e-07,
|
|
"logits/chosen": -2.3812966346740723,
|
|
"logits/rejected": -2.3885483741760254,
|
|
"loss": 1.0979,
|
|
"step": 382
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4209439754486084,
|
|
"beta_dpo/beta_margin_grad_std": 0.07652373611927032,
|
|
"beta_dpo/beta_margin_mean": 0.32738256454467773,
|
|
"beta_dpo/beta_margin_std": 0.3242851793766022,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.201818585395813,
|
|
"beta_dpo/gap_mean": 349.58636474609375,
|
|
"beta_dpo/gap_std": 314.92181396484375,
|
|
"beta_dpo/loss_margin_mean": 327.3825378417969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5624082232011748,
|
|
"grad_norm": 24.684696197509766,
|
|
"learning_rate": 2.4101862075518037e-07,
|
|
"logits/chosen": -2.3738174438476562,
|
|
"logits/rejected": -2.3857202529907227,
|
|
"loss": 1.0963,
|
|
"step": 383
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4266614019870758,
|
|
"beta_dpo/beta_margin_grad_std": 0.0517151914536953,
|
|
"beta_dpo/beta_margin_mean": 0.2990947663784027,
|
|
"beta_dpo/beta_margin_std": 0.214844211935997,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.5872749090194702,
|
|
"beta_dpo/gap_mean": 338.76934814453125,
|
|
"beta_dpo/gap_std": 299.67193603515625,
|
|
"beta_dpo/loss_margin_mean": 299.0947570800781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5638766519823789,
|
|
"grad_norm": 21.041105270385742,
|
|
"learning_rate": 2.397362428170992e-07,
|
|
"logits/chosen": -2.406465768814087,
|
|
"logits/rejected": -2.4182257652282715,
|
|
"loss": 1.1106,
|
|
"step": 384
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5529597401618958,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23210173845291138,
|
|
"beta_dpo/beta_margin_grad_std": 0.24000734090805054,
|
|
"beta_dpo/beta_margin_mean": 192.6968536376953,
|
|
"beta_dpo/beta_margin_std": 267.4351501464844,
|
|
"beta_dpo/beta_used": 0.5529597401618958,
|
|
"beta_dpo/beta_used_raw": 0.1586916744709015,
|
|
"beta_dpo/gap_mean": 334.78759765625,
|
|
"beta_dpo/gap_std": 289.0450744628906,
|
|
"beta_dpo/loss_margin_mean": 311.2054138183594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5653450807635829,
|
|
"grad_norm": 6151.4521484375,
|
|
"learning_rate": 2.3845413533856514e-07,
|
|
"logits/chosen": -2.377863883972168,
|
|
"logits/rejected": -2.3569583892822266,
|
|
"loss": 3.7171,
|
|
"step": 385
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.328173965215683,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24216587841510773,
|
|
"beta_dpo/beta_margin_grad_std": 0.25181594491004944,
|
|
"beta_dpo/beta_margin_mean": 135.31890869140625,
|
|
"beta_dpo/beta_margin_std": 188.71702575683594,
|
|
"beta_dpo/beta_used": 0.328173965215683,
|
|
"beta_dpo/beta_used_raw": -2.265230894088745,
|
|
"beta_dpo/gap_mean": 334.6175231933594,
|
|
"beta_dpo/gap_std": 285.37115478515625,
|
|
"beta_dpo/loss_margin_mean": 366.7230224609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.566813509544787,
|
|
"grad_norm": 7890.92236328125,
|
|
"learning_rate": 2.3717233210430254e-07,
|
|
"logits/chosen": -2.3772597312927246,
|
|
"logits/rejected": -2.3809022903442383,
|
|
"loss": 1.637,
|
|
"step": 386
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4257817566394806,
|
|
"beta_dpo/beta_margin_grad_std": 0.07243922352790833,
|
|
"beta_dpo/beta_margin_mean": 0.3068787157535553,
|
|
"beta_dpo/beta_margin_std": 0.30547431111335754,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.440854072570801,
|
|
"beta_dpo/gap_mean": 335.63763427734375,
|
|
"beta_dpo/gap_std": 289.9078369140625,
|
|
"beta_dpo/loss_margin_mean": 306.8786926269531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5682819383259912,
|
|
"grad_norm": 26.328731536865234,
|
|
"learning_rate": 2.3589086689101889e-07,
|
|
"logits/chosen": -2.427687168121338,
|
|
"logits/rejected": -2.4119105339050293,
|
|
"loss": 1.1311,
|
|
"step": 387
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.6118403673171997,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22462284564971924,
|
|
"beta_dpo/beta_margin_grad_std": 0.23515048623085022,
|
|
"beta_dpo/beta_margin_mean": 676.41845703125,
|
|
"beta_dpo/beta_margin_std": 1056.8880615234375,
|
|
"beta_dpo/beta_used": 1.6118403673171997,
|
|
"beta_dpo/beta_used_raw": -0.10103893280029297,
|
|
"beta_dpo/gap_mean": 341.0576477050781,
|
|
"beta_dpo/gap_std": 298.05126953125,
|
|
"beta_dpo/loss_margin_mean": 380.3453063964844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5697503671071953,
|
|
"grad_norm": 17.152055740356445,
|
|
"learning_rate": 2.3460977346651428e-07,
|
|
"logits/chosen": -2.349907159805298,
|
|
"logits/rejected": -2.363679885864258,
|
|
"loss": 0.5682,
|
|
"step": 388
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4282563030719757,
|
|
"beta_dpo/beta_margin_grad_std": 0.08526238799095154,
|
|
"beta_dpo/beta_margin_mean": 0.2955423891544342,
|
|
"beta_dpo/beta_margin_std": 0.36567941308021545,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.094040870666504,
|
|
"beta_dpo/gap_mean": 335.10137939453125,
|
|
"beta_dpo/gap_std": 306.0793762207031,
|
|
"beta_dpo/loss_margin_mean": 295.5423583984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5712187958883994,
|
|
"grad_norm": 23.256101608276367,
|
|
"learning_rate": 2.3332908558879177e-07,
|
|
"logits/chosen": -2.379899024963379,
|
|
"logits/rejected": -2.373753309249878,
|
|
"loss": 1.1102,
|
|
"step": 389
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6165112257003784,
|
|
"beta_dpo/beta_margin_grad_mean": -0.0921190157532692,
|
|
"beta_dpo/beta_margin_grad_std": 0.28663116693496704,
|
|
"beta_dpo/beta_margin_mean": 242.41787719726562,
|
|
"beta_dpo/beta_margin_std": 333.33306884765625,
|
|
"beta_dpo/beta_used": 0.6165112257003784,
|
|
"beta_dpo/beta_used_raw": 0.6165112257003784,
|
|
"beta_dpo/gap_mean": 338.8628845214844,
|
|
"beta_dpo/gap_std": 316.99127197265625,
|
|
"beta_dpo/loss_margin_mean": 362.915771484375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5726872246696035,
|
|
"grad_norm": 15604.0400390625,
|
|
"learning_rate": 2.320488370051681e-07,
|
|
"logits/chosen": -2.3529043197631836,
|
|
"logits/rejected": -2.3543856143951416,
|
|
"loss": 6.7076,
|
|
"step": 390
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4340205192565918,
|
|
"beta_dpo/beta_margin_grad_std": 0.06161868944764137,
|
|
"beta_dpo/beta_margin_mean": 0.26973778009414673,
|
|
"beta_dpo/beta_margin_std": 0.25571221113204956,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.148557186126709,
|
|
"beta_dpo/gap_mean": 328.48919677734375,
|
|
"beta_dpo/gap_std": 299.50042724609375,
|
|
"beta_dpo/loss_margin_mean": 269.7377624511719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5741556534508077,
|
|
"grad_norm": 21.558528900146484,
|
|
"learning_rate": 2.3076906145138405e-07,
|
|
"logits/chosen": -2.3723154067993164,
|
|
"logits/rejected": -2.3779938220977783,
|
|
"loss": 1.1553,
|
|
"step": 391
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.9164685010910034,
|
|
"beta_dpo/beta_margin_grad_mean": -0.06281369924545288,
|
|
"beta_dpo/beta_margin_grad_std": 0.2419932633638382,
|
|
"beta_dpo/beta_margin_mean": 839.9161376953125,
|
|
"beta_dpo/beta_margin_std": 943.3709716796875,
|
|
"beta_dpo/beta_used": 1.9164685010910034,
|
|
"beta_dpo/beta_used_raw": 1.9164685010910034,
|
|
"beta_dpo/gap_mean": 332.79083251953125,
|
|
"beta_dpo/gap_std": 301.228271484375,
|
|
"beta_dpo/loss_margin_mean": 401.9228515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5756240822320118,
|
|
"grad_norm": 8398.5751953125,
|
|
"learning_rate": 2.294897926507156e-07,
|
|
"logits/chosen": -2.405207633972168,
|
|
"logits/rejected": -2.4119091033935547,
|
|
"loss": 11.169,
|
|
"step": 392
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4241882562637329,
|
|
"beta_dpo/beta_margin_grad_std": 0.07016981393098831,
|
|
"beta_dpo/beta_margin_mean": 0.313728928565979,
|
|
"beta_dpo/beta_margin_std": 0.30085715651512146,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.3607499599456787,
|
|
"beta_dpo/gap_mean": 336.00286865234375,
|
|
"beta_dpo/gap_std": 306.16607666015625,
|
|
"beta_dpo/loss_margin_mean": 313.7289123535156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5770925110132159,
|
|
"grad_norm": 26.408864974975586,
|
|
"learning_rate": 2.2821106431308543e-07,
|
|
"logits/chosen": -2.376577854156494,
|
|
"logits/rejected": -2.37333345413208,
|
|
"loss": 1.1245,
|
|
"step": 393
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.376448392868042,
|
|
"beta_dpo/beta_margin_grad_mean": -0.062322188168764114,
|
|
"beta_dpo/beta_margin_grad_std": 0.24126726388931274,
|
|
"beta_dpo/beta_margin_mean": 556.2864990234375,
|
|
"beta_dpo/beta_margin_std": 614.0519409179688,
|
|
"beta_dpo/beta_used": 1.376448392868042,
|
|
"beta_dpo/beta_used_raw": 1.376448392868042,
|
|
"beta_dpo/gap_mean": 340.67095947265625,
|
|
"beta_dpo/gap_std": 299.90325927734375,
|
|
"beta_dpo/loss_margin_mean": 384.258544921875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.57856093979442,
|
|
"grad_norm": 19108.568359375,
|
|
"learning_rate": 2.2693291013417452e-07,
|
|
"logits/chosen": -2.3865888118743896,
|
|
"logits/rejected": -2.3961262702941895,
|
|
"loss": 3.1294,
|
|
"step": 394
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.42161938548088074,
|
|
"beta_dpo/beta_margin_grad_std": 0.07712607830762863,
|
|
"beta_dpo/beta_margin_mean": 0.32536518573760986,
|
|
"beta_dpo/beta_margin_std": 0.3256779909133911,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.202303886413574,
|
|
"beta_dpo/gap_mean": 342.54486083984375,
|
|
"beta_dpo/gap_std": 305.68255615234375,
|
|
"beta_dpo/loss_margin_mean": 325.36517333984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.580029368575624,
|
|
"grad_norm": 23.2067813873291,
|
|
"learning_rate": 2.2565536379453404e-07,
|
|
"logits/chosen": -2.386712074279785,
|
|
"logits/rejected": -2.381330966949463,
|
|
"loss": 1.153,
|
|
"step": 395
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4167252480983734,
|
|
"beta_dpo/beta_margin_grad_std": 0.07602323591709137,
|
|
"beta_dpo/beta_margin_mean": 0.3457351326942444,
|
|
"beta_dpo/beta_margin_std": 0.32599034905433655,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.017214775085449,
|
|
"beta_dpo/gap_mean": 341.43377685546875,
|
|
"beta_dpo/gap_std": 311.3189697265625,
|
|
"beta_dpo/loss_margin_mean": 345.735107421875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5814977973568282,
|
|
"grad_norm": 22.437475204467773,
|
|
"learning_rate": 2.2437845895869825e-07,
|
|
"logits/chosen": -2.412064552307129,
|
|
"logits/rejected": -2.3995161056518555,
|
|
"loss": 1.112,
|
|
"step": 396
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.7200398445129395,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22126896679401398,
|
|
"beta_dpo/beta_margin_grad_std": 0.2345595508813858,
|
|
"beta_dpo/beta_margin_mean": 340.8731689453125,
|
|
"beta_dpo/beta_margin_std": 545.7304077148438,
|
|
"beta_dpo/beta_used": 0.7200398445129395,
|
|
"beta_dpo/beta_used_raw": 0.6792468428611755,
|
|
"beta_dpo/gap_mean": 355.73583984375,
|
|
"beta_dpo/gap_std": 323.22137451171875,
|
|
"beta_dpo/loss_margin_mean": 421.983154296875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5829662261380323,
|
|
"grad_norm": 17.3627872467041,
|
|
"learning_rate": 2.2310222927429716e-07,
|
|
"logits/chosen": -2.3999505043029785,
|
|
"logits/rejected": -2.3841006755828857,
|
|
"loss": 0.5404,
|
|
"step": 397
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0242469310760498,
|
|
"beta_dpo/beta_margin_grad_mean": -0.14056435227394104,
|
|
"beta_dpo/beta_margin_grad_std": 0.34748461842536926,
|
|
"beta_dpo/beta_margin_mean": 392.9497985839844,
|
|
"beta_dpo/beta_margin_std": 412.2462463378906,
|
|
"beta_dpo/beta_used": 1.0242469310760498,
|
|
"beta_dpo/beta_used_raw": 1.0242469310760498,
|
|
"beta_dpo/gap_mean": 359.8270263671875,
|
|
"beta_dpo/gap_std": 337.1595153808594,
|
|
"beta_dpo/loss_margin_mean": 382.2912292480469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5844346549192364,
|
|
"grad_norm": 17113.068359375,
|
|
"learning_rate": 2.2182670837116972e-07,
|
|
"logits/chosen": -2.438488483428955,
|
|
"logits/rejected": -2.441718101501465,
|
|
"loss": 22.2563,
|
|
"step": 398
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41422000527381897,
|
|
"beta_dpo/beta_margin_grad_std": 0.07931336015462875,
|
|
"beta_dpo/beta_margin_mean": 0.3558643162250519,
|
|
"beta_dpo/beta_margin_std": 0.3342893421649933,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.7503660917282104,
|
|
"beta_dpo/gap_mean": 361.33551025390625,
|
|
"beta_dpo/gap_std": 341.2704162597656,
|
|
"beta_dpo/loss_margin_mean": 355.8642883300781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5859030837004405,
|
|
"grad_norm": 25.74508285522461,
|
|
"learning_rate": 2.2055192986047804e-07,
|
|
"logits/chosen": -2.4001259803771973,
|
|
"logits/rejected": -2.3805899620056152,
|
|
"loss": 1.1024,
|
|
"step": 399
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.27362868189811707,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26691824197769165,
|
|
"beta_dpo/beta_margin_grad_std": 0.2769628167152405,
|
|
"beta_dpo/beta_margin_mean": 99.65248107910156,
|
|
"beta_dpo/beta_margin_std": 166.322265625,
|
|
"beta_dpo/beta_used": 0.27362868189811707,
|
|
"beta_dpo/beta_used_raw": 0.03512483835220337,
|
|
"beta_dpo/gap_mean": 361.3894348144531,
|
|
"beta_dpo/gap_std": 336.9342956542969,
|
|
"beta_dpo/loss_margin_mean": 369.5531921386719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5873715124816447,
|
|
"grad_norm": 8530.029296875,
|
|
"learning_rate": 2.192779273338215e-07,
|
|
"logits/chosen": -2.4256482124328613,
|
|
"logits/rejected": -2.4234414100646973,
|
|
"loss": 3.7878,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5873715124816447,
|
|
"eval_beta_dpo/beta": 0.0010000000474974513,
|
|
"eval_beta_dpo/beta_margin_grad_mean": -0.45058003067970276,
|
|
"eval_beta_dpo/beta_margin_grad_std": 0.08448585122823715,
|
|
"eval_beta_dpo/beta_margin_mean": 0.2054125964641571,
|
|
"eval_beta_dpo/beta_margin_std": 0.35709553956985474,
|
|
"eval_beta_dpo/beta_used": 0.0010000000474974513,
|
|
"eval_beta_dpo/beta_used_raw": -9.301395416259766,
|
|
"eval_beta_dpo/gap_mean": 362.1024475097656,
|
|
"eval_beta_dpo/gap_std": 333.2911682128906,
|
|
"eval_beta_dpo/loss_margin_mean": 205.41256713867188,
|
|
"eval_beta_dpo/mask_keep_frac": 1.0,
|
|
"eval_logits/chosen": -2.443066358566284,
|
|
"eval_logits/rejected": -2.4332497119903564,
|
|
"eval_loss": 0.6121687293052673,
|
|
"eval_runtime": 37.7883,
|
|
"eval_samples_per_second": 61.897,
|
|
"eval_steps_per_second": 1.958,
|
|
"step": 400
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0822142362594604,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27038997411727905,
|
|
"beta_dpo/beta_margin_grad_std": 0.27789729833602905,
|
|
"beta_dpo/beta_margin_mean": 469.0853576660156,
|
|
"beta_dpo/beta_margin_std": 778.7431030273438,
|
|
"beta_dpo/beta_used": 1.0822142362594604,
|
|
"beta_dpo/beta_used_raw": -1.9135266542434692,
|
|
"beta_dpo/gap_mean": 361.82781982421875,
|
|
"beta_dpo/gap_std": 342.1406555175781,
|
|
"beta_dpo/loss_margin_mean": 327.20556640625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5888399412628488,
|
|
"grad_norm": 3323.7548828125,
|
|
"learning_rate": 2.1800473436235136e-07,
|
|
"logits/chosen": -2.4136600494384766,
|
|
"logits/rejected": -2.4133119583129883,
|
|
"loss": 3.0841,
|
|
"step": 401
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41412436962127686,
|
|
"beta_dpo/beta_margin_grad_std": 0.0651504173874855,
|
|
"beta_dpo/beta_margin_mean": 0.35367244482040405,
|
|
"beta_dpo/beta_margin_std": 0.2750694453716278,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.9857521057128906,
|
|
"beta_dpo/gap_mean": 353.83746337890625,
|
|
"beta_dpo/gap_std": 333.810302734375,
|
|
"beta_dpo/loss_margin_mean": 353.67242431640625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5903083700440529,
|
|
"grad_norm": 22.399593353271484,
|
|
"learning_rate": 2.1673238449588665e-07,
|
|
"logits/chosen": -2.428457736968994,
|
|
"logits/rejected": -2.423470973968506,
|
|
"loss": 1.1155,
|
|
"step": 402
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.470536470413208,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2379431277513504,
|
|
"beta_dpo/beta_margin_grad_std": 0.2529750466346741,
|
|
"beta_dpo/beta_margin_mean": 895.9714965820312,
|
|
"beta_dpo/beta_margin_std": 1318.6358642578125,
|
|
"beta_dpo/beta_used": 2.470536470413208,
|
|
"beta_dpo/beta_used_raw": 0.7853410243988037,
|
|
"beta_dpo/gap_mean": 355.718994140625,
|
|
"beta_dpo/gap_std": 329.26959228515625,
|
|
"beta_dpo/loss_margin_mean": 364.1114807128906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.591776798825257,
|
|
"grad_norm": 11.740818977355957,
|
|
"learning_rate": 2.154609112620295e-07,
|
|
"logits/chosen": -2.408144474029541,
|
|
"logits/rejected": -2.403205394744873,
|
|
"loss": 0.5649,
|
|
"step": 403
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4196135401725769,
|
|
"beta_dpo/beta_margin_grad_std": 0.08152054995298386,
|
|
"beta_dpo/beta_margin_mean": 0.3342151939868927,
|
|
"beta_dpo/beta_margin_std": 0.3467388153076172,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.5054094791412354,
|
|
"beta_dpo/gap_mean": 351.60748291015625,
|
|
"beta_dpo/gap_std": 336.61431884765625,
|
|
"beta_dpo/loss_margin_mean": 334.2151794433594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.593245227606461,
|
|
"grad_norm": 22.172229766845703,
|
|
"learning_rate": 2.1419034816528218e-07,
|
|
"logits/chosen": -2.3768763542175293,
|
|
"logits/rejected": -2.3775620460510254,
|
|
"loss": 1.1159,
|
|
"step": 404
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4178047478199005,
|
|
"beta_dpo/beta_margin_grad_std": 0.08822597563266754,
|
|
"beta_dpo/beta_margin_mean": 0.34523844718933105,
|
|
"beta_dpo/beta_margin_std": 0.38265952467918396,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -5.212499618530273,
|
|
"beta_dpo/gap_mean": 349.62945556640625,
|
|
"beta_dpo/gap_std": 341.0721435546875,
|
|
"beta_dpo/loss_margin_mean": 345.2384338378906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5947136563876652,
|
|
"grad_norm": 24.03404426574707,
|
|
"learning_rate": 2.129207286861638e-07,
|
|
"logits/chosen": -2.394247531890869,
|
|
"logits/rejected": -2.390904426574707,
|
|
"loss": 1.1631,
|
|
"step": 405
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40800565481185913,
|
|
"beta_dpo/beta_margin_grad_std": 0.06867733597755432,
|
|
"beta_dpo/beta_margin_mean": 0.3816227912902832,
|
|
"beta_dpo/beta_margin_std": 0.29813575744628906,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0216950178146362,
|
|
"beta_dpo/gap_mean": 353.78961181640625,
|
|
"beta_dpo/gap_std": 335.23284912109375,
|
|
"beta_dpo/loss_margin_mean": 381.6227722167969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5961820851688693,
|
|
"grad_norm": 22.132526397705078,
|
|
"learning_rate": 2.1165208628032861e-07,
|
|
"logits/chosen": -2.3941879272460938,
|
|
"logits/rejected": -2.3966739177703857,
|
|
"loss": 1.0938,
|
|
"step": 406
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.3681471347808838,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2872449457645416,
|
|
"beta_dpo/beta_margin_grad_std": 0.2897031307220459,
|
|
"beta_dpo/beta_margin_mean": 529.73486328125,
|
|
"beta_dpo/beta_margin_std": 880.0147094726562,
|
|
"beta_dpo/beta_used": 1.3681471347808838,
|
|
"beta_dpo/beta_used_raw": -1.0896592140197754,
|
|
"beta_dpo/gap_mean": 344.74383544921875,
|
|
"beta_dpo/gap_std": 335.9972229003906,
|
|
"beta_dpo/loss_margin_mean": 297.04425048828125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5976505139500734,
|
|
"grad_norm": 15591.009765625,
|
|
"learning_rate": 2.1038445437768375e-07,
|
|
"logits/chosen": -2.4210031032562256,
|
|
"logits/rejected": -2.4093196392059326,
|
|
"loss": 7.8766,
|
|
"step": 407
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.12347578257322311,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28025034070014954,
|
|
"beta_dpo/beta_margin_grad_std": 0.2851478159427643,
|
|
"beta_dpo/beta_margin_mean": 42.36891174316406,
|
|
"beta_dpo/beta_margin_std": 62.56731414794922,
|
|
"beta_dpo/beta_used": 0.12347578257322311,
|
|
"beta_dpo/beta_used_raw": -1.5166099071502686,
|
|
"beta_dpo/gap_mean": 342.59320068359375,
|
|
"beta_dpo/gap_std": 329.6495361328125,
|
|
"beta_dpo/loss_margin_mean": 304.6890563964844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.5991189427312775,
|
|
"grad_norm": 2112.055419921875,
|
|
"learning_rate": 2.0911786638150872e-07,
|
|
"logits/chosen": -2.4404101371765137,
|
|
"logits/rejected": -2.420276641845703,
|
|
"loss": 2.3999,
|
|
"step": 408
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4296059310436249,
|
|
"beta_dpo/beta_margin_grad_std": 0.08227775245904922,
|
|
"beta_dpo/beta_margin_mean": 0.29228663444519043,
|
|
"beta_dpo/beta_margin_std": 0.34818896651268005,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.490858793258667,
|
|
"beta_dpo/gap_mean": 331.255126953125,
|
|
"beta_dpo/gap_std": 332.9500732421875,
|
|
"beta_dpo/loss_margin_mean": 292.28662109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6005873715124816,
|
|
"grad_norm": 20.568885803222656,
|
|
"learning_rate": 2.0785235566757517e-07,
|
|
"logits/chosen": -2.4326343536376953,
|
|
"logits/rejected": -2.4223175048828125,
|
|
"loss": 1.1187,
|
|
"step": 409
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.8076552152633667,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24894118309020996,
|
|
"beta_dpo/beta_margin_grad_std": 0.2569856345653534,
|
|
"beta_dpo/beta_margin_mean": 749.4814453125,
|
|
"beta_dpo/beta_margin_std": 1033.7783203125,
|
|
"beta_dpo/beta_used": 1.8076552152633667,
|
|
"beta_dpo/beta_used_raw": 0.11705267429351807,
|
|
"beta_dpo/gap_mean": 334.65582275390625,
|
|
"beta_dpo/gap_std": 324.4977111816406,
|
|
"beta_dpo/loss_margin_mean": 339.3305358886719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6020558002936858,
|
|
"grad_norm": 28801.537109375,
|
|
"learning_rate": 2.065879555832674e-07,
|
|
"logits/chosen": -2.426015853881836,
|
|
"logits/rejected": -2.4410088062286377,
|
|
"loss": 1.872,
|
|
"step": 410
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4142020046710968,
|
|
"beta_dpo/beta_margin_grad_std": 0.06422904133796692,
|
|
"beta_dpo/beta_margin_mean": 0.35382363200187683,
|
|
"beta_dpo/beta_margin_std": 0.27380073070526123,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.1547770500183105,
|
|
"beta_dpo/gap_mean": 335.4295654296875,
|
|
"beta_dpo/gap_std": 315.85736083984375,
|
|
"beta_dpo/loss_margin_mean": 353.8236083984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6035242290748899,
|
|
"grad_norm": 19.184236526489258,
|
|
"learning_rate": 2.0532469944670343e-07,
|
|
"logits/chosen": -2.3894524574279785,
|
|
"logits/rejected": -2.401761293411255,
|
|
"loss": 1.1076,
|
|
"step": 411
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4059164226055145,
|
|
"beta_dpo/beta_margin_grad_std": 0.07964363694190979,
|
|
"beta_dpo/beta_margin_mean": 0.39483025670051575,
|
|
"beta_dpo/beta_margin_std": 0.3526383638381958,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.261457920074463,
|
|
"beta_dpo/gap_mean": 340.3641052246094,
|
|
"beta_dpo/gap_std": 316.4858093261719,
|
|
"beta_dpo/loss_margin_mean": 394.83026123046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.604992657856094,
|
|
"grad_norm": 22.225645065307617,
|
|
"learning_rate": 2.0406262054585738e-07,
|
|
"logits/chosen": -2.419179677963257,
|
|
"logits/rejected": -2.4454574584960938,
|
|
"loss": 1.1086,
|
|
"step": 412
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9109219908714294,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09896637499332428,
|
|
"beta_dpo/beta_margin_grad_std": 0.2925999164581299,
|
|
"beta_dpo/beta_margin_mean": 362.6519775390625,
|
|
"beta_dpo/beta_margin_std": 495.7064514160156,
|
|
"beta_dpo/beta_used": 0.9109219908714294,
|
|
"beta_dpo/beta_used_raw": 0.9109219908714294,
|
|
"beta_dpo/gap_mean": 349.16204833984375,
|
|
"beta_dpo/gap_std": 321.62481689453125,
|
|
"beta_dpo/loss_margin_mean": 370.2857666015625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6064610866372981,
|
|
"grad_norm": 12787.8896484375,
|
|
"learning_rate": 2.0280175213768205e-07,
|
|
"logits/chosen": -2.421027660369873,
|
|
"logits/rejected": -2.4294235706329346,
|
|
"loss": 7.296,
|
|
"step": 413
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.4083163738250732,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23511956632137299,
|
|
"beta_dpo/beta_margin_grad_std": 0.24318645894527435,
|
|
"beta_dpo/beta_margin_mean": 1103.0013427734375,
|
|
"beta_dpo/beta_margin_std": 1460.6190185546875,
|
|
"beta_dpo/beta_used": 2.4083163738250732,
|
|
"beta_dpo/beta_used_raw": -0.48015403747558594,
|
|
"beta_dpo/gap_mean": 347.5014343261719,
|
|
"beta_dpo/gap_std": 315.90374755859375,
|
|
"beta_dpo/loss_margin_mean": 354.2364501953125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6079295154185022,
|
|
"grad_norm": 15.694453239440918,
|
|
"learning_rate": 2.0154212744723247e-07,
|
|
"logits/chosen": -2.4023854732513428,
|
|
"logits/rejected": -2.420386552810669,
|
|
"loss": 0.5842,
|
|
"step": 414
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41483768820762634,
|
|
"beta_dpo/beta_margin_grad_std": 0.07904361933469772,
|
|
"beta_dpo/beta_margin_mean": 0.35519102215766907,
|
|
"beta_dpo/beta_margin_std": 0.3411670923233032,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.372532367706299,
|
|
"beta_dpo/gap_mean": 357.978271484375,
|
|
"beta_dpo/gap_std": 318.06103515625,
|
|
"beta_dpo/loss_margin_mean": 355.1910095214844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6093979441997063,
|
|
"grad_norm": 24.613069534301758,
|
|
"learning_rate": 2.002837796667909e-07,
|
|
"logits/chosen": -2.423779010772705,
|
|
"logits/rejected": -2.4402318000793457,
|
|
"loss": 1.1358,
|
|
"step": 415
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.4572019577026367,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22583799064159393,
|
|
"beta_dpo/beta_margin_grad_std": 0.2351226955652237,
|
|
"beta_dpo/beta_margin_mean": 1054.186767578125,
|
|
"beta_dpo/beta_margin_std": 1374.137451171875,
|
|
"beta_dpo/beta_used": 2.4572019577026367,
|
|
"beta_dpo/beta_used_raw": -8.606910705566406e-05,
|
|
"beta_dpo/gap_mean": 354.178955078125,
|
|
"beta_dpo/gap_std": 311.4892272949219,
|
|
"beta_dpo/loss_margin_mean": 378.5653991699219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6108663729809104,
|
|
"grad_norm": 20.630220413208008,
|
|
"learning_rate": 1.990267419549914e-07,
|
|
"logits/chosen": -2.422940969467163,
|
|
"logits/rejected": -2.435427188873291,
|
|
"loss": 0.5757,
|
|
"step": 416
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.37648269534111023,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27004316449165344,
|
|
"beta_dpo/beta_margin_grad_std": 0.277245432138443,
|
|
"beta_dpo/beta_margin_mean": 133.46795654296875,
|
|
"beta_dpo/beta_margin_std": 200.62649536132812,
|
|
"beta_dpo/beta_used": 0.37648269534111023,
|
|
"beta_dpo/beta_used_raw": -0.30419254302978516,
|
|
"beta_dpo/gap_mean": 356.74456787109375,
|
|
"beta_dpo/gap_std": 307.15887451171875,
|
|
"beta_dpo/loss_margin_mean": 345.98773193359375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6123348017621145,
|
|
"grad_norm": 27230.5078125,
|
|
"learning_rate": 1.9777104743594686e-07,
|
|
"logits/chosen": -2.422550678253174,
|
|
"logits/rejected": -2.4108152389526367,
|
|
"loss": 12.2488,
|
|
"step": 417
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.925933837890625,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21461881697177887,
|
|
"beta_dpo/beta_margin_grad_std": 0.22274993360042572,
|
|
"beta_dpo/beta_margin_mean": 1944.69580078125,
|
|
"beta_dpo/beta_margin_std": 2413.48388671875,
|
|
"beta_dpo/beta_used": 3.925933837890625,
|
|
"beta_dpo/beta_used_raw": 2.9994614124298096,
|
|
"beta_dpo/gap_mean": 356.8074645996094,
|
|
"beta_dpo/gap_std": 310.31988525390625,
|
|
"beta_dpo/loss_margin_mean": 394.4806213378906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6138032305433186,
|
|
"grad_norm": 15.336505889892578,
|
|
"learning_rate": 1.965167291983757e-07,
|
|
"logits/chosen": -2.4672048091888428,
|
|
"logits/rejected": -2.467080593109131,
|
|
"loss": 0.5562,
|
|
"step": 418
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.24180491268634796,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24379843473434448,
|
|
"beta_dpo/beta_margin_grad_std": 0.24562664330005646,
|
|
"beta_dpo/beta_margin_mean": 92.98143005371094,
|
|
"beta_dpo/beta_margin_std": 130.87518310546875,
|
|
"beta_dpo/beta_used": 0.24180491268634796,
|
|
"beta_dpo/beta_used_raw": -0.4406244158744812,
|
|
"beta_dpo/gap_mean": 366.676513671875,
|
|
"beta_dpo/gap_std": 303.807373046875,
|
|
"beta_dpo/loss_margin_mean": 380.661376953125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6152716593245228,
|
|
"grad_norm": 3546.734375,
|
|
"learning_rate": 1.9526382029472988e-07,
|
|
"logits/chosen": -2.4470913410186768,
|
|
"logits/rejected": -2.453092336654663,
|
|
"loss": 0.6635,
|
|
"step": 419
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.377697229385376,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12197641283273697,
|
|
"beta_dpo/beta_margin_grad_std": 0.32349058985710144,
|
|
"beta_dpo/beta_margin_mean": 463.7217102050781,
|
|
"beta_dpo/beta_margin_std": 575.2052612304688,
|
|
"beta_dpo/beta_used": 1.377697229385376,
|
|
"beta_dpo/beta_used_raw": 1.377697229385376,
|
|
"beta_dpo/gap_mean": 362.47210693359375,
|
|
"beta_dpo/gap_std": 298.56646728515625,
|
|
"beta_dpo/loss_margin_mean": 335.21160888671875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6167400881057269,
|
|
"grad_norm": 19327.3125,
|
|
"learning_rate": 1.9401235374032425e-07,
|
|
"logits/chosen": -2.411374092102051,
|
|
"logits/rejected": -2.393552780151367,
|
|
"loss": 14.9604,
|
|
"step": 420
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.411777138710022,
|
|
"beta_dpo/beta_margin_grad_std": 0.07005602866411209,
|
|
"beta_dpo/beta_margin_mean": 0.36520177125930786,
|
|
"beta_dpo/beta_margin_std": 0.29842692613601685,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.6941306591033936,
|
|
"beta_dpo/gap_mean": 360.69580078125,
|
|
"beta_dpo/gap_std": 296.3153076171875,
|
|
"beta_dpo/loss_margin_mean": 365.2017517089844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.618208516886931,
|
|
"grad_norm": 20.577306747436523,
|
|
"learning_rate": 1.9276236251246653e-07,
|
|
"logits/chosen": -2.4650721549987793,
|
|
"logits/rejected": -2.475637912750244,
|
|
"loss": 1.0969,
|
|
"step": 421
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4262285530567169,
|
|
"beta_dpo/beta_margin_grad_std": 0.08390183001756668,
|
|
"beta_dpo/beta_margin_mean": 0.30507245659828186,
|
|
"beta_dpo/beta_margin_std": 0.3505830466747284,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.5142788887023926,
|
|
"beta_dpo/gap_mean": 356.1475524902344,
|
|
"beta_dpo/gap_std": 305.0068664550781,
|
|
"beta_dpo/loss_margin_mean": 305.07244873046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6196769456681351,
|
|
"grad_norm": 27.27686309814453,
|
|
"learning_rate": 1.9151387954958792e-07,
|
|
"logits/chosen": -2.433502674102783,
|
|
"logits/rejected": -2.4503097534179688,
|
|
"loss": 1.1011,
|
|
"step": 422
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.667612075805664,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09069596976041794,
|
|
"beta_dpo/beta_margin_grad_std": 0.2812235653400421,
|
|
"beta_dpo/beta_margin_mean": 1046.2611083984375,
|
|
"beta_dpo/beta_margin_std": 1394.6136474609375,
|
|
"beta_dpo/beta_used": 2.667612075805664,
|
|
"beta_dpo/beta_used_raw": 2.667612075805664,
|
|
"beta_dpo/gap_mean": 355.60418701171875,
|
|
"beta_dpo/gap_std": 305.63763427734375,
|
|
"beta_dpo/loss_margin_mean": 379.52520751953125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6211453744493393,
|
|
"grad_norm": 49413.26953125,
|
|
"learning_rate": 1.902669377503756e-07,
|
|
"logits/chosen": -2.4104795455932617,
|
|
"logits/rejected": -2.43206787109375,
|
|
"loss": 27.8684,
|
|
"step": 423
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.14746440947055817,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2809794247150421,
|
|
"beta_dpo/beta_margin_grad_std": 0.2841648757457733,
|
|
"beta_dpo/beta_margin_mean": 50.7834587097168,
|
|
"beta_dpo/beta_margin_std": 84.9849624633789,
|
|
"beta_dpo/beta_used": 0.14746440947055817,
|
|
"beta_dpo/beta_used_raw": -1.0671793222427368,
|
|
"beta_dpo/gap_mean": 350.1261291503906,
|
|
"beta_dpo/gap_std": 303.9034729003906,
|
|
"beta_dpo/loss_margin_mean": 301.35723876953125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6226138032305433,
|
|
"grad_norm": 2292.713134765625,
|
|
"learning_rate": 1.890215699729057e-07,
|
|
"logits/chosen": -2.436124086380005,
|
|
"logits/rejected": -2.434342861175537,
|
|
"loss": 1.9952,
|
|
"step": 424
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3063303828239441,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2702360153198242,
|
|
"beta_dpo/beta_margin_grad_std": 0.28220611810684204,
|
|
"beta_dpo/beta_margin_mean": 115.15734100341797,
|
|
"beta_dpo/beta_margin_std": 173.1321258544922,
|
|
"beta_dpo/beta_used": 0.3063303828239441,
|
|
"beta_dpo/beta_used_raw": -0.04413789510726929,
|
|
"beta_dpo/gap_mean": 347.618896484375,
|
|
"beta_dpo/gap_std": 309.3476257324219,
|
|
"beta_dpo/loss_margin_mean": 364.8583068847656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6240822320117474,
|
|
"grad_norm": 3245.521240234375,
|
|
"learning_rate": 1.8777780903377732e-07,
|
|
"logits/chosen": -2.4158806800842285,
|
|
"logits/rejected": -2.452991008758545,
|
|
"loss": 2.1748,
|
|
"step": 425
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.3221449851989746,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23410384356975555,
|
|
"beta_dpo/beta_margin_grad_std": 0.2420140653848648,
|
|
"beta_dpo/beta_margin_mean": 937.5267944335938,
|
|
"beta_dpo/beta_margin_std": 1492.68994140625,
|
|
"beta_dpo/beta_used": 2.3221449851989746,
|
|
"beta_dpo/beta_used_raw": 0.11249232292175293,
|
|
"beta_dpo/gap_mean": 343.1509704589844,
|
|
"beta_dpo/gap_std": 309.1755676269531,
|
|
"beta_dpo/loss_margin_mean": 330.6883850097656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6255506607929515,
|
|
"grad_norm": 20.291976928710938,
|
|
"learning_rate": 1.8653568770724803e-07,
|
|
"logits/chosen": -2.4429891109466553,
|
|
"logits/rejected": -2.435697078704834,
|
|
"loss": 0.5759,
|
|
"step": 426
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.05136517435312271,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29277148842811584,
|
|
"beta_dpo/beta_margin_grad_std": 0.290822833776474,
|
|
"beta_dpo/beta_margin_mean": 14.553664207458496,
|
|
"beta_dpo/beta_margin_std": 25.774181365966797,
|
|
"beta_dpo/beta_used": 0.05136517435312271,
|
|
"beta_dpo/beta_used_raw": -2.2542648315429688,
|
|
"beta_dpo/gap_mean": 339.93310546875,
|
|
"beta_dpo/gap_std": 314.22222900390625,
|
|
"beta_dpo/loss_margin_mean": 297.6253662109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6270190895741556,
|
|
"grad_norm": 1669.3848876953125,
|
|
"learning_rate": 1.8529523872436977e-07,
|
|
"logits/chosen": -2.484468460083008,
|
|
"logits/rejected": -2.4828639030456543,
|
|
"loss": 1.797,
|
|
"step": 427
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4063303768634796,
|
|
"beta_dpo/beta_margin_grad_std": 0.08325894176959991,
|
|
"beta_dpo/beta_margin_mean": 0.3934994637966156,
|
|
"beta_dpo/beta_margin_std": 0.36640259623527527,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.5234259366989136,
|
|
"beta_dpo/gap_mean": 344.0052490234375,
|
|
"beta_dpo/gap_std": 324.0714416503906,
|
|
"beta_dpo/loss_margin_mean": 393.49945068359375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6284875183553598,
|
|
"grad_norm": 29.064952850341797,
|
|
"learning_rate": 1.8405649477212697e-07,
|
|
"logits/chosen": -2.44685697555542,
|
|
"logits/rejected": -2.4712371826171875,
|
|
"loss": 1.1086,
|
|
"step": 428
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4477723836898804,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2723529636859894,
|
|
"beta_dpo/beta_margin_grad_std": 0.28279608488082886,
|
|
"beta_dpo/beta_margin_mean": 609.0072631835938,
|
|
"beta_dpo/beta_margin_std": 894.19482421875,
|
|
"beta_dpo/beta_used": 1.4477723836898804,
|
|
"beta_dpo/beta_used_raw": 0.639708936214447,
|
|
"beta_dpo/gap_mean": 350.4301452636719,
|
|
"beta_dpo/gap_std": 327.36663818359375,
|
|
"beta_dpo/loss_margin_mean": 378.28228759765625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6299559471365639,
|
|
"grad_norm": 38852.13671875,
|
|
"learning_rate": 1.828194884925749e-07,
|
|
"logits/chosen": -2.4718470573425293,
|
|
"logits/rejected": -2.4727447032928467,
|
|
"loss": 23.4925,
|
|
"step": 429
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2469240427017212,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27254435420036316,
|
|
"beta_dpo/beta_margin_grad_std": 0.2825857698917389,
|
|
"beta_dpo/beta_margin_mean": 510.65069580078125,
|
|
"beta_dpo/beta_margin_std": 836.8815307617188,
|
|
"beta_dpo/beta_used": 1.2469240427017212,
|
|
"beta_dpo/beta_used_raw": 0.10372543334960938,
|
|
"beta_dpo/gap_mean": 358.3047180175781,
|
|
"beta_dpo/gap_std": 334.221923828125,
|
|
"beta_dpo/loss_margin_mean": 370.95501708984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.631424375917768,
|
|
"grad_norm": 50662.0078125,
|
|
"learning_rate": 1.8158425248197928e-07,
|
|
"logits/chosen": -2.4730615615844727,
|
|
"logits/rejected": -2.498109817504883,
|
|
"loss": 15.693,
|
|
"step": 430
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4051775336265564,
|
|
"beta_dpo/beta_margin_grad_std": 0.07288450002670288,
|
|
"beta_dpo/beta_margin_mean": 0.3952929973602295,
|
|
"beta_dpo/beta_margin_std": 0.3199489414691925,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.4916090965270996,
|
|
"beta_dpo/gap_mean": 365.2705383300781,
|
|
"beta_dpo/gap_std": 334.9329833984375,
|
|
"beta_dpo/loss_margin_mean": 395.29296875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6328928046989721,
|
|
"grad_norm": 24.509140014648438,
|
|
"learning_rate": 1.8035081928995788e-07,
|
|
"logits/chosen": -2.456411361694336,
|
|
"logits/rejected": -2.4808506965637207,
|
|
"loss": 1.1022,
|
|
"step": 431
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41120636463165283,
|
|
"beta_dpo/beta_margin_grad_std": 0.066596120595932,
|
|
"beta_dpo/beta_margin_mean": 0.36704325675964355,
|
|
"beta_dpo/beta_margin_std": 0.28722265362739563,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.3621997833251953,
|
|
"beta_dpo/gap_mean": 362.98553466796875,
|
|
"beta_dpo/gap_std": 325.4151611328125,
|
|
"beta_dpo/loss_margin_mean": 367.0432434082031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6343612334801763,
|
|
"grad_norm": 25.281761169433594,
|
|
"learning_rate": 1.791192214186223e-07,
|
|
"logits/chosen": -2.482973098754883,
|
|
"logits/rejected": -2.493764877319336,
|
|
"loss": 1.1166,
|
|
"step": 432
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.1224658489227295,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2876777946949005,
|
|
"beta_dpo/beta_margin_grad_std": 0.292548805475235,
|
|
"beta_dpo/beta_margin_mean": 841.04052734375,
|
|
"beta_dpo/beta_margin_std": 1226.0413818359375,
|
|
"beta_dpo/beta_used": 2.1224658489227295,
|
|
"beta_dpo/beta_used_raw": 0.5382475852966309,
|
|
"beta_dpo/gap_mean": 363.06561279296875,
|
|
"beta_dpo/gap_std": 322.05609130859375,
|
|
"beta_dpo/loss_margin_mean": 364.42138671875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6358296622613803,
|
|
"grad_norm": 59117.21484375,
|
|
"learning_rate": 1.7788949132172193e-07,
|
|
"logits/chosen": -2.4168553352355957,
|
|
"logits/rejected": -2.4499008655548096,
|
|
"loss": 13.9108,
|
|
"step": 433
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4250311851501465,
|
|
"beta_dpo/beta_margin_grad_std": 0.07213795185089111,
|
|
"beta_dpo/beta_margin_mean": 0.317569375038147,
|
|
"beta_dpo/beta_margin_std": 0.3578694462776184,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.1827993392944336,
|
|
"beta_dpo/gap_mean": 360.7991943359375,
|
|
"beta_dpo/gap_std": 328.626220703125,
|
|
"beta_dpo/loss_margin_mean": 317.5693664550781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6372980910425844,
|
|
"grad_norm": 21.91067123413086,
|
|
"learning_rate": 1.7666166140378853e-07,
|
|
"logits/chosen": -2.5135700702667236,
|
|
"logits/rejected": -2.504258155822754,
|
|
"loss": 1.1136,
|
|
"step": 434
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6958755850791931,
|
|
"beta_dpo/beta_margin_grad_mean": -0.046905096620321274,
|
|
"beta_dpo/beta_margin_grad_std": 0.2113645374774933,
|
|
"beta_dpo/beta_margin_mean": 274.05462646484375,
|
|
"beta_dpo/beta_margin_std": 276.3533935546875,
|
|
"beta_dpo/beta_used": 0.6958755850791931,
|
|
"beta_dpo/beta_used_raw": 0.6958755850791931,
|
|
"beta_dpo/gap_mean": 360.8138427734375,
|
|
"beta_dpo/gap_std": 327.732177734375,
|
|
"beta_dpo/loss_margin_mean": 387.7349548339844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6387665198237885,
|
|
"grad_norm": 10641.4619140625,
|
|
"learning_rate": 1.7543576401928218e-07,
|
|
"logits/chosen": -2.479780673980713,
|
|
"logits/rejected": -2.4990386962890625,
|
|
"loss": 14.9256,
|
|
"step": 435
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.1166325807571411,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22967371344566345,
|
|
"beta_dpo/beta_margin_grad_std": 0.2383747398853302,
|
|
"beta_dpo/beta_margin_mean": 423.3056335449219,
|
|
"beta_dpo/beta_margin_std": 537.5963134765625,
|
|
"beta_dpo/beta_used": 1.1166325807571411,
|
|
"beta_dpo/beta_used_raw": -0.7592837810516357,
|
|
"beta_dpo/gap_mean": 359.64361572265625,
|
|
"beta_dpo/gap_std": 311.7923583984375,
|
|
"beta_dpo/loss_margin_mean": 337.6617736816406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6402349486049926,
|
|
"grad_norm": 35.73384094238281,
|
|
"learning_rate": 1.742118314717391e-07,
|
|
"logits/chosen": -2.4742934703826904,
|
|
"logits/rejected": -2.4709324836730957,
|
|
"loss": 0.5629,
|
|
"step": 436
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.42722299695014954,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26437872648239136,
|
|
"beta_dpo/beta_margin_grad_std": 0.2714844048023224,
|
|
"beta_dpo/beta_margin_mean": 150.46022033691406,
|
|
"beta_dpo/beta_margin_std": 217.8881378173828,
|
|
"beta_dpo/beta_used": 0.42722299695014954,
|
|
"beta_dpo/beta_used_raw": 0.32361477613449097,
|
|
"beta_dpo/gap_mean": 351.33404541015625,
|
|
"beta_dpo/gap_std": 304.4229431152344,
|
|
"beta_dpo/loss_margin_mean": 308.41162109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6417033773861968,
|
|
"grad_norm": 7131.47998046875,
|
|
"learning_rate": 1.7298989601292036e-07,
|
|
"logits/chosen": -2.4637017250061035,
|
|
"logits/rejected": -2.454047203063965,
|
|
"loss": 1.8349,
|
|
"step": 437
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4128943979740143,
|
|
"beta_dpo/beta_margin_grad_std": 0.06244545429944992,
|
|
"beta_dpo/beta_margin_mean": 0.35861438512802124,
|
|
"beta_dpo/beta_margin_std": 0.26432162523269653,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8020297884941101,
|
|
"beta_dpo/gap_mean": 347.9464111328125,
|
|
"beta_dpo/gap_std": 297.93682861328125,
|
|
"beta_dpo/loss_margin_mean": 358.6143798828125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6431718061674009,
|
|
"grad_norm": 25.952686309814453,
|
|
"learning_rate": 1.7176998984196144e-07,
|
|
"logits/chosen": -2.4292826652526855,
|
|
"logits/rejected": -2.4409079551696777,
|
|
"loss": 1.0941,
|
|
"step": 438
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.42735329270362854,
|
|
"beta_dpo/beta_margin_grad_std": 0.0616798959672451,
|
|
"beta_dpo/beta_margin_mean": 0.2987552881240845,
|
|
"beta_dpo/beta_margin_std": 0.26535582542419434,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.115419864654541,
|
|
"beta_dpo/gap_mean": 343.51715087890625,
|
|
"beta_dpo/gap_std": 292.8160400390625,
|
|
"beta_dpo/loss_margin_mean": 298.7552490234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.644640234948605,
|
|
"grad_norm": 27.892019271850586,
|
|
"learning_rate": 1.7055214510452458e-07,
|
|
"logits/chosen": -2.435819149017334,
|
|
"logits/rejected": -2.4549179077148438,
|
|
"loss": 1.1119,
|
|
"step": 439
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4165194034576416,
|
|
"beta_dpo/beta_margin_grad_std": 0.07028567790985107,
|
|
"beta_dpo/beta_margin_mean": 0.34535932540893555,
|
|
"beta_dpo/beta_margin_std": 0.2985744774341583,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.8327579498291016,
|
|
"beta_dpo/gap_mean": 340.54168701171875,
|
|
"beta_dpo/gap_std": 291.77471923828125,
|
|
"beta_dpo/loss_margin_mean": 345.3592834472656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6461086637298091,
|
|
"grad_norm": 34.85447692871094,
|
|
"learning_rate": 1.6933639389195134e-07,
|
|
"logits/chosen": -2.4840898513793945,
|
|
"logits/rejected": -2.5059878826141357,
|
|
"loss": 1.12,
|
|
"step": 440
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4149302840232849,
|
|
"beta_dpo/beta_margin_grad_std": 0.06381641328334808,
|
|
"beta_dpo/beta_margin_mean": 0.3505040109157562,
|
|
"beta_dpo/beta_margin_std": 0.2713123559951782,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8756400346755981,
|
|
"beta_dpo/gap_mean": 340.91888427734375,
|
|
"beta_dpo/gap_std": 290.773681640625,
|
|
"beta_dpo/loss_margin_mean": 350.50396728515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6475770925110133,
|
|
"grad_norm": 23.49580192565918,
|
|
"learning_rate": 1.681227682404166e-07,
|
|
"logits/chosen": -2.4471378326416016,
|
|
"logits/rejected": -2.469273805618286,
|
|
"loss": 1.0976,
|
|
"step": 441
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.011472702026367,
|
|
"beta_dpo/beta_margin_grad_mean": -0.06538188457489014,
|
|
"beta_dpo/beta_margin_grad_std": 0.24239790439605713,
|
|
"beta_dpo/beta_margin_mean": 782.205322265625,
|
|
"beta_dpo/beta_margin_std": 524.3056030273438,
|
|
"beta_dpo/beta_used": 2.011472702026367,
|
|
"beta_dpo/beta_used_raw": 2.011472702026367,
|
|
"beta_dpo/gap_mean": 348.65679931640625,
|
|
"beta_dpo/gap_std": 282.76776123046875,
|
|
"beta_dpo/loss_margin_mean": 388.67510986328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6490455212922174,
|
|
"grad_norm": 25934.462890625,
|
|
"learning_rate": 1.669113001300851e-07,
|
|
"logits/chosen": -2.450514316558838,
|
|
"logits/rejected": -2.4609298706054688,
|
|
"loss": 6.0554,
|
|
"step": 442
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.42931288480758667,
|
|
"beta_dpo/beta_margin_grad_std": 0.07746709883213043,
|
|
"beta_dpo/beta_margin_mean": 0.2938655614852905,
|
|
"beta_dpo/beta_margin_std": 0.33218199014663696,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.1781232357025146,
|
|
"beta_dpo/gap_mean": 344.43426513671875,
|
|
"beta_dpo/gap_std": 289.60614013671875,
|
|
"beta_dpo/loss_margin_mean": 293.86553955078125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6505139500734214,
|
|
"grad_norm": 22.906946182250977,
|
|
"learning_rate": 1.6570202148426815e-07,
|
|
"logits/chosen": -2.47257661819458,
|
|
"logits/rejected": -2.4808216094970703,
|
|
"loss": 1.1169,
|
|
"step": 443
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8307179808616638,
|
|
"beta_dpo/beta_margin_grad_mean": -0.20021703839302063,
|
|
"beta_dpo/beta_margin_grad_std": 0.20530231297016144,
|
|
"beta_dpo/beta_margin_mean": 372.78521728515625,
|
|
"beta_dpo/beta_margin_std": 491.99072265625,
|
|
"beta_dpo/beta_used": 0.8307179808616638,
|
|
"beta_dpo/beta_used_raw": 0.622127115726471,
|
|
"beta_dpo/gap_mean": 353.92706298828125,
|
|
"beta_dpo/gap_std": 291.21728515625,
|
|
"beta_dpo/loss_margin_mean": 429.93499755859375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6519823788546255,
|
|
"grad_norm": 21.445575714111328,
|
|
"learning_rate": 1.6449496416858282e-07,
|
|
"logits/chosen": -2.4195876121520996,
|
|
"logits/rejected": -2.456974983215332,
|
|
"loss": 0.5403,
|
|
"step": 444
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6511656641960144,
|
|
"beta_dpo/beta_margin_grad_mean": -0.292864054441452,
|
|
"beta_dpo/beta_margin_grad_std": 0.29051047563552856,
|
|
"beta_dpo/beta_margin_mean": 210.04498291015625,
|
|
"beta_dpo/beta_margin_std": 451.1952819824219,
|
|
"beta_dpo/beta_used": 0.6511656641960144,
|
|
"beta_dpo/beta_used_raw": -0.10653191804885864,
|
|
"beta_dpo/gap_mean": 353.89892578125,
|
|
"beta_dpo/gap_std": 299.44140625,
|
|
"beta_dpo/loss_margin_mean": 332.807861328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6534508076358296,
|
|
"grad_norm": 12001.302734375,
|
|
"learning_rate": 1.6329015999011182e-07,
|
|
"logits/chosen": -2.465177059173584,
|
|
"logits/rejected": -2.4750633239746094,
|
|
"loss": 23.2322,
|
|
"step": 445
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5344843864440918,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2088708132505417,
|
|
"beta_dpo/beta_margin_grad_std": 0.21212056279182434,
|
|
"beta_dpo/beta_margin_mean": 277.230224609375,
|
|
"beta_dpo/beta_margin_std": 368.5066223144531,
|
|
"beta_dpo/beta_used": 0.5344843864440918,
|
|
"beta_dpo/beta_used_raw": -0.5909568667411804,
|
|
"beta_dpo/gap_mean": 359.41986083984375,
|
|
"beta_dpo/gap_std": 302.03582763671875,
|
|
"beta_dpo/loss_margin_mean": 429.4493713378906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6549192364170338,
|
|
"grad_norm": 36.90497970581055,
|
|
"learning_rate": 1.6208764069656578e-07,
|
|
"logits/chosen": -2.422713041305542,
|
|
"logits/rejected": -2.4601097106933594,
|
|
"loss": 0.5524,
|
|
"step": 446
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4114223122596741,
|
|
"beta_dpo/beta_margin_grad_std": 0.07890864461660385,
|
|
"beta_dpo/beta_margin_mean": 0.3711691200733185,
|
|
"beta_dpo/beta_margin_std": 0.34903499484062195,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.839432716369629,
|
|
"beta_dpo/gap_mean": 369.88690185546875,
|
|
"beta_dpo/gap_std": 312.835205078125,
|
|
"beta_dpo/loss_margin_mean": 371.16912841796875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6563876651982379,
|
|
"grad_norm": 23.881223678588867,
|
|
"learning_rate": 1.608874379754465e-07,
|
|
"logits/chosen": -2.469653606414795,
|
|
"logits/rejected": -2.506166458129883,
|
|
"loss": 1.1173,
|
|
"step": 447
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9949462413787842,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22382524609565735,
|
|
"beta_dpo/beta_margin_grad_std": 0.23410984873771667,
|
|
"beta_dpo/beta_margin_mean": 429.1389465332031,
|
|
"beta_dpo/beta_margin_std": 562.9327392578125,
|
|
"beta_dpo/beta_used": 0.9949462413787842,
|
|
"beta_dpo/beta_used_raw": -2.4666709899902344,
|
|
"beta_dpo/gap_mean": 369.296142578125,
|
|
"beta_dpo/gap_std": 308.22808837890625,
|
|
"beta_dpo/loss_margin_mean": 389.3961181640625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.657856093979442,
|
|
"grad_norm": 19.88318634033203,
|
|
"learning_rate": 1.5968958345321177e-07,
|
|
"logits/chosen": -2.43017578125,
|
|
"logits/rejected": -2.461536407470703,
|
|
"loss": 0.5812,
|
|
"step": 448
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.169326066970825,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2861442565917969,
|
|
"beta_dpo/beta_margin_grad_std": 0.2906011939048767,
|
|
"beta_dpo/beta_margin_mean": 866.29150390625,
|
|
"beta_dpo/beta_margin_std": 1363.672119140625,
|
|
"beta_dpo/beta_used": 2.169326066970825,
|
|
"beta_dpo/beta_used_raw": 1.4625353813171387,
|
|
"beta_dpo/gap_mean": 373.68212890625,
|
|
"beta_dpo/gap_std": 307.01202392578125,
|
|
"beta_dpo/loss_margin_mean": 371.7033996582031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6593245227606461,
|
|
"grad_norm": 50823.05078125,
|
|
"learning_rate": 1.584941086944423e-07,
|
|
"logits/chosen": -2.4476749897003174,
|
|
"logits/rejected": -2.4613916873931885,
|
|
"loss": 54.4296,
|
|
"step": 449
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.47364342212677,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21337755024433136,
|
|
"beta_dpo/beta_margin_grad_std": 0.2242012321949005,
|
|
"beta_dpo/beta_margin_mean": 637.6218872070312,
|
|
"beta_dpo/beta_margin_std": 821.07470703125,
|
|
"beta_dpo/beta_used": 1.47364342212677,
|
|
"beta_dpo/beta_used_raw": 1.1894080638885498,
|
|
"beta_dpo/gap_mean": 380.8287353515625,
|
|
"beta_dpo/gap_std": 297.178955078125,
|
|
"beta_dpo/loss_margin_mean": 431.9281311035156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6607929515418502,
|
|
"grad_norm": 20.199665069580078,
|
|
"learning_rate": 1.573010452010098e-07,
|
|
"logits/chosen": -2.45393705368042,
|
|
"logits/rejected": -2.5022172927856445,
|
|
"loss": 0.5291,
|
|
"step": 450
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.6201918125152588,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2785087823867798,
|
|
"beta_dpo/beta_margin_grad_std": 0.283831387758255,
|
|
"beta_dpo/beta_margin_mean": 560.2830810546875,
|
|
"beta_dpo/beta_margin_std": 958.1199340820312,
|
|
"beta_dpo/beta_used": 1.6201918125152588,
|
|
"beta_dpo/beta_used_raw": -0.5515909194946289,
|
|
"beta_dpo/gap_mean": 371.70465087890625,
|
|
"beta_dpo/gap_std": 297.33319091796875,
|
|
"beta_dpo/loss_margin_mean": 312.59564208984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6622613803230544,
|
|
"grad_norm": 25548.22265625,
|
|
"learning_rate": 1.5611042441124687e-07,
|
|
"logits/chosen": -2.4564037322998047,
|
|
"logits/rejected": -2.468679428100586,
|
|
"loss": 14.8356,
|
|
"step": 451
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4107057452201843,
|
|
"beta_dpo/beta_margin_grad_std": 0.06496448814868927,
|
|
"beta_dpo/beta_margin_mean": 0.36832571029663086,
|
|
"beta_dpo/beta_margin_std": 0.27488240599632263,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.28033387660980225,
|
|
"beta_dpo/gap_mean": 369.5523376464844,
|
|
"beta_dpo/gap_std": 295.28765869140625,
|
|
"beta_dpo/loss_margin_mean": 368.3257141113281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6637298091042585,
|
|
"grad_norm": 38.57242202758789,
|
|
"learning_rate": 1.549222776991186e-07,
|
|
"logits/chosen": -2.464397668838501,
|
|
"logits/rejected": -2.5154190063476562,
|
|
"loss": 1.0722,
|
|
"step": 452
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.47413191199302673,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23885342478752136,
|
|
"beta_dpo/beta_margin_grad_std": 0.25014886260032654,
|
|
"beta_dpo/beta_margin_mean": 179.6550750732422,
|
|
"beta_dpo/beta_margin_std": 257.9681396484375,
|
|
"beta_dpo/beta_used": 0.47413191199302673,
|
|
"beta_dpo/beta_used_raw": -0.7867467999458313,
|
|
"beta_dpo/gap_mean": 369.88800048828125,
|
|
"beta_dpo/gap_std": 293.82562255859375,
|
|
"beta_dpo/loss_margin_mean": 364.253173828125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6651982378854625,
|
|
"grad_norm": 7813.66796875,
|
|
"learning_rate": 1.5373663637339584e-07,
|
|
"logits/chosen": -2.4826607704162598,
|
|
"logits/rejected": -2.501310348510742,
|
|
"loss": 9.4541,
|
|
"step": 453
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6042494773864746,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2383243590593338,
|
|
"beta_dpo/beta_margin_grad_std": 0.25166648626327515,
|
|
"beta_dpo/beta_margin_mean": 243.75340270996094,
|
|
"beta_dpo/beta_margin_std": 361.9959716796875,
|
|
"beta_dpo/beta_used": 0.6042494773864746,
|
|
"beta_dpo/beta_used_raw": -0.8156985640525818,
|
|
"beta_dpo/gap_mean": 371.50396728515625,
|
|
"beta_dpo/gap_std": 297.9644775390625,
|
|
"beta_dpo/loss_margin_mean": 380.0973815917969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6666666666666666,
|
|
"grad_norm": 7821.49853515625,
|
|
"learning_rate": 1.5255353167683017e-07,
|
|
"logits/chosen": -2.459460735321045,
|
|
"logits/rejected": -2.4877805709838867,
|
|
"loss": 12.3928,
|
|
"step": 454
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.558521568775177,
|
|
"beta_dpo/beta_margin_grad_mean": -0.229967400431633,
|
|
"beta_dpo/beta_margin_grad_std": 0.24533946812152863,
|
|
"beta_dpo/beta_margin_mean": 263.99224853515625,
|
|
"beta_dpo/beta_margin_std": 370.77044677734375,
|
|
"beta_dpo/beta_used": 0.558521568775177,
|
|
"beta_dpo/beta_used_raw": 0.07860961556434631,
|
|
"beta_dpo/gap_mean": 381.37921142578125,
|
|
"beta_dpo/gap_std": 305.9591064453125,
|
|
"beta_dpo/loss_margin_mean": 450.9393615722656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6681350954478708,
|
|
"grad_norm": 5878.1142578125,
|
|
"learning_rate": 1.5137299478533064e-07,
|
|
"logits/chosen": -2.476715087890625,
|
|
"logits/rejected": -2.544955253601074,
|
|
"loss": 4.0629,
|
|
"step": 455
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.401821494102478,
|
|
"beta_dpo/beta_margin_grad_std": 0.07234375923871994,
|
|
"beta_dpo/beta_margin_mean": 0.4079440236091614,
|
|
"beta_dpo/beta_margin_std": 0.31001952290534973,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.8649553060531616,
|
|
"beta_dpo/gap_mean": 387.3846435546875,
|
|
"beta_dpo/gap_std": 308.45947265625,
|
|
"beta_dpo/loss_margin_mean": 407.9440002441406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6696035242290749,
|
|
"grad_norm": 28.53789520263672,
|
|
"learning_rate": 1.5019505680714232e-07,
|
|
"logits/chosen": -2.4815030097961426,
|
|
"logits/rejected": -2.5309150218963623,
|
|
"loss": 1.0786,
|
|
"step": 456
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.10005475580692291,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25547903776168823,
|
|
"beta_dpo/beta_margin_grad_std": 0.26753029227256775,
|
|
"beta_dpo/beta_margin_mean": 35.68285369873047,
|
|
"beta_dpo/beta_margin_std": 50.96905517578125,
|
|
"beta_dpo/beta_used": 0.10005475580692291,
|
|
"beta_dpo/beta_used_raw": -0.9987051486968994,
|
|
"beta_dpo/gap_mean": 384.96356201171875,
|
|
"beta_dpo/gap_std": 305.0810546875,
|
|
"beta_dpo/loss_margin_mean": 348.1621398925781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.671071953010279,
|
|
"grad_norm": 23.370983123779297,
|
|
"learning_rate": 1.4901974878202627e-07,
|
|
"logits/chosen": -2.480140209197998,
|
|
"logits/rejected": -2.486316680908203,
|
|
"loss": 0.5419,
|
|
"step": 457
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4038095474243164,
|
|
"beta_dpo/beta_margin_grad_std": 0.06412005424499512,
|
|
"beta_dpo/beta_margin_mean": 0.3973011076450348,
|
|
"beta_dpo/beta_margin_std": 0.2737996578216553,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0926374197006226,
|
|
"beta_dpo/gap_mean": 384.5351257324219,
|
|
"beta_dpo/gap_std": 302.91986083984375,
|
|
"beta_dpo/loss_margin_mean": 397.30108642578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6725403817914831,
|
|
"grad_norm": 28.026363372802734,
|
|
"learning_rate": 1.4784710168044212e-07,
|
|
"logits/chosen": -2.47013783454895,
|
|
"logits/rejected": -2.502309799194336,
|
|
"loss": 1.0673,
|
|
"step": 458
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4167326092720032,
|
|
"beta_dpo/beta_margin_grad_std": 0.06269286572933197,
|
|
"beta_dpo/beta_margin_mean": 0.3419567346572876,
|
|
"beta_dpo/beta_margin_std": 0.26270684599876404,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.917838454246521,
|
|
"beta_dpo/gap_mean": 380.58148193359375,
|
|
"beta_dpo/gap_std": 296.5938720703125,
|
|
"beta_dpo/loss_margin_mean": 341.9566955566406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6740088105726872,
|
|
"grad_norm": 31.750755310058594,
|
|
"learning_rate": 1.466771464027316e-07,
|
|
"logits/chosen": -2.46304988861084,
|
|
"logits/rejected": -2.501253128051758,
|
|
"loss": 1.0655,
|
|
"step": 459
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.6334787607192993,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22203302383422852,
|
|
"beta_dpo/beta_margin_grad_std": 0.23113414645195007,
|
|
"beta_dpo/beta_margin_mean": 714.5479736328125,
|
|
"beta_dpo/beta_margin_std": 986.8130493164062,
|
|
"beta_dpo/beta_used": 1.6334787607192993,
|
|
"beta_dpo/beta_used_raw": -0.6699074506759644,
|
|
"beta_dpo/gap_mean": 378.25775146484375,
|
|
"beta_dpo/gap_std": 291.2427978515625,
|
|
"beta_dpo/loss_margin_mean": 398.70556640625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6754772393538914,
|
|
"grad_norm": 17868.126953125,
|
|
"learning_rate": 1.4550991377830423e-07,
|
|
"logits/chosen": -2.5045523643493652,
|
|
"logits/rejected": -2.5729987621307373,
|
|
"loss": 4.6404,
|
|
"step": 460
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8932273983955383,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2543882429599762,
|
|
"beta_dpo/beta_margin_grad_std": 0.2651003301143646,
|
|
"beta_dpo/beta_margin_mean": 307.47015380859375,
|
|
"beta_dpo/beta_margin_std": 430.9241027832031,
|
|
"beta_dpo/beta_used": 0.8932273983955383,
|
|
"beta_dpo/beta_used_raw": 0.08712393045425415,
|
|
"beta_dpo/gap_mean": 376.22406005859375,
|
|
"beta_dpo/gap_std": 286.36328125,
|
|
"beta_dpo/loss_margin_mean": 346.59515380859375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6769456681350955,
|
|
"grad_norm": 32094.228515625,
|
|
"learning_rate": 1.4434543456482518e-07,
|
|
"logits/chosen": -2.499390125274658,
|
|
"logits/rejected": -2.5347375869750977,
|
|
"loss": 0.7427,
|
|
"step": 461
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.418192595243454,
|
|
"beta_dpo/beta_margin_grad_std": 0.07179278135299683,
|
|
"beta_dpo/beta_margin_mean": 0.3387749493122101,
|
|
"beta_dpo/beta_margin_std": 0.30487060546875,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.4746923446655273,
|
|
"beta_dpo/gap_mean": 369.9292907714844,
|
|
"beta_dpo/gap_std": 287.8791809082031,
|
|
"beta_dpo/loss_margin_mean": 338.7749328613281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6784140969162996,
|
|
"grad_norm": 23.766319274902344,
|
|
"learning_rate": 1.4318373944740484e-07,
|
|
"logits/chosen": -2.5215415954589844,
|
|
"logits/rejected": -2.5495944023132324,
|
|
"loss": 1.1155,
|
|
"step": 462
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6219136714935303,
|
|
"beta_dpo/beta_margin_grad_mean": -0.20181849598884583,
|
|
"beta_dpo/beta_margin_grad_std": 0.2130599319934845,
|
|
"beta_dpo/beta_margin_mean": 281.9193420410156,
|
|
"beta_dpo/beta_margin_std": 356.6446228027344,
|
|
"beta_dpo/beta_used": 0.6219136714935303,
|
|
"beta_dpo/beta_used_raw": -1.7789533138275146,
|
|
"beta_dpo/gap_mean": 376.6002197265625,
|
|
"beta_dpo/gap_std": 303.69659423828125,
|
|
"beta_dpo/loss_margin_mean": 434.07806396484375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6798825256975036,
|
|
"grad_norm": 18.568052291870117,
|
|
"learning_rate": 1.4202485903778976e-07,
|
|
"logits/chosen": -2.4930672645568848,
|
|
"logits/rejected": -2.5247983932495117,
|
|
"loss": 0.5632,
|
|
"step": 463
|
|
},
|
|
{
|
|
"beta_dpo/beta": 5.593118667602539,
|
|
"beta_dpo/beta_margin_grad_mean": -0.078125,
|
|
"beta_dpo/beta_margin_grad_std": 0.2683681845664978,
|
|
"beta_dpo/beta_margin_mean": 2755.906005859375,
|
|
"beta_dpo/beta_margin_std": 1905.5625,
|
|
"beta_dpo/beta_used": 5.593118667602539,
|
|
"beta_dpo/beta_used_raw": 5.593118667602539,
|
|
"beta_dpo/gap_mean": 396.98193359375,
|
|
"beta_dpo/gap_std": 305.0847473144531,
|
|
"beta_dpo/loss_margin_mean": 491.942138671875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6813509544787077,
|
|
"grad_norm": 58260.62890625,
|
|
"learning_rate": 1.4086882387355658e-07,
|
|
"logits/chosen": -2.4916696548461914,
|
|
"logits/rejected": -2.550224781036377,
|
|
"loss": 9.9301,
|
|
"step": 464
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.9029580354690552,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24977104365825653,
|
|
"beta_dpo/beta_margin_grad_std": 0.2622261047363281,
|
|
"beta_dpo/beta_margin_mean": 899.8712768554688,
|
|
"beta_dpo/beta_margin_std": 1295.013916015625,
|
|
"beta_dpo/beta_used": 1.9029580354690552,
|
|
"beta_dpo/beta_used_raw": 1.2743040323257446,
|
|
"beta_dpo/gap_mean": 404.0155029296875,
|
|
"beta_dpo/gap_std": 305.983154296875,
|
|
"beta_dpo/loss_margin_mean": 430.5644226074219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6828193832599119,
|
|
"grad_norm": 30087.87109375,
|
|
"learning_rate": 1.3971566441730714e-07,
|
|
"logits/chosen": -2.516110897064209,
|
|
"logits/rejected": -2.569148540496826,
|
|
"loss": 0.7493,
|
|
"step": 465
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.402956485748291,
|
|
"beta_dpo/beta_margin_grad_std": 0.07183395326137543,
|
|
"beta_dpo/beta_margin_mean": 0.40361106395721436,
|
|
"beta_dpo/beta_margin_std": 0.3109658658504486,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.1695146560668945,
|
|
"beta_dpo/gap_mean": 405.3373107910156,
|
|
"beta_dpo/gap_std": 308.2467041015625,
|
|
"beta_dpo/loss_margin_mean": 403.6110534667969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.684287812041116,
|
|
"grad_norm": 22.92670249938965,
|
|
"learning_rate": 1.3856541105586545e-07,
|
|
"logits/chosen": -2.4996213912963867,
|
|
"logits/rejected": -2.5383553504943848,
|
|
"loss": 1.0607,
|
|
"step": 466
|
|
},
|
|
{
|
|
"beta_dpo/beta": 4.274341583251953,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27562493085861206,
|
|
"beta_dpo/beta_margin_grad_std": 0.2861670255661011,
|
|
"beta_dpo/beta_margin_mean": 2275.166259765625,
|
|
"beta_dpo/beta_margin_std": 4076.872802734375,
|
|
"beta_dpo/beta_used": 4.274341583251953,
|
|
"beta_dpo/beta_used_raw": 2.634734630584717,
|
|
"beta_dpo/gap_mean": 413.9611511230469,
|
|
"beta_dpo/gap_std": 339.21044921875,
|
|
"beta_dpo/loss_margin_mean": 418.0201110839844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6857562408223201,
|
|
"grad_norm": 106529.4140625,
|
|
"learning_rate": 1.3741809409947729e-07,
|
|
"logits/chosen": -2.5394535064697266,
|
|
"logits/rejected": -2.5817573070526123,
|
|
"loss": 2.1242,
|
|
"step": 467
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4308552742004395,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2482784539461136,
|
|
"beta_dpo/beta_margin_grad_std": 0.2650541663169861,
|
|
"beta_dpo/beta_margin_mean": 655.9033203125,
|
|
"beta_dpo/beta_margin_std": 1054.0318603515625,
|
|
"beta_dpo/beta_used": 1.4308552742004395,
|
|
"beta_dpo/beta_used_raw": 0.6839801669120789,
|
|
"beta_dpo/gap_mean": 410.94439697265625,
|
|
"beta_dpo/gap_std": 351.870361328125,
|
|
"beta_dpo/loss_margin_mean": 434.7200012207031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6872246696035242,
|
|
"grad_norm": 32999.328125,
|
|
"learning_rate": 1.362737437810114e-07,
|
|
"logits/chosen": -2.5492682456970215,
|
|
"logits/rejected": -2.587491750717163,
|
|
"loss": 27.315,
|
|
"step": 468
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.8693115711212158,
|
|
"beta_dpo/beta_margin_grad_mean": -0.046875,
|
|
"beta_dpo/beta_margin_grad_std": 0.21137107908725739,
|
|
"beta_dpo/beta_margin_mean": 848.8661499023438,
|
|
"beta_dpo/beta_margin_std": 818.3424072265625,
|
|
"beta_dpo/beta_used": 1.8693115711212158,
|
|
"beta_dpo/beta_used_raw": 1.8693115711212158,
|
|
"beta_dpo/gap_mean": 419.2552490234375,
|
|
"beta_dpo/gap_std": 346.93157958984375,
|
|
"beta_dpo/loss_margin_mean": 452.9244384765625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6886930983847284,
|
|
"grad_norm": 8297.3203125,
|
|
"learning_rate": 1.351323902551631e-07,
|
|
"logits/chosen": -2.5490849018096924,
|
|
"logits/rejected": -2.5901732444763184,
|
|
"loss": 0.8058,
|
|
"step": 469
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3999681770801544,
|
|
"beta_dpo/beta_margin_grad_std": 0.07137025147676468,
|
|
"beta_dpo/beta_margin_mean": 0.4155758321285248,
|
|
"beta_dpo/beta_margin_std": 0.30809733271598816,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.152496337890625,
|
|
"beta_dpo/gap_mean": 421.21563720703125,
|
|
"beta_dpo/gap_std": 342.18878173828125,
|
|
"beta_dpo/loss_margin_mean": 415.5758361816406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6901615271659325,
|
|
"grad_norm": 20.7974853515625,
|
|
"learning_rate": 1.339940635976592e-07,
|
|
"logits/chosen": -2.4830265045166016,
|
|
"logits/rejected": -2.5244758129119873,
|
|
"loss": 1.0437,
|
|
"step": 470
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.478458046913147,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23011885583400726,
|
|
"beta_dpo/beta_margin_grad_std": 0.23751112818717957,
|
|
"beta_dpo/beta_margin_mean": 747.5370483398438,
|
|
"beta_dpo/beta_margin_std": 983.1398315429688,
|
|
"beta_dpo/beta_used": 1.478458046913147,
|
|
"beta_dpo/beta_used_raw": -1.9127386808395386,
|
|
"beta_dpo/gap_mean": 421.38037109375,
|
|
"beta_dpo/gap_std": 329.10821533203125,
|
|
"beta_dpo/loss_margin_mean": 398.04986572265625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6916299559471366,
|
|
"grad_norm": 19.789304733276367,
|
|
"learning_rate": 1.3285879380446563e-07,
|
|
"logits/chosen": -2.537778377532959,
|
|
"logits/rejected": -2.556882858276367,
|
|
"loss": 0.5604,
|
|
"step": 471
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.560537576675415,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2782280147075653,
|
|
"beta_dpo/beta_margin_grad_std": 0.28962796926498413,
|
|
"beta_dpo/beta_margin_mean": 254.7558135986328,
|
|
"beta_dpo/beta_margin_std": 468.81280517578125,
|
|
"beta_dpo/beta_used": 0.560537576675415,
|
|
"beta_dpo/beta_used_raw": -2.741248846054077,
|
|
"beta_dpo/gap_mean": 418.555908203125,
|
|
"beta_dpo/gap_std": 343.5157165527344,
|
|
"beta_dpo/loss_margin_mean": 437.1629943847656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6930983847283406,
|
|
"grad_norm": 11883.9248046875,
|
|
"learning_rate": 1.317266107909975e-07,
|
|
"logits/chosen": -2.5896711349487305,
|
|
"logits/rejected": -2.6116137504577637,
|
|
"loss": 14.0955,
|
|
"step": 472
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.42513933777809143,
|
|
"beta_dpo/beta_margin_grad_std": 0.07835263758897781,
|
|
"beta_dpo/beta_margin_mean": 0.3119952082633972,
|
|
"beta_dpo/beta_margin_std": 0.3434739410877228,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.782205581665039,
|
|
"beta_dpo/gap_mean": 403.55072021484375,
|
|
"beta_dpo/gap_std": 345.77349853515625,
|
|
"beta_dpo/loss_margin_mean": 311.9952087402344,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6945668135095447,
|
|
"grad_norm": 28.633747100830078,
|
|
"learning_rate": 1.3059754439133002e-07,
|
|
"logits/chosen": -2.514747381210327,
|
|
"logits/rejected": -2.5092692375183105,
|
|
"loss": 1.1064,
|
|
"step": 473
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.020935971289873123,
|
|
"beta_dpo/beta_margin_grad_mean": -0.08262869715690613,
|
|
"beta_dpo/beta_margin_grad_std": 0.23250208795070648,
|
|
"beta_dpo/beta_margin_mean": 8.29855728149414,
|
|
"beta_dpo/beta_margin_std": 6.226876258850098,
|
|
"beta_dpo/beta_used": 0.020935971289873123,
|
|
"beta_dpo/beta_used_raw": 0.020935971289873123,
|
|
"beta_dpo/gap_mean": 397.82635498046875,
|
|
"beta_dpo/gap_std": 337.72735595703125,
|
|
"beta_dpo/loss_margin_mean": 396.6646728515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6960352422907489,
|
|
"grad_norm": 188.52386474609375,
|
|
"learning_rate": 1.2947162435741277e-07,
|
|
"logits/chosen": -2.512279987335205,
|
|
"logits/rejected": -2.5492238998413086,
|
|
"loss": 0.13,
|
|
"step": 474
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4154300093650818,
|
|
"beta_dpo/beta_margin_grad_std": 0.0788368210196495,
|
|
"beta_dpo/beta_margin_mean": 0.3546939194202423,
|
|
"beta_dpo/beta_margin_std": 0.35067981481552124,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.2628698348999023,
|
|
"beta_dpo/gap_mean": 390.2703857421875,
|
|
"beta_dpo/gap_std": 338.8818359375,
|
|
"beta_dpo/loss_margin_mean": 354.69390869140625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.697503671071953,
|
|
"grad_norm": 24.54264259338379,
|
|
"learning_rate": 1.2834888035828596e-07,
|
|
"logits/chosen": -2.537545680999756,
|
|
"logits/rejected": -2.5808987617492676,
|
|
"loss": 1.0798,
|
|
"step": 475
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41025692224502563,
|
|
"beta_dpo/beta_margin_grad_std": 0.0702584832906723,
|
|
"beta_dpo/beta_margin_mean": 0.37235116958618164,
|
|
"beta_dpo/beta_margin_std": 0.3026776611804962,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.833922863006592,
|
|
"beta_dpo/gap_mean": 388.89324951171875,
|
|
"beta_dpo/gap_std": 336.2122497558594,
|
|
"beta_dpo/loss_margin_mean": 372.3511657714844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.6989720998531571,
|
|
"grad_norm": 33.014305114746094,
|
|
"learning_rate": 1.2722934197929802e-07,
|
|
"logits/chosen": -2.5620553493499756,
|
|
"logits/rejected": -2.5980238914489746,
|
|
"loss": 1.0909,
|
|
"step": 476
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2425551414489746,
|
|
"beta_dpo/beta_margin_grad_mean": -0.28471827507019043,
|
|
"beta_dpo/beta_margin_grad_std": 0.29025498032569885,
|
|
"beta_dpo/beta_margin_mean": 494.6365661621094,
|
|
"beta_dpo/beta_margin_std": 732.710693359375,
|
|
"beta_dpo/beta_used": 1.2425551414489746,
|
|
"beta_dpo/beta_used_raw": 0.43174052238464355,
|
|
"beta_dpo/gap_mean": 384.2132568359375,
|
|
"beta_dpo/gap_std": 325.43865966796875,
|
|
"beta_dpo/loss_margin_mean": 377.49932861328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7004405286343612,
|
|
"grad_norm": 9603.7666015625,
|
|
"learning_rate": 1.2611303872132631e-07,
|
|
"logits/chosen": -2.513644218444824,
|
|
"logits/rejected": -2.5187504291534424,
|
|
"loss": 4.2004,
|
|
"step": 477
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4097580313682556,
|
|
"beta_dpo/beta_margin_grad_std": 0.06587263941764832,
|
|
"beta_dpo/beta_margin_mean": 0.3733612298965454,
|
|
"beta_dpo/beta_margin_std": 0.28482842445373535,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.8485658168792725,
|
|
"beta_dpo/gap_mean": 381.9994812011719,
|
|
"beta_dpo/gap_std": 319.3729248046875,
|
|
"beta_dpo/loss_margin_mean": 373.3612060546875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7019089574155654,
|
|
"grad_norm": 30.411699295043945,
|
|
"learning_rate": 1.2500000000000005e-07,
|
|
"logits/chosen": -2.530264139175415,
|
|
"logits/rejected": -2.573782205581665,
|
|
"loss": 1.094,
|
|
"step": 478
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9422978758811951,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2220228761434555,
|
|
"beta_dpo/beta_margin_grad_std": 0.23495404422283173,
|
|
"beta_dpo/beta_margin_mean": 474.04681396484375,
|
|
"beta_dpo/beta_margin_std": 649.1632080078125,
|
|
"beta_dpo/beta_used": 0.9422978758811951,
|
|
"beta_dpo/beta_used_raw": -1.9218789339065552,
|
|
"beta_dpo/gap_mean": 387.78704833984375,
|
|
"beta_dpo/gap_std": 326.0636291503906,
|
|
"beta_dpo/loss_margin_mean": 436.54107666015625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7033773861967695,
|
|
"grad_norm": 22.1717472076416,
|
|
"learning_rate": 1.2389025514492456e-07,
|
|
"logits/chosen": -2.5045852661132812,
|
|
"logits/rejected": -2.568833112716675,
|
|
"loss": 0.5685,
|
|
"step": 479
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2717182636260986,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2628205120563507,
|
|
"beta_dpo/beta_margin_grad_std": 0.26975834369659424,
|
|
"beta_dpo/beta_margin_mean": 558.4786376953125,
|
|
"beta_dpo/beta_margin_std": 861.1541748046875,
|
|
"beta_dpo/beta_used": 1.2717182636260986,
|
|
"beta_dpo/beta_used_raw": -0.5218298435211182,
|
|
"beta_dpo/gap_mean": 385.14337158203125,
|
|
"beta_dpo/gap_std": 321.2519836425781,
|
|
"beta_dpo/loss_margin_mean": 358.7318420410156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7048458149779736,
|
|
"grad_norm": 34886.80859375,
|
|
"learning_rate": 1.227838333989088e-07,
|
|
"logits/chosen": -2.5319626331329346,
|
|
"logits/rejected": -2.5537118911743164,
|
|
"loss": 41.8083,
|
|
"step": 480
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40268126130104065,
|
|
"beta_dpo/beta_margin_grad_std": 0.08109249919652939,
|
|
"beta_dpo/beta_margin_mean": 0.4087076485157013,
|
|
"beta_dpo/beta_margin_std": 0.3610389828681946,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.2213141918182373,
|
|
"beta_dpo/gap_mean": 391.42706298828125,
|
|
"beta_dpo/gap_std": 332.341552734375,
|
|
"beta_dpo/loss_margin_mean": 408.7076416015625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7063142437591777,
|
|
"grad_norm": 25.161544799804688,
|
|
"learning_rate": 1.2168076391719489e-07,
|
|
"logits/chosen": -2.5190436840057373,
|
|
"logits/rejected": -2.566678285598755,
|
|
"loss": 1.0755,
|
|
"step": 481
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4173520803451538,
|
|
"beta_dpo/beta_margin_grad_std": 0.07615692913532257,
|
|
"beta_dpo/beta_margin_mean": 0.34368330240249634,
|
|
"beta_dpo/beta_margin_std": 0.32845625281333923,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.8063464164733887,
|
|
"beta_dpo/gap_mean": 385.44921875,
|
|
"beta_dpo/gap_std": 331.2301025390625,
|
|
"beta_dpo/loss_margin_mean": 343.68328857421875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7077826725403817,
|
|
"grad_norm": 26.447629928588867,
|
|
"learning_rate": 1.2058107576668938e-07,
|
|
"logits/chosen": -2.5346310138702393,
|
|
"logits/rejected": -2.5641980171203613,
|
|
"loss": 1.0925,
|
|
"step": 482
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39801958203315735,
|
|
"beta_dpo/beta_margin_grad_std": 0.08198197931051254,
|
|
"beta_dpo/beta_margin_mean": 0.4265040457248688,
|
|
"beta_dpo/beta_margin_std": 0.35432857275009155,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8292474746704102,
|
|
"beta_dpo/gap_mean": 391.0637512207031,
|
|
"beta_dpo/gap_std": 335.705322265625,
|
|
"beta_dpo/loss_margin_mean": 426.5040283203125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7092511013215859,
|
|
"grad_norm": 27.4080810546875,
|
|
"learning_rate": 1.194847979251979e-07,
|
|
"logits/chosen": -2.523871898651123,
|
|
"logits/rejected": -2.5619630813598633,
|
|
"loss": 1.0621,
|
|
"step": 483
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4196456968784332,
|
|
"beta_dpo/beta_margin_grad_std": 0.08170211315155029,
|
|
"beta_dpo/beta_margin_mean": 0.33604294061660767,
|
|
"beta_dpo/beta_margin_std": 0.3569084405899048,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.1099014282226562,
|
|
"beta_dpo/gap_mean": 378.6564636230469,
|
|
"beta_dpo/gap_std": 334.283935546875,
|
|
"beta_dpo/loss_margin_mean": 336.04290771484375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.71071953010279,
|
|
"grad_norm": 27.6585636138916,
|
|
"learning_rate": 1.1839195928066101e-07,
|
|
"logits/chosen": -2.5312819480895996,
|
|
"logits/rejected": -2.561755657196045,
|
|
"loss": 1.1056,
|
|
"step": 484
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.6319665908813477,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12443296611309052,
|
|
"beta_dpo/beta_margin_grad_std": 0.3292010724544525,
|
|
"beta_dpo/beta_margin_mean": 740.8789672851562,
|
|
"beta_dpo/beta_margin_std": 888.0823364257812,
|
|
"beta_dpo/beta_used": 1.6319665908813477,
|
|
"beta_dpo/beta_used_raw": 1.6319665908813477,
|
|
"beta_dpo/gap_mean": 391.543212890625,
|
|
"beta_dpo/gap_std": 351.833740234375,
|
|
"beta_dpo/loss_margin_mean": 453.990478515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7121879588839941,
|
|
"grad_norm": 43631.0625,
|
|
"learning_rate": 1.1730258863039347e-07,
|
|
"logits/chosen": -2.533259153366089,
|
|
"logits/rejected": -2.598179340362549,
|
|
"loss": 48.0713,
|
|
"step": 485
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40132319927215576,
|
|
"beta_dpo/beta_margin_grad_std": 0.08668551594018936,
|
|
"beta_dpo/beta_margin_mean": 0.41604912281036377,
|
|
"beta_dpo/beta_margin_std": 0.3828808665275574,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.8486566543579102,
|
|
"beta_dpo/gap_mean": 397.93218994140625,
|
|
"beta_dpo/gap_std": 359.19091796875,
|
|
"beta_dpo/loss_margin_mean": 416.0491027832031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7136563876651982,
|
|
"grad_norm": 32.61040115356445,
|
|
"learning_rate": 1.1621671468032493e-07,
|
|
"logits/chosen": -2.5248947143554688,
|
|
"logits/rejected": -2.5619006156921387,
|
|
"loss": 1.0617,
|
|
"step": 486
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.366377353668213,
|
|
"beta_dpo/beta_margin_grad_mean": -0.078125,
|
|
"beta_dpo/beta_margin_grad_std": 0.2683681845664978,
|
|
"beta_dpo/beta_margin_mean": 1427.7965087890625,
|
|
"beta_dpo/beta_margin_std": 970.3147583007812,
|
|
"beta_dpo/beta_used": 3.366377353668213,
|
|
"beta_dpo/beta_used_raw": 3.366377353668213,
|
|
"beta_dpo/gap_mean": 402.7051696777344,
|
|
"beta_dpo/gap_std": 350.80084228515625,
|
|
"beta_dpo/loss_margin_mean": 424.222900390625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7151248164464024,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1.1513436604424378e-07,
|
|
"logits/chosen": -2.5087289810180664,
|
|
"logits/rejected": -2.566622257232666,
|
|
"loss": 0.0,
|
|
"step": 487
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41341128945350647,
|
|
"beta_dpo/beta_margin_grad_std": 0.07426728308200836,
|
|
"beta_dpo/beta_margin_mean": 0.3592289388179779,
|
|
"beta_dpo/beta_margin_std": 0.314900279045105,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.6400840282440186,
|
|
"beta_dpo/gap_mean": 395.1749267578125,
|
|
"beta_dpo/gap_std": 344.27288818359375,
|
|
"beta_dpo/loss_margin_mean": 359.2289123535156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7165932452276065,
|
|
"grad_norm": 28.298213958740234,
|
|
"learning_rate": 1.1405557124304335e-07,
|
|
"logits/chosen": -2.5287461280822754,
|
|
"logits/rejected": -2.5617752075195312,
|
|
"loss": 1.1019,
|
|
"step": 488
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41154804825782776,
|
|
"beta_dpo/beta_margin_grad_std": 0.06765101850032806,
|
|
"beta_dpo/beta_margin_mean": 0.3656145930290222,
|
|
"beta_dpo/beta_margin_std": 0.2879979908466339,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.769493579864502,
|
|
"beta_dpo/gap_mean": 391.515625,
|
|
"beta_dpo/gap_std": 335.818359375,
|
|
"beta_dpo/loss_margin_mean": 365.6145935058594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7180616740088106,
|
|
"grad_norm": 24.00615692138672,
|
|
"learning_rate": 1.1298035870396985e-07,
|
|
"logits/chosen": -2.5616259574890137,
|
|
"logits/rejected": -2.594806432723999,
|
|
"loss": 1.1112,
|
|
"step": 489
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4056692123413086,
|
|
"beta_dpo/beta_margin_grad_std": 0.10516832768917084,
|
|
"beta_dpo/beta_margin_mean": 0.40864163637161255,
|
|
"beta_dpo/beta_margin_std": 0.48387810587882996,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -5.209046363830566,
|
|
"beta_dpo/gap_mean": 392.92205810546875,
|
|
"beta_dpo/gap_std": 358.7939453125,
|
|
"beta_dpo/loss_margin_mean": 408.6416015625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7195301027900147,
|
|
"grad_norm": 26.91084098815918,
|
|
"learning_rate": 1.1190875675987355e-07,
|
|
"logits/chosen": -2.6055257320404053,
|
|
"logits/rejected": -2.6725831031799316,
|
|
"loss": 1.1313,
|
|
"step": 490
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41814425587654114,
|
|
"beta_dpo/beta_margin_grad_std": 0.07073578983545303,
|
|
"beta_dpo/beta_margin_mean": 0.3384140133857727,
|
|
"beta_dpo/beta_margin_std": 0.2998770475387573,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0293617248535156,
|
|
"beta_dpo/gap_mean": 385.5333251953125,
|
|
"beta_dpo/gap_std": 351.7672119140625,
|
|
"beta_dpo/loss_margin_mean": 338.41400146484375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7209985315712188,
|
|
"grad_norm": 27.26293182373047,
|
|
"learning_rate": 1.1084079364846241e-07,
|
|
"logits/chosen": -2.5373287200927734,
|
|
"logits/rejected": -2.5686709880828857,
|
|
"loss": 1.0702,
|
|
"step": 491
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.420479416847229,
|
|
"beta_dpo/beta_margin_grad_std": 0.072646863758564,
|
|
"beta_dpo/beta_margin_mean": 0.3299318253993988,
|
|
"beta_dpo/beta_margin_std": 0.31079521775245667,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.52386474609375,
|
|
"beta_dpo/gap_mean": 376.2781677246094,
|
|
"beta_dpo/gap_std": 346.640380859375,
|
|
"beta_dpo/loss_margin_mean": 329.93182373046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7224669603524229,
|
|
"grad_norm": 26.98293685913086,
|
|
"learning_rate": 1.097764975115576e-07,
|
|
"logits/chosen": -2.5452327728271484,
|
|
"logits/rejected": -2.5773203372955322,
|
|
"loss": 1.1255,
|
|
"step": 492
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.19389446079730988,
|
|
"beta_dpo/beta_margin_grad_mean": -0.30418387055397034,
|
|
"beta_dpo/beta_margin_grad_std": 0.3022761046886444,
|
|
"beta_dpo/beta_margin_mean": 63.774085998535156,
|
|
"beta_dpo/beta_margin_std": 104.5694351196289,
|
|
"beta_dpo/beta_used": 0.19389446079730988,
|
|
"beta_dpo/beta_used_raw": -0.4551483392715454,
|
|
"beta_dpo/gap_mean": 366.77899169921875,
|
|
"beta_dpo/gap_std": 340.6854553222656,
|
|
"beta_dpo/loss_margin_mean": 331.5319519042969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.723935389133627,
|
|
"grad_norm": 4613.84375,
|
|
"learning_rate": 1.0871589639435203e-07,
|
|
"logits/chosen": -2.5896708965301514,
|
|
"logits/rejected": -2.6086065769195557,
|
|
"loss": 3.1786,
|
|
"step": 493
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.5493483543395996,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09375,
|
|
"beta_dpo/beta_margin_grad_std": 0.2914806008338928,
|
|
"beta_dpo/beta_margin_mean": 1450.2022705078125,
|
|
"beta_dpo/beta_margin_std": 1306.51318359375,
|
|
"beta_dpo/beta_used": 3.5493483543395996,
|
|
"beta_dpo/beta_used_raw": 3.5493483543395996,
|
|
"beta_dpo/gap_mean": 373.76806640625,
|
|
"beta_dpo/gap_std": 341.1934814453125,
|
|
"beta_dpo/loss_margin_mean": 413.81048583984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7254038179148311,
|
|
"grad_norm": 1680.6007080078125,
|
|
"learning_rate": 1.0765901824467166e-07,
|
|
"logits/chosen": -2.5582916736602783,
|
|
"logits/rejected": -2.609689474105835,
|
|
"loss": 0.9905,
|
|
"step": 494
|
|
},
|
|
{
|
|
"beta_dpo/beta": 4.066619873046875,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21904002130031586,
|
|
"beta_dpo/beta_margin_grad_std": 0.23028062283992767,
|
|
"beta_dpo/beta_margin_mean": 1956.0379638671875,
|
|
"beta_dpo/beta_margin_std": 2361.65234375,
|
|
"beta_dpo/beta_used": 4.066619873046875,
|
|
"beta_dpo/beta_used_raw": 3.1382861137390137,
|
|
"beta_dpo/gap_mean": 380.26708984375,
|
|
"beta_dpo/gap_std": 335.47235107421875,
|
|
"beta_dpo/loss_margin_mean": 433.75335693359375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7268722466960352,
|
|
"grad_norm": 22.30237579345703,
|
|
"learning_rate": 1.0660589091223854e-07,
|
|
"logits/chosen": -2.5691702365875244,
|
|
"logits/rejected": -2.6163458824157715,
|
|
"loss": 0.5429,
|
|
"step": 495
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.409112811088562,
|
|
"beta_dpo/beta_margin_grad_std": 0.06478798389434814,
|
|
"beta_dpo/beta_margin_mean": 0.37426888942718506,
|
|
"beta_dpo/beta_margin_std": 0.27219101786613464,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.4662165641784668,
|
|
"beta_dpo/gap_mean": 383.7269287109375,
|
|
"beta_dpo/gap_std": 323.4496765136719,
|
|
"beta_dpo/loss_margin_mean": 374.26885986328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7283406754772394,
|
|
"grad_norm": 27.432092666625977,
|
|
"learning_rate": 1.0555654214793722e-07,
|
|
"logits/chosen": -2.5857880115509033,
|
|
"logits/rejected": -2.6112606525421143,
|
|
"loss": 1.0738,
|
|
"step": 496
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.4332939088344574,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2696186900138855,
|
|
"beta_dpo/beta_margin_grad_std": 0.27868661284446716,
|
|
"beta_dpo/beta_margin_mean": 175.2734375,
|
|
"beta_dpo/beta_margin_std": 240.9627685546875,
|
|
"beta_dpo/beta_used": 0.4332939088344574,
|
|
"beta_dpo/beta_used_raw": -1.5341229438781738,
|
|
"beta_dpo/gap_mean": 384.20770263671875,
|
|
"beta_dpo/gap_std": 314.7350158691406,
|
|
"beta_dpo/loss_margin_mean": 381.3740234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7298091042584435,
|
|
"grad_norm": 12841.9072265625,
|
|
"learning_rate": 1.0451099960308374e-07,
|
|
"logits/chosen": -2.5560591220855713,
|
|
"logits/rejected": -2.60593318939209,
|
|
"loss": 3.4472,
|
|
"step": 497
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41190412640571594,
|
|
"beta_dpo/beta_margin_grad_std": 0.08984668552875519,
|
|
"beta_dpo/beta_margin_mean": 0.37027707695961,
|
|
"beta_dpo/beta_margin_std": 0.39214304089546204,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.132285237312317,
|
|
"beta_dpo/gap_mean": 380.50390625,
|
|
"beta_dpo/gap_std": 325.7791442871094,
|
|
"beta_dpo/loss_margin_mean": 370.2770690917969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7312775330396476,
|
|
"grad_norm": 28.40172004699707,
|
|
"learning_rate": 1.0346929082869641e-07,
|
|
"logits/chosen": -2.5905566215515137,
|
|
"logits/rejected": -2.6150288581848145,
|
|
"loss": 1.0755,
|
|
"step": 498
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.1377925872802734,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25142624974250793,
|
|
"beta_dpo/beta_margin_grad_std": 0.2667090594768524,
|
|
"beta_dpo/beta_margin_mean": 950.3165283203125,
|
|
"beta_dpo/beta_margin_std": 1399.9261474609375,
|
|
"beta_dpo/beta_used": 2.1377925872802734,
|
|
"beta_dpo/beta_used_raw": -0.2245168685913086,
|
|
"beta_dpo/gap_mean": 386.89794921875,
|
|
"beta_dpo/gap_std": 335.3507080078125,
|
|
"beta_dpo/loss_margin_mean": 415.10186767578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7327459618208517,
|
|
"grad_norm": 27508.541015625,
|
|
"learning_rate": 1.0243144327477013e-07,
|
|
"logits/chosen": -2.5723462104797363,
|
|
"logits/rejected": -2.636953830718994,
|
|
"loss": 25.3095,
|
|
"step": 499
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8406103849411011,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21563133597373962,
|
|
"beta_dpo/beta_margin_grad_std": 0.27382031083106995,
|
|
"beta_dpo/beta_margin_mean": 361.0441589355469,
|
|
"beta_dpo/beta_margin_std": 578.6198120117188,
|
|
"beta_dpo/beta_used": 0.8406103849411011,
|
|
"beta_dpo/beta_used_raw": 0.8406103849411011,
|
|
"beta_dpo/gap_mean": 390.808837890625,
|
|
"beta_dpo/gap_std": 344.6997375488281,
|
|
"beta_dpo/loss_margin_mean": 407.3626403808594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7342143906020558,
|
|
"grad_norm": 12742.4658203125,
|
|
"learning_rate": 1.0139748428955333e-07,
|
|
"logits/chosen": -2.5479164123535156,
|
|
"logits/rejected": -2.6190683841705322,
|
|
"loss": 6.7444,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.7342143906020558,
|
|
"eval_beta_dpo/beta": 0.0010000000474974513,
|
|
"eval_beta_dpo/beta_margin_grad_mean": -0.4440717399120331,
|
|
"eval_beta_dpo/beta_margin_grad_std": 0.09194090217351913,
|
|
"eval_beta_dpo/beta_margin_mean": 0.23392271995544434,
|
|
"eval_beta_dpo/beta_margin_std": 0.3909527361392975,
|
|
"eval_beta_dpo/beta_used": 0.0010000000474974513,
|
|
"eval_beta_dpo/beta_used_raw": -9.295312881469727,
|
|
"eval_beta_dpo/gap_mean": 390.51129150390625,
|
|
"eval_beta_dpo/gap_std": 345.8570556640625,
|
|
"eval_beta_dpo/loss_margin_mean": 233.9227294921875,
|
|
"eval_beta_dpo/mask_keep_frac": 1.0,
|
|
"eval_logits/chosen": -2.6421258449554443,
|
|
"eval_logits/rejected": -2.6563594341278076,
|
|
"eval_loss": 0.602572500705719,
|
|
"eval_runtime": 37.7828,
|
|
"eval_samples_per_second": 61.907,
|
|
"eval_steps_per_second": 1.959,
|
|
"step": 500
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40555235743522644,
|
|
"beta_dpo/beta_margin_grad_std": 0.0859164372086525,
|
|
"beta_dpo/beta_margin_mean": 0.39426204562187195,
|
|
"beta_dpo/beta_margin_std": 0.3646033704280853,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.6879664659500122,
|
|
"beta_dpo/gap_mean": 391.73577880859375,
|
|
"beta_dpo/gap_std": 350.0826416015625,
|
|
"beta_dpo/loss_margin_mean": 394.26202392578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.73568281938326,
|
|
"grad_norm": 30.496671676635742,
|
|
"learning_rate": 1.0036744111882672e-07,
|
|
"logits/chosen": -2.519293785095215,
|
|
"logits/rejected": -2.5676636695861816,
|
|
"loss": 1.0742,
|
|
"step": 501
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4322647154331207,
|
|
"beta_dpo/beta_margin_grad_std": 0.07655175775289536,
|
|
"beta_dpo/beta_margin_mean": 0.27862733602523804,
|
|
"beta_dpo/beta_margin_std": 0.3172290623188019,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.7983059883117676,
|
|
"beta_dpo/gap_mean": 375.173095703125,
|
|
"beta_dpo/gap_std": 344.82647705078125,
|
|
"beta_dpo/loss_margin_mean": 278.6273193359375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.737151248164464,
|
|
"grad_norm": 28.89243507385254,
|
|
"learning_rate": 9.934134090518592e-08,
|
|
"logits/chosen": -2.5925819873809814,
|
|
"logits/rejected": -2.604139804840088,
|
|
"loss": 1.1023,
|
|
"step": 502
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4080732762813568,
|
|
"beta_dpo/beta_margin_grad_std": 0.07349276542663574,
|
|
"beta_dpo/beta_margin_mean": 0.3811602294445038,
|
|
"beta_dpo/beta_margin_std": 0.31262895464897156,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.2302193641662598,
|
|
"beta_dpo/gap_mean": 371.81048583984375,
|
|
"beta_dpo/gap_std": 340.92041015625,
|
|
"beta_dpo/loss_margin_mean": 381.16021728515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7386196769456681,
|
|
"grad_norm": 29.317602157592773,
|
|
"learning_rate": 9.831921068732571e-08,
|
|
"logits/chosen": -2.562480926513672,
|
|
"logits/rejected": -2.6049766540527344,
|
|
"loss": 1.0943,
|
|
"step": 503
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.8556745052337646,
|
|
"beta_dpo/beta_margin_grad_mean": -0.07770384848117828,
|
|
"beta_dpo/beta_margin_grad_std": 0.2669384777545929,
|
|
"beta_dpo/beta_margin_mean": 1844.1893310546875,
|
|
"beta_dpo/beta_margin_std": 1516.7772216796875,
|
|
"beta_dpo/beta_used": 3.8556745052337646,
|
|
"beta_dpo/beta_used_raw": 3.8556745052337646,
|
|
"beta_dpo/gap_mean": 387.630126953125,
|
|
"beta_dpo/gap_std": 345.894775390625,
|
|
"beta_dpo/loss_margin_mean": 475.9586181640625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7400881057268722,
|
|
"grad_norm": 30736.17578125,
|
|
"learning_rate": 9.730107739932805e-08,
|
|
"logits/chosen": -2.5559229850769043,
|
|
"logits/rejected": -2.630270481109619,
|
|
"loss": 0.1445,
|
|
"step": 504
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.16585735976696014,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2709311544895172,
|
|
"beta_dpo/beta_margin_grad_std": 0.27986839413642883,
|
|
"beta_dpo/beta_margin_mean": 50.6553840637207,
|
|
"beta_dpo/beta_margin_std": 122.78081512451172,
|
|
"beta_dpo/beta_used": 0.16585735976696014,
|
|
"beta_dpo/beta_used_raw": 0.008618459105491638,
|
|
"beta_dpo/gap_mean": 381.24481201171875,
|
|
"beta_dpo/gap_std": 361.23394775390625,
|
|
"beta_dpo/loss_margin_mean": 317.9100036621094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7415565345080763,
|
|
"grad_norm": 4374.5302734375,
|
|
"learning_rate": 9.628696786995188e-08,
|
|
"logits/chosen": -2.6019692420959473,
|
|
"logits/rejected": -2.6386215686798096,
|
|
"loss": 5.8766,
|
|
"step": 505
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.0099172592163086,
|
|
"beta_dpo/beta_margin_grad_mean": -0.267021119594574,
|
|
"beta_dpo/beta_margin_grad_std": 0.27634257078170776,
|
|
"beta_dpo/beta_margin_mean": 805.3701171875,
|
|
"beta_dpo/beta_margin_std": 1344.735107421875,
|
|
"beta_dpo/beta_used": 2.0099172592163086,
|
|
"beta_dpo/beta_used_raw": 1.2173224687576294,
|
|
"beta_dpo/gap_mean": 379.2687072753906,
|
|
"beta_dpo/gap_std": 362.82806396484375,
|
|
"beta_dpo/loss_margin_mean": 386.8839416503906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7430249632892805,
|
|
"grad_norm": 4789.49853515625,
|
|
"learning_rate": 9.527690882192635e-08,
|
|
"logits/chosen": -2.577932357788086,
|
|
"logits/rejected": -2.627098560333252,
|
|
"loss": 5.6618,
|
|
"step": 506
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4179416000843048,
|
|
"beta_dpo/beta_margin_grad_std": 0.08015478402376175,
|
|
"beta_dpo/beta_margin_mean": 0.34152480959892273,
|
|
"beta_dpo/beta_margin_std": 0.33954671025276184,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.6754093170166016,
|
|
"beta_dpo/gap_mean": 374.45867919921875,
|
|
"beta_dpo/gap_std": 354.2435607910156,
|
|
"beta_dpo/loss_margin_mean": 341.5248107910156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7444933920704846,
|
|
"grad_norm": 29.211889266967773,
|
|
"learning_rate": 9.427092687124691e-08,
|
|
"logits/chosen": -2.6004366874694824,
|
|
"logits/rejected": -2.6304922103881836,
|
|
"loss": 1.1038,
|
|
"step": 507
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4112531542778015,
|
|
"beta_dpo/beta_margin_grad_std": 0.09117571264505386,
|
|
"beta_dpo/beta_margin_mean": 0.3736902177333832,
|
|
"beta_dpo/beta_margin_std": 0.3952173888683319,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.8720993995666504,
|
|
"beta_dpo/gap_mean": 369.85711669921875,
|
|
"beta_dpo/gap_std": 362.80743408203125,
|
|
"beta_dpo/loss_margin_mean": 373.6902160644531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7459618208516887,
|
|
"grad_norm": 28.369312286376953,
|
|
"learning_rate": 9.326904852647344e-08,
|
|
"logits/chosen": -2.6036999225616455,
|
|
"logits/rejected": -2.642367362976074,
|
|
"loss": 1.1269,
|
|
"step": 508
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41093164682388306,
|
|
"beta_dpo/beta_margin_grad_std": 0.07275278866291046,
|
|
"beta_dpo/beta_margin_mean": 0.36885690689086914,
|
|
"beta_dpo/beta_margin_std": 0.30931705236434937,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.5401596426963806,
|
|
"beta_dpo/gap_mean": 372.418212890625,
|
|
"beta_dpo/gap_std": 355.0517272949219,
|
|
"beta_dpo/loss_margin_mean": 368.8569030761719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7474302496328928,
|
|
"grad_norm": 26.87523651123047,
|
|
"learning_rate": 9.227130018803195e-08,
|
|
"logits/chosen": -2.550929307937622,
|
|
"logits/rejected": -2.5829358100891113,
|
|
"loss": 1.0726,
|
|
"step": 509
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.996654748916626,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21876604855060577,
|
|
"beta_dpo/beta_margin_grad_std": 0.2280726283788681,
|
|
"beta_dpo/beta_margin_mean": 924.012939453125,
|
|
"beta_dpo/beta_margin_std": 1230.9832763671875,
|
|
"beta_dpo/beta_used": 1.996654748916626,
|
|
"beta_dpo/beta_used_raw": 0.9159380197525024,
|
|
"beta_dpo/gap_mean": 381.06939697265625,
|
|
"beta_dpo/gap_std": 340.1996154785156,
|
|
"beta_dpo/loss_margin_mean": 424.53790283203125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.748898678414097,
|
|
"grad_norm": 15.459576606750488,
|
|
"learning_rate": 9.127770814751932e-08,
|
|
"logits/chosen": -2.52266788482666,
|
|
"logits/rejected": -2.6167397499084473,
|
|
"loss": 0.5411,
|
|
"step": 510
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6738762259483337,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26766470074653625,
|
|
"beta_dpo/beta_margin_grad_std": 0.2765524685382843,
|
|
"beta_dpo/beta_margin_mean": 276.3068542480469,
|
|
"beta_dpo/beta_margin_std": 440.9131164550781,
|
|
"beta_dpo/beta_used": 0.6738762259483337,
|
|
"beta_dpo/beta_used_raw": -0.26501893997192383,
|
|
"beta_dpo/gap_mean": 381.2438659667969,
|
|
"beta_dpo/gap_std": 331.12408447265625,
|
|
"beta_dpo/loss_margin_mean": 387.6205139160156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.750367107195301,
|
|
"grad_norm": 6812.96630859375,
|
|
"learning_rate": 9.028829858700973e-08,
|
|
"logits/chosen": -2.5795559883117676,
|
|
"logits/rejected": -2.6349778175354004,
|
|
"loss": 0.6808,
|
|
"step": 511
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.29571211338043213,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21608670055866241,
|
|
"beta_dpo/beta_margin_grad_std": 0.2315717190504074,
|
|
"beta_dpo/beta_margin_mean": 118.46554565429688,
|
|
"beta_dpo/beta_margin_std": 172.0571746826172,
|
|
"beta_dpo/beta_used": 0.29571211338043213,
|
|
"beta_dpo/beta_used_raw": -0.0050897300243377686,
|
|
"beta_dpo/gap_mean": 386.8934020996094,
|
|
"beta_dpo/gap_std": 338.73162841796875,
|
|
"beta_dpo/loss_margin_mean": 408.7341003417969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7518355359765051,
|
|
"grad_norm": 8849.5224609375,
|
|
"learning_rate": 8.930309757836516e-08,
|
|
"logits/chosen": -2.5242481231689453,
|
|
"logits/rejected": -2.5922141075134277,
|
|
"loss": 4.6625,
|
|
"step": 512
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41406044363975525,
|
|
"beta_dpo/beta_margin_grad_std": 0.07585693150758743,
|
|
"beta_dpo/beta_margin_mean": 0.35993170738220215,
|
|
"beta_dpo/beta_margin_std": 0.3399631977081299,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.504025936126709,
|
|
"beta_dpo/gap_mean": 387.1397705078125,
|
|
"beta_dpo/gap_std": 338.95806884765625,
|
|
"beta_dpo/loss_margin_mean": 359.9316711425781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7533039647577092,
|
|
"grad_norm": 31.581235885620117,
|
|
"learning_rate": 8.832213108254863e-08,
|
|
"logits/chosen": -2.545290231704712,
|
|
"logits/rejected": -2.597968578338623,
|
|
"loss": 1.0993,
|
|
"step": 513
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4150505065917969,
|
|
"beta_dpo/beta_margin_grad_std": 0.07808467000722885,
|
|
"beta_dpo/beta_margin_mean": 0.3535368740558624,
|
|
"beta_dpo/beta_margin_std": 0.3357268273830414,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.028075695037842,
|
|
"beta_dpo/gap_mean": 375.724365234375,
|
|
"beta_dpo/gap_std": 334.5943908691406,
|
|
"beta_dpo/loss_margin_mean": 353.536865234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7547723935389133,
|
|
"grad_norm": 32.19744873046875,
|
|
"learning_rate": 8.734542494893954e-08,
|
|
"logits/chosen": -2.555717706680298,
|
|
"logits/rejected": -2.603783369064331,
|
|
"loss": 1.0886,
|
|
"step": 514
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.737210988998413,
|
|
"beta_dpo/beta_margin_grad_mean": -0.14027747511863708,
|
|
"beta_dpo/beta_margin_grad_std": 0.3467850983142853,
|
|
"beta_dpo/beta_margin_mean": 1422.5458984375,
|
|
"beta_dpo/beta_margin_std": 1979.079345703125,
|
|
"beta_dpo/beta_used": 3.737210988998413,
|
|
"beta_dpo/beta_used_raw": 3.737210988998413,
|
|
"beta_dpo/gap_mean": 374.7799987792969,
|
|
"beta_dpo/gap_std": 340.4820556640625,
|
|
"beta_dpo/loss_margin_mean": 349.0910949707031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7562408223201175,
|
|
"grad_norm": 7053.50537109375,
|
|
"learning_rate": 8.637300491465272e-08,
|
|
"logits/chosen": -2.5503506660461426,
|
|
"logits/rejected": -2.6203575134277344,
|
|
"loss": 3.1348,
|
|
"step": 515
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.4134178161621094,
|
|
"beta_dpo/beta_margin_grad_mean": -0.279133677482605,
|
|
"beta_dpo/beta_margin_grad_std": 0.2908896803855896,
|
|
"beta_dpo/beta_margin_mean": 205.34104919433594,
|
|
"beta_dpo/beta_margin_std": 399.5401611328125,
|
|
"beta_dpo/beta_used": 0.4134178161621094,
|
|
"beta_dpo/beta_used_raw": -0.5360516309738159,
|
|
"beta_dpo/gap_mean": 384.8233642578125,
|
|
"beta_dpo/gap_std": 366.90264892578125,
|
|
"beta_dpo/loss_margin_mean": 453.2371520996094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7577092511013216,
|
|
"grad_norm": 11149.91796875,
|
|
"learning_rate": 8.540489660386064e-08,
|
|
"logits/chosen": -2.595862627029419,
|
|
"logits/rejected": -2.688004493713379,
|
|
"loss": 8.3798,
|
|
"step": 516
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.5258046388626099,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2544478476047516,
|
|
"beta_dpo/beta_margin_grad_std": 0.26925018429756165,
|
|
"beta_dpo/beta_margin_mean": 720.7048950195312,
|
|
"beta_dpo/beta_margin_std": 1094.427001953125,
|
|
"beta_dpo/beta_used": 1.5258046388626099,
|
|
"beta_dpo/beta_used_raw": 1.4570624828338623,
|
|
"beta_dpo/gap_mean": 392.5709533691406,
|
|
"beta_dpo/gap_std": 370.2559509277344,
|
|
"beta_dpo/loss_margin_mean": 411.7127685546875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7591776798825257,
|
|
"grad_norm": 21861.8203125,
|
|
"learning_rate": 8.444112552711752e-08,
|
|
"logits/chosen": -2.5828640460968018,
|
|
"logits/rejected": -2.6403748989105225,
|
|
"loss": 18.2512,
|
|
"step": 517
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4097965955734253,
|
|
"beta_dpo/beta_margin_grad_std": 0.0833662748336792,
|
|
"beta_dpo/beta_margin_mean": 0.3765363395214081,
|
|
"beta_dpo/beta_margin_std": 0.3559281527996063,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.791685104370117,
|
|
"beta_dpo/gap_mean": 390.30194091796875,
|
|
"beta_dpo/gap_std": 370.6991882324219,
|
|
"beta_dpo/loss_margin_mean": 376.53631591796875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7606461086637298,
|
|
"grad_norm": 35.12870788574219,
|
|
"learning_rate": 8.348171708068747e-08,
|
|
"logits/chosen": -2.565255880355835,
|
|
"logits/rejected": -2.629263162612915,
|
|
"loss": 1.0891,
|
|
"step": 518
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4017196595668793,
|
|
"beta_dpo/beta_margin_grad_std": 0.0956096351146698,
|
|
"beta_dpo/beta_margin_mean": 0.4176686108112335,
|
|
"beta_dpo/beta_margin_std": 0.4266796112060547,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.772631883621216,
|
|
"beta_dpo/gap_mean": 392.6380615234375,
|
|
"beta_dpo/gap_std": 373.5154113769531,
|
|
"beta_dpo/loss_margin_mean": 417.6685791015625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.762114537444934,
|
|
"grad_norm": 29.76219367980957,
|
|
"learning_rate": 8.25266965458755e-08,
|
|
"logits/chosen": -2.607625961303711,
|
|
"logits/rejected": -2.6605143547058105,
|
|
"loss": 1.0877,
|
|
"step": 519
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.1038464307785034,
|
|
"beta_dpo/beta_margin_grad_mean": -0.07864061743021011,
|
|
"beta_dpo/beta_margin_grad_std": 0.26824915409088135,
|
|
"beta_dpo/beta_margin_mean": 507.0424499511719,
|
|
"beta_dpo/beta_margin_std": 722.092041015625,
|
|
"beta_dpo/beta_used": 1.1038464307785034,
|
|
"beta_dpo/beta_used_raw": 1.1038464307785034,
|
|
"beta_dpo/gap_mean": 399.25439453125,
|
|
"beta_dpo/gap_std": 378.07611083984375,
|
|
"beta_dpo/loss_margin_mean": 422.9557189941406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7635829662261381,
|
|
"grad_norm": 16670.017578125,
|
|
"learning_rate": 8.15760890883607e-08,
|
|
"logits/chosen": -2.5519027709960938,
|
|
"logits/rejected": -2.6170506477355957,
|
|
"loss": 6.8461,
|
|
"step": 520
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2119355201721191,
|
|
"beta_dpo/beta_margin_grad_mean": -0.1504676640033722,
|
|
"beta_dpo/beta_margin_grad_std": 0.35239845514297485,
|
|
"beta_dpo/beta_margin_mean": 429.1769104003906,
|
|
"beta_dpo/beta_margin_std": 439.9748229980469,
|
|
"beta_dpo/beta_used": 1.2119355201721191,
|
|
"beta_dpo/beta_used_raw": 1.2119355201721191,
|
|
"beta_dpo/gap_mean": 391.2038879394531,
|
|
"beta_dpo/gap_std": 371.0228271484375,
|
|
"beta_dpo/loss_margin_mean": 357.31500244140625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7650513950073421,
|
|
"grad_norm": 20681.9140625,
|
|
"learning_rate": 8.062991975753378e-08,
|
|
"logits/chosen": -2.595400333404541,
|
|
"logits/rejected": -2.645524024963379,
|
|
"loss": 13.2254,
|
|
"step": 521
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.42150411009788513,
|
|
"beta_dpo/beta_margin_grad_std": 0.09007870405912399,
|
|
"beta_dpo/beta_margin_mean": 0.3268641233444214,
|
|
"beta_dpo/beta_margin_std": 0.3883510231971741,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.027829647064209,
|
|
"beta_dpo/gap_mean": 379.1689147949219,
|
|
"beta_dpo/gap_std": 368.09771728515625,
|
|
"beta_dpo/loss_margin_mean": 326.8641052246094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7665198237885462,
|
|
"grad_norm": 25.660398483276367,
|
|
"learning_rate": 7.968821348583643e-08,
|
|
"logits/chosen": -2.570929527282715,
|
|
"logits/rejected": -2.6126561164855957,
|
|
"loss": 1.1038,
|
|
"step": 522
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4145914912223816,
|
|
"beta_dpo/beta_margin_grad_std": 0.08707987517118454,
|
|
"beta_dpo/beta_margin_mean": 0.3594570457935333,
|
|
"beta_dpo/beta_margin_std": 0.3804139494895935,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.161890029907227,
|
|
"beta_dpo/gap_mean": 377.38458251953125,
|
|
"beta_dpo/gap_std": 372.1881103515625,
|
|
"beta_dpo/loss_margin_mean": 359.45703125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7679882525697503,
|
|
"grad_norm": 29.40875244140625,
|
|
"learning_rate": 7.875099508810484e-08,
|
|
"logits/chosen": -2.5962986946105957,
|
|
"logits/rejected": -2.658698081970215,
|
|
"loss": 1.1252,
|
|
"step": 523
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.29657244682312,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24990545213222504,
|
|
"beta_dpo/beta_margin_grad_std": 0.2622232139110565,
|
|
"beta_dpo/beta_margin_mean": 990.7221069335938,
|
|
"beta_dpo/beta_margin_std": 1473.548583984375,
|
|
"beta_dpo/beta_used": 2.29657244682312,
|
|
"beta_dpo/beta_used_raw": 1.6613166332244873,
|
|
"beta_dpo/gap_mean": 380.9888916015625,
|
|
"beta_dpo/gap_std": 365.5908203125,
|
|
"beta_dpo/loss_margin_mean": 409.36083984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7694566813509545,
|
|
"grad_norm": 44968.94140625,
|
|
"learning_rate": 7.781828926091535e-08,
|
|
"logits/chosen": -2.635100841522217,
|
|
"logits/rejected": -2.6668195724487305,
|
|
"loss": 25.215,
|
|
"step": 524
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3882511854171753,
|
|
"beta_dpo/beta_margin_grad_std": 0.08840624243021011,
|
|
"beta_dpo/beta_margin_mean": 0.4734336733818054,
|
|
"beta_dpo/beta_margin_std": 0.39495328068733215,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.465731620788574,
|
|
"beta_dpo/gap_mean": 395.54022216796875,
|
|
"beta_dpo/gap_std": 364.6204833984375,
|
|
"beta_dpo/loss_margin_mean": 473.43365478515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7709251101321586,
|
|
"grad_norm": 29.38068199157715,
|
|
"learning_rate": 7.689012058193384e-08,
|
|
"logits/chosen": -2.5895333290100098,
|
|
"logits/rejected": -2.678103446960449,
|
|
"loss": 1.0849,
|
|
"step": 525
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39801308512687683,
|
|
"beta_dpo/beta_margin_grad_std": 0.069210484623909,
|
|
"beta_dpo/beta_margin_mean": 0.42354100942611694,
|
|
"beta_dpo/beta_margin_std": 0.29874640703201294,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.39645516872406,
|
|
"beta_dpo/gap_mean": 402.06280517578125,
|
|
"beta_dpo/gap_std": 358.2450256347656,
|
|
"beta_dpo/loss_margin_mean": 423.5409851074219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7723935389133627,
|
|
"grad_norm": 25.767820358276367,
|
|
"learning_rate": 7.596651350926836e-08,
|
|
"logits/chosen": -2.546776294708252,
|
|
"logits/rejected": -2.6113803386688232,
|
|
"loss": 1.0586,
|
|
"step": 526
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2439494132995605,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2362092137336731,
|
|
"beta_dpo/beta_margin_grad_std": 0.24737012386322021,
|
|
"beta_dpo/beta_margin_mean": 454.62713623046875,
|
|
"beta_dpo/beta_margin_std": 778.6525268554688,
|
|
"beta_dpo/beta_used": 1.2439494132995605,
|
|
"beta_dpo/beta_used_raw": 1.1723434925079346,
|
|
"beta_dpo/gap_mean": 398.12164306640625,
|
|
"beta_dpo/gap_std": 351.50225830078125,
|
|
"beta_dpo/loss_margin_mean": 367.6454772949219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7738619676945668,
|
|
"grad_norm": 16.68294334411621,
|
|
"learning_rate": 7.504749238082414e-08,
|
|
"logits/chosen": -2.6491198539733887,
|
|
"logits/rejected": -2.690948009490967,
|
|
"loss": 0.5226,
|
|
"step": 527
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8667228817939758,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26016074419021606,
|
|
"beta_dpo/beta_margin_grad_std": 0.27118903398513794,
|
|
"beta_dpo/beta_margin_mean": 354.2501220703125,
|
|
"beta_dpo/beta_margin_std": 623.5271606445312,
|
|
"beta_dpo/beta_used": 0.8667228817939758,
|
|
"beta_dpo/beta_used_raw": 0.277756929397583,
|
|
"beta_dpo/gap_mean": 398.49981689453125,
|
|
"beta_dpo/gap_std": 356.4637451171875,
|
|
"beta_dpo/loss_margin_mean": 409.8684997558594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.775330396475771,
|
|
"grad_norm": 1461.6524658203125,
|
|
"learning_rate": 7.413308141366254e-08,
|
|
"logits/chosen": -2.6200203895568848,
|
|
"logits/rejected": -2.6761021614074707,
|
|
"loss": 0.5897,
|
|
"step": 528
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.020879942923784256,
|
|
"beta_dpo/beta_margin_grad_mean": -0.297647625207901,
|
|
"beta_dpo/beta_margin_grad_std": 0.2874828577041626,
|
|
"beta_dpo/beta_margin_mean": 7.774229049682617,
|
|
"beta_dpo/beta_margin_std": 13.420893669128418,
|
|
"beta_dpo/beta_used": 0.020879942923784256,
|
|
"beta_dpo/beta_used_raw": -1.6348989009857178,
|
|
"beta_dpo/gap_mean": 395.13067626953125,
|
|
"beta_dpo/gap_std": 357.2676696777344,
|
|
"beta_dpo/loss_margin_mean": 370.92144775390625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7767988252569751,
|
|
"grad_norm": 690.0531616210938,
|
|
"learning_rate": 7.322330470336313e-08,
|
|
"logits/chosen": -2.5948238372802734,
|
|
"logits/rejected": -2.6651344299316406,
|
|
"loss": 1.5042,
|
|
"step": 529
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4054359197616577,
|
|
"beta_dpo/beta_margin_grad_std": 0.08558761328458786,
|
|
"beta_dpo/beta_margin_mean": 0.39957693219184875,
|
|
"beta_dpo/beta_margin_std": 0.38682180643081665,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.191973090171814,
|
|
"beta_dpo/gap_mean": 393.1609802246094,
|
|
"beta_dpo/gap_std": 356.60321044921875,
|
|
"beta_dpo/loss_margin_mean": 399.576904296875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7782672540381792,
|
|
"grad_norm": 31.403076171875,
|
|
"learning_rate": 7.231818622338822e-08,
|
|
"logits/chosen": -2.56978440284729,
|
|
"logits/rejected": -2.6270089149475098,
|
|
"loss": 1.0674,
|
|
"step": 530
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4135720729827881,
|
|
"beta_dpo/beta_margin_grad_std": 0.06900368630886078,
|
|
"beta_dpo/beta_margin_mean": 0.35677656531333923,
|
|
"beta_dpo/beta_margin_std": 0.29285889863967896,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.2725701332092285,
|
|
"beta_dpo/gap_mean": 389.8656005859375,
|
|
"beta_dpo/gap_std": 350.780517578125,
|
|
"beta_dpo/loss_margin_mean": 356.77655029296875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7797356828193832,
|
|
"grad_norm": 32.01457595825195,
|
|
"learning_rate": 7.141774982445147e-08,
|
|
"logits/chosen": -2.617753028869629,
|
|
"logits/rejected": -2.669069290161133,
|
|
"loss": 1.0808,
|
|
"step": 531
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.38395339250564575,
|
|
"beta_dpo/beta_margin_grad_mean": -0.279400497674942,
|
|
"beta_dpo/beta_margin_grad_std": 0.2882375121116638,
|
|
"beta_dpo/beta_margin_mean": 182.82447814941406,
|
|
"beta_dpo/beta_margin_std": 285.89324951171875,
|
|
"beta_dpo/beta_used": 0.38395339250564575,
|
|
"beta_dpo/beta_used_raw": -0.1769140660762787,
|
|
"beta_dpo/gap_mean": 390.6937255859375,
|
|
"beta_dpo/gap_std": 356.8406066894531,
|
|
"beta_dpo/loss_margin_mean": 376.4180908203125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7812041116005873,
|
|
"grad_norm": 17459.1015625,
|
|
"learning_rate": 7.052201923388953e-08,
|
|
"logits/chosen": -2.6145548820495605,
|
|
"logits/rejected": -2.652617931365967,
|
|
"loss": 15.0837,
|
|
"step": 532
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4183679223060608,
|
|
"beta_dpo/beta_margin_grad_std": 0.08154841512441635,
|
|
"beta_dpo/beta_margin_mean": 0.34022700786590576,
|
|
"beta_dpo/beta_margin_std": 0.35058268904685974,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.292230129241943,
|
|
"beta_dpo/gap_mean": 374.2669677734375,
|
|
"beta_dpo/gap_std": 358.24761962890625,
|
|
"beta_dpo/loss_margin_mean": 340.22698974609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7826725403817915,
|
|
"grad_norm": 28.039445877075195,
|
|
"learning_rate": 6.963101805503646e-08,
|
|
"logits/chosen": -2.640665054321289,
|
|
"logits/rejected": -2.666677474975586,
|
|
"loss": 1.1201,
|
|
"step": 533
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.3452858924865723,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23946087062358856,
|
|
"beta_dpo/beta_margin_grad_std": 0.25099116563796997,
|
|
"beta_dpo/beta_margin_mean": 1004.8426513671875,
|
|
"beta_dpo/beta_margin_std": 1333.9495849609375,
|
|
"beta_dpo/beta_used": 2.3452858924865723,
|
|
"beta_dpo/beta_used_raw": 1.948282241821289,
|
|
"beta_dpo/gap_mean": 380.38348388671875,
|
|
"beta_dpo/gap_std": 344.8291015625,
|
|
"beta_dpo/loss_margin_mean": 386.6946105957031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7841409691629956,
|
|
"grad_norm": 23.792999267578125,
|
|
"learning_rate": 6.874476976660184e-08,
|
|
"logits/chosen": -2.6000583171844482,
|
|
"logits/rejected": -2.6584179401397705,
|
|
"loss": 0.5378,
|
|
"step": 534
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.4307881295681,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2031729370355606,
|
|
"beta_dpo/beta_margin_grad_std": 0.2144741714000702,
|
|
"beta_dpo/beta_margin_mean": 199.35324096679688,
|
|
"beta_dpo/beta_margin_std": 241.94654846191406,
|
|
"beta_dpo/beta_used": 0.4307881295681,
|
|
"beta_dpo/beta_used_raw": -1.0114219188690186,
|
|
"beta_dpo/gap_mean": 384.7388610839844,
|
|
"beta_dpo/gap_std": 346.4287109375,
|
|
"beta_dpo/loss_margin_mean": 431.2344970703125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7856093979441997,
|
|
"grad_norm": 19.05939292907715,
|
|
"learning_rate": 6.786329772205246e-08,
|
|
"logits/chosen": -2.6111888885498047,
|
|
"logits/rejected": -2.6767916679382324,
|
|
"loss": 0.5469,
|
|
"step": 535
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0523020029067993,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2505837380886078,
|
|
"beta_dpo/beta_margin_grad_std": 0.2626131772994995,
|
|
"beta_dpo/beta_margin_mean": 488.0250244140625,
|
|
"beta_dpo/beta_margin_std": 711.1327514648438,
|
|
"beta_dpo/beta_used": 1.0523020029067993,
|
|
"beta_dpo/beta_used_raw": -0.6562622785568237,
|
|
"beta_dpo/gap_mean": 391.91632080078125,
|
|
"beta_dpo/gap_std": 333.91339111328125,
|
|
"beta_dpo/loss_margin_mean": 422.7450256347656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7870778267254038,
|
|
"grad_norm": 29718.033203125,
|
|
"learning_rate": 6.698662514899638e-08,
|
|
"logits/chosen": -2.6316137313842773,
|
|
"logits/rejected": -2.7104220390319824,
|
|
"loss": 17.1121,
|
|
"step": 536
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.34959638118743896,
|
|
"beta_dpo/beta_margin_grad_mean": -0.22656311094760895,
|
|
"beta_dpo/beta_margin_grad_std": 0.2316565066576004,
|
|
"beta_dpo/beta_margin_mean": 147.22267150878906,
|
|
"beta_dpo/beta_margin_std": 205.2282257080078,
|
|
"beta_dpo/beta_used": 0.34959638118743896,
|
|
"beta_dpo/beta_used_raw": -1.9871588945388794,
|
|
"beta_dpo/gap_mean": 391.814208984375,
|
|
"beta_dpo/gap_std": 327.1514892578125,
|
|
"beta_dpo/loss_margin_mean": 381.2959899902344,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.788546255506608,
|
|
"grad_norm": 6313.296875,
|
|
"learning_rate": 6.611477514857114e-08,
|
|
"logits/chosen": -2.599724531173706,
|
|
"logits/rejected": -2.6477694511413574,
|
|
"loss": 8.0099,
|
|
"step": 537
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2048133611679077,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2943846881389618,
|
|
"beta_dpo/beta_margin_grad_std": 0.2998571991920471,
|
|
"beta_dpo/beta_margin_mean": 482.0516662597656,
|
|
"beta_dpo/beta_margin_std": 875.5813598632812,
|
|
"beta_dpo/beta_used": 1.2048133611679077,
|
|
"beta_dpo/beta_used_raw": 1.1537326574325562,
|
|
"beta_dpo/gap_mean": 395.1236572265625,
|
|
"beta_dpo/gap_std": 327.88568115234375,
|
|
"beta_dpo/loss_margin_mean": 404.2270202636719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7900146842878121,
|
|
"grad_norm": 41323.671875,
|
|
"learning_rate": 6.524777069483525e-08,
|
|
"logits/chosen": -2.661776542663574,
|
|
"logits/rejected": -2.7106385231018066,
|
|
"loss": 14.8493,
|
|
"step": 538
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.1724320650100708,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25635582208633423,
|
|
"beta_dpo/beta_margin_grad_std": 0.2684703767299652,
|
|
"beta_dpo/beta_margin_mean": 555.3789672851562,
|
|
"beta_dpo/beta_margin_std": 792.5721435546875,
|
|
"beta_dpo/beta_used": 1.1724320650100708,
|
|
"beta_dpo/beta_used_raw": -1.5928860902786255,
|
|
"beta_dpo/gap_mean": 399.99566650390625,
|
|
"beta_dpo/gap_std": 335.73211669921875,
|
|
"beta_dpo/loss_margin_mean": 406.2193298339844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7914831130690162,
|
|
"grad_norm": 28027.939453125,
|
|
"learning_rate": 6.438563463416221e-08,
|
|
"logits/chosen": -2.6445555686950684,
|
|
"logits/rejected": -2.700587511062622,
|
|
"loss": 4.6975,
|
|
"step": 539
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.8483712673187256,
|
|
"beta_dpo/beta_margin_grad_mean": -0.15959151089191437,
|
|
"beta_dpo/beta_margin_grad_std": 0.36039698123931885,
|
|
"beta_dpo/beta_margin_mean": 1546.208740234375,
|
|
"beta_dpo/beta_margin_std": 2491.53271484375,
|
|
"beta_dpo/beta_used": 2.8483712673187256,
|
|
"beta_dpo/beta_used_raw": 2.8483712673187256,
|
|
"beta_dpo/gap_mean": 402.3629150390625,
|
|
"beta_dpo/gap_std": 344.58099365234375,
|
|
"beta_dpo/loss_margin_mean": 461.46783447265625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7929515418502202,
|
|
"grad_norm": 125367.3359375,
|
|
"learning_rate": 6.352838968463919e-08,
|
|
"logits/chosen": -2.636195659637451,
|
|
"logits/rejected": -2.7234175205230713,
|
|
"loss": 106.2179,
|
|
"step": 540
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.7488507628440857,
|
|
"beta_dpo/beta_margin_grad_mean": -0.09352385252714157,
|
|
"beta_dpo/beta_margin_grad_std": 0.2907818555831909,
|
|
"beta_dpo/beta_margin_mean": 317.4271545410156,
|
|
"beta_dpo/beta_margin_std": 262.6924133300781,
|
|
"beta_dpo/beta_used": 0.7488507628440857,
|
|
"beta_dpo/beta_used_raw": 0.7488507628440857,
|
|
"beta_dpo/gap_mean": 415.33734130859375,
|
|
"beta_dpo/gap_std": 352.0528869628906,
|
|
"beta_dpo/loss_margin_mean": 423.2877502441406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7944199706314243,
|
|
"grad_norm": 10829.640625,
|
|
"learning_rate": 6.267605843546767e-08,
|
|
"logits/chosen": -2.6342930793762207,
|
|
"logits/rejected": -2.6951427459716797,
|
|
"loss": 11.3344,
|
|
"step": 541
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9092043042182922,
|
|
"beta_dpo/beta_margin_grad_mean": -0.20870338380336761,
|
|
"beta_dpo/beta_margin_grad_std": 0.21450397372245789,
|
|
"beta_dpo/beta_margin_mean": 514.0123291015625,
|
|
"beta_dpo/beta_margin_std": 732.8487548828125,
|
|
"beta_dpo/beta_used": 0.9092043042182922,
|
|
"beta_dpo/beta_used_raw": 0.6870288848876953,
|
|
"beta_dpo/gap_mean": 422.68658447265625,
|
|
"beta_dpo/gap_std": 355.011474609375,
|
|
"beta_dpo/loss_margin_mean": 452.62939453125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7958883994126285,
|
|
"grad_norm": 18.31541633605957,
|
|
"learning_rate": 6.182866334636888e-08,
|
|
"logits/chosen": -2.609865188598633,
|
|
"logits/rejected": -2.7070038318634033,
|
|
"loss": 0.5163,
|
|
"step": 542
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4074610471725464,
|
|
"beta_dpo/beta_margin_grad_std": 0.09081309288740158,
|
|
"beta_dpo/beta_margin_mean": 0.3921317458152771,
|
|
"beta_dpo/beta_margin_std": 0.4030529856681824,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.4958415031433105,
|
|
"beta_dpo/gap_mean": 415.68060302734375,
|
|
"beta_dpo/gap_std": 361.8984375,
|
|
"beta_dpo/loss_margin_mean": 392.1317443847656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7973568281938326,
|
|
"grad_norm": 31.554765701293945,
|
|
"learning_rate": 6.098622674699147e-08,
|
|
"logits/chosen": -2.6560518741607666,
|
|
"logits/rejected": -2.747661828994751,
|
|
"loss": 1.0673,
|
|
"step": 543
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39683008193969727,
|
|
"beta_dpo/beta_margin_grad_std": 0.09721162915229797,
|
|
"beta_dpo/beta_margin_mean": 0.4478076100349426,
|
|
"beta_dpo/beta_margin_std": 0.4668709635734558,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.288872003555298,
|
|
"beta_dpo/gap_mean": 416.6801452636719,
|
|
"beta_dpo/gap_std": 379.8859558105469,
|
|
"beta_dpo/loss_margin_mean": 447.8075866699219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.7988252569750367,
|
|
"grad_norm": 30.542661666870117,
|
|
"learning_rate": 6.01487708363232e-08,
|
|
"logits/chosen": -2.613583564758301,
|
|
"logits/rejected": -2.71563458442688,
|
|
"loss": 1.0614,
|
|
"step": 544
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.401055246591568,
|
|
"beta_dpo/beta_margin_grad_std": 0.08540057390928268,
|
|
"beta_dpo/beta_margin_mean": 0.4177744388580322,
|
|
"beta_dpo/beta_margin_std": 0.38359367847442627,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.404287576675415,
|
|
"beta_dpo/gap_mean": 419.52044677734375,
|
|
"beta_dpo/gap_std": 381.7796630859375,
|
|
"beta_dpo/loss_margin_mean": 417.7744140625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8002936857562408,
|
|
"grad_norm": 28.606706619262695,
|
|
"learning_rate": 5.9316317682106294e-08,
|
|
"logits/chosen": -2.57968807220459,
|
|
"logits/rejected": -2.652435302734375,
|
|
"loss": 1.0454,
|
|
"step": 545
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4062955677509308,
|
|
"beta_dpo/beta_margin_grad_std": 0.08303419500589371,
|
|
"beta_dpo/beta_margin_mean": 0.39024630188941956,
|
|
"beta_dpo/beta_margin_std": 0.3554038107395172,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.077838897705078,
|
|
"beta_dpo/gap_mean": 417.3631591796875,
|
|
"beta_dpo/gap_std": 376.4284362792969,
|
|
"beta_dpo/loss_margin_mean": 390.24627685546875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.801762114537445,
|
|
"grad_norm": 27.95216941833496,
|
|
"learning_rate": 5.848888922025552e-08,
|
|
"logits/chosen": -2.5938525199890137,
|
|
"logits/rejected": -2.6582956314086914,
|
|
"loss": 1.0875,
|
|
"step": 546
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.1992263793945312,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2521184980869293,
|
|
"beta_dpo/beta_margin_grad_std": 0.264556348323822,
|
|
"beta_dpo/beta_margin_mean": 901.9774169921875,
|
|
"beta_dpo/beta_margin_std": 1424.427490234375,
|
|
"beta_dpo/beta_used": 2.1992263793945312,
|
|
"beta_dpo/beta_used_raw": 0.5545767545700073,
|
|
"beta_dpo/gap_mean": 409.3026123046875,
|
|
"beta_dpo/gap_std": 369.95867919921875,
|
|
"beta_dpo/loss_margin_mean": 390.19146728515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8032305433186491,
|
|
"grad_norm": 20749.9375,
|
|
"learning_rate": 5.7666507254280265e-08,
|
|
"logits/chosen": -2.629915714263916,
|
|
"logits/rejected": -2.6946840286254883,
|
|
"loss": 15.5121,
|
|
"step": 547
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3962554335594177,
|
|
"beta_dpo/beta_margin_grad_std": 0.07508466392755508,
|
|
"beta_dpo/beta_margin_mean": 0.4344782829284668,
|
|
"beta_dpo/beta_margin_std": 0.3340926766395569,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.715212345123291,
|
|
"beta_dpo/gap_mean": 413.58660888671875,
|
|
"beta_dpo/gap_std": 367.1708984375,
|
|
"beta_dpo/loss_margin_mean": 434.478271484375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8046989720998532,
|
|
"grad_norm": 31.523386001586914,
|
|
"learning_rate": 5.684919345471029e-08,
|
|
"logits/chosen": -2.636038064956665,
|
|
"logits/rejected": -2.7150168418884277,
|
|
"loss": 1.0832,
|
|
"step": 548
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41863563656806946,
|
|
"beta_dpo/beta_margin_grad_std": 0.07815483957529068,
|
|
"beta_dpo/beta_margin_mean": 0.3384191393852234,
|
|
"beta_dpo/beta_margin_std": 0.33370572328567505,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.80178689956665,
|
|
"beta_dpo/gap_mean": 403.1805419921875,
|
|
"beta_dpo/gap_std": 359.09490966796875,
|
|
"beta_dpo/loss_margin_mean": 338.41912841796875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8061674008810573,
|
|
"grad_norm": 27.266035079956055,
|
|
"learning_rate": 5.603696935852426e-08,
|
|
"logits/chosen": -2.6075119972229004,
|
|
"logits/rejected": -2.672619342803955,
|
|
"loss": 1.1066,
|
|
"step": 549
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0455665588378906,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2663308084011078,
|
|
"beta_dpo/beta_margin_grad_std": 0.2776106595993042,
|
|
"beta_dpo/beta_margin_mean": 477.6100769042969,
|
|
"beta_dpo/beta_margin_std": 690.04931640625,
|
|
"beta_dpo/beta_used": 1.0455665588378906,
|
|
"beta_dpo/beta_used_raw": 0.7292245626449585,
|
|
"beta_dpo/gap_mean": 404.16845703125,
|
|
"beta_dpo/gap_std": 352.69635009765625,
|
|
"beta_dpo/loss_margin_mean": 419.5482177734375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8076358296622613,
|
|
"grad_norm": 23331.697265625,
|
|
"learning_rate": 5.5229856368582376e-08,
|
|
"logits/chosen": -2.585789680480957,
|
|
"logits/rejected": -2.6722607612609863,
|
|
"loss": 21.6382,
|
|
"step": 550
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.5201354026794434,
|
|
"beta_dpo/beta_margin_grad_mean": -0.078125,
|
|
"beta_dpo/beta_margin_grad_std": 0.2683681845664978,
|
|
"beta_dpo/beta_margin_mean": 1171.402099609375,
|
|
"beta_dpo/beta_margin_std": 1031.3978271484375,
|
|
"beta_dpo/beta_used": 2.5201354026794434,
|
|
"beta_dpo/beta_used_raw": 2.5201354026794434,
|
|
"beta_dpo/gap_mean": 412.00518798828125,
|
|
"beta_dpo/gap_std": 351.75830078125,
|
|
"beta_dpo/loss_margin_mean": 468.22454833984375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8091042584434655,
|
|
"grad_norm": 50141.44140625,
|
|
"learning_rate": 5.4427875753062734e-08,
|
|
"logits/chosen": -2.585766553878784,
|
|
"logits/rejected": -2.6833677291870117,
|
|
"loss": 40.5927,
|
|
"step": 551
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.441541910171509,
|
|
"beta_dpo/beta_margin_grad_mean": -0.0625,
|
|
"beta_dpo/beta_margin_grad_std": 0.24206146597862244,
|
|
"beta_dpo/beta_margin_mean": 1086.489013671875,
|
|
"beta_dpo/beta_margin_std": 1131.8115234375,
|
|
"beta_dpo/beta_used": 2.441541910171509,
|
|
"beta_dpo/beta_used_raw": 2.441541910171509,
|
|
"beta_dpo/gap_mean": 425.93841552734375,
|
|
"beta_dpo/gap_std": 354.64801025390625,
|
|
"beta_dpo/loss_margin_mean": 469.59014892578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8105726872246696,
|
|
"grad_norm": 5.776141833273076e-18,
|
|
"learning_rate": 5.363104864490034e-08,
|
|
"logits/chosen": -2.6029000282287598,
|
|
"logits/rejected": -2.7190818786621094,
|
|
"loss": 0.0,
|
|
"step": 552
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4171546697616577,
|
|
"beta_dpo/beta_margin_grad_std": 0.08783596754074097,
|
|
"beta_dpo/beta_margin_mean": 0.35056599974632263,
|
|
"beta_dpo/beta_margin_std": 0.3956577479839325,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -5.028049945831299,
|
|
"beta_dpo/gap_mean": 416.62200927734375,
|
|
"beta_dpo/gap_std": 366.66143798828125,
|
|
"beta_dpo/loss_margin_mean": 350.56597900390625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8120411160058737,
|
|
"grad_norm": 38.534873962402344,
|
|
"learning_rate": 5.2839396041230415e-08,
|
|
"logits/chosen": -2.6273856163024902,
|
|
"logits/rejected": -2.6760740280151367,
|
|
"loss": 1.0993,
|
|
"step": 553
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2997931241989136,
|
|
"beta_dpo/beta_margin_grad_mean": -0.109375,
|
|
"beta_dpo/beta_margin_grad_std": 0.3121091425418854,
|
|
"beta_dpo/beta_margin_mean": 508.1010437011719,
|
|
"beta_dpo/beta_margin_std": 419.82977294921875,
|
|
"beta_dpo/beta_used": 1.2997931241989136,
|
|
"beta_dpo/beta_used_raw": 1.2997931241989136,
|
|
"beta_dpo/gap_mean": 408.51751708984375,
|
|
"beta_dpo/gap_std": 352.59368896484375,
|
|
"beta_dpo/loss_margin_mean": 392.65533447265625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8135095447870778,
|
|
"grad_norm": 1126.18505859375,
|
|
"learning_rate": 5.205293880283551e-08,
|
|
"logits/chosen": -2.5999350547790527,
|
|
"logits/rejected": -2.641538143157959,
|
|
"loss": 1.1067,
|
|
"step": 554
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.38637006282806396,
|
|
"beta_dpo/beta_margin_grad_std": 0.08780043572187424,
|
|
"beta_dpo/beta_margin_mean": 0.4845529794692993,
|
|
"beta_dpo/beta_margin_std": 0.4055006504058838,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.4782108068466187,
|
|
"beta_dpo/gap_mean": 418.3115234375,
|
|
"beta_dpo/gap_std": 359.09735107421875,
|
|
"beta_dpo/loss_margin_mean": 484.5529479980469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8149779735682819,
|
|
"grad_norm": 34.80107116699219,
|
|
"learning_rate": 5.127169765359515e-08,
|
|
"logits/chosen": -2.5728185176849365,
|
|
"logits/rejected": -2.6831090450286865,
|
|
"loss": 1.0492,
|
|
"step": 555
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.1738041341304779,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2653172016143799,
|
|
"beta_dpo/beta_margin_grad_std": 0.27693215012550354,
|
|
"beta_dpo/beta_margin_mean": 74.29708862304688,
|
|
"beta_dpo/beta_margin_std": 117.45230102539062,
|
|
"beta_dpo/beta_used": 0.1738041341304779,
|
|
"beta_dpo/beta_used_raw": -0.14080658555030823,
|
|
"beta_dpo/gap_mean": 420.77178955078125,
|
|
"beta_dpo/gap_std": 361.7735595703125,
|
|
"beta_dpo/loss_margin_mean": 409.55499267578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8164464023494861,
|
|
"grad_norm": 9489.583984375,
|
|
"learning_rate": 5.049569317994012e-08,
|
|
"logits/chosen": -2.5879225730895996,
|
|
"logits/rejected": -2.687049388885498,
|
|
"loss": 0.8618,
|
|
"step": 556
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4377208948135376,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2533913552761078,
|
|
"beta_dpo/beta_margin_grad_std": 0.26882457733154297,
|
|
"beta_dpo/beta_margin_mean": 662.1341552734375,
|
|
"beta_dpo/beta_margin_std": 1010.3662109375,
|
|
"beta_dpo/beta_used": 1.4377208948135376,
|
|
"beta_dpo/beta_used_raw": -2.254744052886963,
|
|
"beta_dpo/gap_mean": 420.5530090332031,
|
|
"beta_dpo/gap_std": 363.3053283691406,
|
|
"beta_dpo/loss_margin_mean": 413.83026123046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8179148311306902,
|
|
"grad_norm": 39387.4296875,
|
|
"learning_rate": 4.9724945830310144e-08,
|
|
"logits/chosen": -2.614232063293457,
|
|
"logits/rejected": -2.698105812072754,
|
|
"loss": 1.1225,
|
|
"step": 557
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.3792362213134766,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2171439528465271,
|
|
"beta_dpo/beta_margin_grad_std": 0.23049600422382355,
|
|
"beta_dpo/beta_margin_mean": 1273.4932861328125,
|
|
"beta_dpo/beta_margin_std": 1737.8341064453125,
|
|
"beta_dpo/beta_used": 2.3792362213134766,
|
|
"beta_dpo/beta_used_raw": 2.243035316467285,
|
|
"beta_dpo/gap_mean": 433.89971923828125,
|
|
"beta_dpo/gap_std": 363.2912902832031,
|
|
"beta_dpo/loss_margin_mean": 526.4646606445312,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8193832599118943,
|
|
"grad_norm": 42691.87109375,
|
|
"learning_rate": 4.8959475914614554e-08,
|
|
"logits/chosen": -2.583843231201172,
|
|
"logits/rejected": -2.682178497314453,
|
|
"loss": 0.5778,
|
|
"step": 558
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.7704762816429138,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26277267932891846,
|
|
"beta_dpo/beta_margin_grad_std": 0.2761915922164917,
|
|
"beta_dpo/beta_margin_mean": 319.6239013671875,
|
|
"beta_dpo/beta_margin_std": 505.88055419921875,
|
|
"beta_dpo/beta_used": 0.7704762816429138,
|
|
"beta_dpo/beta_used_raw": -1.242068886756897,
|
|
"beta_dpo/gap_mean": 435.0184326171875,
|
|
"beta_dpo/gap_std": 358.98150634765625,
|
|
"beta_dpo/loss_margin_mean": 415.4324645996094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8208516886930984,
|
|
"grad_norm": 10755.6123046875,
|
|
"learning_rate": 4.8199303603697614e-08,
|
|
"logits/chosen": -2.5926883220672607,
|
|
"logits/rejected": -2.6858973503112793,
|
|
"loss": 19.0269,
|
|
"step": 559
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41588789224624634,
|
|
"beta_dpo/beta_margin_grad_std": 0.07136394083499908,
|
|
"beta_dpo/beta_margin_mean": 0.3474079966545105,
|
|
"beta_dpo/beta_margin_std": 0.30195000767707825,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -5.194782257080078,
|
|
"beta_dpo/gap_mean": 420.5264587402344,
|
|
"beta_dpo/gap_std": 349.8380126953125,
|
|
"beta_dpo/loss_margin_mean": 347.4079895019531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8223201174743024,
|
|
"grad_norm": 30.960620880126953,
|
|
"learning_rate": 4.7444448928806615e-08,
|
|
"logits/chosen": -2.601827621459961,
|
|
"logits/rejected": -2.647773265838623,
|
|
"loss": 1.0969,
|
|
"step": 560
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4009363651275635,
|
|
"beta_dpo/beta_margin_grad_std": 0.08546130359172821,
|
|
"beta_dpo/beta_margin_mean": 0.41753634810447693,
|
|
"beta_dpo/beta_margin_std": 0.37727662920951843,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.459141254425049,
|
|
"beta_dpo/gap_mean": 415.37933349609375,
|
|
"beta_dpo/gap_std": 348.2698974609375,
|
|
"beta_dpo/loss_margin_mean": 417.53631591796875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8237885462555066,
|
|
"grad_norm": 37.56061935424805,
|
|
"learning_rate": 4.669493178106432e-08,
|
|
"logits/chosen": -2.5700759887695312,
|
|
"logits/rejected": -2.675487995147705,
|
|
"loss": 1.0767,
|
|
"step": 561
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2502177059650421,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21572551131248474,
|
|
"beta_dpo/beta_margin_grad_std": 0.2213025540113449,
|
|
"beta_dpo/beta_margin_mean": 141.0299835205078,
|
|
"beta_dpo/beta_margin_std": 179.341552734375,
|
|
"beta_dpo/beta_used": 0.2502177059650421,
|
|
"beta_dpo/beta_used_raw": -3.710040330886841,
|
|
"beta_dpo/gap_mean": 425.248779296875,
|
|
"beta_dpo/gap_std": 348.01812744140625,
|
|
"beta_dpo/loss_margin_mean": 423.1015930175781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8252569750367107,
|
|
"grad_norm": 24.252979278564453,
|
|
"learning_rate": 4.5950771910944596e-08,
|
|
"logits/chosen": -2.571836233139038,
|
|
"logits/rejected": -2.639535427093506,
|
|
"loss": 0.5656,
|
|
"step": 562
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2640678286552429,
|
|
"beta_dpo/beta_margin_grad_mean": -0.29870668053627014,
|
|
"beta_dpo/beta_margin_grad_std": 0.29806166887283325,
|
|
"beta_dpo/beta_margin_mean": 106.81857299804688,
|
|
"beta_dpo/beta_margin_std": 167.0647430419922,
|
|
"beta_dpo/beta_used": 0.2640678286552429,
|
|
"beta_dpo/beta_used_raw": -3.6815316677093506,
|
|
"beta_dpo/gap_mean": 400.4122314453125,
|
|
"beta_dpo/gap_std": 342.2979736328125,
|
|
"beta_dpo/loss_margin_mean": 323.61669921875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8267254038179148,
|
|
"grad_norm": 2278.085205078125,
|
|
"learning_rate": 4.521198892775202e-08,
|
|
"logits/chosen": -2.5583033561706543,
|
|
"logits/rejected": -2.625527858734131,
|
|
"loss": 1.711,
|
|
"step": 563
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4716358184814453,
|
|
"beta_dpo/beta_margin_grad_mean": -0.06238251551985741,
|
|
"beta_dpo/beta_margin_grad_std": 0.2377287596464157,
|
|
"beta_dpo/beta_margin_mean": 640.9151611328125,
|
|
"beta_dpo/beta_margin_std": 831.071044921875,
|
|
"beta_dpo/beta_used": 1.4716358184814453,
|
|
"beta_dpo/beta_used_raw": 1.4716358184814453,
|
|
"beta_dpo/gap_mean": 404.2149658203125,
|
|
"beta_dpo/gap_std": 335.1101989746094,
|
|
"beta_dpo/loss_margin_mean": 427.81158447265625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8281938325991189,
|
|
"grad_norm": 40077.27734375,
|
|
"learning_rate": 4.447860229910544e-08,
|
|
"logits/chosen": -2.6260743141174316,
|
|
"logits/rejected": -2.693694591522217,
|
|
"loss": 9.7535,
|
|
"step": 564
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.3616304397583008,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2510206699371338,
|
|
"beta_dpo/beta_margin_grad_std": 0.26485851407051086,
|
|
"beta_dpo/beta_margin_mean": 526.1046752929688,
|
|
"beta_dpo/beta_margin_std": 818.9513549804688,
|
|
"beta_dpo/beta_used": 1.3616304397583008,
|
|
"beta_dpo/beta_used_raw": 0.21367371082305908,
|
|
"beta_dpo/gap_mean": 402.90283203125,
|
|
"beta_dpo/gap_std": 332.8077392578125,
|
|
"beta_dpo/loss_margin_mean": 384.8162841796875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8296622613803231,
|
|
"grad_norm": 25.107654571533203,
|
|
"learning_rate": 4.375063135042445e-08,
|
|
"logits/chosen": -2.5877442359924316,
|
|
"logits/rejected": -2.6708731651306152,
|
|
"loss": 0.5359,
|
|
"step": 565
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.1117289066314697,
|
|
"beta_dpo/beta_margin_grad_mean": -0.20614933967590332,
|
|
"beta_dpo/beta_margin_grad_std": 0.21492846310138702,
|
|
"beta_dpo/beta_margin_mean": 586.4969482421875,
|
|
"beta_dpo/beta_margin_std": 798.0072631835938,
|
|
"beta_dpo/beta_used": 1.1117289066314697,
|
|
"beta_dpo/beta_used_raw": -2.902581214904785,
|
|
"beta_dpo/gap_mean": 405.11932373046875,
|
|
"beta_dpo/gap_std": 338.6856994628906,
|
|
"beta_dpo/loss_margin_mean": 448.7620544433594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8311306901615272,
|
|
"grad_norm": 25.039459228515625,
|
|
"learning_rate": 4.3028095264420525e-08,
|
|
"logits/chosen": -2.568953037261963,
|
|
"logits/rejected": -2.6582655906677246,
|
|
"loss": 0.5761,
|
|
"step": 566
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.7229585647583008,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2360886186361313,
|
|
"beta_dpo/beta_margin_grad_std": 0.24904760718345642,
|
|
"beta_dpo/beta_margin_mean": 289.8537902832031,
|
|
"beta_dpo/beta_margin_std": 402.62164306640625,
|
|
"beta_dpo/beta_used": 0.7229585647583008,
|
|
"beta_dpo/beta_used_raw": -2.865267753601074,
|
|
"beta_dpo/gap_mean": 407.5924072265625,
|
|
"beta_dpo/gap_std": 333.9209289550781,
|
|
"beta_dpo/loss_margin_mean": 388.1324768066406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8325991189427313,
|
|
"grad_norm": 17.503690719604492,
|
|
"learning_rate": 4.231101308059165e-08,
|
|
"logits/chosen": -2.597644329071045,
|
|
"logits/rejected": -2.676164150238037,
|
|
"loss": 0.5655,
|
|
"step": 567
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4946538209915161,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2185162454843521,
|
|
"beta_dpo/beta_margin_grad_std": 0.23013083636760712,
|
|
"beta_dpo/beta_margin_mean": 696.1226196289062,
|
|
"beta_dpo/beta_margin_std": 929.7108154296875,
|
|
"beta_dpo/beta_used": 1.4946538209915161,
|
|
"beta_dpo/beta_used_raw": 0.43433964252471924,
|
|
"beta_dpo/gap_mean": 412.0642395019531,
|
|
"beta_dpo/gap_std": 327.5071105957031,
|
|
"beta_dpo/loss_margin_mean": 429.1242370605469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8340675477239354,
|
|
"grad_norm": 25.224254608154297,
|
|
"learning_rate": 4.1599403694720145e-08,
|
|
"logits/chosen": -2.5394160747528076,
|
|
"logits/rejected": -2.6383776664733887,
|
|
"loss": 0.5296,
|
|
"step": 568
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4124916195869446,
|
|
"beta_dpo/beta_margin_grad_std": 0.07660473138093948,
|
|
"beta_dpo/beta_margin_mean": 0.36489877104759216,
|
|
"beta_dpo/beta_margin_std": 0.3362376391887665,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.51121187210083,
|
|
"beta_dpo/gap_mean": 404.7514953613281,
|
|
"beta_dpo/gap_std": 331.2776794433594,
|
|
"beta_dpo/loss_margin_mean": 364.8987731933594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8355359765051396,
|
|
"grad_norm": 38.54376983642578,
|
|
"learning_rate": 4.089328585837512e-08,
|
|
"logits/chosen": -2.577268123626709,
|
|
"logits/rejected": -2.6315019130706787,
|
|
"loss": 1.0865,
|
|
"step": 569
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4066663384437561,
|
|
"beta_dpo/beta_margin_grad_std": 0.07038101553916931,
|
|
"beta_dpo/beta_margin_mean": 0.38760435581207275,
|
|
"beta_dpo/beta_margin_std": 0.30543091893196106,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.0776772499084473,
|
|
"beta_dpo/gap_mean": 402.11566162109375,
|
|
"beta_dpo/gap_std": 327.2163391113281,
|
|
"beta_dpo/loss_margin_mean": 387.6043395996094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8370044052863436,
|
|
"grad_norm": 30.418283462524414,
|
|
"learning_rate": 4.019267817841834e-08,
|
|
"logits/chosen": -2.6355581283569336,
|
|
"logits/rejected": -2.698420286178589,
|
|
"loss": 1.0645,
|
|
"step": 570
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9894961714744568,
|
|
"beta_dpo/beta_margin_grad_mean": -0.015625763684511185,
|
|
"beta_dpo/beta_margin_grad_std": 0.12401949614286423,
|
|
"beta_dpo/beta_margin_mean": 509.4083251953125,
|
|
"beta_dpo/beta_margin_std": 569.6787719726562,
|
|
"beta_dpo/beta_used": 0.9894961714744568,
|
|
"beta_dpo/beta_used_raw": 0.9894961714744568,
|
|
"beta_dpo/gap_mean": 413.31024169921875,
|
|
"beta_dpo/gap_std": 324.23162841796875,
|
|
"beta_dpo/loss_margin_mean": 487.08660888671875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8384728340675477,
|
|
"grad_norm": 0.0953197032213211,
|
|
"learning_rate": 3.9497599116513705e-08,
|
|
"logits/chosen": -2.5648531913757324,
|
|
"logits/rejected": -2.6703522205352783,
|
|
"loss": 0.0,
|
|
"step": 571
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.384432792663574,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26681461930274963,
|
|
"beta_dpo/beta_margin_grad_std": 0.2802523076534271,
|
|
"beta_dpo/beta_margin_mean": 1058.479736328125,
|
|
"beta_dpo/beta_margin_std": 1517.7403564453125,
|
|
"beta_dpo/beta_used": 2.384432792663574,
|
|
"beta_dpo/beta_used_raw": 1.8693180084228516,
|
|
"beta_dpo/gap_mean": 416.3902587890625,
|
|
"beta_dpo/gap_std": 328.5089111328125,
|
|
"beta_dpo/loss_margin_mean": 412.9160461425781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8399412628487518,
|
|
"grad_norm": 131773.4375,
|
|
"learning_rate": 3.880806698864086e-08,
|
|
"logits/chosen": -2.548858642578125,
|
|
"logits/rejected": -2.6395535469055176,
|
|
"loss": 10.9273,
|
|
"step": 572
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.410361111164093,
|
|
"beta_dpo/beta_margin_grad_std": 0.0751841589808464,
|
|
"beta_dpo/beta_margin_mean": 0.37288060784339905,
|
|
"beta_dpo/beta_margin_std": 0.32199469208717346,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.7826108932495117,
|
|
"beta_dpo/gap_mean": 408.19134521484375,
|
|
"beta_dpo/gap_std": 329.8096008300781,
|
|
"beta_dpo/loss_margin_mean": 372.8805847167969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8414096916299559,
|
|
"grad_norm": 31.714269638061523,
|
|
"learning_rate": 3.812409996461275e-08,
|
|
"logits/chosen": -2.6157898902893066,
|
|
"logits/rejected": -2.681966781616211,
|
|
"loss": 1.064,
|
|
"step": 573
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.0706517696380615,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23131898045539856,
|
|
"beta_dpo/beta_margin_grad_std": 0.2463950663805008,
|
|
"beta_dpo/beta_margin_mean": 1551.0074462890625,
|
|
"beta_dpo/beta_margin_std": 2098.472900390625,
|
|
"beta_dpo/beta_used": 3.0706517696380615,
|
|
"beta_dpo/beta_used_raw": 2.929409980773926,
|
|
"beta_dpo/gap_mean": 412.53070068359375,
|
|
"beta_dpo/gap_std": 330.5635986328125,
|
|
"beta_dpo/loss_margin_mean": 460.5701904296875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8428781204111601,
|
|
"grad_norm": 21.132007598876953,
|
|
"learning_rate": 3.74457160675965e-08,
|
|
"logits/chosen": -2.6238203048706055,
|
|
"logits/rejected": -2.720996856689453,
|
|
"loss": 0.5214,
|
|
"step": 574
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.38933318853378296,
|
|
"beta_dpo/beta_margin_grad_std": 0.0830337256193161,
|
|
"beta_dpo/beta_margin_mean": 0.4672733247280121,
|
|
"beta_dpo/beta_margin_std": 0.37139537930488586,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.8129258155822754,
|
|
"beta_dpo/gap_mean": 423.2646484375,
|
|
"beta_dpo/gap_std": 333.3087463378906,
|
|
"beta_dpo/loss_margin_mean": 467.2732849121094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8443465491923642,
|
|
"grad_norm": 36.160743713378906,
|
|
"learning_rate": 3.677293317363864e-08,
|
|
"logits/chosen": -2.5244140625,
|
|
"logits/rejected": -2.6159071922302246,
|
|
"loss": 1.0514,
|
|
"step": 575
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.7232588529586792,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2747720777988434,
|
|
"beta_dpo/beta_margin_grad_std": 0.2860935628414154,
|
|
"beta_dpo/beta_margin_mean": 359.212646484375,
|
|
"beta_dpo/beta_margin_std": 551.1680297851562,
|
|
"beta_dpo/beta_used": 0.7232588529586792,
|
|
"beta_dpo/beta_used_raw": -2.2491068840026855,
|
|
"beta_dpo/gap_mean": 428.517578125,
|
|
"beta_dpo/gap_std": 351.6317443847656,
|
|
"beta_dpo/loss_margin_mean": 409.20550537109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8458149779735683,
|
|
"grad_norm": 22067.658203125,
|
|
"learning_rate": 3.6105769011194224e-08,
|
|
"logits/chosen": -2.6149516105651855,
|
|
"logits/rejected": -2.7341456413269043,
|
|
"loss": 7.0247,
|
|
"step": 576
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4043275713920593,
|
|
"beta_dpo/beta_margin_grad_std": 0.06338826566934586,
|
|
"beta_dpo/beta_margin_mean": 0.3954284191131592,
|
|
"beta_dpo/beta_margin_std": 0.27391013503074646,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.11868953704834,
|
|
"beta_dpo/gap_mean": 419.32061767578125,
|
|
"beta_dpo/gap_std": 346.9593505859375,
|
|
"beta_dpo/loss_margin_mean": 395.42840576171875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8472834067547724,
|
|
"grad_norm": 23.889705657958984,
|
|
"learning_rate": 3.5444241160659304e-08,
|
|
"logits/chosen": -2.5963869094848633,
|
|
"logits/rejected": -2.666391372680664,
|
|
"loss": 1.0657,
|
|
"step": 577
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4147895574569702,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23017671704292297,
|
|
"beta_dpo/beta_margin_grad_std": 0.24086488783359528,
|
|
"beta_dpo/beta_margin_mean": 697.8129272460938,
|
|
"beta_dpo/beta_margin_std": 934.8630981445312,
|
|
"beta_dpo/beta_used": 1.4147895574569702,
|
|
"beta_dpo/beta_used_raw": 0.5740838050842285,
|
|
"beta_dpo/gap_mean": 418.50299072265625,
|
|
"beta_dpo/gap_std": 338.3753662109375,
|
|
"beta_dpo/loss_margin_mean": 392.7145690917969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8487518355359766,
|
|
"grad_norm": 17.308809280395508,
|
|
"learning_rate": 3.478836705390808e-08,
|
|
"logits/chosen": -2.5874085426330566,
|
|
"logits/rejected": -2.6705245971679688,
|
|
"loss": 0.5284,
|
|
"step": 578
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4038708806037903,
|
|
"beta_dpo/beta_margin_grad_std": 0.06529239565134048,
|
|
"beta_dpo/beta_margin_mean": 0.39867639541625977,
|
|
"beta_dpo/beta_margin_std": 0.2885708808898926,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.684361457824707,
|
|
"beta_dpo/gap_mean": 410.989501953125,
|
|
"beta_dpo/gap_std": 332.41058349609375,
|
|
"beta_dpo/loss_margin_mean": 398.6763610839844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8502202643171806,
|
|
"grad_norm": 27.025897979736328,
|
|
"learning_rate": 3.41381639738331e-08,
|
|
"logits/chosen": -2.5982184410095215,
|
|
"logits/rejected": -2.6704506874084473,
|
|
"loss": 1.064,
|
|
"step": 579
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4143833518028259,
|
|
"beta_dpo/beta_margin_grad_std": 0.09059581905603409,
|
|
"beta_dpo/beta_margin_mean": 0.36092129349708557,
|
|
"beta_dpo/beta_margin_std": 0.3993350863456726,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.0408852100372314,
|
|
"beta_dpo/gap_mean": 404.3271484375,
|
|
"beta_dpo/gap_std": 339.910888671875,
|
|
"beta_dpo/loss_margin_mean": 360.9212646484375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8516886930983847,
|
|
"grad_norm": 35.905391693115234,
|
|
"learning_rate": 3.349364905389032e-08,
|
|
"logits/chosen": -2.6305480003356934,
|
|
"logits/rejected": -2.701788902282715,
|
|
"loss": 1.0811,
|
|
"step": 580
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.7418603897094727,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24978549778461456,
|
|
"beta_dpo/beta_margin_grad_std": 0.26566624641418457,
|
|
"beta_dpo/beta_margin_mean": 1087.9808349609375,
|
|
"beta_dpo/beta_margin_std": 1690.84423828125,
|
|
"beta_dpo/beta_used": 2.7418603897094727,
|
|
"beta_dpo/beta_used_raw": 1.8317091464996338,
|
|
"beta_dpo/gap_mean": 399.22796630859375,
|
|
"beta_dpo/gap_std": 348.1463928222656,
|
|
"beta_dpo/loss_margin_mean": 397.565673828125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8531571218795888,
|
|
"grad_norm": 18.974281311035156,
|
|
"learning_rate": 3.285483927764726e-08,
|
|
"logits/chosen": -2.6319580078125,
|
|
"logits/rejected": -2.72849702835083,
|
|
"loss": 0.5351,
|
|
"step": 581
|
|
},
|
|
{
|
|
"beta_dpo/beta": 4.0522236824035645,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23803767561912537,
|
|
"beta_dpo/beta_margin_grad_std": 0.25662490725517273,
|
|
"beta_dpo/beta_margin_mean": 2030.54248046875,
|
|
"beta_dpo/beta_margin_std": 2940.37939453125,
|
|
"beta_dpo/beta_used": 4.0522236824035645,
|
|
"beta_dpo/beta_used_raw": 2.1622235774993896,
|
|
"beta_dpo/gap_mean": 401.3326416015625,
|
|
"beta_dpo/gap_std": 360.9919738769531,
|
|
"beta_dpo/loss_margin_mean": 435.0095520019531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8546255506607929,
|
|
"grad_norm": 60133.34765625,
|
|
"learning_rate": 3.222175147833556e-08,
|
|
"logits/chosen": -2.6616077423095703,
|
|
"logits/rejected": -2.7627522945404053,
|
|
"loss": 28.4446,
|
|
"step": 582
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.0656654834747314,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25362730026245117,
|
|
"beta_dpo/beta_margin_grad_std": 0.26769015192985535,
|
|
"beta_dpo/beta_margin_mean": 971.8246459960938,
|
|
"beta_dpo/beta_margin_std": 1321.0806884765625,
|
|
"beta_dpo/beta_used": 2.0656654834747314,
|
|
"beta_dpo/beta_used_raw": -2.4300012588500977,
|
|
"beta_dpo/gap_mean": 404.298828125,
|
|
"beta_dpo/gap_std": 363.2397155761719,
|
|
"beta_dpo/loss_margin_mean": 354.9598693847656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.856093979441997,
|
|
"grad_norm": 24.137685775756836,
|
|
"learning_rate": 3.159440233840763e-08,
|
|
"logits/chosen": -2.6524295806884766,
|
|
"logits/rejected": -2.708967685699463,
|
|
"loss": 0.589,
|
|
"step": 583
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.7489781379699707,
|
|
"beta_dpo/beta_margin_grad_mean": -0.03125,
|
|
"beta_dpo/beta_margin_grad_std": 0.17399263381958008,
|
|
"beta_dpo/beta_margin_mean": 1806.1541748046875,
|
|
"beta_dpo/beta_margin_std": 1009.447021484375,
|
|
"beta_dpo/beta_used": 3.7489781379699707,
|
|
"beta_dpo/beta_used_raw": 3.7489781379699707,
|
|
"beta_dpo/gap_mean": 409.24481201171875,
|
|
"beta_dpo/gap_std": 357.03033447265625,
|
|
"beta_dpo/loss_margin_mean": 484.4200134277344,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8575624082232012,
|
|
"grad_norm": 42800.1015625,
|
|
"learning_rate": 3.0972808389096635e-08,
|
|
"logits/chosen": -2.6379153728485107,
|
|
"logits/rejected": -2.73630428314209,
|
|
"loss": 29.1861,
|
|
"step": 584
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4875493049621582,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21340592205524445,
|
|
"beta_dpo/beta_margin_grad_std": 0.22890949249267578,
|
|
"beta_dpo/beta_margin_mean": 652.2798461914062,
|
|
"beta_dpo/beta_margin_std": 899.9537963867188,
|
|
"beta_dpo/beta_used": 1.4875493049621582,
|
|
"beta_dpo/beta_used_raw": 1.2730743885040283,
|
|
"beta_dpo/gap_mean": 416.50323486328125,
|
|
"beta_dpo/gap_std": 353.55279541015625,
|
|
"beta_dpo/loss_margin_mean": 440.5109558105469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8590308370044053,
|
|
"grad_norm": 190.09495544433594,
|
|
"learning_rate": 3.035698600998121e-08,
|
|
"logits/chosen": -2.620150566101074,
|
|
"logits/rejected": -2.7122976779937744,
|
|
"loss": 0.5226,
|
|
"step": 585
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40226635336875916,
|
|
"beta_dpo/beta_margin_grad_std": 0.08389244973659515,
|
|
"beta_dpo/beta_margin_mean": 0.41126564145088196,
|
|
"beta_dpo/beta_margin_std": 0.3721611201763153,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.403745174407959,
|
|
"beta_dpo/gap_mean": 414.5293884277344,
|
|
"beta_dpo/gap_std": 352.2566833496094,
|
|
"beta_dpo/loss_margin_mean": 411.265625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8604992657856094,
|
|
"grad_norm": 30.519203186035156,
|
|
"learning_rate": 2.974695142855388e-08,
|
|
"logits/chosen": -2.6606011390686035,
|
|
"logits/rejected": -2.7575459480285645,
|
|
"loss": 1.0402,
|
|
"step": 586
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4073527753353119,
|
|
"beta_dpo/beta_margin_grad_std": 0.08573532849550247,
|
|
"beta_dpo/beta_margin_mean": 0.3906094431877136,
|
|
"beta_dpo/beta_margin_std": 0.37780997157096863,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.9087142944335938,
|
|
"beta_dpo/gap_mean": 414.48919677734375,
|
|
"beta_dpo/gap_std": 359.83526611328125,
|
|
"beta_dpo/loss_margin_mean": 390.6094055175781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8619676945668135,
|
|
"grad_norm": 31.185678482055664,
|
|
"learning_rate": 2.9142720719793122e-08,
|
|
"logits/chosen": -2.6660566329956055,
|
|
"logits/rejected": -2.7582545280456543,
|
|
"loss": 1.0746,
|
|
"step": 587
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.2841373682022095,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24702578783035278,
|
|
"beta_dpo/beta_margin_grad_std": 0.26337432861328125,
|
|
"beta_dpo/beta_margin_mean": 613.1337890625,
|
|
"beta_dpo/beta_margin_std": 1012.4978637695312,
|
|
"beta_dpo/beta_used": 1.2841373682022095,
|
|
"beta_dpo/beta_used_raw": 0.5301622748374939,
|
|
"beta_dpo/gap_mean": 418.21356201171875,
|
|
"beta_dpo/gap_std": 369.077392578125,
|
|
"beta_dpo/loss_margin_mean": 447.8412170410156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8634361233480177,
|
|
"grad_norm": 156.63604736328125,
|
|
"learning_rate": 2.8544309805740018e-08,
|
|
"logits/chosen": -2.622619152069092,
|
|
"logits/rejected": -2.741431951522827,
|
|
"loss": 0.5288,
|
|
"step": 588
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.8131837844848633,
|
|
"beta_dpo/beta_margin_grad_mean": -0.15622615814208984,
|
|
"beta_dpo/beta_margin_grad_std": 0.3630368411540985,
|
|
"beta_dpo/beta_margin_mean": 1222.51513671875,
|
|
"beta_dpo/beta_margin_std": 1587.3143310546875,
|
|
"beta_dpo/beta_used": 2.8131837844848633,
|
|
"beta_dpo/beta_used_raw": 2.8131837844848633,
|
|
"beta_dpo/gap_mean": 424.3757629394531,
|
|
"beta_dpo/gap_std": 373.8643798828125,
|
|
"beta_dpo/loss_margin_mean": 476.1635437011719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8649045521292217,
|
|
"grad_norm": 21344.30859375,
|
|
"learning_rate": 2.7951734455078786e-08,
|
|
"logits/chosen": -2.665593147277832,
|
|
"logits/rejected": -2.7899739742279053,
|
|
"loss": 19.9603,
|
|
"step": 589
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3272712528705597,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24900765717029572,
|
|
"beta_dpo/beta_margin_grad_std": 0.27214398980140686,
|
|
"beta_dpo/beta_margin_mean": 128.85853576660156,
|
|
"beta_dpo/beta_margin_std": 206.27391052246094,
|
|
"beta_dpo/beta_used": 0.3272712528705597,
|
|
"beta_dpo/beta_used_raw": -0.018486618995666504,
|
|
"beta_dpo/gap_mean": 432.82989501953125,
|
|
"beta_dpo/gap_std": 383.5555725097656,
|
|
"beta_dpo/loss_margin_mean": 477.1200256347656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8663729809104258,
|
|
"grad_norm": 12794.5634765625,
|
|
"learning_rate": 2.736501028272095e-08,
|
|
"logits/chosen": -2.6336288452148438,
|
|
"logits/rejected": -2.7674968242645264,
|
|
"loss": 4.8078,
|
|
"step": 590
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0653685331344604,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24658845365047455,
|
|
"beta_dpo/beta_margin_grad_std": 0.2630532681941986,
|
|
"beta_dpo/beta_margin_mean": 457.1188049316406,
|
|
"beta_dpo/beta_margin_std": 694.888916015625,
|
|
"beta_dpo/beta_used": 1.0653685331344604,
|
|
"beta_dpo/beta_used_raw": 0.8892130255699158,
|
|
"beta_dpo/gap_mean": 437.3492126464844,
|
|
"beta_dpo/gap_std": 384.0495300292969,
|
|
"beta_dpo/loss_margin_mean": 425.42724609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8678414096916299,
|
|
"grad_norm": 21.35018539428711,
|
|
"learning_rate": 2.678415274939408e-08,
|
|
"logits/chosen": -2.6317532062530518,
|
|
"logits/rejected": -2.708348512649536,
|
|
"loss": 0.5106,
|
|
"step": 591
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4068998694419861,
|
|
"beta_dpo/beta_margin_grad_std": 0.07823016494512558,
|
|
"beta_dpo/beta_margin_mean": 0.3857780396938324,
|
|
"beta_dpo/beta_margin_std": 0.328925758600235,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.6076627969741821,
|
|
"beta_dpo/gap_mean": 429.1395568847656,
|
|
"beta_dpo/gap_std": 375.99981689453125,
|
|
"beta_dpo/loss_margin_mean": 385.77801513671875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.869309838472834,
|
|
"grad_norm": 33.770408630371094,
|
|
"learning_rate": 2.6209177161234442e-08,
|
|
"logits/chosen": -2.661647319793701,
|
|
"logits/rejected": -2.7471630573272705,
|
|
"loss": 1.0442,
|
|
"step": 592
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4293622672557831,
|
|
"beta_dpo/beta_margin_grad_std": 0.08562355488538742,
|
|
"beta_dpo/beta_margin_mean": 0.2922646403312683,
|
|
"beta_dpo/beta_margin_std": 0.3565959632396698,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.2156394720077515,
|
|
"beta_dpo/gap_mean": 407.1551513671875,
|
|
"beta_dpo/gap_std": 372.3304443359375,
|
|
"beta_dpo/loss_margin_mean": 292.2646179199219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8707782672540382,
|
|
"grad_norm": 36.524818420410156,
|
|
"learning_rate": 2.564009866938349e-08,
|
|
"logits/chosen": -2.611865997314453,
|
|
"logits/rejected": -2.6518046855926514,
|
|
"loss": 1.0559,
|
|
"step": 593
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8479436039924622,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2311505377292633,
|
|
"beta_dpo/beta_margin_grad_std": 0.2402058094739914,
|
|
"beta_dpo/beta_margin_mean": 418.7370910644531,
|
|
"beta_dpo/beta_margin_std": 590.7848510742188,
|
|
"beta_dpo/beta_used": 0.8479436039924622,
|
|
"beta_dpo/beta_used_raw": -2.808560848236084,
|
|
"beta_dpo/gap_mean": 395.17657470703125,
|
|
"beta_dpo/gap_std": 363.5038146972656,
|
|
"beta_dpo/loss_margin_mean": 389.0055847167969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8722466960352423,
|
|
"grad_norm": 34.29524612426758,
|
|
"learning_rate": 2.5076932269588708e-08,
|
|
"logits/chosen": -2.6745779514312744,
|
|
"logits/rejected": -2.7460973262786865,
|
|
"loss": 0.5787,
|
|
"step": 594
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40659037232398987,
|
|
"beta_dpo/beta_margin_grad_std": 0.08061845600605011,
|
|
"beta_dpo/beta_margin_mean": 0.3967190682888031,
|
|
"beta_dpo/beta_margin_std": 0.38002651929855347,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.7765114307403564,
|
|
"beta_dpo/gap_mean": 399.8134765625,
|
|
"beta_dpo/gap_std": 366.6833190917969,
|
|
"beta_dpo/loss_margin_mean": 396.71905517578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8737151248164464,
|
|
"grad_norm": 24.00165557861328,
|
|
"learning_rate": 2.451969280180849e-08,
|
|
"logits/chosen": -2.631199359893799,
|
|
"logits/rejected": -2.704611301422119,
|
|
"loss": 1.092,
|
|
"step": 595
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2762553095817566,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24379974603652954,
|
|
"beta_dpo/beta_margin_grad_std": 0.25649240612983704,
|
|
"beta_dpo/beta_margin_mean": 115.0361557006836,
|
|
"beta_dpo/beta_margin_std": 168.74607849121094,
|
|
"beta_dpo/beta_used": 0.2762553095817566,
|
|
"beta_dpo/beta_used_raw": -2.552454948425293,
|
|
"beta_dpo/gap_mean": 392.48553466796875,
|
|
"beta_dpo/gap_std": 366.8974304199219,
|
|
"beta_dpo/loss_margin_mean": 365.7869873046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8751835535976505,
|
|
"grad_norm": 7005.2744140625,
|
|
"learning_rate": 2.396839494982103e-08,
|
|
"logits/chosen": -2.6771159172058105,
|
|
"logits/rejected": -2.754608631134033,
|
|
"loss": 1.0769,
|
|
"step": 596
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9559941291809082,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2519912123680115,
|
|
"beta_dpo/beta_margin_grad_std": 0.26672062277793884,
|
|
"beta_dpo/beta_margin_mean": 528.5262451171875,
|
|
"beta_dpo/beta_margin_std": 805.1343383789062,
|
|
"beta_dpo/beta_used": 0.9559941291809082,
|
|
"beta_dpo/beta_used_raw": -0.41350656747817993,
|
|
"beta_dpo/gap_mean": 407.9326171875,
|
|
"beta_dpo/gap_std": 373.6595764160156,
|
|
"beta_dpo/loss_margin_mean": 463.8984680175781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8766519823788547,
|
|
"grad_norm": 22720.951171875,
|
|
"learning_rate": 2.3423053240837514e-08,
|
|
"logits/chosen": -2.669362783432007,
|
|
"logits/rejected": -2.770700693130493,
|
|
"loss": 10.1259,
|
|
"step": 597
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.963083267211914,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24513758718967438,
|
|
"beta_dpo/beta_margin_grad_std": 0.25738582015037537,
|
|
"beta_dpo/beta_margin_mean": 2212.566650390625,
|
|
"beta_dpo/beta_margin_std": 3305.775634765625,
|
|
"beta_dpo/beta_used": 3.963083267211914,
|
|
"beta_dpo/beta_used_raw": 3.8194849491119385,
|
|
"beta_dpo/gap_mean": 415.28436279296875,
|
|
"beta_dpo/gap_std": 379.65228271484375,
|
|
"beta_dpo/loss_margin_mean": 428.660400390625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8781204111600588,
|
|
"grad_norm": 26.368377685546875,
|
|
"learning_rate": 2.2883682045119062e-08,
|
|
"logits/chosen": -2.6603126525878906,
|
|
"logits/rejected": -2.7383084297180176,
|
|
"loss": 0.5217,
|
|
"step": 598
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40531477332115173,
|
|
"beta_dpo/beta_margin_grad_std": 0.07651591300964355,
|
|
"beta_dpo/beta_margin_mean": 0.3960907757282257,
|
|
"beta_dpo/beta_margin_std": 0.3420655429363251,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.9786413908004761,
|
|
"beta_dpo/gap_mean": 409.6473388671875,
|
|
"beta_dpo/gap_std": 375.72576904296875,
|
|
"beta_dpo/loss_margin_mean": 396.09075927734375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8795888399412628,
|
|
"grad_norm": 25.34820556640625,
|
|
"learning_rate": 2.2350295575598367e-08,
|
|
"logits/chosen": -2.662172317504883,
|
|
"logits/rejected": -2.7468762397766113,
|
|
"loss": 1.0483,
|
|
"step": 599
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6657280325889587,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2325766235589981,
|
|
"beta_dpo/beta_margin_grad_std": 0.24632975459098816,
|
|
"beta_dpo/beta_margin_mean": 258.60589599609375,
|
|
"beta_dpo/beta_margin_std": 383.517333984375,
|
|
"beta_dpo/beta_used": 0.6657280325889587,
|
|
"beta_dpo/beta_used_raw": -0.8385080099105835,
|
|
"beta_dpo/gap_mean": 404.39202880859375,
|
|
"beta_dpo/gap_std": 360.17938232421875,
|
|
"beta_dpo/loss_margin_mean": 396.5282287597656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8810572687224669,
|
|
"grad_norm": 21.900676727294922,
|
|
"learning_rate": 2.1822907887504932e-08,
|
|
"logits/chosen": -2.691469669342041,
|
|
"logits/rejected": -2.772876262664795,
|
|
"loss": 0.5388,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8810572687224669,
|
|
"eval_beta_dpo/beta": 0.0010000000474974513,
|
|
"eval_beta_dpo/beta_margin_grad_mean": -0.44217321276664734,
|
|
"eval_beta_dpo/beta_margin_grad_std": 0.09827572852373123,
|
|
"eval_beta_dpo/beta_margin_mean": 0.24340428411960602,
|
|
"eval_beta_dpo/beta_margin_std": 0.4217238128185272,
|
|
"eval_beta_dpo/beta_used": 0.0010000000474974513,
|
|
"eval_beta_dpo/beta_used_raw": -9.559965133666992,
|
|
"eval_beta_dpo/gap_mean": 404.4036560058594,
|
|
"eval_beta_dpo/gap_std": 357.40692138671875,
|
|
"eval_beta_dpo/loss_margin_mean": 243.40426635742188,
|
|
"eval_beta_dpo/mask_keep_frac": 1.0,
|
|
"eval_logits/chosen": -2.781342029571533,
|
|
"eval_logits/rejected": -2.8108348846435547,
|
|
"eval_loss": 0.6014874577522278,
|
|
"eval_runtime": 37.9176,
|
|
"eval_samples_per_second": 61.686,
|
|
"eval_steps_per_second": 1.952,
|
|
"step": 600
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.2707443237304688,
|
|
"beta_dpo/beta_margin_grad_mean": -0.1272067278623581,
|
|
"beta_dpo/beta_margin_grad_std": 0.3303484618663788,
|
|
"beta_dpo/beta_margin_mean": 1377.1533203125,
|
|
"beta_dpo/beta_margin_std": 1483.408203125,
|
|
"beta_dpo/beta_used": 3.2707443237304688,
|
|
"beta_dpo/beta_used_raw": 3.2707443237304688,
|
|
"beta_dpo/gap_mean": 407.78009033203125,
|
|
"beta_dpo/gap_std": 361.7393798828125,
|
|
"beta_dpo/loss_margin_mean": 442.43011474609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.882525697503671,
|
|
"grad_norm": 115072.109375,
|
|
"learning_rate": 2.1301532877994742e-08,
|
|
"logits/chosen": -2.629629611968994,
|
|
"logits/rejected": -2.7460412979125977,
|
|
"loss": 73.7058,
|
|
"step": 601
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6206492781639099,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23634099960327148,
|
|
"beta_dpo/beta_margin_grad_std": 0.25701799988746643,
|
|
"beta_dpo/beta_margin_mean": 263.51983642578125,
|
|
"beta_dpo/beta_margin_std": 366.13446044921875,
|
|
"beta_dpo/beta_used": 0.6206492781639099,
|
|
"beta_dpo/beta_used_raw": 0.1565786600112915,
|
|
"beta_dpo/gap_mean": 418.2040100097656,
|
|
"beta_dpo/gap_std": 355.448486328125,
|
|
"beta_dpo/loss_margin_mean": 467.5210876464844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8839941262848752,
|
|
"grad_norm": 17.351716995239258,
|
|
"learning_rate": 2.0786184285784298e-08,
|
|
"logits/chosen": -2.7129225730895996,
|
|
"logits/rejected": -2.8077144622802734,
|
|
"loss": 0.5206,
|
|
"step": 602
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40567564964294434,
|
|
"beta_dpo/beta_margin_grad_std": 0.08365360647439957,
|
|
"beta_dpo/beta_margin_mean": 0.39521723985671997,
|
|
"beta_dpo/beta_margin_std": 0.362171471118927,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.66328501701355,
|
|
"beta_dpo/gap_mean": 419.609375,
|
|
"beta_dpo/gap_std": 357.5216064453125,
|
|
"beta_dpo/loss_margin_mean": 395.21722412109375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8854625550660793,
|
|
"grad_norm": 28.13165283203125,
|
|
"learning_rate": 2.0276875690788204e-08,
|
|
"logits/chosen": -2.724299430847168,
|
|
"logits/rejected": -2.7903506755828857,
|
|
"loss": 1.0665,
|
|
"step": 603
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39339369535446167,
|
|
"beta_dpo/beta_margin_grad_std": 0.08459168672561646,
|
|
"beta_dpo/beta_margin_mean": 0.45500051975250244,
|
|
"beta_dpo/beta_margin_std": 0.4009777903556824,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.187976598739624,
|
|
"beta_dpo/gap_mean": 422.8662109375,
|
|
"beta_dpo/gap_std": 361.38897705078125,
|
|
"beta_dpo/loss_margin_mean": 455.00048828125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8869309838472834,
|
|
"grad_norm": 29.62197494506836,
|
|
"learning_rate": 1.977362051376158e-08,
|
|
"logits/chosen": -2.675366163253784,
|
|
"logits/rejected": -2.7828691005706787,
|
|
"loss": 1.0659,
|
|
"step": 604
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3992781937122345,
|
|
"beta_dpo/beta_margin_grad_std": 0.09205874055624008,
|
|
"beta_dpo/beta_margin_mean": 0.4294624328613281,
|
|
"beta_dpo/beta_margin_std": 0.4166560769081116,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.5361804962158203,
|
|
"beta_dpo/gap_mean": 425.74029541015625,
|
|
"beta_dpo/gap_std": 377.2355651855469,
|
|
"beta_dpo/loss_margin_mean": 429.46240234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8883994126284875,
|
|
"grad_norm": 26.918848037719727,
|
|
"learning_rate": 1.9276432015946446e-08,
|
|
"logits/chosen": -2.671755790710449,
|
|
"logits/rejected": -2.7664794921875,
|
|
"loss": 1.061,
|
|
"step": 605
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.22745679318904877,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2854154407978058,
|
|
"beta_dpo/beta_margin_grad_std": 0.29264795780181885,
|
|
"beta_dpo/beta_margin_mean": 94.71522521972656,
|
|
"beta_dpo/beta_margin_std": 168.86326599121094,
|
|
"beta_dpo/beta_used": 0.22745679318904877,
|
|
"beta_dpo/beta_used_raw": -0.09920136630535126,
|
|
"beta_dpo/gap_mean": 419.8639221191406,
|
|
"beta_dpo/gap_std": 381.78704833984375,
|
|
"beta_dpo/loss_margin_mean": 393.2106018066406,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8898678414096917,
|
|
"grad_norm": 2858.609375,
|
|
"learning_rate": 1.8785323298722093e-08,
|
|
"logits/chosen": -2.6562769412994385,
|
|
"logits/rejected": -2.7424442768096924,
|
|
"loss": 2.9883,
|
|
"step": 606
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.7348406314849854,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27311357855796814,
|
|
"beta_dpo/beta_margin_grad_std": 0.28208261728286743,
|
|
"beta_dpo/beta_margin_mean": 661.6439819335938,
|
|
"beta_dpo/beta_margin_std": 1191.2918701171875,
|
|
"beta_dpo/beta_used": 1.7348406314849854,
|
|
"beta_dpo/beta_used_raw": -0.1976158618927002,
|
|
"beta_dpo/gap_mean": 408.81640625,
|
|
"beta_dpo/gap_std": 380.6749572753906,
|
|
"beta_dpo/loss_margin_mean": 353.4035949707031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8913362701908958,
|
|
"grad_norm": 22962.126953125,
|
|
"learning_rate": 1.8300307303259904e-08,
|
|
"logits/chosen": -2.70379900932312,
|
|
"logits/rejected": -2.7581634521484375,
|
|
"loss": 13.9436,
|
|
"step": 607
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39080411195755005,
|
|
"beta_dpo/beta_margin_grad_std": 0.07455727458000183,
|
|
"beta_dpo/beta_margin_mean": 0.4569474458694458,
|
|
"beta_dpo/beta_margin_std": 0.328748881816864,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0561343431472778,
|
|
"beta_dpo/gap_mean": 413.77947998046875,
|
|
"beta_dpo/gap_std": 373.3118896484375,
|
|
"beta_dpo/loss_margin_mean": 456.9474182128906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8928046989720999,
|
|
"grad_norm": 33.10684585571289,
|
|
"learning_rate": 1.7821396810182437e-08,
|
|
"logits/chosen": -2.6324472427368164,
|
|
"logits/rejected": -2.7429165840148926,
|
|
"loss": 1.0452,
|
|
"step": 608
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39637741446495056,
|
|
"beta_dpo/beta_margin_grad_std": 0.07506369054317474,
|
|
"beta_dpo/beta_margin_mean": 0.43244999647140503,
|
|
"beta_dpo/beta_margin_std": 0.3281441330909729,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.2443511486053467,
|
|
"beta_dpo/gap_mean": 418.79693603515625,
|
|
"beta_dpo/gap_std": 365.2125244140625,
|
|
"beta_dpo/loss_margin_mean": 432.449951171875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8942731277533039,
|
|
"grad_norm": 32.35358810424805,
|
|
"learning_rate": 1.7348604439226617e-08,
|
|
"logits/chosen": -2.6647348403930664,
|
|
"logits/rejected": -2.755319595336914,
|
|
"loss": 1.0498,
|
|
"step": 609
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5392435193061829,
|
|
"beta_dpo/beta_margin_grad_mean": -0.13582731783390045,
|
|
"beta_dpo/beta_margin_grad_std": 0.33346015214920044,
|
|
"beta_dpo/beta_margin_mean": 214.91415405273438,
|
|
"beta_dpo/beta_margin_std": 323.2533874511719,
|
|
"beta_dpo/beta_used": 0.5392435193061829,
|
|
"beta_dpo/beta_used_raw": 0.5392435193061829,
|
|
"beta_dpo/gap_mean": 417.583740234375,
|
|
"beta_dpo/gap_std": 368.72650146484375,
|
|
"beta_dpo/loss_margin_mean": 401.9151916503906,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.895741556534508,
|
|
"grad_norm": 11989.935546875,
|
|
"learning_rate": 1.6881942648911074e-08,
|
|
"logits/chosen": -2.679978370666504,
|
|
"logits/rejected": -2.7630457878112793,
|
|
"loss": 2.1845,
|
|
"step": 610
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9980809688568115,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24803996086120605,
|
|
"beta_dpo/beta_margin_grad_std": 0.261429101228714,
|
|
"beta_dpo/beta_margin_mean": 542.123046875,
|
|
"beta_dpo/beta_margin_std": 784.00927734375,
|
|
"beta_dpo/beta_used": 0.9980809688568115,
|
|
"beta_dpo/beta_used_raw": -1.0866873264312744,
|
|
"beta_dpo/gap_mean": 409.5821838378906,
|
|
"beta_dpo/gap_std": 377.1724853515625,
|
|
"beta_dpo/loss_margin_mean": 410.0389709472656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8972099853157122,
|
|
"grad_norm": 32357.361328125,
|
|
"learning_rate": 1.6421423736208e-08,
|
|
"logits/chosen": -2.6723227500915527,
|
|
"logits/rejected": -2.769993305206299,
|
|
"loss": 16.8196,
|
|
"step": 611
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.3185596466064453,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23413825035095215,
|
|
"beta_dpo/beta_margin_grad_std": 0.24901816248893738,
|
|
"beta_dpo/beta_margin_mean": 1160.555908203125,
|
|
"beta_dpo/beta_margin_std": 1747.4896240234375,
|
|
"beta_dpo/beta_used": 2.3185596466064453,
|
|
"beta_dpo/beta_used_raw": -0.5901319980621338,
|
|
"beta_dpo/gap_mean": 418.91546630859375,
|
|
"beta_dpo/gap_std": 378.8981628417969,
|
|
"beta_dpo/loss_margin_mean": 448.5855407714844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.8986784140969163,
|
|
"grad_norm": 61496.91796875,
|
|
"learning_rate": 1.5967059836219042e-08,
|
|
"logits/chosen": -2.6705970764160156,
|
|
"logits/rejected": -2.763137102127075,
|
|
"loss": 96.682,
|
|
"step": 612
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.020970344543457,
|
|
"beta_dpo/beta_margin_grad_mean": -0.19436858594417572,
|
|
"beta_dpo/beta_margin_grad_std": 0.20417682826519012,
|
|
"beta_dpo/beta_margin_mean": 1071.8077392578125,
|
|
"beta_dpo/beta_margin_std": 1328.2359619140625,
|
|
"beta_dpo/beta_used": 2.020970344543457,
|
|
"beta_dpo/beta_used_raw": 1.5588773488998413,
|
|
"beta_dpo/gap_mean": 433.0877685546875,
|
|
"beta_dpo/gap_std": 376.868408203125,
|
|
"beta_dpo/loss_margin_mean": 500.5608825683594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9001468428781204,
|
|
"grad_norm": 18.278871536254883,
|
|
"learning_rate": 1.551886292185553e-08,
|
|
"logits/chosen": -2.6771979331970215,
|
|
"logits/rejected": -2.7864818572998047,
|
|
"loss": 0.5219,
|
|
"step": 613
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.24590833485126495,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2588002383708954,
|
|
"beta_dpo/beta_margin_grad_std": 0.2748665511608124,
|
|
"beta_dpo/beta_margin_mean": 115.03885650634766,
|
|
"beta_dpo/beta_margin_std": 184.78355407714844,
|
|
"beta_dpo/beta_used": 0.24590833485126495,
|
|
"beta_dpo/beta_used_raw": -0.8744754195213318,
|
|
"beta_dpo/gap_mean": 440.6515197753906,
|
|
"beta_dpo/gap_std": 371.03680419921875,
|
|
"beta_dpo/loss_margin_mean": 458.093017578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9016152716593245,
|
|
"grad_norm": 4608.9931640625,
|
|
"learning_rate": 1.507684480352292e-08,
|
|
"logits/chosen": -2.6669607162475586,
|
|
"logits/rejected": -2.793607234954834,
|
|
"loss": 4.2601,
|
|
"step": 614
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40423691272735596,
|
|
"beta_dpo/beta_margin_grad_std": 0.09169073402881622,
|
|
"beta_dpo/beta_margin_mean": 0.405151903629303,
|
|
"beta_dpo/beta_margin_std": 0.404344379901886,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.9845547676086426,
|
|
"beta_dpo/gap_mean": 434.8990173339844,
|
|
"beta_dpo/gap_std": 375.1392822265625,
|
|
"beta_dpo/loss_margin_mean": 405.1518859863281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9030837004405287,
|
|
"grad_norm": 32.88970184326172,
|
|
"learning_rate": 1.4641017128809801e-08,
|
|
"logits/chosen": -2.6823935508728027,
|
|
"logits/rejected": -2.760178327560425,
|
|
"loss": 1.0597,
|
|
"step": 615
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4224950969219208,
|
|
"beta_dpo/beta_margin_grad_std": 0.09631507843732834,
|
|
"beta_dpo/beta_margin_mean": 0.32495495676994324,
|
|
"beta_dpo/beta_margin_std": 0.4116884171962738,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -6.70911169052124,
|
|
"beta_dpo/gap_mean": 418.6253662109375,
|
|
"beta_dpo/gap_std": 380.8779296875,
|
|
"beta_dpo/loss_margin_mean": 324.9549255371094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9045521292217328,
|
|
"grad_norm": 30.849700927734375,
|
|
"learning_rate": 1.4211391382180637e-08,
|
|
"logits/chosen": -2.723806381225586,
|
|
"logits/rejected": -2.776388168334961,
|
|
"loss": 1.125,
|
|
"step": 616
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.43244972825050354,
|
|
"beta_dpo/beta_margin_grad_std": 0.0854559987783432,
|
|
"beta_dpo/beta_margin_mean": 0.28103265166282654,
|
|
"beta_dpo/beta_margin_std": 0.3670780658721924,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -5.807326793670654,
|
|
"beta_dpo/gap_mean": 394.8181457519531,
|
|
"beta_dpo/gap_std": 384.7374267578125,
|
|
"beta_dpo/loss_margin_mean": 281.0326232910156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9060205580029369,
|
|
"grad_norm": 39.271549224853516,
|
|
"learning_rate": 1.378797888467345e-08,
|
|
"logits/chosen": -2.678978443145752,
|
|
"logits/rejected": -2.696861982345581,
|
|
"loss": 1.1238,
|
|
"step": 617
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.199589967727661,
|
|
"beta_dpo/beta_margin_grad_mean": -0.255416601896286,
|
|
"beta_dpo/beta_margin_grad_std": 0.2696244418621063,
|
|
"beta_dpo/beta_margin_mean": 1545.2900390625,
|
|
"beta_dpo/beta_margin_std": 2523.5302734375,
|
|
"beta_dpo/beta_used": 3.199589967727661,
|
|
"beta_dpo/beta_used_raw": 2.9987692832946777,
|
|
"beta_dpo/gap_mean": 389.8605041503906,
|
|
"beta_dpo/gap_std": 387.74383544921875,
|
|
"beta_dpo/loss_margin_mean": 414.118408203125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9074889867841409,
|
|
"grad_norm": 48797.57421875,
|
|
"learning_rate": 1.3370790793601371e-08,
|
|
"logits/chosen": -2.6983931064605713,
|
|
"logits/rejected": -2.777221918106079,
|
|
"loss": 29.3627,
|
|
"step": 618
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.616031527519226,
|
|
"beta_dpo/beta_margin_grad_mean": -0.21139781177043915,
|
|
"beta_dpo/beta_margin_grad_std": 0.22920767962932587,
|
|
"beta_dpo/beta_margin_mean": 684.386962890625,
|
|
"beta_dpo/beta_margin_std": 936.7298583984375,
|
|
"beta_dpo/beta_used": 1.616031527519226,
|
|
"beta_dpo/beta_used_raw": 1.383022665977478,
|
|
"beta_dpo/gap_mean": 402.82708740234375,
|
|
"beta_dpo/gap_std": 391.6029052734375,
|
|
"beta_dpo/loss_margin_mean": 444.0133972167969,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.908957415565345,
|
|
"grad_norm": 21.482271194458008,
|
|
"learning_rate": 1.2959838102258535e-08,
|
|
"logits/chosen": -2.6790366172790527,
|
|
"logits/rejected": -2.7796411514282227,
|
|
"loss": 0.5328,
|
|
"step": 619
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5794578790664673,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27765581011772156,
|
|
"beta_dpo/beta_margin_grad_std": 0.2895503342151642,
|
|
"beta_dpo/beta_margin_mean": 221.4326934814453,
|
|
"beta_dpo/beta_margin_std": 396.32781982421875,
|
|
"beta_dpo/beta_used": 0.5794578790664673,
|
|
"beta_dpo/beta_used_raw": -1.1368510723114014,
|
|
"beta_dpo/gap_mean": 396.19281005859375,
|
|
"beta_dpo/gap_std": 392.14300537109375,
|
|
"beta_dpo/loss_margin_mean": 336.462646484375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9104258443465492,
|
|
"grad_norm": 11751.0576171875,
|
|
"learning_rate": 1.2555131639630567e-08,
|
|
"logits/chosen": -2.7585866451263428,
|
|
"logits/rejected": -2.8155956268310547,
|
|
"loss": 1.5936,
|
|
"step": 620
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.6563297510147095,
|
|
"beta_dpo/beta_margin_grad_mean": -0.12106994539499283,
|
|
"beta_dpo/beta_margin_grad_std": 0.32161012291908264,
|
|
"beta_dpo/beta_margin_mean": 764.5799560546875,
|
|
"beta_dpo/beta_margin_std": 776.7243041992188,
|
|
"beta_dpo/beta_used": 1.6563297510147095,
|
|
"beta_dpo/beta_used_raw": 1.6563297510147095,
|
|
"beta_dpo/gap_mean": 396.5146789550781,
|
|
"beta_dpo/gap_std": 397.0887451171875,
|
|
"beta_dpo/loss_margin_mean": 447.8368225097656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9118942731277533,
|
|
"grad_norm": 31736.255859375,
|
|
"learning_rate": 1.2156682070109086e-08,
|
|
"logits/chosen": -2.6869962215423584,
|
|
"logits/rejected": -2.7847015857696533,
|
|
"loss": 14.073,
|
|
"step": 621
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.41488850116729736,
|
|
"beta_dpo/beta_margin_grad_std": 0.07999890297651291,
|
|
"beta_dpo/beta_margin_mean": 0.355562299489975,
|
|
"beta_dpo/beta_margin_std": 0.34979644417762756,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.0101447105407715,
|
|
"beta_dpo/gap_mean": 397.34625244140625,
|
|
"beta_dpo/gap_std": 393.946533203125,
|
|
"beta_dpo/loss_margin_mean": 355.5622863769531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9133627019089574,
|
|
"grad_norm": 31.479806900024414,
|
|
"learning_rate": 1.1764499893210878e-08,
|
|
"logits/chosen": -2.645836353302002,
|
|
"logits/rejected": -2.7034900188446045,
|
|
"loss": 1.1033,
|
|
"step": 622
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4140608608722687,
|
|
"beta_dpo/beta_margin_grad_std": 0.09136771410703659,
|
|
"beta_dpo/beta_margin_mean": 0.3609658181667328,
|
|
"beta_dpo/beta_margin_std": 0.39438700675964355,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.0260062217712402,
|
|
"beta_dpo/gap_mean": 391.07568359375,
|
|
"beta_dpo/gap_std": 391.4143981933594,
|
|
"beta_dpo/loss_margin_mean": 360.9658203125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9148311306901615,
|
|
"grad_norm": 28.536666870117188,
|
|
"learning_rate": 1.1378595443300998e-08,
|
|
"logits/chosen": -2.684234619140625,
|
|
"logits/rejected": -2.754396677017212,
|
|
"loss": 1.0963,
|
|
"step": 623
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.6343820095062256,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24445195496082306,
|
|
"beta_dpo/beta_margin_grad_std": 0.26650404930114746,
|
|
"beta_dpo/beta_margin_mean": 1267.75537109375,
|
|
"beta_dpo/beta_margin_std": 1948.750732421875,
|
|
"beta_dpo/beta_used": 2.6343820095062256,
|
|
"beta_dpo/beta_used_raw": 0.8192785978317261,
|
|
"beta_dpo/gap_mean": 396.8379821777344,
|
|
"beta_dpo/gap_std": 400.57037353515625,
|
|
"beta_dpo/loss_margin_mean": 465.2864990234375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9162995594713657,
|
|
"grad_norm": 90469.3125,
|
|
"learning_rate": 1.0998978889320582e-08,
|
|
"logits/chosen": -2.734670400619507,
|
|
"logits/rejected": -2.8197638988494873,
|
|
"loss": 3.3046,
|
|
"step": 624
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0586000680923462,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23547331988811493,
|
|
"beta_dpo/beta_margin_grad_std": 0.24736563861370087,
|
|
"beta_dpo/beta_margin_mean": 472.75970458984375,
|
|
"beta_dpo/beta_margin_std": 665.16015625,
|
|
"beta_dpo/beta_used": 1.0586000680923462,
|
|
"beta_dpo/beta_used_raw": -0.7660439014434814,
|
|
"beta_dpo/gap_mean": 401.3667297363281,
|
|
"beta_dpo/gap_std": 384.7314453125,
|
|
"beta_dpo/loss_margin_mean": 412.4743957519531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9177679882525698,
|
|
"grad_norm": 26121.599609375,
|
|
"learning_rate": 1.0625660234518913e-08,
|
|
"logits/chosen": -2.6922459602355957,
|
|
"logits/rejected": -2.767746925354004,
|
|
"loss": 17.1082,
|
|
"step": 625
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40667426586151123,
|
|
"beta_dpo/beta_margin_grad_std": 0.09596937894821167,
|
|
"beta_dpo/beta_margin_mean": 0.39498090744018555,
|
|
"beta_dpo/beta_margin_std": 0.4217798709869385,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.5139915943145752,
|
|
"beta_dpo/gap_mean": 402.25384521484375,
|
|
"beta_dpo/gap_std": 386.7181091308594,
|
|
"beta_dpo/loss_margin_mean": 394.98089599609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9192364170337739,
|
|
"grad_norm": 34.07730484008789,
|
|
"learning_rate": 1.0258649316189721e-08,
|
|
"logits/chosen": -2.66159725189209,
|
|
"logits/rejected": -2.7381627559661865,
|
|
"loss": 1.0654,
|
|
"step": 626
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.29160737991333,
|
|
"beta_dpo/beta_margin_grad_mean": -0.07827726006507874,
|
|
"beta_dpo/beta_margin_grad_std": 0.26832348108291626,
|
|
"beta_dpo/beta_margin_mean": 1679.4234619140625,
|
|
"beta_dpo/beta_margin_std": 2345.806884765625,
|
|
"beta_dpo/beta_used": 3.29160737991333,
|
|
"beta_dpo/beta_used_raw": 3.29160737991333,
|
|
"beta_dpo/gap_mean": 412.205810546875,
|
|
"beta_dpo/gap_std": 392.6907958984375,
|
|
"beta_dpo/loss_margin_mean": 481.80487060546875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.920704845814978,
|
|
"grad_norm": 38.47300720214844,
|
|
"learning_rate": 9.897955805412e-09,
|
|
"logits/chosen": -2.662278413772583,
|
|
"logits/rejected": -2.77905011177063,
|
|
"loss": 0.0004,
|
|
"step": 627
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.16923931241035461,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25576457381248474,
|
|
"beta_dpo/beta_margin_grad_std": 0.2738649845123291,
|
|
"beta_dpo/beta_margin_mean": 74.43370819091797,
|
|
"beta_dpo/beta_margin_std": 140.65432739257812,
|
|
"beta_dpo/beta_used": 0.16923931241035461,
|
|
"beta_dpo/beta_used_raw": -0.19239750504493713,
|
|
"beta_dpo/gap_mean": 422.8117370605469,
|
|
"beta_dpo/gap_std": 404.112548828125,
|
|
"beta_dpo/loss_margin_mean": 459.7860107421875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.922173274596182,
|
|
"grad_norm": 5785.26611328125,
|
|
"learning_rate": 9.543589206795238e-09,
|
|
"logits/chosen": -2.7000410556793213,
|
|
"logits/rejected": -2.7991576194763184,
|
|
"loss": 6.3099,
|
|
"step": 628
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.24494610726833344,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25123071670532227,
|
|
"beta_dpo/beta_margin_grad_std": 0.26800966262817383,
|
|
"beta_dpo/beta_margin_mean": 103.84109497070312,
|
|
"beta_dpo/beta_margin_std": 145.34765625,
|
|
"beta_dpo/beta_used": 0.24494610726833344,
|
|
"beta_dpo/beta_used_raw": -1.8342968225479126,
|
|
"beta_dpo/gap_mean": 422.31903076171875,
|
|
"beta_dpo/gap_std": 400.2264404296875,
|
|
"beta_dpo/loss_margin_mean": 406.9854736328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9236417033773862,
|
|
"grad_norm": 3084.765380859375,
|
|
"learning_rate": 9.19555885822887e-09,
|
|
"logits/chosen": -2.698057174682617,
|
|
"logits/rejected": -2.772855281829834,
|
|
"loss": 1.5217,
|
|
"step": 629
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.010254332795739174,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2662602663040161,
|
|
"beta_dpo/beta_margin_grad_std": 0.2255883365869522,
|
|
"beta_dpo/beta_margin_mean": 4.899608135223389,
|
|
"beta_dpo/beta_margin_std": 7.351168155670166,
|
|
"beta_dpo/beta_used": 0.010254332795739174,
|
|
"beta_dpo/beta_used_raw": -1.358794927597046,
|
|
"beta_dpo/gap_mean": 418.43365478515625,
|
|
"beta_dpo/gap_std": 380.6859436035156,
|
|
"beta_dpo/loss_margin_mean": 416.1902160644531,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9251101321585903,
|
|
"grad_norm": 128.09698486328125,
|
|
"learning_rate": 8.85387393063622e-09,
|
|
"logits/chosen": -2.66994571685791,
|
|
"logits/rejected": -2.7564034461975098,
|
|
"loss": 0.6538,
|
|
"step": 630
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.572502613067627,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25285497307777405,
|
|
"beta_dpo/beta_margin_grad_std": 0.25762128829956055,
|
|
"beta_dpo/beta_margin_mean": 263.9234619140625,
|
|
"beta_dpo/beta_margin_std": 379.5833435058594,
|
|
"beta_dpo/beta_used": 0.572502613067627,
|
|
"beta_dpo/beta_used_raw": -2.5124354362487793,
|
|
"beta_dpo/gap_mean": 411.2584228515625,
|
|
"beta_dpo/gap_std": 380.47344970703125,
|
|
"beta_dpo/loss_margin_mean": 375.1467590332031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9265785609397944,
|
|
"grad_norm": 6237.07177734375,
|
|
"learning_rate": 8.518543427732949e-09,
|
|
"logits/chosen": -2.7391209602355957,
|
|
"logits/rejected": -2.8207530975341797,
|
|
"loss": 0.5817,
|
|
"step": 631
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.6863118410110474,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2555917203426361,
|
|
"beta_dpo/beta_margin_grad_std": 0.2693924009799957,
|
|
"beta_dpo/beta_margin_mean": 278.4541931152344,
|
|
"beta_dpo/beta_margin_std": 425.826416015625,
|
|
"beta_dpo/beta_used": 0.6863118410110474,
|
|
"beta_dpo/beta_used_raw": -3.893587350845337,
|
|
"beta_dpo/gap_mean": 409.9851989746094,
|
|
"beta_dpo/gap_std": 375.94549560546875,
|
|
"beta_dpo/loss_margin_mean": 378.54730224609375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9280469897209985,
|
|
"grad_norm": 13932.6142578125,
|
|
"learning_rate": 8.189576185789637e-09,
|
|
"logits/chosen": -2.712128162384033,
|
|
"logits/rejected": -2.77974009513855,
|
|
"loss": 20.7515,
|
|
"step": 632
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8220680356025696,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2414667308330536,
|
|
"beta_dpo/beta_margin_grad_std": 0.25375181436538696,
|
|
"beta_dpo/beta_margin_mean": 374.5078125,
|
|
"beta_dpo/beta_margin_std": 545.6969604492188,
|
|
"beta_dpo/beta_used": 0.8220680356025696,
|
|
"beta_dpo/beta_used_raw": -0.9592093229293823,
|
|
"beta_dpo/gap_mean": 402.13140869140625,
|
|
"beta_dpo/gap_std": 372.6382141113281,
|
|
"beta_dpo/loss_margin_mean": 393.3555908203125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9295154185022027,
|
|
"grad_norm": 38270.734375,
|
|
"learning_rate": 7.866980873399015e-09,
|
|
"logits/chosen": -2.7510485649108887,
|
|
"logits/rejected": -2.832095146179199,
|
|
"loss": 33.6173,
|
|
"step": 633
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.9843794107437134,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23467524349689484,
|
|
"beta_dpo/beta_margin_grad_std": 0.25170350074768066,
|
|
"beta_dpo/beta_margin_mean": 1058.7847900390625,
|
|
"beta_dpo/beta_margin_std": 1698.30029296875,
|
|
"beta_dpo/beta_used": 1.9843794107437134,
|
|
"beta_dpo/beta_used_raw": -1.2706066370010376,
|
|
"beta_dpo/gap_mean": 416.826171875,
|
|
"beta_dpo/gap_std": 384.57305908203125,
|
|
"beta_dpo/loss_margin_mean": 465.8406982421875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9309838472834068,
|
|
"grad_norm": 23538.87109375,
|
|
"learning_rate": 7.550765991247654e-09,
|
|
"logits/chosen": -2.7076034545898438,
|
|
"logits/rejected": -2.8257126808166504,
|
|
"loss": 3.3576,
|
|
"step": 634
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4157167077064514,
|
|
"beta_dpo/beta_margin_grad_std": 0.08389006555080414,
|
|
"beta_dpo/beta_margin_mean": 0.3560367524623871,
|
|
"beta_dpo/beta_margin_std": 0.37797629833221436,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -5.190537452697754,
|
|
"beta_dpo/gap_mean": 407.5589904785156,
|
|
"beta_dpo/gap_std": 385.02099609375,
|
|
"beta_dpo/loss_margin_mean": 356.0367431640625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9324522760646109,
|
|
"grad_norm": 27.796987533569336,
|
|
"learning_rate": 7.240939871891699e-09,
|
|
"logits/chosen": -2.733865261077881,
|
|
"logits/rejected": -2.7787771224975586,
|
|
"loss": 1.1092,
|
|
"step": 635
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4153151214122772,
|
|
"beta_dpo/beta_margin_grad_std": 0.08367358148097992,
|
|
"beta_dpo/beta_margin_mean": 0.3526773154735565,
|
|
"beta_dpo/beta_margin_std": 0.35728541016578674,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.042080402374268,
|
|
"beta_dpo/gap_mean": 395.2179870605469,
|
|
"beta_dpo/gap_std": 383.86175537109375,
|
|
"beta_dpo/loss_margin_mean": 352.67730712890625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.933920704845815,
|
|
"grad_norm": 32.076175689697266,
|
|
"learning_rate": 6.937510679537628e-09,
|
|
"logits/chosen": -2.7010793685913086,
|
|
"logits/rejected": -2.7489147186279297,
|
|
"loss": 1.1087,
|
|
"step": 636
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39836955070495605,
|
|
"beta_dpo/beta_margin_grad_std": 0.09108960628509521,
|
|
"beta_dpo/beta_margin_mean": 0.43129873275756836,
|
|
"beta_dpo/beta_margin_std": 0.4063197672367096,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.6715822219848633,
|
|
"beta_dpo/gap_mean": 400.29400634765625,
|
|
"beta_dpo/gap_std": 386.3330993652344,
|
|
"beta_dpo/loss_margin_mean": 431.2987060546875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9353891336270191,
|
|
"grad_norm": 30.193471908569336,
|
|
"learning_rate": 6.640486409826785e-09,
|
|
"logits/chosen": -2.697937250137329,
|
|
"logits/rejected": -2.79691219329834,
|
|
"loss": 1.098,
|
|
"step": 637
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0422019958496094,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2520017921924591,
|
|
"beta_dpo/beta_margin_grad_std": 0.26553109288215637,
|
|
"beta_dpo/beta_margin_mean": 457.24200439453125,
|
|
"beta_dpo/beta_margin_std": 703.8070678710938,
|
|
"beta_dpo/beta_used": 1.0422019958496094,
|
|
"beta_dpo/beta_used_raw": -1.928234577178955,
|
|
"beta_dpo/gap_mean": 401.11383056640625,
|
|
"beta_dpo/gap_std": 381.48553466796875,
|
|
"beta_dpo/loss_margin_mean": 406.8208923339844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9368575624082232,
|
|
"grad_norm": 4143.97802734375,
|
|
"learning_rate": 6.349874889624962e-09,
|
|
"logits/chosen": -2.663649320602417,
|
|
"logits/rejected": -2.739919662475586,
|
|
"loss": 2.9598,
|
|
"step": 638
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8174912929534912,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27413713932037354,
|
|
"beta_dpo/beta_margin_grad_std": 0.2775159478187561,
|
|
"beta_dpo/beta_margin_mean": 371.14276123046875,
|
|
"beta_dpo/beta_margin_std": 529.0587768554688,
|
|
"beta_dpo/beta_used": 0.8174912929534912,
|
|
"beta_dpo/beta_used_raw": -3.1657114028930664,
|
|
"beta_dpo/gap_mean": 384.8955078125,
|
|
"beta_dpo/gap_std": 367.0290832519531,
|
|
"beta_dpo/loss_margin_mean": 319.404052734375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9383259911894273,
|
|
"grad_norm": 12486.169921875,
|
|
"learning_rate": 6.065683776815933e-09,
|
|
"logits/chosen": -2.637770652770996,
|
|
"logits/rejected": -2.6620218753814697,
|
|
"loss": 5.479,
|
|
"step": 639
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.4668891727924347,
|
|
"beta_dpo/beta_margin_grad_mean": -0.19809961318969727,
|
|
"beta_dpo/beta_margin_grad_std": 0.21003539860248566,
|
|
"beta_dpo/beta_margin_mean": 256.6402893066406,
|
|
"beta_dpo/beta_margin_std": 346.9462890625,
|
|
"beta_dpo/beta_used": 0.4668891727924347,
|
|
"beta_dpo/beta_used_raw": 0.036036670207977295,
|
|
"beta_dpo/gap_mean": 402.61187744140625,
|
|
"beta_dpo/gap_std": 376.2877197265625,
|
|
"beta_dpo/loss_margin_mean": 500.78619384765625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9397944199706314,
|
|
"grad_norm": 27.00104522705078,
|
|
"learning_rate": 5.7879205600998296e-09,
|
|
"logits/chosen": -2.6643662452697754,
|
|
"logits/rejected": -2.7640390396118164,
|
|
"loss": 0.5343,
|
|
"step": 640
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.7024071216583252,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2545595169067383,
|
|
"beta_dpo/beta_margin_grad_std": 0.2657167315483093,
|
|
"beta_dpo/beta_margin_mean": 737.9752197265625,
|
|
"beta_dpo/beta_margin_std": 1113.3966064453125,
|
|
"beta_dpo/beta_used": 1.7024071216583252,
|
|
"beta_dpo/beta_used_raw": -0.05958735942840576,
|
|
"beta_dpo/gap_mean": 413.01031494140625,
|
|
"beta_dpo/gap_std": 378.18206787109375,
|
|
"beta_dpo/loss_margin_mean": 426.4712829589844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9412628487518355,
|
|
"grad_norm": 37319.4296875,
|
|
"learning_rate": 5.516592558795746e-09,
|
|
"logits/chosen": -2.7087669372558594,
|
|
"logits/rejected": -2.7869772911071777,
|
|
"loss": 33.9035,
|
|
"step": 641
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.8232624530792236,
|
|
"beta_dpo/beta_margin_grad_mean": -0.20511434972286224,
|
|
"beta_dpo/beta_margin_grad_std": 0.21880197525024414,
|
|
"beta_dpo/beta_margin_mean": 1559.873291015625,
|
|
"beta_dpo/beta_margin_std": 2155.26123046875,
|
|
"beta_dpo/beta_used": 2.8232624530792236,
|
|
"beta_dpo/beta_used_raw": 0.7488954067230225,
|
|
"beta_dpo/gap_mean": 417.9560852050781,
|
|
"beta_dpo/gap_std": 394.83203125,
|
|
"beta_dpo/loss_margin_mean": 470.5616455078125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9427312775330396,
|
|
"grad_norm": 23.94333839416504,
|
|
"learning_rate": 5.251706922648868e-09,
|
|
"logits/chosen": -2.6790223121643066,
|
|
"logits/rejected": -2.778899908065796,
|
|
"loss": 0.554,
|
|
"step": 642
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40317976474761963,
|
|
"beta_dpo/beta_margin_grad_std": 0.07390031218528748,
|
|
"beta_dpo/beta_margin_mean": 0.40420055389404297,
|
|
"beta_dpo/beta_margin_std": 0.3269351124763489,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0951257944107056,
|
|
"beta_dpo/gap_mean": 421.8905944824219,
|
|
"beta_dpo/gap_std": 384.7071228027344,
|
|
"beta_dpo/loss_margin_mean": 404.2005310058594,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9441997063142438,
|
|
"grad_norm": 28.286523818969727,
|
|
"learning_rate": 4.993270631642038e-09,
|
|
"logits/chosen": -2.6818013191223145,
|
|
"logits/rejected": -2.7871358394622803,
|
|
"loss": 1.0385,
|
|
"step": 643
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4079779386520386,
|
|
"beta_dpo/beta_margin_grad_std": 0.08079829812049866,
|
|
"beta_dpo/beta_margin_mean": 0.38570505380630493,
|
|
"beta_dpo/beta_margin_std": 0.35576584935188293,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -1.0808398723602295,
|
|
"beta_dpo/gap_mean": 418.0806884765625,
|
|
"beta_dpo/gap_std": 381.2316589355469,
|
|
"beta_dpo/loss_margin_mean": 385.7050476074219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9456681350954479,
|
|
"grad_norm": 33.19130325317383,
|
|
"learning_rate": 4.741290495811873e-09,
|
|
"logits/chosen": -2.66062068939209,
|
|
"logits/rejected": -2.7422003746032715,
|
|
"loss": 1.0446,
|
|
"step": 644
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4123542308807373,
|
|
"beta_dpo/beta_margin_grad_std": 0.09075028449296951,
|
|
"beta_dpo/beta_margin_mean": 0.37113770842552185,
|
|
"beta_dpo/beta_margin_std": 0.40058884024620056,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.672241687774658,
|
|
"beta_dpo/gap_mean": 409.40924072265625,
|
|
"beta_dpo/gap_std": 381.2076416015625,
|
|
"beta_dpo/loss_margin_mean": 371.1376953125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.947136563876652,
|
|
"grad_norm": 28.928796768188477,
|
|
"learning_rate": 4.495773155069299e-09,
|
|
"logits/chosen": -2.667562484741211,
|
|
"logits/rejected": -2.756563663482666,
|
|
"loss": 1.1056,
|
|
"step": 645
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.956828236579895,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24536260962486267,
|
|
"beta_dpo/beta_margin_grad_std": 0.25645798444747925,
|
|
"beta_dpo/beta_margin_mean": 412.48236083984375,
|
|
"beta_dpo/beta_margin_std": 568.2828979492188,
|
|
"beta_dpo/beta_used": 0.956828236579895,
|
|
"beta_dpo/beta_used_raw": -0.878277599811554,
|
|
"beta_dpo/gap_mean": 402.3372802734375,
|
|
"beta_dpo/gap_std": 367.37091064453125,
|
|
"beta_dpo/loss_margin_mean": 363.10498046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9486049926578561,
|
|
"grad_norm": 9614.8828125,
|
|
"learning_rate": 4.256725079024553e-09,
|
|
"logits/chosen": -2.677600383758545,
|
|
"logits/rejected": -2.7275893688201904,
|
|
"loss": 6.6603,
|
|
"step": 646
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4117124080657959,
|
|
"beta_dpo/beta_margin_grad_std": 0.06875015795230865,
|
|
"beta_dpo/beta_margin_mean": 0.3639964163303375,
|
|
"beta_dpo/beta_margin_std": 0.28941068053245544,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.700502395629883,
|
|
"beta_dpo/gap_mean": 389.188232421875,
|
|
"beta_dpo/gap_std": 354.70404052734375,
|
|
"beta_dpo/loss_margin_mean": 363.99639892578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9500734214390602,
|
|
"grad_norm": 31.953645706176758,
|
|
"learning_rate": 4.024152566816791e-09,
|
|
"logits/chosen": -2.6596388816833496,
|
|
"logits/rejected": -2.7517409324645996,
|
|
"loss": 1.1028,
|
|
"step": 647
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.5703181624412537,
|
|
"beta_dpo/beta_margin_grad_mean": -0.24799123406410217,
|
|
"beta_dpo/beta_margin_grad_std": 0.264789342880249,
|
|
"beta_dpo/beta_margin_mean": 255.6763916015625,
|
|
"beta_dpo/beta_margin_std": 416.8722229003906,
|
|
"beta_dpo/beta_used": 0.5703181624412537,
|
|
"beta_dpo/beta_used_raw": -1.4812275171279907,
|
|
"beta_dpo/gap_mean": 395.81396484375,
|
|
"beta_dpo/gap_std": 356.5727233886719,
|
|
"beta_dpo/loss_margin_mean": 429.5019836425781,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9515418502202643,
|
|
"grad_norm": 27103.84375,
|
|
"learning_rate": 3.798061746947995e-09,
|
|
"logits/chosen": -2.6732091903686523,
|
|
"logits/rejected": -2.8048720359802246,
|
|
"loss": 31.5114,
|
|
"step": 648
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4284921884536743,
|
|
"beta_dpo/beta_margin_grad_std": 0.09621818363666534,
|
|
"beta_dpo/beta_margin_mean": 0.3019213378429413,
|
|
"beta_dpo/beta_margin_std": 0.4140036404132843,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.014761924743652,
|
|
"beta_dpo/gap_mean": 385.2996826171875,
|
|
"beta_dpo/gap_std": 365.2505798339844,
|
|
"beta_dpo/loss_margin_mean": 301.92132568359375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9530102790014684,
|
|
"grad_norm": 31.426589965820312,
|
|
"learning_rate": 3.5784585771215235e-09,
|
|
"logits/chosen": -2.712721347808838,
|
|
"logits/rejected": -2.750772476196289,
|
|
"loss": 1.1226,
|
|
"step": 649
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.429518222808838,
|
|
"beta_dpo/beta_margin_grad_mean": -0.15641333162784576,
|
|
"beta_dpo/beta_margin_grad_std": 0.3630194067955017,
|
|
"beta_dpo/beta_margin_mean": 1180.8328857421875,
|
|
"beta_dpo/beta_margin_std": 1849.820556640625,
|
|
"beta_dpo/beta_used": 2.429518222808838,
|
|
"beta_dpo/beta_used_raw": 2.429518222808838,
|
|
"beta_dpo/gap_mean": 391.6793212890625,
|
|
"beta_dpo/gap_std": 385.2804870605469,
|
|
"beta_dpo/loss_margin_mean": 466.9042663574219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9544787077826725,
|
|
"grad_norm": 76912.0078125,
|
|
"learning_rate": 3.3653488440851253e-09,
|
|
"logits/chosen": -2.6398544311523438,
|
|
"logits/rejected": -2.746957540512085,
|
|
"loss": 46.2267,
|
|
"step": 650
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.192887306213379,
|
|
"beta_dpo/beta_margin_grad_mean": -0.14100220799446106,
|
|
"beta_dpo/beta_margin_grad_std": 0.33823278546333313,
|
|
"beta_dpo/beta_margin_mean": 1079.196533203125,
|
|
"beta_dpo/beta_margin_std": 1474.2933349609375,
|
|
"beta_dpo/beta_used": 2.192887306213379,
|
|
"beta_dpo/beta_used_raw": 2.192887306213379,
|
|
"beta_dpo/gap_mean": 405.810546875,
|
|
"beta_dpo/gap_std": 392.9246826171875,
|
|
"beta_dpo/loss_margin_mean": 439.4105529785156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9559471365638766,
|
|
"grad_norm": 72114.8203125,
|
|
"learning_rate": 3.158738163478475e-09,
|
|
"logits/chosen": -2.726027011871338,
|
|
"logits/rejected": -2.8322134017944336,
|
|
"loss": 14.6975,
|
|
"step": 651
|
|
},
|
|
{
|
|
"beta_dpo/beta": 3.001312732696533,
|
|
"beta_dpo/beta_margin_grad_mean": -0.15693050622940063,
|
|
"beta_dpo/beta_margin_grad_std": 0.3628394305706024,
|
|
"beta_dpo/beta_margin_mean": 1289.045654296875,
|
|
"beta_dpo/beta_margin_std": 2198.432861328125,
|
|
"beta_dpo/beta_used": 3.001312732696533,
|
|
"beta_dpo/beta_used_raw": 3.001312732696533,
|
|
"beta_dpo/gap_mean": 408.07373046875,
|
|
"beta_dpo/gap_std": 401.121337890625,
|
|
"beta_dpo/loss_margin_mean": 430.41156005859375,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9574155653450808,
|
|
"grad_norm": 58857.55859375,
|
|
"learning_rate": 2.9586319796851555e-09,
|
|
"logits/chosen": -2.7092862129211426,
|
|
"logits/rejected": -2.830409049987793,
|
|
"loss": 64.9825,
|
|
"step": 652
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.4299049377441406,
|
|
"beta_dpo/beta_margin_grad_mean": -0.23352642357349396,
|
|
"beta_dpo/beta_margin_grad_std": 0.25692373514175415,
|
|
"beta_dpo/beta_margin_mean": 643.51318359375,
|
|
"beta_dpo/beta_margin_std": 985.338623046875,
|
|
"beta_dpo/beta_used": 1.4299049377441406,
|
|
"beta_dpo/beta_used_raw": 0.3514312505722046,
|
|
"beta_dpo/gap_mean": 420.1622314453125,
|
|
"beta_dpo/gap_std": 404.7218933105469,
|
|
"beta_dpo/loss_margin_mean": 500.4862365722656,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9588839941262849,
|
|
"grad_norm": 35622.0390625,
|
|
"learning_rate": 2.7650355656892166e-09,
|
|
"logits/chosen": -2.677021026611328,
|
|
"logits/rejected": -2.806358814239502,
|
|
"loss": 23.9097,
|
|
"step": 653
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9755141735076904,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2645564079284668,
|
|
"beta_dpo/beta_margin_grad_std": 0.2780976891517639,
|
|
"beta_dpo/beta_margin_mean": 399.7411804199219,
|
|
"beta_dpo/beta_margin_std": 707.6503295898438,
|
|
"beta_dpo/beta_used": 0.9755141735076904,
|
|
"beta_dpo/beta_used_raw": -0.04932612180709839,
|
|
"beta_dpo/gap_mean": 423.97796630859375,
|
|
"beta_dpo/gap_std": 405.8636474609375,
|
|
"beta_dpo/loss_margin_mean": 406.8052062988281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.960352422907489,
|
|
"grad_norm": 18767.052734375,
|
|
"learning_rate": 2.577954022936174e-09,
|
|
"logits/chosen": -2.706268787384033,
|
|
"logits/rejected": -2.816140651702881,
|
|
"loss": 4.9214,
|
|
"step": 654
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.0244628190994263,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25356224179267883,
|
|
"beta_dpo/beta_margin_grad_std": 0.2714630663394928,
|
|
"beta_dpo/beta_margin_mean": 510.8900451660156,
|
|
"beta_dpo/beta_margin_std": 796.9216918945312,
|
|
"beta_dpo/beta_used": 1.0244628190994263,
|
|
"beta_dpo/beta_used_raw": 0.2029227614402771,
|
|
"beta_dpo/gap_mean": 427.8386535644531,
|
|
"beta_dpo/gap_std": 412.89520263671875,
|
|
"beta_dpo/loss_margin_mean": 435.32635498046875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9618208516886931,
|
|
"grad_norm": 18119.2421875,
|
|
"learning_rate": 2.397392281198729e-09,
|
|
"logits/chosen": -2.680917263031006,
|
|
"logits/rejected": -2.7932283878326416,
|
|
"loss": 3.7187,
|
|
"step": 655
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.7679896354675293,
|
|
"beta_dpo/beta_margin_grad_mean": -0.08084268122911453,
|
|
"beta_dpo/beta_margin_grad_std": 0.26834091544151306,
|
|
"beta_dpo/beta_margin_mean": 1555.604248046875,
|
|
"beta_dpo/beta_margin_std": 1787.474853515625,
|
|
"beta_dpo/beta_used": 2.7679896354675293,
|
|
"beta_dpo/beta_used_raw": 2.7679896354675293,
|
|
"beta_dpo/gap_mean": 440.5277404785156,
|
|
"beta_dpo/gap_std": 414.5347900390625,
|
|
"beta_dpo/loss_margin_mean": 540.3606567382812,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9632892804698973,
|
|
"grad_norm": 6175.9521484375,
|
|
"learning_rate": 2.223355098446622e-09,
|
|
"logits/chosen": -2.654201030731201,
|
|
"logits/rejected": -2.7860169410705566,
|
|
"loss": 3.1373,
|
|
"step": 656
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.38836878538131714,
|
|
"beta_dpo/beta_margin_grad_std": 0.08058323711156845,
|
|
"beta_dpo/beta_margin_mean": 0.4706077575683594,
|
|
"beta_dpo/beta_margin_std": 0.3598029613494873,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.769383430480957,
|
|
"beta_dpo/gap_mean": 452.3384094238281,
|
|
"beta_dpo/gap_std": 401.4930725097656,
|
|
"beta_dpo/loss_margin_mean": 470.60772705078125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9647577092511013,
|
|
"grad_norm": 29.89715003967285,
|
|
"learning_rate": 2.055847060721566e-09,
|
|
"logits/chosen": -2.7141106128692627,
|
|
"logits/rejected": -2.818988084793091,
|
|
"loss": 1.0684,
|
|
"step": 657
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3125349283218384,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2526739537715912,
|
|
"beta_dpo/beta_margin_grad_std": 0.2634952664375305,
|
|
"beta_dpo/beta_margin_mean": 149.250732421875,
|
|
"beta_dpo/beta_margin_std": 212.36988830566406,
|
|
"beta_dpo/beta_used": 0.3125349283218384,
|
|
"beta_dpo/beta_used_raw": -3.3986754417419434,
|
|
"beta_dpo/gap_mean": 444.14215087890625,
|
|
"beta_dpo/gap_std": 383.74365234375,
|
|
"beta_dpo/loss_margin_mean": 420.6098937988281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9662261380323054,
|
|
"grad_norm": 15924.490234375,
|
|
"learning_rate": 1.8948725820160662e-09,
|
|
"logits/chosen": -2.6853952407836914,
|
|
"logits/rejected": -2.801093578338623,
|
|
"loss": 8.8263,
|
|
"step": 658
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4057837426662445,
|
|
"beta_dpo/beta_margin_grad_std": 0.08504177629947662,
|
|
"beta_dpo/beta_margin_mean": 0.3961847126483917,
|
|
"beta_dpo/beta_margin_std": 0.37236061692237854,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.6913902759552,
|
|
"beta_dpo/gap_mean": 439.3583679199219,
|
|
"beta_dpo/gap_std": 378.472412109375,
|
|
"beta_dpo/loss_margin_mean": 396.1846923828125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9676945668135095,
|
|
"grad_norm": 33.702049255371094,
|
|
"learning_rate": 1.7404359041573723e-09,
|
|
"logits/chosen": -2.692453145980835,
|
|
"logits/rejected": -2.7410359382629395,
|
|
"loss": 1.0658,
|
|
"step": 659
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.3935113251209259,
|
|
"beta_dpo/beta_margin_grad_std": 0.07827742397785187,
|
|
"beta_dpo/beta_margin_mean": 0.4469809830188751,
|
|
"beta_dpo/beta_margin_std": 0.3453811705112457,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.6651883125305176,
|
|
"beta_dpo/gap_mean": 438.81976318359375,
|
|
"beta_dpo/gap_std": 376.5857849121094,
|
|
"beta_dpo/loss_margin_mean": 446.9809875488281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9691629955947136,
|
|
"grad_norm": 25.069368362426758,
|
|
"learning_rate": 1.592541096695571e-09,
|
|
"logits/chosen": -2.6554760932922363,
|
|
"logits/rejected": -2.734593391418457,
|
|
"loss": 1.0505,
|
|
"step": 660
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40231382846832275,
|
|
"beta_dpo/beta_margin_grad_std": 0.0794859230518341,
|
|
"beta_dpo/beta_margin_mean": 0.4129953980445862,
|
|
"beta_dpo/beta_margin_std": 0.3699604570865631,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.9038610458374023,
|
|
"beta_dpo/gap_mean": 431.3531494140625,
|
|
"beta_dpo/gap_std": 369.5039367675781,
|
|
"beta_dpo/loss_margin_mean": 412.9953918457031,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9706314243759178,
|
|
"grad_norm": 28.522525787353516,
|
|
"learning_rate": 1.4511920567963908e-09,
|
|
"logits/chosen": -2.6922972202301025,
|
|
"logits/rejected": -2.764153242111206,
|
|
"loss": 1.0573,
|
|
"step": 661
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.1851645708084106,
|
|
"beta_dpo/beta_margin_grad_mean": -0.10670406371355057,
|
|
"beta_dpo/beta_margin_grad_std": 0.29752376675605774,
|
|
"beta_dpo/beta_margin_mean": 517.0401000976562,
|
|
"beta_dpo/beta_margin_std": 781.3206787109375,
|
|
"beta_dpo/beta_used": 1.1851645708084106,
|
|
"beta_dpo/beta_used_raw": 1.1851645708084106,
|
|
"beta_dpo/gap_mean": 432.52001953125,
|
|
"beta_dpo/gap_std": 375.65093994140625,
|
|
"beta_dpo/loss_margin_mean": 417.3674011230469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9720998531571219,
|
|
"grad_norm": 10333.798828125,
|
|
"learning_rate": 1.3163925091384532e-09,
|
|
"logits/chosen": -2.6792619228363037,
|
|
"logits/rejected": -2.7577905654907227,
|
|
"loss": 4.327,
|
|
"step": 662
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39322903752326965,
|
|
"beta_dpo/beta_margin_grad_std": 0.10028216242790222,
|
|
"beta_dpo/beta_margin_mean": 0.46100181341171265,
|
|
"beta_dpo/beta_margin_std": 0.46238890290260315,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.2035021781921387,
|
|
"beta_dpo/gap_mean": 433.63494873046875,
|
|
"beta_dpo/gap_std": 389.35064697265625,
|
|
"beta_dpo/loss_margin_mean": 461.0018005371094,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.973568281938326,
|
|
"grad_norm": 36.53968811035156,
|
|
"learning_rate": 1.1881460058152382e-09,
|
|
"logits/chosen": -2.7297868728637695,
|
|
"logits/rejected": -2.8309314250946045,
|
|
"loss": 1.0669,
|
|
"step": 663
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.8343223929405212,
|
|
"beta_dpo/beta_margin_grad_mean": -0.20197950303554535,
|
|
"beta_dpo/beta_margin_grad_std": 0.22016191482543945,
|
|
"beta_dpo/beta_margin_mean": 431.966064453125,
|
|
"beta_dpo/beta_margin_std": 605.8634643554688,
|
|
"beta_dpo/beta_used": 0.8343223929405212,
|
|
"beta_dpo/beta_used_raw": -0.03142118453979492,
|
|
"beta_dpo/gap_mean": 449.9844970703125,
|
|
"beta_dpo/gap_std": 394.67535400390625,
|
|
"beta_dpo/loss_margin_mean": 532.53369140625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9750367107195301,
|
|
"grad_norm": 37235.65625,
|
|
"learning_rate": 1.066455926241383e-09,
|
|
"logits/chosen": -2.619297504425049,
|
|
"logits/rejected": -2.753797769546509,
|
|
"loss": 5.8712,
|
|
"step": 664
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.3265571594238281,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2155420482158661,
|
|
"beta_dpo/beta_margin_grad_std": 0.23008538782596588,
|
|
"beta_dpo/beta_margin_mean": 137.3500518798828,
|
|
"beta_dpo/beta_margin_std": 180.5250701904297,
|
|
"beta_dpo/beta_used": 0.3265571594238281,
|
|
"beta_dpo/beta_used_raw": -1.14801025390625,
|
|
"beta_dpo/gap_mean": 450.2773742675781,
|
|
"beta_dpo/gap_std": 383.91949462890625,
|
|
"beta_dpo/loss_margin_mean": 422.32843017578125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9765051395007343,
|
|
"grad_norm": 31.23434829711914,
|
|
"learning_rate": 9.513254770636137e-10,
|
|
"logits/chosen": -2.6968719959259033,
|
|
"logits/rejected": -2.7933096885681152,
|
|
"loss": 0.5228,
|
|
"step": 665
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40940189361572266,
|
|
"beta_dpo/beta_margin_grad_std": 0.08813069015741348,
|
|
"beta_dpo/beta_margin_mean": 0.3812684416770935,
|
|
"beta_dpo/beta_margin_std": 0.38156983256340027,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.0323309898376465,
|
|
"beta_dpo/gap_mean": 441.28741455078125,
|
|
"beta_dpo/gap_std": 383.9112548828125,
|
|
"beta_dpo/loss_margin_mean": 381.2684326171875,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9779735682819384,
|
|
"grad_norm": 33.36760330200195,
|
|
"learning_rate": 8.427576920763956e-10,
|
|
"logits/chosen": -2.680263042449951,
|
|
"logits/rejected": -2.7376909255981445,
|
|
"loss": 1.0748,
|
|
"step": 666
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.045063599944114685,
|
|
"beta_dpo/beta_margin_grad_mean": -0.26865431666374207,
|
|
"beta_dpo/beta_margin_grad_std": 0.2749733626842499,
|
|
"beta_dpo/beta_margin_mean": 20.753093719482422,
|
|
"beta_dpo/beta_margin_std": 34.60969543457031,
|
|
"beta_dpo/beta_used": 0.045063599944114685,
|
|
"beta_dpo/beta_used_raw": -0.0253823921084404,
|
|
"beta_dpo/gap_mean": 433.1988525390625,
|
|
"beta_dpo/gap_std": 388.9822998046875,
|
|
"beta_dpo/loss_margin_mean": 400.0951843261719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9794419970631424,
|
|
"grad_norm": 1199.2330322265625,
|
|
"learning_rate": 7.407554321417764e-10,
|
|
"logits/chosen": -2.7143611907958984,
|
|
"logits/rejected": -2.784897804260254,
|
|
"loss": 0.5765,
|
|
"step": 667
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4063694179058075,
|
|
"beta_dpo/beta_margin_grad_std": 0.0748078003525734,
|
|
"beta_dpo/beta_margin_mean": 0.3895391523838043,
|
|
"beta_dpo/beta_margin_std": 0.3211991488933563,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.1987266540527344,
|
|
"beta_dpo/gap_mean": 420.1002197265625,
|
|
"beta_dpo/gap_std": 376.9314880371094,
|
|
"beta_dpo/loss_margin_mean": 389.53912353515625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9809104258443465,
|
|
"grad_norm": 25.63221549987793,
|
|
"learning_rate": 6.453213851142225e-10,
|
|
"logits/chosen": -2.7238945960998535,
|
|
"logits/rejected": -2.78692889213562,
|
|
"loss": 1.07,
|
|
"step": 668
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.40521949529647827,
|
|
"beta_dpo/beta_margin_grad_std": 0.08128622174263,
|
|
"beta_dpo/beta_margin_mean": 0.3963834047317505,
|
|
"beta_dpo/beta_margin_std": 0.35128095746040344,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.542426109313965,
|
|
"beta_dpo/gap_mean": 417.46697998046875,
|
|
"beta_dpo/gap_std": 373.0389404296875,
|
|
"beta_dpo/loss_margin_mean": 396.3833923339844,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9823788546255506,
|
|
"grad_norm": 26.949857711791992,
|
|
"learning_rate": 5.564580657695939e-10,
|
|
"logits/chosen": -2.7172703742980957,
|
|
"logits/rejected": -2.7912344932556152,
|
|
"loss": 1.0688,
|
|
"step": 669
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.05918378010392189,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2792292535305023,
|
|
"beta_dpo/beta_margin_grad_std": 0.29437491297721863,
|
|
"beta_dpo/beta_margin_mean": 25.40052604675293,
|
|
"beta_dpo/beta_margin_std": 43.54753112792969,
|
|
"beta_dpo/beta_used": 0.05918378010392189,
|
|
"beta_dpo/beta_used_raw": -2.199822187423706,
|
|
"beta_dpo/gap_mean": 417.28021240234375,
|
|
"beta_dpo/gap_std": 383.91400146484375,
|
|
"beta_dpo/loss_margin_mean": 422.6147155761719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9838472834067548,
|
|
"grad_norm": 3242.469482421875,
|
|
"learning_rate": 4.741678157389739e-10,
|
|
"logits/chosen": -2.631437063217163,
|
|
"logits/rejected": -2.709677219390869,
|
|
"loss": 3.7298,
|
|
"step": 670
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.2660801410675049,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2764192223548889,
|
|
"beta_dpo/beta_margin_grad_std": 0.29103344678878784,
|
|
"beta_dpo/beta_margin_mean": 98.8425064086914,
|
|
"beta_dpo/beta_margin_std": 195.78741455078125,
|
|
"beta_dpo/beta_used": 0.2660801410675049,
|
|
"beta_dpo/beta_used_raw": -1.3270782232284546,
|
|
"beta_dpo/gap_mean": 414.47650146484375,
|
|
"beta_dpo/gap_std": 394.97772216796875,
|
|
"beta_dpo/loss_margin_mean": 404.5050964355469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9853157121879589,
|
|
"grad_norm": 9340.4814453125,
|
|
"learning_rate": 3.9845280344705245e-10,
|
|
"logits/chosen": -2.6861400604248047,
|
|
"logits/rejected": -2.7837982177734375,
|
|
"loss": 6.7776,
|
|
"step": 671
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4036671817302704,
|
|
"beta_dpo/beta_margin_grad_std": 0.09355795383453369,
|
|
"beta_dpo/beta_margin_mean": 0.40639743208885193,
|
|
"beta_dpo/beta_margin_std": 0.4057061970233917,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -2.5519256591796875,
|
|
"beta_dpo/gap_mean": 413.4635009765625,
|
|
"beta_dpo/gap_std": 395.70111083984375,
|
|
"beta_dpo/loss_margin_mean": 406.3974304199219,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.986784140969163,
|
|
"grad_norm": 35.250362396240234,
|
|
"learning_rate": 3.293150240547549e-10,
|
|
"logits/chosen": -2.658691167831421,
|
|
"logits/rejected": -2.7492518424987793,
|
|
"loss": 1.077,
|
|
"step": 672
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9086623191833496,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2269849181175232,
|
|
"beta_dpo/beta_margin_grad_std": 0.24361680448055267,
|
|
"beta_dpo/beta_margin_mean": 426.99517822265625,
|
|
"beta_dpo/beta_margin_std": 589.7709350585938,
|
|
"beta_dpo/beta_used": 0.9086623191833496,
|
|
"beta_dpo/beta_used_raw": -1.017942190170288,
|
|
"beta_dpo/gap_mean": 421.73614501953125,
|
|
"beta_dpo/gap_std": 390.53887939453125,
|
|
"beta_dpo/loss_margin_mean": 465.11651611328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9882525697503671,
|
|
"grad_norm": 38753.8828125,
|
|
"learning_rate": 2.6675629940689504e-10,
|
|
"logits/chosen": -2.6246771812438965,
|
|
"logits/rejected": -2.71297287940979,
|
|
"loss": 3.2363,
|
|
"step": 673
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4113142192363739,
|
|
"beta_dpo/beta_margin_grad_std": 0.0917758122086525,
|
|
"beta_dpo/beta_margin_mean": 0.3718118667602539,
|
|
"beta_dpo/beta_margin_std": 0.39119818806648254,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -0.3440747559070587,
|
|
"beta_dpo/gap_mean": 413.60247802734375,
|
|
"beta_dpo/gap_std": 390.3453674316406,
|
|
"beta_dpo/loss_margin_mean": 371.81182861328125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9897209985315712,
|
|
"grad_norm": 26.38853645324707,
|
|
"learning_rate": 2.1077827798404725e-10,
|
|
"logits/chosen": -2.6841392517089844,
|
|
"logits/rejected": -2.7508318424224854,
|
|
"loss": 1.0447,
|
|
"step": 674
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.39887651801109314,
|
|
"beta_dpo/beta_margin_grad_std": 0.08341260254383087,
|
|
"beta_dpo/beta_margin_mean": 0.42533355951309204,
|
|
"beta_dpo/beta_margin_std": 0.3670670986175537,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -3.083004951477051,
|
|
"beta_dpo/gap_mean": 418.37017822265625,
|
|
"beta_dpo/gap_std": 389.00335693359375,
|
|
"beta_dpo/loss_margin_mean": 425.3335266113281,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9911894273127754,
|
|
"grad_norm": 22.43824005126953,
|
|
"learning_rate": 1.6138243485910863e-10,
|
|
"logits/chosen": -2.626479148864746,
|
|
"logits/rejected": -2.709594249725342,
|
|
"loss": 1.0741,
|
|
"step": 675
|
|
},
|
|
{
|
|
"beta_dpo/beta": 1.3021284341812134,
|
|
"beta_dpo/beta_margin_grad_mean": -0.27147722244262695,
|
|
"beta_dpo/beta_margin_grad_std": 0.28175440430641174,
|
|
"beta_dpo/beta_margin_mean": 523.4214477539062,
|
|
"beta_dpo/beta_margin_std": 893.8363647460938,
|
|
"beta_dpo/beta_used": 1.3021284341812134,
|
|
"beta_dpo/beta_used_raw": 0.7559030055999756,
|
|
"beta_dpo/gap_mean": 408.2674255371094,
|
|
"beta_dpo/gap_std": 382.36383056640625,
|
|
"beta_dpo/loss_margin_mean": 371.8644714355469,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9926578560939795,
|
|
"grad_norm": 25749.53515625,
|
|
"learning_rate": 1.1857007165852472e-10,
|
|
"logits/chosen": -2.6717138290405273,
|
|
"logits/rejected": -2.7089462280273438,
|
|
"loss": 3.2861,
|
|
"step": 676
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.582223653793335,
|
|
"beta_dpo/beta_margin_grad_mean": -0.047779832035303116,
|
|
"beta_dpo/beta_margin_grad_std": 0.21129241585731506,
|
|
"beta_dpo/beta_margin_mean": 1187.6051025390625,
|
|
"beta_dpo/beta_margin_std": 1076.4876708984375,
|
|
"beta_dpo/beta_used": 2.582223653793335,
|
|
"beta_dpo/beta_used_raw": 2.582223653793335,
|
|
"beta_dpo/gap_mean": 415.4775695800781,
|
|
"beta_dpo/gap_std": 378.42694091796875,
|
|
"beta_dpo/loss_margin_mean": 460.754150390625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9941262848751835,
|
|
"grad_norm": 50163.39453125,
|
|
"learning_rate": 8.23423165278725e-11,
|
|
"logits/chosen": -2.6218011379241943,
|
|
"logits/rejected": -2.706326961517334,
|
|
"loss": 5.7215,
|
|
"step": 677
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.4946085810661316,
|
|
"beta_dpo/beta_margin_grad_mean": -0.264606773853302,
|
|
"beta_dpo/beta_margin_grad_std": 0.2809548079967499,
|
|
"beta_dpo/beta_margin_mean": 201.0238800048828,
|
|
"beta_dpo/beta_margin_std": 325.8772277832031,
|
|
"beta_dpo/beta_used": 0.4946085810661316,
|
|
"beta_dpo/beta_used_raw": -0.259298712015152,
|
|
"beta_dpo/gap_mean": 415.98052978515625,
|
|
"beta_dpo/gap_std": 376.99005126953125,
|
|
"beta_dpo/loss_margin_mean": 404.6373291015625,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9955947136563876,
|
|
"grad_norm": 6914.888671875,
|
|
"learning_rate": 5.270012410216185e-11,
|
|
"logits/chosen": -2.6387507915496826,
|
|
"logits/rejected": -2.7189407348632812,
|
|
"loss": 4.2516,
|
|
"step": 678
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.0010000000474974513,
|
|
"beta_dpo/beta_margin_grad_mean": -0.4169081151485443,
|
|
"beta_dpo/beta_margin_grad_std": 0.07902750372886658,
|
|
"beta_dpo/beta_margin_mean": 0.34554827213287354,
|
|
"beta_dpo/beta_margin_std": 0.3380792737007141,
|
|
"beta_dpo/beta_used": 0.0010000000474974513,
|
|
"beta_dpo/beta_used_raw": -4.045163631439209,
|
|
"beta_dpo/gap_mean": 405.5008850097656,
|
|
"beta_dpo/gap_std": 373.1779479980469,
|
|
"beta_dpo/loss_margin_mean": 345.5482482910156,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9970631424375918,
|
|
"grad_norm": 32.21836471557617,
|
|
"learning_rate": 2.9644275480772416e-11,
|
|
"logits/chosen": -2.7105813026428223,
|
|
"logits/rejected": -2.7737274169921875,
|
|
"loss": 1.0921,
|
|
"step": 679
|
|
},
|
|
{
|
|
"beta_dpo/beta": 2.237010955810547,
|
|
"beta_dpo/beta_margin_grad_mean": -0.25544458627700806,
|
|
"beta_dpo/beta_margin_grad_std": 0.27421900629997253,
|
|
"beta_dpo/beta_margin_mean": 1121.7413330078125,
|
|
"beta_dpo/beta_margin_std": 1776.8070068359375,
|
|
"beta_dpo/beta_used": 2.237010955810547,
|
|
"beta_dpo/beta_used_raw": 1.5813672542572021,
|
|
"beta_dpo/gap_mean": 415.71636962890625,
|
|
"beta_dpo/gap_std": 377.8543395996094,
|
|
"beta_dpo/loss_margin_mean": 493.08428955078125,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 0.9985315712187959,
|
|
"grad_norm": 100621.9609375,
|
|
"learning_rate": 1.31753782067201e-11,
|
|
"logits/chosen": -2.689682722091675,
|
|
"logits/rejected": -2.7969093322753906,
|
|
"loss": 49.8028,
|
|
"step": 680
|
|
},
|
|
{
|
|
"beta_dpo/beta": 0.9950853586196899,
|
|
"beta_dpo/beta_margin_grad_mean": -0.2544940412044525,
|
|
"beta_dpo/beta_margin_grad_std": 0.2666052281856537,
|
|
"beta_dpo/beta_margin_mean": 497.1185607910156,
|
|
"beta_dpo/beta_margin_std": 809.5509643554688,
|
|
"beta_dpo/beta_used": 0.9950853586196899,
|
|
"beta_dpo/beta_used_raw": -0.9259518980979919,
|
|
"beta_dpo/gap_mean": 416.84112548828125,
|
|
"beta_dpo/gap_std": 377.1964416503906,
|
|
"beta_dpo/loss_margin_mean": 425.1723327636719,
|
|
"beta_dpo/mask_keep_frac": 0.78125,
|
|
"epoch": 1.0,
|
|
"grad_norm": 16219.0615234375,
|
|
"learning_rate": 3.2938662507808745e-12,
|
|
"logits/chosen": -2.7180047035217285,
|
|
"logits/rejected": -2.8164992332458496,
|
|
"loss": 11.4723,
|
|
"step": 681
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 681,
|
|
"total_flos": 0.0,
|
|
"train_loss": 4.511853932745809,
|
|
"train_runtime": 2770.5528,
|
|
"train_samples_per_second": 15.736,
|
|
"train_steps_per_second": 0.246
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 681,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 200,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|