W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.5
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC 25ce01a846 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.5
Source: Original Platform
2026-05-13 18:55:34 +08:00

15707 lines
589 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.51216888427734,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.19176483154297,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": 0.011138916015625,
"KL/mean": 0.01624371111392975,
"KL/rejected_KL_mean": 0.021350860595703125,
"KL/std": 0.2411436140537262,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.010211020708084106,
"fcm_dpo/q_t": 0.5002557039260864,
"grad_norm": 71.00299072265625,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.48165208101272583,
"logits/rejected": -0.44220495223999023,
"logps/chosen": -60.970458984375,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.6512451171875,
"loss": 1.3877,
"margin_dpo/margin_mean": -0.010210752487182617,
"margin_dpo/margin_std": 0.36640018224716187,
"step": 3
},
{
"KL/chosen_KL_mean": 0.000141143798828125,
"KL/mean": 0.032934680581092834,
"KL/rejected_KL_mean": 0.06572723388671875,
"KL/std": 0.26030904054641724,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06558361649513245,
"fcm_dpo/q_t": 0.501638650894165,
"grad_norm": 72.05967712402344,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.46887367963790894,
"logits/rejected": -0.4411189556121826,
"logps/chosen": -56.76757049560547,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.58138275146484,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06558331847190857,
"margin_dpo/margin_std": 0.38221287727355957,
"step": 4
},
{
"KL/chosen_KL_mean": -0.008298873901367188,
"KL/mean": 0.015575245022773743,
"KL/rejected_KL_mean": 0.03945159912109375,
"KL/std": 0.26084285974502563,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04775431752204895,
"fcm_dpo/q_t": 0.5011929273605347,
"grad_norm": 89.86566925048828,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.5145574808120728,
"logits/rejected": -0.47081369161605835,
"logps/chosen": -53.86767578125,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.10973358154297,
"loss": 1.3914,
"margin_dpo/margin_mean": -0.04775422811508179,
"margin_dpo/margin_std": 0.37420204281806946,
"step": 5
},
{
"KL/chosen_KL_mean": -0.02581787109375,
"KL/mean": 0.0013223886489868164,
"KL/rejected_KL_mean": 0.02846527099609375,
"KL/std": 0.254623144865036,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.05428045988082886,
"fcm_dpo/q_t": 0.5013556480407715,
"grad_norm": 91.9102554321289,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.5099787712097168,
"logits/rejected": -0.4680352807044983,
"logps/chosen": -63.033302307128906,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.61688232421875,
"loss": 1.3921,
"margin_dpo/margin_mean": -0.05428001284599304,
"margin_dpo/margin_std": 0.3631099462509155,
"step": 6
},
{
"KL/chosen_KL_mean": 0.014619827270507812,
"KL/mean": 0.007828325033187866,
"KL/rejected_KL_mean": 0.00103759765625,
"KL/std": 0.28590360283851624,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.013581156730651855,
"fcm_dpo/q_t": 0.4996616840362549,
"grad_norm": 82.45801544189453,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.5054452419281006,
"logits/rejected": -0.47154390811920166,
"logps/chosen": -57.76020050048828,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.9195556640625,
"loss": 1.3854,
"margin_dpo/margin_mean": 0.01358148455619812,
"margin_dpo/margin_std": 0.4422546327114105,
"step": 7
},
{
"KL/chosen_KL_mean": 0.01812744140625,
"KL/mean": 0.019948050379753113,
"KL/rejected_KL_mean": 0.021778106689453125,
"KL/std": 0.27578145265579224,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.003651231527328491,
"fcm_dpo/q_t": 0.5000911951065063,
"grad_norm": 79.03565216064453,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.49302494525909424,
"logits/rejected": -0.46536964178085327,
"logps/chosen": -58.697906494140625,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.28964233398438,
"loss": 1.387,
"margin_dpo/margin_mean": -0.0036510825157165527,
"margin_dpo/margin_std": 0.3407641053199768,
"step": 8
},
{
"KL/chosen_KL_mean": 0.00136566162109375,
"KL/mean": 0.0017981678247451782,
"KL/rejected_KL_mean": 0.002231597900390625,
"KL/std": 0.2728290259838104,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.000867694616317749,
"fcm_dpo/q_t": 0.5000215768814087,
"grad_norm": 85.38078308105469,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.48713457584381104,
"logits/rejected": -0.4398614168167114,
"logps/chosen": -69.865478515625,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.60043334960938,
"loss": 1.3868,
"margin_dpo/margin_mean": -0.0008689463138580322,
"margin_dpo/margin_std": 0.41773098707199097,
"step": 9
},
{
"KL/chosen_KL_mean": -0.0075225830078125,
"KL/mean": -0.013112656772136688,
"KL/rejected_KL_mean": -0.018695831298828125,
"KL/std": 0.253761887550354,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01117675006389618,
"fcm_dpo/q_t": 0.4997211694717407,
"grad_norm": 70.45758819580078,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.498451828956604,
"logits/rejected": -0.4553406834602356,
"logps/chosen": -48.36520767211914,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.39076232910156,
"loss": 1.3855,
"margin_dpo/margin_mean": 0.011177659034729004,
"margin_dpo/margin_std": 0.3267936706542969,
"step": 10
},
{
"KL/chosen_KL_mean": 0.0040302276611328125,
"KL/mean": -0.007809385657310486,
"KL/rejected_KL_mean": -0.019649505615234375,
"KL/std": 0.23007726669311523,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.023678749799728394,
"fcm_dpo/q_t": 0.49940839409828186,
"grad_norm": 68.75594329833984,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.45761120319366455,
"logits/rejected": -0.4324725270271301,
"logps/chosen": -53.01282501220703,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.8000259399414,
"loss": 1.3842,
"margin_dpo/margin_mean": 0.02367931604385376,
"margin_dpo/margin_std": 0.3247877359390259,
"step": 11
},
{
"KL/chosen_KL_mean": -0.022077560424804688,
"KL/mean": -0.030066579580307007,
"KL/rejected_KL_mean": -0.038055419921875,
"KL/std": 0.2843082547187805,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.015976309776306152,
"fcm_dpo/q_t": 0.4996001422405243,
"grad_norm": 89.8901596069336,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.5207371711730957,
"logits/rejected": -0.48408442735671997,
"logps/chosen": -61.8275146484375,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.89633178710938,
"loss": 1.3851,
"margin_dpo/margin_mean": 0.015975654125213623,
"margin_dpo/margin_std": 0.39000314474105835,
"step": 12
},
{
"KL/chosen_KL_mean": 0.018079757690429688,
"KL/mean": 0.03311806917190552,
"KL/rejected_KL_mean": 0.04815673828125,
"KL/std": 0.299828439950943,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.030076265335083008,
"fcm_dpo/q_t": 0.5007518529891968,
"grad_norm": 79.48653411865234,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.46800822019577026,
"logits/rejected": -0.4389190673828125,
"logps/chosen": -64.24227905273438,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.15492248535156,
"loss": 1.3898,
"margin_dpo/margin_mean": -0.030076295137405396,
"margin_dpo/margin_std": 0.40763989090919495,
"step": 13
},
{
"KL/chosen_KL_mean": -0.027566909790039062,
"KL/mean": -0.022734537720680237,
"KL/rejected_KL_mean": -0.01790618896484375,
"KL/std": 0.27561646699905396,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.009662479162216187,
"fcm_dpo/q_t": 0.5002409219741821,
"grad_norm": 85.24372863769531,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.46077555418014526,
"logits/rejected": -0.420840322971344,
"logps/chosen": -58.137779235839844,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.06498718261719,
"loss": 1.3878,
"margin_dpo/margin_mean": -0.009661853313446045,
"margin_dpo/margin_std": 0.4318341016769409,
"step": 14
},
{
"KL/chosen_KL_mean": -0.0386199951171875,
"KL/mean": -0.045700281858444214,
"KL/rejected_KL_mean": -0.052783966064453125,
"KL/std": 0.23048102855682373,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014169454574584961,
"fcm_dpo/q_t": 0.49964454770088196,
"grad_norm": 64.31354522705078,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.5125108361244202,
"logits/rejected": -0.49486109614372253,
"logps/chosen": -57.005531311035156,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.8614273071289,
"loss": 1.3852,
"margin_dpo/margin_mean": 0.014169484376907349,
"margin_dpo/margin_std": 0.3576519191265106,
"step": 15
},
{
"KL/chosen_KL_mean": 0.019407272338867188,
"KL/mean": -0.03246238827705383,
"KL/rejected_KL_mean": -0.08432769775390625,
"KL/std": 0.27561452984809875,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10373848676681519,
"fcm_dpo/q_t": 0.49740785360336304,
"grad_norm": 83.85110473632812,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.4948147237300873,
"logits/rejected": -0.45085063576698303,
"logps/chosen": -61.72048568725586,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.45379638671875,
"loss": 1.3763,
"margin_dpo/margin_mean": 0.10373827815055847,
"margin_dpo/margin_std": 0.38368722796440125,
"step": 16
},
{
"KL/chosen_KL_mean": 0.032100677490234375,
"KL/mean": -0.02624189853668213,
"KL/rejected_KL_mean": -0.08458328247070312,
"KL/std": 0.2859901189804077,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11668473482131958,
"fcm_dpo/q_t": 0.497084379196167,
"grad_norm": 78.15997314453125,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.511766791343689,
"logits/rejected": -0.475982666015625,
"logps/chosen": -67.67823791503906,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.46324157714844,
"loss": 1.3751,
"margin_dpo/margin_mean": 0.11668440699577332,
"margin_dpo/margin_std": 0.39858338236808777,
"step": 17
},
{
"KL/chosen_KL_mean": 0.030401229858398438,
"KL/mean": -0.01864929497241974,
"KL/rejected_KL_mean": -0.06769943237304688,
"KL/std": 0.24296830594539642,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09810277819633484,
"fcm_dpo/q_t": 0.4975478947162628,
"grad_norm": 81.9465560913086,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.5076760053634644,
"logits/rejected": -0.4538722038269043,
"logps/chosen": -47.70909118652344,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.53998565673828,
"loss": 1.3768,
"margin_dpo/margin_mean": 0.09810280799865723,
"margin_dpo/margin_std": 0.3186327815055847,
"step": 18
},
{
"KL/chosen_KL_mean": 0.016094207763671875,
"KL/mean": -0.009688735008239746,
"KL/rejected_KL_mean": -0.035465240478515625,
"KL/std": 0.24112674593925476,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05155247449874878,
"fcm_dpo/q_t": 0.49871158599853516,
"grad_norm": 73.13494110107422,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.5065875053405762,
"logits/rejected": -0.4579053223133087,
"logps/chosen": -70.18927001953125,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.79306030273438,
"loss": 1.3814,
"margin_dpo/margin_mean": 0.05155327916145325,
"margin_dpo/margin_std": 0.3446183204650879,
"step": 19
},
{
"KL/chosen_KL_mean": -0.03119659423828125,
"KL/mean": -0.04489029943943024,
"KL/rejected_KL_mean": -0.0585784912109375,
"KL/std": 0.27683231234550476,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.027387112379074097,
"fcm_dpo/q_t": 0.4993155002593994,
"grad_norm": 73.76658630371094,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5554596185684204,
"logits/rejected": -0.4994776248931885,
"logps/chosen": -50.83443832397461,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.88192749023438,
"loss": 1.3839,
"margin_dpo/margin_mean": 0.027386903762817383,
"margin_dpo/margin_std": 0.36115431785583496,
"step": 20
},
{
"KL/chosen_KL_mean": 0.008577346801757812,
"KL/mean": -0.03955095261335373,
"KL/rejected_KL_mean": -0.087677001953125,
"KL/std": 0.25458982586860657,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0962568074464798,
"fcm_dpo/q_t": 0.49759435653686523,
"grad_norm": 77.66507720947266,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.4872417449951172,
"logits/rejected": -0.46275877952575684,
"logps/chosen": -50.054439544677734,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -77.95646667480469,
"loss": 1.3769,
"margin_dpo/margin_mean": 0.09625642001628876,
"margin_dpo/margin_std": 0.3011889159679413,
"step": 21
},
{
"KL/chosen_KL_mean": 0.05830574035644531,
"KL/mean": -0.05371597409248352,
"KL/rejected_KL_mean": -0.16573715209960938,
"KL/std": 0.32429108023643494,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.22404277324676514,
"fcm_dpo/q_t": 0.494402676820755,
"grad_norm": 84.32189178466797,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.502210259437561,
"logits/rejected": -0.459368497133255,
"logps/chosen": -58.99932861328125,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.67041015625,
"loss": 1.3646,
"margin_dpo/margin_mean": 0.22404247522354126,
"margin_dpo/margin_std": 0.4611518979072571,
"step": 22
},
{
"KL/chosen_KL_mean": 0.014989852905273438,
"KL/mean": -0.061570554971694946,
"KL/rejected_KL_mean": -0.13812637329101562,
"KL/std": 0.32376909255981445,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1531180739402771,
"fcm_dpo/q_t": 0.49617335200309753,
"grad_norm": 80.89447021484375,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.4944395422935486,
"logits/rejected": -0.4721882939338684,
"logps/chosen": -60.062705993652344,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.27767944335938,
"loss": 1.3716,
"margin_dpo/margin_mean": 0.1531185805797577,
"margin_dpo/margin_std": 0.4450337886810303,
"step": 23
},
{
"KL/chosen_KL_mean": 0.009403228759765625,
"KL/mean": -0.08805283904075623,
"KL/rejected_KL_mean": -0.18550872802734375,
"KL/std": 0.26540276408195496,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19491511583328247,
"fcm_dpo/q_t": 0.4951290786266327,
"grad_norm": 82.0745849609375,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.49009591341018677,
"logits/rejected": -0.47387370467185974,
"logps/chosen": -44.28163146972656,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.31072235107422,
"loss": 1.3672,
"margin_dpo/margin_mean": 0.19491589069366455,
"margin_dpo/margin_std": 0.3483674228191376,
"step": 24
},
{
"KL/chosen_KL_mean": 0.0049686431884765625,
"KL/mean": -0.09683172404766083,
"KL/rejected_KL_mean": -0.19862747192382812,
"KL/std": 0.31956106424331665,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20359697937965393,
"fcm_dpo/q_t": 0.4949125647544861,
"grad_norm": 73.71135711669922,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.48128461837768555,
"logits/rejected": -0.45070409774780273,
"logps/chosen": -52.532081604003906,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.54081726074219,
"loss": 1.3664,
"margin_dpo/margin_mean": 0.20359739661216736,
"margin_dpo/margin_std": 0.36989957094192505,
"step": 25
},
{
"KL/chosen_KL_mean": 0.07361602783203125,
"KL/mean": -0.10151402652263641,
"KL/rejected_KL_mean": -0.2766456604003906,
"KL/std": 0.41175055503845215,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3502626121044159,
"fcm_dpo/q_t": 0.49125543236732483,
"grad_norm": 87.7682876586914,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5210403800010681,
"logits/rejected": -0.48897308111190796,
"logps/chosen": -53.84918975830078,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.63636016845703,
"loss": 1.3523,
"margin_dpo/margin_mean": 0.3502630591392517,
"margin_dpo/margin_std": 0.549583911895752,
"step": 26
},
{
"KL/chosen_KL_mean": 0.12030792236328125,
"KL/mean": -0.14729268848896027,
"KL/rejected_KL_mean": -0.4149017333984375,
"KL/std": 0.4725148677825928,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5352023839950562,
"fcm_dpo/q_t": 0.4866352677345276,
"grad_norm": 93.7070541381836,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.4835624098777771,
"logits/rejected": -0.4467238783836365,
"logps/chosen": -42.778221130371094,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.13909912109375,
"loss": 1.3343,
"margin_dpo/margin_mean": 0.535202145576477,
"margin_dpo/margin_std": 0.5412513017654419,
"step": 27
},
{
"KL/chosen_KL_mean": -0.009973526000976562,
"KL/mean": -0.14437878131866455,
"KL/rejected_KL_mean": -0.27878570556640625,
"KL/std": 0.4356634020805359,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.26881152391433716,
"fcm_dpo/q_t": 0.4932839870452881,
"grad_norm": 75.4567642211914,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.5053049921989441,
"logits/rejected": -0.4499465525150299,
"logps/chosen": -60.56647491455078,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.67990112304688,
"loss": 1.3606,
"margin_dpo/margin_mean": 0.26881229877471924,
"margin_dpo/margin_std": 0.6206564903259277,
"step": 28
},
{
"KL/chosen_KL_mean": 0.102294921875,
"KL/mean": -0.17882275581359863,
"KL/rejected_KL_mean": -0.4599418640136719,
"KL/std": 0.5050041079521179,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5622385144233704,
"fcm_dpo/q_t": 0.4859647750854492,
"grad_norm": 90.32099151611328,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.5919088125228882,
"logits/rejected": -0.5483555793762207,
"logps/chosen": -57.70549011230469,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.85429382324219,
"loss": 1.3318,
"margin_dpo/margin_mean": 0.562239408493042,
"margin_dpo/margin_std": 0.5976601839065552,
"step": 29
},
{
"KL/chosen_KL_mean": 0.1593170166015625,
"KL/mean": -0.19309324026107788,
"KL/rejected_KL_mean": -0.5455055236816406,
"KL/std": 0.6001869440078735,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7048233151435852,
"fcm_dpo/q_t": 0.48240745067596436,
"grad_norm": 87.2791519165039,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.4931980073451996,
"logits/rejected": -0.4624643921852112,
"logps/chosen": -52.418052673339844,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -99.03471374511719,
"loss": 1.3181,
"margin_dpo/margin_mean": 0.7048235535621643,
"margin_dpo/margin_std": 0.6384134292602539,
"step": 30
},
{
"KL/chosen_KL_mean": 0.028833389282226562,
"KL/mean": -0.1849493831396103,
"KL/rejected_KL_mean": -0.39873504638671875,
"KL/std": 0.5692216157913208,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.42756929993629456,
"fcm_dpo/q_t": 0.489329993724823,
"grad_norm": 67.7244873046875,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.5044924020767212,
"logits/rejected": -0.4595579504966736,
"logps/chosen": -63.7780876159668,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.29273986816406,
"loss": 1.3454,
"margin_dpo/margin_mean": 0.42756909132003784,
"margin_dpo/margin_std": 0.738195538520813,
"step": 31
},
{
"KL/chosen_KL_mean": 0.25072669982910156,
"KL/mean": -0.14465753734111786,
"KL/rejected_KL_mean": -0.5400428771972656,
"KL/std": 0.7525659799575806,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7907640933990479,
"fcm_dpo/q_t": 0.4803106188774109,
"grad_norm": 83.765625,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.5086926221847534,
"logits/rejected": -0.46712368726730347,
"logps/chosen": -62.488800048828125,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.8575439453125,
"loss": 1.311,
"margin_dpo/margin_mean": 0.790764331817627,
"margin_dpo/margin_std": 0.9426593780517578,
"step": 32
},
{
"KL/chosen_KL_mean": 0.07011795043945312,
"KL/mean": -0.21620376408100128,
"KL/rejected_KL_mean": -0.5025215148925781,
"KL/std": 0.5353639125823975,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.572637677192688,
"fcm_dpo/q_t": 0.4857015609741211,
"grad_norm": 72.43000030517578,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.46579599380493164,
"logits/rejected": -0.4388483166694641,
"logps/chosen": -53.19085693359375,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.38765716552734,
"loss": 1.3308,
"margin_dpo/margin_mean": 0.5726377964019775,
"margin_dpo/margin_std": 0.5959737300872803,
"step": 33
},
{
"KL/chosen_KL_mean": 0.08471107482910156,
"KL/mean": -0.3118361830711365,
"KL/rejected_KL_mean": -0.7083892822265625,
"KL/std": 0.7339533567428589,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7930940389633179,
"fcm_dpo/q_t": 0.48022860288619995,
"grad_norm": 77.76030731201172,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.45824792981147766,
"logits/rejected": -0.4408394694328308,
"logps/chosen": -50.73261642456055,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.63023376464844,
"loss": 1.3103,
"margin_dpo/margin_mean": 0.7930936813354492,
"margin_dpo/margin_std": 0.8268696069717407,
"step": 34
},
{
"KL/chosen_KL_mean": 0.15523910522460938,
"KL/mean": -0.4550577700138092,
"KL/rejected_KL_mean": -1.0653495788574219,
"KL/std": 1.036551594734192,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2205886840820312,
"fcm_dpo/q_t": 0.46972745656967163,
"grad_norm": 82.48926544189453,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.5093280076980591,
"logits/rejected": -0.47247546911239624,
"logps/chosen": -50.8692512512207,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.88977813720703,
"loss": 1.2718,
"margin_dpo/margin_mean": 1.2205885648727417,
"margin_dpo/margin_std": 1.2264792919158936,
"step": 35
},
{
"KL/chosen_KL_mean": 0.03851318359375,
"KL/mean": -0.5172837972640991,
"KL/rejected_KL_mean": -1.0730781555175781,
"KL/std": 1.0524473190307617,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1115931272506714,
"fcm_dpo/q_t": 0.47235026955604553,
"grad_norm": 73.26398468017578,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5541160106658936,
"logits/rejected": -0.5175681114196777,
"logps/chosen": -51.952980041503906,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.11370849609375,
"loss": 1.282,
"margin_dpo/margin_mean": 1.1115922927856445,
"margin_dpo/margin_std": 1.172994613647461,
"step": 36
},
{
"KL/chosen_KL_mean": 0.0073757171630859375,
"KL/mean": -0.5105187892913818,
"KL/rejected_KL_mean": -1.0284156799316406,
"KL/std": 1.0426149368286133,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0357892513275146,
"fcm_dpo/q_t": 0.4743063151836395,
"grad_norm": 61.77984619140625,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.511062502861023,
"logits/rejected": -0.4673815965652466,
"logps/chosen": -62.79973220825195,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.92349243164062,
"loss": 1.2901,
"margin_dpo/margin_mean": 1.0357893705368042,
"margin_dpo/margin_std": 1.3421692848205566,
"step": 37
},
{
"KL/chosen_KL_mean": 0.18421363830566406,
"KL/mean": -0.5241457223892212,
"KL/rejected_KL_mean": -1.2325019836425781,
"KL/std": 1.3683629035949707,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4167163372039795,
"fcm_dpo/q_t": 0.4650591313838959,
"grad_norm": 69.80412292480469,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5317132472991943,
"logits/rejected": -0.4989948868751526,
"logps/chosen": -48.20630645751953,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.14494323730469,
"loss": 1.2566,
"margin_dpo/margin_mean": 1.4167169332504272,
"margin_dpo/margin_std": 1.6653265953063965,
"step": 38
},
{
"KL/chosen_KL_mean": 0.10546493530273438,
"KL/mean": -0.7446496486663818,
"KL/rejected_KL_mean": -1.5947608947753906,
"KL/std": 1.3329132795333862,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.7002233266830444,
"fcm_dpo/q_t": 0.45783674716949463,
"grad_norm": 76.12843322753906,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.5526484847068787,
"logits/rejected": -0.5124742984771729,
"logps/chosen": -50.6450080871582,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.16427612304688,
"loss": 1.2289,
"margin_dpo/margin_mean": 1.7002232074737549,
"margin_dpo/margin_std": 1.4578522443771362,
"step": 39
},
{
"KL/chosen_KL_mean": 0.2315082550048828,
"KL/mean": -0.5488898754119873,
"KL/rejected_KL_mean": -1.3292922973632812,
"KL/std": 1.390199065208435,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5607997179031372,
"fcm_dpo/q_t": 0.4614887237548828,
"grad_norm": 59.96656036376953,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5169125199317932,
"logits/rejected": -0.48675721883773804,
"logps/chosen": -57.75355911254883,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.62936401367188,
"loss": 1.2437,
"margin_dpo/margin_mean": 1.560800313949585,
"margin_dpo/margin_std": 1.693495750427246,
"step": 40
},
{
"KL/chosen_KL_mean": 0.028055191040039062,
"KL/mean": -0.925218403339386,
"KL/rejected_KL_mean": -1.8784866333007812,
"KL/std": 1.867018699645996,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9065409898757935,
"fcm_dpo/q_t": 0.4531286358833313,
"grad_norm": 67.948974609375,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.5261883735656738,
"logits/rejected": -0.4887946844100952,
"logps/chosen": -62.667762756347656,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.90201568603516,
"loss": 1.2148,
"margin_dpo/margin_mean": 1.9065414667129517,
"margin_dpo/margin_std": 1.9881553649902344,
"step": 41
},
{
"KL/chosen_KL_mean": 0.22861289978027344,
"KL/mean": -1.08327054977417,
"KL/rejected_KL_mean": -2.3951492309570312,
"KL/std": 2.1879308223724365,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.11069603264331818,
"fcm_dpo/delta": 0.12672916054725647,
"fcm_dpo/margin": 2.623763084411621,
"fcm_dpo/q_t": 0.43353042006492615,
"grad_norm": 85.34500122070312,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.5527105331420898,
"logits/rejected": -0.5064513683319092,
"logps/chosen": -58.737815856933594,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.30352783203125,
"loss": 1.134,
"margin_dpo/margin_mean": 2.623763084411621,
"margin_dpo/margin_std": 2.4292526245117188,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5411643981933594,
"KL/mean": -0.7341057062149048,
"KL/rejected_KL_mean": -2.0093841552734375,
"KL/std": 1.8598041534423828,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.12222427129745483,
"fcm_dpo/delta": 0.10482510924339294,
"fcm_dpo/margin": 2.550539255142212,
"fcm_dpo/q_t": 0.4277779459953308,
"grad_norm": 83.47020721435547,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.5464547276496887,
"logits/rejected": -0.5218390226364136,
"logps/chosen": -53.614830017089844,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.48957824707031,
"loss": 1.1131,
"margin_dpo/margin_mean": 2.550539255142212,
"margin_dpo/margin_std": 1.9405533075332642,
"step": 43
},
{
"KL/chosen_KL_mean": 0.24042892456054688,
"KL/mean": -1.1693687438964844,
"KL/rejected_KL_mean": -2.5791664123535156,
"KL/std": 2.276883602142334,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.12953037023544312,
"fcm_dpo/delta": 0.042644016444683075,
"fcm_dpo/margin": 2.819594383239746,
"fcm_dpo/q_t": 0.41355597972869873,
"grad_norm": 95.25187683105469,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.4547986388206482,
"logits/rejected": -0.4352615475654602,
"logps/chosen": -49.838069915771484,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.36293029785156,
"loss": 1.0737,
"margin_dpo/margin_mean": 2.819593906402588,
"margin_dpo/margin_std": 2.199711322784424,
"step": 44
},
{
"KL/chosen_KL_mean": 0.1290454864501953,
"KL/mean": -1.0186073780059814,
"KL/rejected_KL_mean": -2.166259765625,
"KL/std": 2.0731301307678223,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.1417458951473236,
"fcm_dpo/delta": 0.08769898861646652,
"fcm_dpo/margin": 2.295309066772461,
"fcm_dpo/q_t": 0.42574048042297363,
"grad_norm": 79.44340515136719,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.4766472280025482,
"logits/rejected": -0.4639527201652527,
"logps/chosen": -48.28588104248047,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.10269165039062,
"loss": 1.1186,
"margin_dpo/margin_mean": 2.2953085899353027,
"margin_dpo/margin_std": 2.5507259368896484,
"step": 45
},
{
"KL/chosen_KL_mean": 0.21740150451660156,
"KL/mean": -1.253394365310669,
"KL/rejected_KL_mean": -2.724193572998047,
"KL/std": 2.6467161178588867,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.14241422712802887,
"fcm_dpo/delta": -0.024083826690912247,
"fcm_dpo/margin": 2.941596031188965,
"fcm_dpo/q_t": 0.4018367528915405,
"grad_norm": 87.27326965332031,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.5110729932785034,
"logits/rejected": -0.4591823220252991,
"logps/chosen": -55.78202438354492,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.37678527832031,
"loss": 1.0582,
"margin_dpo/margin_mean": 2.9415955543518066,
"margin_dpo/margin_std": 3.2013213634490967,
"step": 46
},
{
"KL/chosen_KL_mean": 0.37536048889160156,
"KL/mean": -1.1281816959381104,
"KL/rejected_KL_mean": -2.6317214965820312,
"KL/std": 2.644698143005371,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.14004012942314148,
"fcm_dpo/delta": -0.02779523655772209,
"fcm_dpo/margin": 3.007082462310791,
"fcm_dpo/q_t": 0.39907705783843994,
"grad_norm": 82.21805572509766,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.5815363526344299,
"logits/rejected": -0.5293323397636414,
"logps/chosen": -57.55072021484375,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.31092834472656,
"loss": 1.046,
"margin_dpo/margin_mean": 3.007082462310791,
"margin_dpo/margin_std": 2.746891975402832,
"step": 47
},
{
"KL/chosen_KL_mean": 0.05398750305175781,
"KL/mean": -1.5238816738128662,
"KL/rejected_KL_mean": -3.1017494201660156,
"KL/std": 2.673588514328003,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.1313745081424713,
"fcm_dpo/delta": -0.02106369659304619,
"fcm_dpo/margin": 3.1557374000549316,
"fcm_dpo/q_t": 0.4008999466896057,
"grad_norm": 87.41127014160156,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.5923665761947632,
"logits/rejected": -0.5343451499938965,
"logps/chosen": -57.13408660888672,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -91.11834716796875,
"loss": 1.0545,
"margin_dpo/margin_mean": 3.15573787689209,
"margin_dpo/margin_std": 2.916724681854248,
"step": 48
},
{
"KL/chosen_KL_mean": 0.3004131317138672,
"KL/mean": -1.5658769607543945,
"KL/rejected_KL_mean": -3.4321632385253906,
"KL/std": 3.3690695762634277,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.12684862315654755,
"fcm_dpo/delta": -0.09592577815055847,
"fcm_dpo/margin": 3.7325761318206787,
"fcm_dpo/q_t": 0.38731497526168823,
"grad_norm": 71.87007141113281,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.5684317350387573,
"logits/rejected": -0.5121915936470032,
"logps/chosen": -61.384857177734375,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.19963836669922,
"loss": 1.025,
"margin_dpo/margin_mean": 3.7325758934020996,
"margin_dpo/margin_std": 3.874316930770874,
"step": 49
},
{
"KL/chosen_KL_mean": -0.1456451416015625,
"KL/mean": -2.1883692741394043,
"KL/rejected_KL_mean": -4.231086730957031,
"KL/std": 3.4617319107055664,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.11314891278743744,
"fcm_dpo/delta": -0.08234786987304688,
"fcm_dpo/margin": 4.085442066192627,
"fcm_dpo/q_t": 0.3882601261138916,
"grad_norm": 68.02591705322266,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5450817346572876,
"logits/rejected": -0.5092841982841492,
"logps/chosen": -58.869781494140625,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.5892333984375,
"loss": 1.0227,
"margin_dpo/margin_mean": 4.085441589355469,
"margin_dpo/margin_std": 3.960975408554077,
"step": 50
},
{
"KL/chosen_KL_mean": -0.2403240203857422,
"KL/mean": -2.285604953765869,
"KL/rejected_KL_mean": -4.330890655517578,
"KL/std": 4.1982645988464355,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.10526888072490692,
"fcm_dpo/delta": -0.04377248138189316,
"fcm_dpo/margin": 4.090564250946045,
"fcm_dpo/q_t": 0.4009581208229065,
"grad_norm": 53.3188591003418,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.5137035250663757,
"logits/rejected": -0.4802321493625641,
"logps/chosen": -61.613990783691406,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.33289337158203,
"loss": 1.0685,
"margin_dpo/margin_mean": 4.090563774108887,
"margin_dpo/margin_std": 5.139671325683594,
"step": 51
},
{
"KL/chosen_KL_mean": 0.37200355529785156,
"KL/mean": -2.7163591384887695,
"KL/rejected_KL_mean": -5.804725646972656,
"KL/std": 4.783724784851074,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.0902351513504982,
"fcm_dpo/delta": -0.22192168235778809,
"fcm_dpo/margin": 6.176729202270508,
"fcm_dpo/q_t": 0.35875505208969116,
"grad_norm": 56.78388595581055,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.5432096719741821,
"logits/rejected": -0.48703598976135254,
"logps/chosen": -51.965354919433594,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.77864074707031,
"loss": 0.9533,
"margin_dpo/margin_mean": 6.176729202270508,
"margin_dpo/margin_std": 5.141578197479248,
"step": 52
},
{
"KL/chosen_KL_mean": -0.20578384399414062,
"KL/mean": -3.3196845054626465,
"KL/rejected_KL_mean": -6.433582305908203,
"KL/std": 5.414214611053467,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.08034509420394897,
"fcm_dpo/delta": -0.13391053676605225,
"fcm_dpo/margin": 6.227805137634277,
"fcm_dpo/q_t": 0.38188230991363525,
"grad_norm": 52.141883850097656,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.6204248666763306,
"logits/rejected": -0.5989758968353271,
"logps/chosen": -53.52043533325195,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.21717834472656,
"loss": 1.0126,
"margin_dpo/margin_mean": 6.227805137634277,
"margin_dpo/margin_std": 6.215023517608643,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3681602478027344,
"KL/mean": -3.0784969329833984,
"KL/rejected_KL_mean": -5.788829803466797,
"KL/std": 5.0799665451049805,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.07495428621768951,
"fcm_dpo/delta": -0.008042382076382637,
"fcm_dpo/margin": 5.420670032501221,
"fcm_dpo/q_t": 0.4037601351737976,
"grad_norm": 46.64132308959961,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.5916078090667725,
"logits/rejected": -0.5381488800048828,
"logps/chosen": -51.05682373046875,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.50422668457031,
"loss": 1.0548,
"margin_dpo/margin_mean": 5.420670032501221,
"margin_dpo/margin_std": 5.109761714935303,
"step": 54
},
{
"KL/chosen_KL_mean": -0.9354953765869141,
"KL/mean": -4.157511234283447,
"KL/rejected_KL_mean": -7.379520416259766,
"KL/std": 6.1912150382995605,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.07125985622406006,
"fcm_dpo/delta": -0.07659684866666794,
"fcm_dpo/margin": 6.444026470184326,
"fcm_dpo/q_t": 0.39212772250175476,
"grad_norm": 44.093040466308594,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6305919885635376,
"logits/rejected": -0.5667720437049866,
"logps/chosen": -63.55072784423828,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.3730239868164,
"loss": 1.0454,
"margin_dpo/margin_mean": 6.44402551651001,
"margin_dpo/margin_std": 7.479229927062988,
"step": 55
},
{
"KL/chosen_KL_mean": -0.7067909240722656,
"KL/mean": -3.802842140197754,
"KL/rejected_KL_mean": -6.898895263671875,
"KL/std": 6.042604446411133,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.0678362250328064,
"fcm_dpo/delta": -0.02534756436944008,
"fcm_dpo/margin": 6.192107677459717,
"fcm_dpo/q_t": 0.40231359004974365,
"grad_norm": 39.32019805908203,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.6268018484115601,
"logits/rejected": -0.5851395130157471,
"logps/chosen": -58.63951873779297,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -101.07333374023438,
"loss": 1.0686,
"margin_dpo/margin_mean": 6.192108154296875,
"margin_dpo/margin_std": 7.3723554611206055,
"step": 56
},
{
"KL/chosen_KL_mean": -0.7017936706542969,
"KL/mean": -4.175287246704102,
"KL/rejected_KL_mean": -7.648784637451172,
"KL/std": 5.941527366638184,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.06503595411777496,
"fcm_dpo/delta": -0.06703046709299088,
"fcm_dpo/margin": 6.946985244750977,
"fcm_dpo/q_t": 0.3904920220375061,
"grad_norm": 43.75485610961914,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.588129997253418,
"logits/rejected": -0.5603782534599304,
"logps/chosen": -71.19708251953125,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -103.21424102783203,
"loss": 1.0297,
"margin_dpo/margin_mean": 6.946985721588135,
"margin_dpo/margin_std": 6.547435283660889,
"step": 57
},
{
"KL/chosen_KL_mean": -1.027395248413086,
"KL/mean": -4.887474060058594,
"KL/rejected_KL_mean": -8.747550964355469,
"KL/std": 6.817049980163574,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.06007317081093788,
"fcm_dpo/delta": -0.08280885219573975,
"fcm_dpo/margin": 7.720156192779541,
"fcm_dpo/q_t": 0.3894719183444977,
"grad_norm": 42.88956069946289,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.5993902683258057,
"logits/rejected": -0.5237987041473389,
"logps/chosen": -63.160335540771484,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -93.3648452758789,
"loss": 1.0325,
"margin_dpo/margin_mean": 7.720156669616699,
"margin_dpo/margin_std": 8.126496315002441,
"step": 58
},
{
"KL/chosen_KL_mean": -1.5261516571044922,
"KL/mean": -5.766140460968018,
"KL/rejected_KL_mean": -10.00613021850586,
"KL/std": 7.495608806610107,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.053062424063682556,
"fcm_dpo/delta": -0.07194283604621887,
"fcm_dpo/margin": 8.47997760772705,
"fcm_dpo/q_t": 0.39142686128616333,
"grad_norm": 40.191532135009766,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.6348840594291687,
"logits/rejected": -0.595291793346405,
"logps/chosen": -53.45867919921875,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.89132690429688,
"loss": 1.0373,
"margin_dpo/margin_mean": 8.479978561401367,
"margin_dpo/margin_std": 8.641672134399414,
"step": 59
},
{
"KL/chosen_KL_mean": -2.7480087280273438,
"KL/mean": -6.107456684112549,
"KL/rejected_KL_mean": -9.466899871826172,
"KL/std": 6.831831455230713,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.054739855229854584,
"fcm_dpo/delta": 0.039379529654979706,
"fcm_dpo/margin": 6.718890190124512,
"fcm_dpo/q_t": 0.4139224886894226,
"grad_norm": 41.76387023925781,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.6037277579307556,
"logits/rejected": -0.5432413816452026,
"logps/chosen": -63.6901969909668,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.86029815673828,
"loss": 1.096,
"margin_dpo/margin_mean": 6.718890190124512,
"margin_dpo/margin_std": 7.675610065460205,
"step": 60
},
{
"KL/chosen_KL_mean": -1.6316070556640625,
"KL/mean": -5.852381706237793,
"KL/rejected_KL_mean": -10.073162078857422,
"KL/std": 9.248079299926758,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.05262649431824684,
"fcm_dpo/delta": -0.059413738548755646,
"fcm_dpo/margin": 8.441549301147461,
"fcm_dpo/q_t": 0.3997190594673157,
"grad_norm": 35.92430877685547,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.597582995891571,
"logits/rejected": -0.5633736848831177,
"logps/chosen": -62.26512908935547,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -99.9256591796875,
"loss": 1.0687,
"margin_dpo/margin_mean": 8.441547393798828,
"margin_dpo/margin_std": 11.492226600646973,
"step": 61
},
{
"KL/chosen_KL_mean": -1.7598438262939453,
"KL/mean": -4.921261787414551,
"KL/rejected_KL_mean": -8.082679748535156,
"KL/std": 6.796797275543213,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.05579136312007904,
"fcm_dpo/delta": 0.055100131779909134,
"fcm_dpo/margin": 6.3228349685668945,
"fcm_dpo/q_t": 0.4199411869049072,
"grad_norm": 35.19557189941406,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.611149251461029,
"logits/rejected": -0.5770503282546997,
"logps/chosen": -57.910614013671875,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -83.64888000488281,
"loss": 1.1166,
"margin_dpo/margin_mean": 6.322835922241211,
"margin_dpo/margin_std": 8.347599029541016,
"step": 62
},
{
"KL/chosen_KL_mean": -2.888803482055664,
"KL/mean": -7.189858436584473,
"KL/rejected_KL_mean": -11.490913391113281,
"KL/std": 8.740306854248047,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.051797665655612946,
"fcm_dpo/delta": -0.06353449821472168,
"fcm_dpo/margin": 8.602106094360352,
"fcm_dpo/q_t": 0.3924814760684967,
"grad_norm": 37.61563491821289,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.6047888398170471,
"logits/rejected": -0.5606403350830078,
"logps/chosen": -76.03619384765625,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -109.1009750366211,
"loss": 1.0442,
"margin_dpo/margin_mean": 8.602106094360352,
"margin_dpo/margin_std": 9.156990051269531,
"step": 63
},
{
"KL/chosen_KL_mean": -1.1866321563720703,
"KL/mean": -6.277315139770508,
"KL/rejected_KL_mean": -11.368000030517578,
"KL/std": 9.430252075195312,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.0463373102247715,
"fcm_dpo/delta": -0.10717649757862091,
"fcm_dpo/margin": 10.181361198425293,
"fcm_dpo/q_t": 0.38632842898368835,
"grad_norm": 33.78566360473633,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.5959369540214539,
"logits/rejected": -0.5642282366752625,
"logps/chosen": -55.18523406982422,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -104.89820098876953,
"loss": 1.0364,
"margin_dpo/margin_mean": 10.18136215209961,
"margin_dpo/margin_std": 11.540030479431152,
"step": 64
},
{
"KL/chosen_KL_mean": -3.5648746490478516,
"KL/mean": -8.770621299743652,
"KL/rejected_KL_mean": -13.976367950439453,
"KL/std": 10.70554256439209,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.043465256690979004,
"fcm_dpo/delta": -0.07250865548849106,
"fcm_dpo/margin": 10.411500930786133,
"fcm_dpo/q_t": 0.39109233021736145,
"grad_norm": 33.85374450683594,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6580019593238831,
"logits/rejected": -0.6455116271972656,
"logps/chosen": -68.40087890625,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -123.9228286743164,
"loss": 1.0464,
"margin_dpo/margin_mean": 10.411500930786133,
"margin_dpo/margin_std": 11.7676362991333,
"step": 65
},
{
"KL/chosen_KL_mean": -3.3290576934814453,
"KL/mean": -8.054281234741211,
"KL/rejected_KL_mean": -12.77950668334961,
"KL/std": 9.668466567993164,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.042388349771499634,
"fcm_dpo/delta": -0.004084587097167969,
"fcm_dpo/margin": 9.450448989868164,
"fcm_dpo/q_t": 0.4077484607696533,
"grad_norm": 29.53615951538086,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.6404643058776855,
"logits/rejected": -0.6079626083374023,
"logps/chosen": -54.7725830078125,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -88.41580200195312,
"loss": 1.0842,
"margin_dpo/margin_mean": 9.450448989868164,
"margin_dpo/margin_std": 11.738363265991211,
"step": 66
},
{
"KL/chosen_KL_mean": -3.056062698364258,
"KL/mean": -7.95568323135376,
"KL/rejected_KL_mean": -12.855304718017578,
"KL/std": 9.883098602294922,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.043049998581409454,
"fcm_dpo/delta": -0.02787696197628975,
"fcm_dpo/margin": 9.79924201965332,
"fcm_dpo/q_t": 0.4025185704231262,
"grad_norm": 30.74718475341797,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.5991535186767578,
"logits/rejected": -0.5557255744934082,
"logps/chosen": -62.39686584472656,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -85.64259338378906,
"loss": 1.0678,
"margin_dpo/margin_mean": 9.799242973327637,
"margin_dpo/margin_std": 11.892528533935547,
"step": 67
},
{
"KL/chosen_KL_mean": -3.8343505859375,
"KL/mean": -8.516582489013672,
"KL/rejected_KL_mean": -13.198814392089844,
"KL/std": 8.895482063293457,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.04273660480976105,
"fcm_dpo/delta": -0.000813731923699379,
"fcm_dpo/margin": 9.364463806152344,
"fcm_dpo/q_t": 0.40617847442626953,
"grad_norm": 30.00288963317871,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6549866199493408,
"logits/rejected": -0.5948320627212524,
"logps/chosen": -69.0401840209961,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -90.40606689453125,
"loss": 1.0692,
"margin_dpo/margin_mean": 9.364463806152344,
"margin_dpo/margin_std": 10.025825500488281,
"step": 68
},
{
"KL/chosen_KL_mean": -4.661190032958984,
"KL/mean": -10.855348587036133,
"KL/rejected_KL_mean": -17.04950714111328,
"KL/std": 11.315301895141602,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.03835180401802063,
"fcm_dpo/delta": -0.10325458645820618,
"fcm_dpo/margin": 12.388309478759766,
"fcm_dpo/q_t": 0.3859746754169464,
"grad_norm": 31.52351188659668,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.6151754856109619,
"logits/rejected": -0.5926914215087891,
"logps/chosen": -64.48043060302734,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -120.43836975097656,
"loss": 1.0203,
"margin_dpo/margin_mean": 12.388309478759766,
"margin_dpo/margin_std": 12.643640518188477,
"step": 69
},
{
"KL/chosen_KL_mean": -7.054567337036133,
"KL/mean": -13.696563720703125,
"KL/rejected_KL_mean": -20.338558197021484,
"KL/std": 14.29193115234375,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.03588715195655823,
"fcm_dpo/delta": -0.10018091648817062,
"fcm_dpo/margin": 13.283990859985352,
"fcm_dpo/q_t": 0.38754719495773315,
"grad_norm": 31.2943115234375,
"learning_rate": 5e-07,
"logits/chosen": -0.614654004573822,
"logits/rejected": -0.5801568627357483,
"logps/chosen": -68.9852066040039,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -111.39933776855469,
"loss": 1.0364,
"margin_dpo/margin_mean": 13.283990859985352,
"margin_dpo/margin_std": 15.29912281036377,
"step": 70
},
{
"KL/chosen_KL_mean": -7.13560676574707,
"KL/mean": -15.221653938293457,
"KL/rejected_KL_mean": -23.307708740234375,
"KL/std": 15.503862380981445,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.030674291774630547,
"fcm_dpo/delta": -0.13285793364048004,
"fcm_dpo/margin": 16.172096252441406,
"fcm_dpo/q_t": 0.3823116719722748,
"grad_norm": 28.419921875,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.649775505065918,
"logits/rejected": -0.6120765805244446,
"logps/chosen": -68.88594055175781,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -120.64433288574219,
"loss": 1.0137,
"margin_dpo/margin_mean": 16.172096252441406,
"margin_dpo/margin_std": 17.033647537231445,
"step": 71
},
{
"KL/chosen_KL_mean": -8.378385543823242,
"KL/mean": -16.67862892150879,
"KL/rejected_KL_mean": -24.978866577148438,
"KL/std": 16.624849319458008,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.028072141110897064,
"fcm_dpo/delta": -0.08738085627555847,
"fcm_dpo/margin": 16.600479125976562,
"fcm_dpo/q_t": 0.38960447907447815,
"grad_norm": 30.47901153564453,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.6570615172386169,
"logits/rejected": -0.6202565431594849,
"logps/chosen": -74.43179321289062,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -120.26585388183594,
"loss": 1.0429,
"margin_dpo/margin_mean": 16.600479125976562,
"margin_dpo/margin_std": 19.797344207763672,
"step": 72
},
{
"KL/chosen_KL_mean": -10.47215461730957,
"KL/mean": -18.878131866455078,
"KL/rejected_KL_mean": -27.284114837646484,
"KL/std": 21.38900375366211,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.026148542761802673,
"fcm_dpo/delta": -0.0531650111079216,
"fcm_dpo/margin": 16.811960220336914,
"fcm_dpo/q_t": 0.40286535024642944,
"grad_norm": 26.106237411499023,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.7131662368774414,
"logits/rejected": -0.6752426624298096,
"logps/chosen": -76.72842407226562,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -117.74024963378906,
"loss": 1.1056,
"margin_dpo/margin_mean": 16.811960220336914,
"margin_dpo/margin_std": 27.132373809814453,
"step": 73
},
{
"KL/chosen_KL_mean": -10.981948852539062,
"KL/mean": -20.437519073486328,
"KL/rejected_KL_mean": -29.893081665039062,
"KL/std": 23.818477630615234,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.024446729570627213,
"fcm_dpo/delta": -0.08252619206905365,
"fcm_dpo/margin": 18.911136627197266,
"fcm_dpo/q_t": 0.39837193489074707,
"grad_norm": 26.160118103027344,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.6098172664642334,
"logits/rejected": -0.6038111448287964,
"logps/chosen": -64.40682983398438,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -125.84001922607422,
"loss": 1.0783,
"margin_dpo/margin_mean": 18.911136627197266,
"margin_dpo/margin_std": 29.8692569732666,
"step": 74
},
{
"KL/chosen_KL_mean": -10.932317733764648,
"KL/mean": -24.136571884155273,
"KL/rejected_KL_mean": -37.3408203125,
"KL/std": 24.335262298583984,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.021133817732334137,
"fcm_dpo/delta": -0.21684792637825012,
"fcm_dpo/margin": 26.408512115478516,
"fcm_dpo/q_t": 0.3661307096481323,
"grad_norm": 24.379812240600586,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6909558773040771,
"logits/rejected": -0.6851760149002075,
"logps/chosen": -62.79397964477539,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -148.5948028564453,
"loss": 0.9864,
"margin_dpo/margin_mean": 26.408510208129883,
"margin_dpo/margin_std": 27.935894012451172,
"step": 75
},
{
"KL/chosen_KL_mean": -13.196050643920898,
"KL/mean": -21.37349510192871,
"KL/rejected_KL_mean": -29.55093765258789,
"KL/std": 19.751428604125977,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.02066803351044655,
"fcm_dpo/delta": 0.07377178966999054,
"fcm_dpo/margin": 16.354894638061523,
"fcm_dpo/q_t": 0.4222240149974823,
"grad_norm": 23.001667022705078,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.6944067478179932,
"logits/rejected": -0.6538985967636108,
"logps/chosen": -66.46208953857422,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -107.76756286621094,
"loss": 1.1248,
"margin_dpo/margin_mean": 16.354894638061523,
"margin_dpo/margin_std": 21.762468338012695,
"step": 76
},
{
"KL/chosen_KL_mean": -12.05851936340332,
"KL/mean": -25.64178466796875,
"KL/rejected_KL_mean": -39.22505187988281,
"KL/std": 27.021289825439453,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.018379777669906616,
"fcm_dpo/delta": -0.14052104949951172,
"fcm_dpo/margin": 27.166534423828125,
"fcm_dpo/q_t": 0.3777894973754883,
"grad_norm": 23.42577362060547,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.7054200172424316,
"logits/rejected": -0.6716604232788086,
"logps/chosen": -70.15519714355469,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -132.9986572265625,
"loss": 1.0196,
"margin_dpo/margin_mean": 27.166534423828125,
"margin_dpo/margin_std": 30.322921752929688,
"step": 77
},
{
"KL/chosen_KL_mean": -12.313678741455078,
"KL/mean": -22.357601165771484,
"KL/rejected_KL_mean": -32.401519775390625,
"KL/std": 22.272029876708984,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.01876748539507389,
"fcm_dpo/delta": 0.02741411328315735,
"fcm_dpo/margin": 20.087839126586914,
"fcm_dpo/q_t": 0.41380858421325684,
"grad_norm": 21.392480850219727,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.6116062998771667,
"logits/rejected": -0.587343156337738,
"logps/chosen": -67.92745971679688,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -117.33588409423828,
"loss": 1.1009,
"margin_dpo/margin_mean": 20.087839126586914,
"margin_dpo/margin_std": 25.943618774414062,
"step": 78
},
{
"KL/chosen_KL_mean": -12.862058639526367,
"KL/mean": -23.823688507080078,
"KL/rejected_KL_mean": -34.78531265258789,
"KL/std": 24.85503578186035,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.018557265400886536,
"fcm_dpo/delta": -0.008593140169978142,
"fcm_dpo/margin": 21.92325210571289,
"fcm_dpo/q_t": 0.40805089473724365,
"grad_norm": 20.17791748046875,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6797605752944946,
"logits/rejected": -0.6534685492515564,
"logps/chosen": -68.31254577636719,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -122.43287658691406,
"loss": 1.0936,
"margin_dpo/margin_mean": 21.92325210571289,
"margin_dpo/margin_std": 30.41646385192871,
"step": 79
},
{
"KL/chosen_KL_mean": -16.53986358642578,
"KL/mean": -27.252384185791016,
"KL/rejected_KL_mean": -37.964908599853516,
"KL/std": 27.36221694946289,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.01815933734178543,
"fcm_dpo/delta": 0.010311007499694824,
"fcm_dpo/margin": 21.425039291381836,
"fcm_dpo/q_t": 0.4146670699119568,
"grad_norm": 22.17724609375,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.7165290117263794,
"logits/rejected": -0.6780725717544556,
"logps/chosen": -75.05915832519531,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -125.51240539550781,
"loss": 1.1159,
"margin_dpo/margin_mean": 21.42504119873047,
"margin_dpo/margin_std": 31.943340301513672,
"step": 80
},
{
"KL/chosen_KL_mean": -15.941177368164062,
"KL/mean": -30.589588165283203,
"KL/rejected_KL_mean": -45.237998962402344,
"KL/std": 32.461692810058594,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.016676858067512512,
"fcm_dpo/delta": -0.1239388957619667,
"fcm_dpo/margin": 29.296829223632812,
"fcm_dpo/q_t": 0.38681286573410034,
"grad_norm": 21.989290237426758,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.7118068337440491,
"logits/rejected": -0.68656325340271,
"logps/chosen": -82.39004516601562,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -174.90069580078125,
"loss": 1.0461,
"margin_dpo/margin_mean": 29.296829223632812,
"margin_dpo/margin_std": 38.70458984375,
"step": 81
},
{
"KL/chosen_KL_mean": -18.573457717895508,
"KL/mean": -30.226879119873047,
"KL/rejected_KL_mean": -41.88030242919922,
"KL/std": 29.723350524902344,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.017191056162118912,
"fcm_dpo/delta": -0.009201280772686005,
"fcm_dpo/margin": 23.30685043334961,
"fcm_dpo/q_t": 0.41180986166000366,
"grad_norm": 23.152570724487305,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.6191596984863281,
"logits/rejected": -0.6112022995948792,
"logps/chosen": -70.80584716796875,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -132.62355041503906,
"loss": 1.1455,
"margin_dpo/margin_mean": 23.30685043334961,
"margin_dpo/margin_std": 41.297733306884766,
"step": 82
},
{
"KL/chosen_KL_mean": -19.169687271118164,
"KL/mean": -33.314022064208984,
"KL/rejected_KL_mean": -47.4583625793457,
"KL/std": 30.860076904296875,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.01552935317158699,
"fcm_dpo/delta": -0.05240233987569809,
"fcm_dpo/margin": 28.28866958618164,
"fcm_dpo/q_t": 0.3983423411846161,
"grad_norm": 21.990673065185547,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.6577206254005432,
"logits/rejected": -0.6294866800308228,
"logps/chosen": -74.9970703125,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -151.17425537109375,
"loss": 1.0706,
"margin_dpo/margin_mean": 28.288667678833008,
"margin_dpo/margin_std": 36.98453903198242,
"step": 83
},
{
"KL/chosen_KL_mean": -18.952524185180664,
"KL/mean": -30.40399169921875,
"KL/rejected_KL_mean": -41.855464935302734,
"KL/std": 26.464885711669922,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.01563824899494648,
"fcm_dpo/delta": 0.048310477286577225,
"fcm_dpo/margin": 22.902938842773438,
"fcm_dpo/q_t": 0.4166381359100342,
"grad_norm": 20.835840225219727,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.6432810425758362,
"logits/rejected": -0.6044536828994751,
"logps/chosen": -86.12869262695312,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -129.15406799316406,
"loss": 1.1085,
"margin_dpo/margin_mean": 22.902938842773438,
"margin_dpo/margin_std": 26.787670135498047,
"step": 84
},
{
"KL/chosen_KL_mean": -19.004091262817383,
"KL/mean": -31.023822784423828,
"KL/rejected_KL_mean": -43.043556213378906,
"KL/std": 27.054758071899414,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.016782496124505997,
"fcm_dpo/delta": -0.009143978357315063,
"fcm_dpo/margin": 24.039470672607422,
"fcm_dpo/q_t": 0.40713611245155334,
"grad_norm": 21.613723754882812,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6534522175788879,
"logits/rejected": -0.622204065322876,
"logps/chosen": -77.41070556640625,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -121.68235778808594,
"loss": 1.0991,
"margin_dpo/margin_mean": 24.039470672607422,
"margin_dpo/margin_std": 32.703369140625,
"step": 85
},
{
"KL/chosen_KL_mean": -24.901784896850586,
"KL/mean": -37.69927978515625,
"KL/rejected_KL_mean": -50.49677276611328,
"KL/std": 38.232906341552734,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.01626906730234623,
"fcm_dpo/delta": -0.02235548198223114,
"fcm_dpo/margin": 25.594993591308594,
"fcm_dpo/q_t": 0.41634446382522583,
"grad_norm": 25.438549041748047,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.6633529663085938,
"logits/rejected": -0.6544677019119263,
"logps/chosen": -81.03924560546875,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -138.61842346191406,
"loss": 1.1718,
"margin_dpo/margin_mean": 25.594993591308594,
"margin_dpo/margin_std": 53.408973693847656,
"step": 86
},
{
"KL/chosen_KL_mean": -24.417339324951172,
"KL/mean": -40.079750061035156,
"KL/rejected_KL_mean": -55.74216079711914,
"KL/std": 38.12931823730469,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.014551367610692978,
"fcm_dpo/delta": -0.07786225527524948,
"fcm_dpo/margin": 31.324817657470703,
"fcm_dpo/q_t": 0.39422881603240967,
"grad_norm": 21.019412994384766,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.6794058084487915,
"logits/rejected": -0.6601496934890747,
"logps/chosen": -80.05343627929688,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -151.20974731445312,
"loss": 1.0915,
"margin_dpo/margin_mean": 31.324819564819336,
"margin_dpo/margin_std": 47.0085334777832,
"step": 87
},
{
"KL/chosen_KL_mean": -30.980731964111328,
"KL/mean": -42.74275207519531,
"KL/rejected_KL_mean": -54.50476837158203,
"KL/std": 40.58386993408203,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.014943553134799004,
"fcm_dpo/delta": 0.057977911084890366,
"fcm_dpo/margin": 23.52403450012207,
"fcm_dpo/q_t": 0.42546147108078003,
"grad_norm": 21.812320709228516,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.6842135787010193,
"logits/rejected": -0.6828039288520813,
"logps/chosen": -104.65188598632812,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -161.21327209472656,
"loss": 1.1737,
"margin_dpo/margin_mean": 23.52403450012207,
"margin_dpo/margin_std": 43.78386306762695,
"step": 88
},
{
"KL/chosen_KL_mean": -19.886613845825195,
"KL/mean": -33.26313781738281,
"KL/rejected_KL_mean": -46.63965606689453,
"KL/std": 34.684234619140625,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.015584435313940048,
"fcm_dpo/delta": -0.02295827493071556,
"fcm_dpo/margin": 26.753049850463867,
"fcm_dpo/q_t": 0.40534842014312744,
"grad_norm": 21.010114669799805,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.7116603851318359,
"logits/rejected": -0.6837852001190186,
"logps/chosen": -80.51152801513672,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -128.72320556640625,
"loss": 1.1014,
"margin_dpo/margin_mean": 26.753049850463867,
"margin_dpo/margin_std": 40.44371032714844,
"step": 89
},
{
"KL/chosen_KL_mean": -24.155946731567383,
"KL/mean": -43.333587646484375,
"KL/rejected_KL_mean": -62.511234283447266,
"KL/std": 47.100379943847656,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.013927513733506203,
"fcm_dpo/delta": -0.18348340690135956,
"fcm_dpo/margin": 38.355281829833984,
"fcm_dpo/q_t": 0.38429707288742065,
"grad_norm": 21.59499740600586,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.6261708736419678,
"logits/rejected": -0.646406888961792,
"logps/chosen": -77.44125366210938,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -174.05593872070312,
"loss": 1.0798,
"margin_dpo/margin_mean": 38.35528564453125,
"margin_dpo/margin_std": 59.089813232421875,
"step": 90
},
{
"KL/chosen_KL_mean": -25.931039810180664,
"KL/mean": -40.36906814575195,
"KL/rejected_KL_mean": -54.807098388671875,
"KL/std": 36.47618103027344,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.013058323413133621,
"fcm_dpo/delta": 0.026791641488671303,
"fcm_dpo/margin": 28.876054763793945,
"fcm_dpo/q_t": 0.4173550009727478,
"grad_norm": 19.14020347595215,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.6626990437507629,
"logits/rejected": -0.6330728530883789,
"logps/chosen": -87.73399353027344,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -142.68106079101562,
"loss": 1.1368,
"margin_dpo/margin_mean": 28.876056671142578,
"margin_dpo/margin_std": 48.72987365722656,
"step": 91
},
{
"KL/chosen_KL_mean": -22.967710494995117,
"KL/mean": -38.062904357910156,
"KL/rejected_KL_mean": -53.15810012817383,
"KL/std": 38.03881072998047,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.013261953368782997,
"fcm_dpo/delta": -0.0033196136355400085,
"fcm_dpo/margin": 30.190397262573242,
"fcm_dpo/q_t": 0.4104510247707367,
"grad_norm": 18.81719398498535,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.6928216218948364,
"logits/rejected": -0.6727030277252197,
"logps/chosen": -74.60847473144531,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -131.03927612304688,
"loss": 1.1136,
"margin_dpo/margin_mean": 30.190399169921875,
"margin_dpo/margin_std": 46.062583923339844,
"step": 92
},
{
"KL/chosen_KL_mean": -26.070533752441406,
"KL/mean": -40.96013641357422,
"KL/rejected_KL_mean": -55.8497314453125,
"KL/std": 34.986263275146484,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.013152128085494041,
"fcm_dpo/delta": 0.009675238281488419,
"fcm_dpo/margin": 29.77920150756836,
"fcm_dpo/q_t": 0.4116442799568176,
"grad_norm": 19.744487762451172,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.7134292125701904,
"logits/rejected": -0.6725969910621643,
"logps/chosen": -78.59977722167969,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -133.01048278808594,
"loss": 1.0986,
"margin_dpo/margin_mean": 29.77920150756836,
"margin_dpo/margin_std": 41.22077941894531,
"step": 93
},
{
"KL/chosen_KL_mean": -27.902204513549805,
"KL/mean": -45.52301025390625,
"KL/rejected_KL_mean": -63.14381790161133,
"KL/std": 41.25383758544922,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.012278085574507713,
"fcm_dpo/delta": -0.0462297722697258,
"fcm_dpo/margin": 35.24161148071289,
"fcm_dpo/q_t": 0.39923009276390076,
"grad_norm": 18.810970306396484,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.645513653755188,
"logits/rejected": -0.637039303779602,
"logps/chosen": -89.12481689453125,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -162.74285888671875,
"loss": 1.0683,
"margin_dpo/margin_mean": 35.24161148071289,
"margin_dpo/margin_std": 43.92166519165039,
"step": 94
},
{
"KL/chosen_KL_mean": -27.263856887817383,
"KL/mean": -41.50673294067383,
"KL/rejected_KL_mean": -55.749610900878906,
"KL/std": 41.475608825683594,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.012666614726185799,
"fcm_dpo/delta": 0.047285325825214386,
"fcm_dpo/margin": 28.485750198364258,
"fcm_dpo/q_t": 0.41799068450927734,
"grad_norm": 18.525175094604492,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.6651968359947205,
"logits/rejected": -0.6414941549301147,
"logps/chosen": -79.78750610351562,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -131.6299591064453,
"loss": 1.131,
"margin_dpo/margin_mean": 28.48575210571289,
"margin_dpo/margin_std": 43.52128601074219,
"step": 95
},
{
"KL/chosen_KL_mean": -27.1774845123291,
"KL/mean": -46.510169982910156,
"KL/rejected_KL_mean": -65.84285736083984,
"KL/std": 44.56120300292969,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.011798870749771595,
"fcm_dpo/delta": -0.08383983373641968,
"fcm_dpo/margin": 38.665374755859375,
"fcm_dpo/q_t": 0.39412879943847656,
"grad_norm": 18.564252853393555,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.6842184066772461,
"logits/rejected": -0.662356972694397,
"logps/chosen": -89.33445739746094,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -162.43887329101562,
"loss": 1.0612,
"margin_dpo/margin_mean": 38.665374755859375,
"margin_dpo/margin_std": 50.07075500488281,
"step": 96
},
{
"KL/chosen_KL_mean": -28.180774688720703,
"KL/mean": -42.7745361328125,
"KL/rejected_KL_mean": -57.36829376220703,
"KL/std": 38.20796203613281,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.011734258383512497,
"fcm_dpo/delta": -0.04553954675793648,
"fcm_dpo/margin": 29.187524795532227,
"fcm_dpo/q_t": 0.4178887903690338,
"grad_norm": 19.07831573486328,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.6238841414451599,
"logits/rejected": -0.588768720626831,
"logps/chosen": -82.82714080810547,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -134.33303833007812,
"loss": 1.1271,
"margin_dpo/margin_mean": 29.187524795532227,
"margin_dpo/margin_std": 38.123573303222656,
"step": 97
},
{
"KL/chosen_KL_mean": -35.863746643066406,
"KL/mean": -52.72106170654297,
"KL/rejected_KL_mean": -69.57838439941406,
"KL/std": 44.223392486572266,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.011502566747367382,
"fcm_dpo/delta": 0.014971842989325523,
"fcm_dpo/margin": 33.714637756347656,
"fcm_dpo/q_t": 0.41005343198776245,
"grad_norm": 20.55078887939453,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.6732558012008667,
"logits/rejected": -0.6637331247329712,
"logps/chosen": -101.12237548828125,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -166.10586547851562,
"loss": 1.1266,
"margin_dpo/margin_mean": 33.714637756347656,
"margin_dpo/margin_std": 53.72822570800781,
"step": 98
},
{
"KL/chosen_KL_mean": -27.312252044677734,
"KL/mean": -48.77324676513672,
"KL/rejected_KL_mean": -70.2342300415039,
"KL/std": 46.53744125366211,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.010750599205493927,
"fcm_dpo/delta": -0.08328957110643387,
"fcm_dpo/margin": 42.921993255615234,
"fcm_dpo/q_t": 0.39549580216407776,
"grad_norm": 17.073204040527344,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.6471402049064636,
"logits/rejected": -0.651877224445343,
"logps/chosen": -72.95073699951172,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -156.6721649169922,
"loss": 1.0633,
"margin_dpo/margin_mean": 42.921993255615234,
"margin_dpo/margin_std": 58.77742004394531,
"step": 99
},
{
"KL/chosen_KL_mean": -31.377222061157227,
"KL/mean": -42.55781555175781,
"KL/rejected_KL_mean": -53.738407135009766,
"KL/std": 38.19342803955078,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.010938813909888268,
"fcm_dpo/delta": 0.041864871978759766,
"fcm_dpo/margin": 22.361183166503906,
"fcm_dpo/q_t": 0.4438517093658447,
"grad_norm": 18.95781898498535,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6145943403244019,
"logits/rejected": -0.5761110782623291,
"logps/chosen": -88.97120666503906,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -127.79861450195312,
"loss": 1.2277,
"margin_dpo/margin_mean": 22.361183166503906,
"margin_dpo/margin_std": 49.1837158203125,
"step": 100
},
{
"KL/chosen_KL_mean": -38.442962646484375,
"KL/mean": -52.74586486816406,
"KL/rejected_KL_mean": -67.04876708984375,
"KL/std": 47.44648742675781,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.011103025637567043,
"fcm_dpo/delta": -0.0058130607940256596,
"fcm_dpo/margin": 28.60578727722168,
"fcm_dpo/q_t": 0.4297153949737549,
"grad_norm": 19.153141021728516,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.6148165464401245,
"logits/rejected": -0.5886383056640625,
"logps/chosen": -100.09181213378906,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -150.2384490966797,
"loss": 1.1865,
"margin_dpo/margin_mean": 28.605789184570312,
"margin_dpo/margin_std": 56.565982818603516,
"step": 101
},
{
"KL/chosen_KL_mean": -40.61670684814453,
"KL/mean": -50.855247497558594,
"KL/rejected_KL_mean": -61.093788146972656,
"KL/std": 44.045799255371094,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.011240271851420403,
"fcm_dpo/delta": 0.024421557784080505,
"fcm_dpo/margin": 20.477081298828125,
"fcm_dpo/q_t": 0.44768211245536804,
"grad_norm": 22.304677963256836,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.673102080821991,
"logits/rejected": -0.6373000144958496,
"logps/chosen": -104.69558715820312,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -129.28086853027344,
"loss": 1.2568,
"margin_dpo/margin_mean": 20.477081298828125,
"margin_dpo/margin_std": 53.06024169921875,
"step": 102
},
{
"KL/chosen_KL_mean": -34.13248825073242,
"KL/mean": -55.830631256103516,
"KL/rejected_KL_mean": -77.52876281738281,
"KL/std": 49.93252182006836,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.010603921487927437,
"fcm_dpo/delta": -0.07997814565896988,
"fcm_dpo/margin": 43.39628601074219,
"fcm_dpo/q_t": 0.3960493803024292,
"grad_norm": 18.789276123046875,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.6219948530197144,
"logits/rejected": -0.59108966588974,
"logps/chosen": -95.4317626953125,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -171.10147094726562,
"loss": 1.0721,
"margin_dpo/margin_mean": 43.39628601074219,
"margin_dpo/margin_std": 62.30549621582031,
"step": 103
},
{
"KL/chosen_KL_mean": -35.155120849609375,
"KL/mean": -55.93231964111328,
"KL/rejected_KL_mean": -76.70951080322266,
"KL/std": 53.614723205566406,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.010277830064296722,
"fcm_dpo/delta": -0.03438165411353111,
"fcm_dpo/margin": 41.55438995361328,
"fcm_dpo/q_t": 0.40735965967178345,
"grad_norm": 18.074642181396484,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.6393266916275024,
"logits/rejected": -0.6158906817436218,
"logps/chosen": -89.52789306640625,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -166.27423095703125,
"loss": 1.1084,
"margin_dpo/margin_mean": 41.55438995361328,
"margin_dpo/margin_std": 66.71173095703125,
"step": 104
},
{
"KL/chosen_KL_mean": -33.67083740234375,
"KL/mean": -63.1674919128418,
"KL/rejected_KL_mean": -92.66414642333984,
"KL/std": 50.66936492919922,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.008944441564381123,
"fcm_dpo/delta": -0.17402513325214386,
"fcm_dpo/margin": 58.99330139160156,
"fcm_dpo/q_t": 0.36932459473609924,
"grad_norm": 18.142919540405273,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.5875688791275024,
"logits/rejected": -0.5673741102218628,
"logps/chosen": -88.30978393554688,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -190.63766479492188,
"loss": 0.9869,
"margin_dpo/margin_mean": 58.99330139160156,
"margin_dpo/margin_std": 57.22583770751953,
"step": 105
},
{
"KL/chosen_KL_mean": -38.18315124511719,
"KL/mean": -56.45439910888672,
"KL/rejected_KL_mean": -74.72565460205078,
"KL/std": 45.731204986572266,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.009024987928569317,
"fcm_dpo/delta": 0.08437706530094147,
"fcm_dpo/margin": 36.542503356933594,
"fcm_dpo/q_t": 0.42528021335601807,
"grad_norm": 16.702926635742188,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.535358190536499,
"logits/rejected": -0.5128501653671265,
"logps/chosen": -93.01604461669922,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -159.9502716064453,
"loss": 1.1339,
"margin_dpo/margin_mean": 36.542503356933594,
"margin_dpo/margin_std": 49.62269592285156,
"step": 106
},
{
"KL/chosen_KL_mean": -45.14348220825195,
"KL/mean": -65.80903625488281,
"KL/rejected_KL_mean": -86.47457885742188,
"KL/std": 56.287105560302734,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.009254291653633118,
"fcm_dpo/delta": 0.020608151331543922,
"fcm_dpo/margin": 41.33110427856445,
"fcm_dpo/q_t": 0.4126184582710266,
"grad_norm": 17.756500244140625,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.6197066307067871,
"logits/rejected": -0.580697774887085,
"logps/chosen": -114.8512954711914,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -181.21408081054688,
"loss": 1.115,
"margin_dpo/margin_mean": 41.33110046386719,
"margin_dpo/margin_std": 59.82461166381836,
"step": 107
},
{
"KL/chosen_KL_mean": -39.11261749267578,
"KL/mean": -65.50827026367188,
"KL/rejected_KL_mean": -91.90391540527344,
"KL/std": 62.87705993652344,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.008883420377969742,
"fcm_dpo/delta": -0.09007138013839722,
"fcm_dpo/margin": 52.791290283203125,
"fcm_dpo/q_t": 0.39717066287994385,
"grad_norm": 17.294050216674805,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.5953981280326843,
"logits/rejected": -0.5820811986923218,
"logps/chosen": -95.12250518798828,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -187.69992065429688,
"loss": 1.0879,
"margin_dpo/margin_mean": 52.791290283203125,
"margin_dpo/margin_std": 81.55433654785156,
"step": 108
},
{
"KL/chosen_KL_mean": -37.30243682861328,
"KL/mean": -64.34707641601562,
"KL/rejected_KL_mean": -91.39170837402344,
"KL/std": 59.183998107910156,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.008149518631398678,
"fcm_dpo/delta": -0.05536172538995743,
"fcm_dpo/margin": 54.08927536010742,
"fcm_dpo/q_t": 0.3969169855117798,
"grad_norm": 18.013111114501953,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.5478425025939941,
"logits/rejected": -0.528139054775238,
"logps/chosen": -100.18792724609375,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -190.07745361328125,
"loss": 1.0772,
"margin_dpo/margin_mean": 54.08927536010742,
"margin_dpo/margin_std": 73.88963317871094,
"step": 109
},
{
"KL/chosen_KL_mean": -39.2655029296875,
"KL/mean": -63.571868896484375,
"KL/rejected_KL_mean": -87.87823486328125,
"KL/std": 59.547828674316406,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.007406959775835276,
"fcm_dpo/delta": -0.10038627684116364,
"fcm_dpo/margin": 48.61272430419922,
"fcm_dpo/q_t": 0.4130057692527771,
"grad_norm": 13.945056915283203,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.5412209033966064,
"logits/rejected": -0.5109491348266602,
"logps/chosen": -98.0191879272461,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -167.6282501220703,
"loss": 1.1332,
"margin_dpo/margin_mean": 48.61272430419922,
"margin_dpo/margin_std": 72.85327911376953,
"step": 110
},
{
"KL/chosen_KL_mean": -43.91405487060547,
"KL/mean": -72.29114532470703,
"KL/rejected_KL_mean": -100.66822052001953,
"KL/std": 67.76360321044922,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.0071860793977975845,
"fcm_dpo/delta": -0.012220162898302078,
"fcm_dpo/margin": 56.754173278808594,
"fcm_dpo/q_t": 0.4068235754966736,
"grad_norm": 15.937190055847168,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.5931087732315063,
"logits/rejected": -0.5733714699745178,
"logps/chosen": -112.53816223144531,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -199.09707641601562,
"loss": 1.1115,
"margin_dpo/margin_mean": 56.754173278808594,
"margin_dpo/margin_std": 86.54757690429688,
"step": 111
},
{
"KL/chosen_KL_mean": -40.607540130615234,
"KL/mean": -57.865150451660156,
"KL/rejected_KL_mean": -75.12275695800781,
"KL/std": 45.083778381347656,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.007618125528097153,
"fcm_dpo/delta": 0.04033050686120987,
"fcm_dpo/margin": 34.51521682739258,
"fcm_dpo/q_t": 0.43914544582366943,
"grad_norm": 15.417643547058105,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.5523747801780701,
"logits/rejected": -0.5217946767807007,
"logps/chosen": -90.857177734375,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -139.89718627929688,
"loss": 1.1866,
"margin_dpo/margin_mean": 34.51521682739258,
"margin_dpo/margin_std": 56.524749755859375,
"step": 112
},
{
"KL/chosen_KL_mean": -53.0630989074707,
"KL/mean": -70.5762939453125,
"KL/rejected_KL_mean": -88.08948516845703,
"KL/std": 50.8385009765625,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.008391564711928368,
"fcm_dpo/delta": 0.12340584397315979,
"fcm_dpo/margin": 35.0263786315918,
"fcm_dpo/q_t": 0.4351421892642975,
"grad_norm": 17.364025115966797,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.5283902883529663,
"logits/rejected": -0.4908086061477661,
"logps/chosen": -119.77604675292969,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -166.05819702148438,
"loss": 1.1661,
"margin_dpo/margin_mean": 35.0263786315918,
"margin_dpo/margin_std": 56.08738327026367,
"step": 113
},
{
"KL/chosen_KL_mean": -47.82563018798828,
"KL/mean": -77.25653076171875,
"KL/rejected_KL_mean": -106.68740844726562,
"KL/std": 72.02836608886719,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.007831193506717682,
"fcm_dpo/delta": -0.08718626201152802,
"fcm_dpo/margin": 58.861785888671875,
"fcm_dpo/q_t": 0.39837566018104553,
"grad_norm": 19.08915138244629,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.5314050912857056,
"logits/rejected": -0.5059875845909119,
"logps/chosen": -105.6107177734375,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -193.7970733642578,
"loss": 1.0774,
"margin_dpo/margin_mean": 58.861785888671875,
"margin_dpo/margin_std": 88.37593078613281,
"step": 114
},
{
"KL/chosen_KL_mean": -60.48192596435547,
"KL/mean": -80.31732177734375,
"KL/rejected_KL_mean": -100.15271759033203,
"KL/std": 69.82859802246094,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.00848847534507513,
"fcm_dpo/delta": 0.0731273964047432,
"fcm_dpo/margin": 39.67078399658203,
"fcm_dpo/q_t": 0.43038254976272583,
"grad_norm": 25.075674057006836,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.5371870398521423,
"logits/rejected": -0.5258318185806274,
"logps/chosen": -126.06456756591797,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -198.71823120117188,
"loss": 1.2184,
"margin_dpo/margin_mean": 39.67078399658203,
"margin_dpo/margin_std": 93.87147521972656,
"step": 115
},
{
"KL/chosen_KL_mean": -43.80780792236328,
"KL/mean": -66.62762451171875,
"KL/rejected_KL_mean": -89.44744110107422,
"KL/std": 57.25769805908203,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.008733630180358887,
"fcm_dpo/delta": -0.0017695911228656769,
"fcm_dpo/margin": 45.639625549316406,
"fcm_dpo/q_t": 0.4090723991394043,
"grad_norm": 21.11087417602539,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.5265697836875916,
"logits/rejected": -0.5241330862045288,
"logps/chosen": -95.20811462402344,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -169.96929931640625,
"loss": 1.1005,
"margin_dpo/margin_mean": 45.639625549316406,
"margin_dpo/margin_std": 63.12974548339844,
"step": 116
},
{
"KL/chosen_KL_mean": -56.89840316772461,
"KL/mean": -74.12403869628906,
"KL/rejected_KL_mean": -91.34967041015625,
"KL/std": 59.87590789794922,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.009167262353003025,
"fcm_dpo/delta": 0.09936673939228058,
"fcm_dpo/margin": 34.45126724243164,
"fcm_dpo/q_t": 0.43182051181793213,
"grad_norm": 28.09205436706543,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.5017322301864624,
"logits/rejected": -0.4683611989021301,
"logps/chosen": -126.19680786132812,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -157.93365478515625,
"loss": 1.1893,
"margin_dpo/margin_mean": 34.451271057128906,
"margin_dpo/margin_std": 69.84674072265625,
"step": 117
},
{
"KL/chosen_KL_mean": -43.14284133911133,
"KL/mean": -65.05967712402344,
"KL/rejected_KL_mean": -86.97651672363281,
"KL/std": 53.93627166748047,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.009347407147288322,
"fcm_dpo/delta": -0.012348111718893051,
"fcm_dpo/margin": 43.83367919921875,
"fcm_dpo/q_t": 0.4077704846858978,
"grad_norm": 19.32469367980957,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.5494426488876343,
"logits/rejected": -0.5325976014137268,
"logps/chosen": -98.78382110595703,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -162.64556884765625,
"loss": 1.0957,
"margin_dpo/margin_mean": 43.83367919921875,
"margin_dpo/margin_std": 62.52387237548828,
"step": 118
},
{
"KL/chosen_KL_mean": -59.46000289916992,
"KL/mean": -80.9298095703125,
"KL/rejected_KL_mean": -102.39959716796875,
"KL/std": 60.10303497314453,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.008993204683065414,
"fcm_dpo/delta": 0.011231735348701477,
"fcm_dpo/margin": 42.93959426879883,
"fcm_dpo/q_t": 0.4158446788787842,
"grad_norm": 21.934490203857422,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.5536962151527405,
"logits/rejected": -0.5457164645195007,
"logps/chosen": -132.97019958496094,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -205.37689208984375,
"loss": 1.1473,
"margin_dpo/margin_mean": 42.939598083496094,
"margin_dpo/margin_std": 75.34599304199219,
"step": 119
},
{
"KL/chosen_KL_mean": -58.99882888793945,
"KL/mean": -89.11383056640625,
"KL/rejected_KL_mean": -119.22883605957031,
"KL/std": 73.892822265625,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.008354677818715572,
"fcm_dpo/delta": -0.1406194120645523,
"fcm_dpo/margin": 60.230003356933594,
"fcm_dpo/q_t": 0.3826143443584442,
"grad_norm": 22.082054138183594,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.567842960357666,
"logits/rejected": -0.5426411032676697,
"logps/chosen": -135.7796630859375,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -227.25257873535156,
"loss": 1.0416,
"margin_dpo/margin_mean": 60.230003356933594,
"margin_dpo/margin_std": 79.12581634521484,
"step": 120
},
{
"KL/chosen_KL_mean": -55.65784454345703,
"KL/mean": -89.1383056640625,
"KL/rejected_KL_mean": -122.61875915527344,
"KL/std": 71.20858764648438,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.007244884967803955,
"fcm_dpo/delta": -0.11996462941169739,
"fcm_dpo/margin": 66.9609375,
"fcm_dpo/q_t": 0.3862529993057251,
"grad_norm": 18.99169158935547,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.5176138877868652,
"logits/rejected": -0.5163988471031189,
"logps/chosen": -117.44773864746094,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -232.61334228515625,
"loss": 1.0532,
"margin_dpo/margin_mean": 66.9609375,
"margin_dpo/margin_std": 89.08659362792969,
"step": 121
},
{
"KL/chosen_KL_mean": -50.142337799072266,
"KL/mean": -91.1891098022461,
"KL/rejected_KL_mean": -132.2358856201172,
"KL/std": 79.52822875976562,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.0063782669603824615,
"fcm_dpo/delta": -0.16817903518676758,
"fcm_dpo/margin": 82.09354400634766,
"fcm_dpo/q_t": 0.3728944957256317,
"grad_norm": 18.489049911499023,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.46613478660583496,
"logits/rejected": -0.48410728573799133,
"logps/chosen": -97.04454803466797,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -238.9500732421875,
"loss": 0.9968,
"margin_dpo/margin_mean": 82.09354400634766,
"margin_dpo/margin_std": 84.7960205078125,
"step": 122
},
{
"KL/chosen_KL_mean": -57.29137420654297,
"KL/mean": -83.6397476196289,
"KL/rejected_KL_mean": -109.98812866210938,
"KL/std": 68.29986572265625,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.006330414209514856,
"fcm_dpo/delta": 0.07942930608987808,
"fcm_dpo/margin": 52.696746826171875,
"fcm_dpo/q_t": 0.4271467626094818,
"grad_norm": 17.179462432861328,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.4523668885231018,
"logits/rejected": -0.43804460763931274,
"logps/chosen": -118.63002014160156,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -197.76351928710938,
"loss": 1.1663,
"margin_dpo/margin_mean": 52.69674301147461,
"margin_dpo/margin_std": 93.03097534179688,
"step": 123
},
{
"KL/chosen_KL_mean": -66.95594787597656,
"KL/mean": -107.28865051269531,
"KL/rejected_KL_mean": -147.62136840820312,
"KL/std": 95.69377899169922,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.005985944531857967,
"fcm_dpo/delta": -0.11251755058765411,
"fcm_dpo/margin": 80.66542053222656,
"fcm_dpo/q_t": 0.39224308729171753,
"grad_norm": 17.14952278137207,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.46528786420822144,
"logits/rejected": -0.47065889835357666,
"logps/chosen": -138.40428161621094,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -265.2019348144531,
"loss": 1.0778,
"margin_dpo/margin_mean": 80.66542053222656,
"margin_dpo/margin_std": 120.44160461425781,
"step": 124
},
{
"KL/chosen_KL_mean": -54.70975112915039,
"KL/mean": -89.0744857788086,
"KL/rejected_KL_mean": -123.43922424316406,
"KL/std": 80.83454895019531,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.0059068650007247925,
"fcm_dpo/delta": -0.007670434191823006,
"fcm_dpo/margin": 68.72946166992188,
"fcm_dpo/q_t": 0.40873342752456665,
"grad_norm": 14.995677947998047,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.36938369274139404,
"logits/rejected": -0.37332677841186523,
"logps/chosen": -104.84669494628906,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -207.42782592773438,
"loss": 1.1061,
"margin_dpo/margin_mean": 68.72946166992188,
"margin_dpo/margin_std": 102.3861083984375,
"step": 125
},
{
"KL/chosen_KL_mean": -59.12432861328125,
"KL/mean": -92.44718933105469,
"KL/rejected_KL_mean": -125.77005767822266,
"KL/std": 74.66473388671875,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.005859220400452614,
"fcm_dpo/delta": 0.011660443618893623,
"fcm_dpo/margin": 66.6457290649414,
"fcm_dpo/q_t": 0.4124431610107422,
"grad_norm": 16.17157554626465,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.44136887788772583,
"logits/rejected": -0.441866010427475,
"logps/chosen": -114.79139709472656,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -223.89981079101562,
"loss": 1.1071,
"margin_dpo/margin_mean": 66.64572143554688,
"margin_dpo/margin_std": 96.18585968017578,
"step": 126
},
{
"KL/chosen_KL_mean": -58.726707458496094,
"KL/mean": -81.67306518554688,
"KL/rejected_KL_mean": -104.61940002441406,
"KL/std": 65.82872009277344,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.006503104232251644,
"fcm_dpo/delta": 0.11829523742198944,
"fcm_dpo/margin": 45.892696380615234,
"fcm_dpo/q_t": 0.43302494287490845,
"grad_norm": 19.481531143188477,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.459053099155426,
"logits/rejected": -0.44523656368255615,
"logps/chosen": -115.28138732910156,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -181.41519165039062,
"loss": 1.1793,
"margin_dpo/margin_mean": 45.8927001953125,
"margin_dpo/margin_std": 83.20150756835938,
"step": 127
},
{
"KL/chosen_KL_mean": -61.33363342285156,
"KL/mean": -86.33102416992188,
"KL/rejected_KL_mean": -111.32841491699219,
"KL/std": 76.6578598022461,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.007103352341800928,
"fcm_dpo/delta": 0.04982195049524307,
"fcm_dpo/margin": 49.994781494140625,
"fcm_dpo/q_t": 0.42171311378479004,
"grad_norm": 28.149051666259766,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.4325607419013977,
"logits/rejected": -0.42009297013282776,
"logps/chosen": -119.45458984375,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -187.7673797607422,
"loss": 1.1761,
"margin_dpo/margin_mean": 49.994781494140625,
"margin_dpo/margin_std": 97.5225830078125,
"step": 128
},
{
"KL/chosen_KL_mean": -75.2124252319336,
"KL/mean": -101.20868682861328,
"KL/rejected_KL_mean": -127.2049560546875,
"KL/std": 87.85305786132812,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.007200577296316624,
"fcm_dpo/delta": 0.03126451000571251,
"fcm_dpo/margin": 51.99251174926758,
"fcm_dpo/q_t": 0.4244125485420227,
"grad_norm": 21.352294921875,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.4686553478240967,
"logits/rejected": -0.46320652961730957,
"logps/chosen": -142.12879943847656,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -223.84716796875,
"loss": 1.1947,
"margin_dpo/margin_mean": 51.992515563964844,
"margin_dpo/margin_std": 116.50946044921875,
"step": 129
},
{
"KL/chosen_KL_mean": -54.154781341552734,
"KL/mean": -93.47927856445312,
"KL/rejected_KL_mean": -132.8037567138672,
"KL/std": 83.20738983154297,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.006708543747663498,
"fcm_dpo/delta": -0.17494350671768188,
"fcm_dpo/margin": 78.64897918701172,
"fcm_dpo/q_t": 0.379627525806427,
"grad_norm": 17.628036499023438,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.44320395588874817,
"logits/rejected": -0.4385804831981659,
"logps/chosen": -98.8216323852539,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -215.58541870117188,
"loss": 1.0494,
"margin_dpo/margin_mean": 78.64897918701172,
"margin_dpo/margin_std": 104.21408081054688,
"step": 130
},
{
"KL/chosen_KL_mean": -50.51225280761719,
"KL/mean": -91.10379791259766,
"KL/rejected_KL_mean": -131.69534301757812,
"KL/std": 87.86686706542969,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.005672046914696693,
"fcm_dpo/delta": -0.07984572649002075,
"fcm_dpo/margin": 81.18309020996094,
"fcm_dpo/q_t": 0.38995444774627686,
"grad_norm": 26.029661178588867,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.4229122996330261,
"logits/rejected": -0.4288443326950073,
"logps/chosen": -95.43684387207031,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -220.1393585205078,
"loss": 1.0467,
"margin_dpo/margin_mean": 81.18309020996094,
"margin_dpo/margin_std": 95.4244384765625,
"step": 131
},
{
"KL/chosen_KL_mean": -65.28816986083984,
"KL/mean": -97.25325012207031,
"KL/rejected_KL_mean": -129.2183380126953,
"KL/std": 84.85006713867188,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.0057223797775805,
"fcm_dpo/delta": 0.04182172939181328,
"fcm_dpo/margin": 63.93017578125,
"fcm_dpo/q_t": 0.4215155839920044,
"grad_norm": 17.443683624267578,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.4490656852722168,
"logits/rejected": -0.45410823822021484,
"logps/chosen": -124.28925323486328,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -217.11048889160156,
"loss": 1.1448,
"margin_dpo/margin_mean": 63.93017578125,
"margin_dpo/margin_std": 110.97628784179688,
"step": 132
},
{
"KL/chosen_KL_mean": -76.72837829589844,
"KL/mean": -104.55119323730469,
"KL/rejected_KL_mean": -132.37400817871094,
"KL/std": 75.45304107666016,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.006116401869803667,
"fcm_dpo/delta": 0.07216479629278183,
"fcm_dpo/margin": 55.64562225341797,
"fcm_dpo/q_t": 0.4264053702354431,
"grad_norm": 22.552947998046875,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.42806553840637207,
"logits/rejected": -0.4164883494377136,
"logps/chosen": -143.3328857421875,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -228.7075653076172,
"loss": 1.1563,
"margin_dpo/margin_mean": 55.64562225341797,
"margin_dpo/margin_std": 97.3055191040039,
"step": 133
},
{
"KL/chosen_KL_mean": -58.90048599243164,
"KL/mean": -88.99266815185547,
"KL/rejected_KL_mean": -119.08485412597656,
"KL/std": 71.10643005371094,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.006456311792135239,
"fcm_dpo/delta": 0.012345774099230766,
"fcm_dpo/margin": 60.18437194824219,
"fcm_dpo/q_t": 0.4125699996948242,
"grad_norm": 18.19769287109375,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.4092817008495331,
"logits/rejected": -0.3999384641647339,
"logps/chosen": -110.96974182128906,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -206.7393798828125,
"loss": 1.1018,
"margin_dpo/margin_mean": 60.18437194824219,
"margin_dpo/margin_std": 84.06219482421875,
"step": 134
},
{
"KL/chosen_KL_mean": -64.12809753417969,
"KL/mean": -109.45974731445312,
"KL/rejected_KL_mean": -154.7913818359375,
"KL/std": 97.96502685546875,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.005646052770316601,
"fcm_dpo/delta": -0.15241768956184387,
"fcm_dpo/margin": 90.66329193115234,
"fcm_dpo/q_t": 0.38228270411491394,
"grad_norm": 22.45749855041504,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.4356979727745056,
"logits/rejected": -0.46872130036354065,
"logps/chosen": -114.48196411132812,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -270.7711181640625,
"loss": 1.0405,
"margin_dpo/margin_mean": 90.66329193115234,
"margin_dpo/margin_std": 120.07051086425781,
"step": 135
},
{
"KL/chosen_KL_mean": -76.92674255371094,
"KL/mean": -106.96867370605469,
"KL/rejected_KL_mean": -137.01058959960938,
"KL/std": 91.17539978027344,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.005636686459183693,
"fcm_dpo/delta": 0.07331253588199615,
"fcm_dpo/margin": 60.0838623046875,
"fcm_dpo/q_t": 0.4317668080329895,
"grad_norm": 19.86505699157715,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.42861732840538025,
"logits/rejected": -0.42438289523124695,
"logps/chosen": -141.99925231933594,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -233.33181762695312,
"loss": 1.1772,
"margin_dpo/margin_mean": 60.083858489990234,
"margin_dpo/margin_std": 119.13693237304688,
"step": 136
},
{
"KL/chosen_KL_mean": -70.731201171875,
"KL/mean": -117.82597351074219,
"KL/rejected_KL_mean": -164.92074584960938,
"KL/std": 116.82910919189453,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.005281469784677029,
"fcm_dpo/delta": -0.1319933831691742,
"fcm_dpo/margin": 94.18954467773438,
"fcm_dpo/q_t": 0.3907054662704468,
"grad_norm": 16.0644588470459,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.38837531208992004,
"logits/rejected": -0.4134828746318817,
"logps/chosen": -119.49031829833984,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -278.7845153808594,
"loss": 1.0696,
"margin_dpo/margin_mean": 94.1895523071289,
"margin_dpo/margin_std": 142.3530731201172,
"step": 137
},
{
"KL/chosen_KL_mean": -75.57490539550781,
"KL/mean": -110.52081298828125,
"KL/rejected_KL_mean": -145.46673583984375,
"KL/std": 88.41027069091797,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.005329763051122427,
"fcm_dpo/delta": 0.032860539853572845,
"fcm_dpo/margin": 69.89181518554688,
"fcm_dpo/q_t": 0.41438084840774536,
"grad_norm": 17.347095489501953,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.4172437787055969,
"logits/rejected": -0.40940582752227783,
"logps/chosen": -136.09454345703125,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -238.66368103027344,
"loss": 1.1021,
"margin_dpo/margin_mean": 69.89181518554688,
"margin_dpo/margin_std": 89.81456756591797,
"step": 138
},
{
"KL/chosen_KL_mean": -63.819305419921875,
"KL/mean": -104.35844421386719,
"KL/rejected_KL_mean": -144.8975830078125,
"KL/std": 82.37762451171875,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.005249645560979843,
"fcm_dpo/delta": -0.032708242535591125,
"fcm_dpo/margin": 81.07829284667969,
"fcm_dpo/q_t": 0.402879536151886,
"grad_norm": 16.59463882446289,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.35705071687698364,
"logits/rejected": -0.3492419719696045,
"logps/chosen": -110.710693359375,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -224.62557983398438,
"loss": 1.0796,
"margin_dpo/margin_mean": 81.07829284667969,
"margin_dpo/margin_std": 109.54811096191406,
"step": 139
},
{
"KL/chosen_KL_mean": -76.06951904296875,
"KL/mean": -112.28123474121094,
"KL/rejected_KL_mean": -148.49295043945312,
"KL/std": 90.1976318359375,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.0052453503012657166,
"fcm_dpo/delta": 0.02480418235063553,
"fcm_dpo/margin": 72.42343139648438,
"fcm_dpo/q_t": 0.41421765089035034,
"grad_norm": 17.3253116607666,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.35899558663368225,
"logits/rejected": -0.34745436906814575,
"logps/chosen": -135.0442352294922,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -231.77706909179688,
"loss": 1.1193,
"margin_dpo/margin_mean": 72.42343139648438,
"margin_dpo/margin_std": 110.29766845703125,
"step": 140
},
{
"KL/chosen_KL_mean": -86.08993530273438,
"KL/mean": -120.1361312866211,
"KL/rejected_KL_mean": -154.1823272705078,
"KL/std": 103.17254638671875,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.0054466212168335915,
"fcm_dpo/delta": 0.03537699952721596,
"fcm_dpo/margin": 68.09239196777344,
"fcm_dpo/q_t": 0.4169376492500305,
"grad_norm": 22.393936157226562,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.36837026476860046,
"logits/rejected": -0.35952502489089966,
"logps/chosen": -161.16558837890625,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -252.3745880126953,
"loss": 1.148,
"margin_dpo/margin_mean": 68.09239196777344,
"margin_dpo/margin_std": 120.2028579711914,
"step": 141
},
{
"KL/chosen_KL_mean": -84.57060241699219,
"KL/mean": -123.71681213378906,
"KL/rejected_KL_mean": -162.86300659179688,
"KL/std": 108.09979248046875,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.005447980482131243,
"fcm_dpo/delta": -0.03504451364278793,
"fcm_dpo/margin": 78.29240417480469,
"fcm_dpo/q_t": 0.40442773699760437,
"grad_norm": 23.142438888549805,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.34042251110076904,
"logits/rejected": -0.341153621673584,
"logps/chosen": -142.59854125976562,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -257.44525146484375,
"loss": 1.1208,
"margin_dpo/margin_mean": 78.29240417480469,
"margin_dpo/margin_std": 126.79087829589844,
"step": 142
},
{
"KL/chosen_KL_mean": -86.95822143554688,
"KL/mean": -112.62377166748047,
"KL/rejected_KL_mean": -138.28932189941406,
"KL/std": 93.756591796875,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.005426798947155476,
"fcm_dpo/delta": 0.027737241238355637,
"fcm_dpo/margin": 51.33110046386719,
"fcm_dpo/q_t": 0.4396059513092041,
"grad_norm": 20.347646713256836,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.3267577290534973,
"logits/rejected": -0.3233267068862915,
"logps/chosen": -144.55467224121094,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -217.2888946533203,
"loss": 1.2111,
"margin_dpo/margin_mean": 51.33110046386719,
"margin_dpo/margin_std": 109.97335815429688,
"step": 143
},
{
"KL/chosen_KL_mean": -79.65751647949219,
"KL/mean": -109.87149810791016,
"KL/rejected_KL_mean": -140.08547973632812,
"KL/std": 80.81317138671875,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.005699482746422291,
"fcm_dpo/delta": 0.06714989244937897,
"fcm_dpo/margin": 60.42796325683594,
"fcm_dpo/q_t": 0.42339879274368286,
"grad_norm": 19.846458435058594,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.3399592638015747,
"logits/rejected": -0.3357587456703186,
"logps/chosen": -139.56387329101562,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -222.0857391357422,
"loss": 1.1396,
"margin_dpo/margin_mean": 60.427955627441406,
"margin_dpo/margin_std": 94.39094543457031,
"step": 144
},
{
"KL/chosen_KL_mean": -76.95576477050781,
"KL/mean": -109.23151397705078,
"KL/rejected_KL_mean": -141.50726318359375,
"KL/std": 79.54987335205078,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.0058981748297810555,
"fcm_dpo/delta": 0.02381444163620472,
"fcm_dpo/margin": 64.55150604248047,
"fcm_dpo/q_t": 0.4141959547996521,
"grad_norm": 23.242799758911133,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.37550991773605347,
"logits/rejected": -0.3661358058452606,
"logps/chosen": -133.55642700195312,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -219.3735809326172,
"loss": 1.1192,
"margin_dpo/margin_mean": 64.55149841308594,
"margin_dpo/margin_std": 98.04264831542969,
"step": 145
},
{
"KL/chosen_KL_mean": -103.22034454345703,
"KL/mean": -128.1830596923828,
"KL/rejected_KL_mean": -153.14578247070312,
"KL/std": 87.43064880371094,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.006305292714387178,
"fcm_dpo/delta": 0.10162100195884705,
"fcm_dpo/margin": 49.925445556640625,
"fcm_dpo/q_t": 0.43169236183166504,
"grad_norm": 39.6645622253418,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.32967615127563477,
"logits/rejected": -0.30988651514053345,
"logps/chosen": -169.22079467773438,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -234.84857177734375,
"loss": 1.206,
"margin_dpo/margin_mean": 49.925445556640625,
"margin_dpo/margin_std": 104.19798278808594,
"step": 146
},
{
"KL/chosen_KL_mean": -75.02632904052734,
"KL/mean": -108.63874816894531,
"KL/rejected_KL_mean": -142.25115966796875,
"KL/std": 86.88672637939453,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.0063508497551083565,
"fcm_dpo/delta": -0.035061098635196686,
"fcm_dpo/margin": 67.22482299804688,
"fcm_dpo/q_t": 0.4050912857055664,
"grad_norm": 20.40033721923828,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.3828004002571106,
"logits/rejected": -0.3709457516670227,
"logps/chosen": -128.43182373046875,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -213.6417694091797,
"loss": 1.1127,
"margin_dpo/margin_mean": 67.22482299804688,
"margin_dpo/margin_std": 109.10481262207031,
"step": 147
},
{
"KL/chosen_KL_mean": -73.86956787109375,
"KL/mean": -103.73741149902344,
"KL/rejected_KL_mean": -133.60525512695312,
"KL/std": 89.116455078125,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.0060739642940461636,
"fcm_dpo/delta": -0.0918317511677742,
"fcm_dpo/margin": 59.735694885253906,
"fcm_dpo/q_t": 0.41868463158607483,
"grad_norm": 18.132606506347656,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.41148072481155396,
"logits/rejected": -0.4058622121810913,
"logps/chosen": -138.806640625,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -236.6990966796875,
"loss": 1.1443,
"margin_dpo/margin_mean": 59.735694885253906,
"margin_dpo/margin_std": 94.41209411621094,
"step": 148
},
{
"KL/chosen_KL_mean": -69.35594940185547,
"KL/mean": -106.29493713378906,
"KL/rejected_KL_mean": -143.23391723632812,
"KL/std": 77.87938690185547,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.005501337815076113,
"fcm_dpo/delta": -0.0162881538271904,
"fcm_dpo/margin": 73.87796020507812,
"fcm_dpo/q_t": 0.40519750118255615,
"grad_norm": 16.045988082885742,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.428414911031723,
"logits/rejected": -0.4166084825992584,
"logps/chosen": -127.8297119140625,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -242.5486602783203,
"loss": 1.0887,
"margin_dpo/margin_mean": 73.87796020507812,
"margin_dpo/margin_std": 93.21217346191406,
"step": 149
},
{
"KL/chosen_KL_mean": -60.34211349487305,
"KL/mean": -97.03437042236328,
"KL/rejected_KL_mean": -133.72662353515625,
"KL/std": 91.03880310058594,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.005468531046062708,
"fcm_dpo/delta": -0.007947839796543121,
"fcm_dpo/margin": 73.38451385498047,
"fcm_dpo/q_t": 0.411196231842041,
"grad_norm": 18.763927459716797,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.4295479655265808,
"logits/rejected": -0.4361698627471924,
"logps/chosen": -106.04792785644531,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -217.07421875,
"loss": 1.1058,
"margin_dpo/margin_mean": 73.38451385498047,
"margin_dpo/margin_std": 108.1720199584961,
"step": 150
},
{
"KL/chosen_KL_mean": -77.08334350585938,
"KL/mean": -112.62428283691406,
"KL/rejected_KL_mean": -148.16522216796875,
"KL/std": 86.95265197753906,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.005633922293782234,
"fcm_dpo/delta": -0.0006460570730268955,
"fcm_dpo/margin": 71.08187866210938,
"fcm_dpo/q_t": 0.40812182426452637,
"grad_norm": 22.76422882080078,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.47929656505584717,
"logits/rejected": -0.4552541971206665,
"logps/chosen": -147.6541748046875,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -248.62904357910156,
"loss": 1.0926,
"margin_dpo/margin_mean": 71.0818862915039,
"margin_dpo/margin_std": 95.64910888671875,
"step": 151
},
{
"KL/chosen_KL_mean": -66.67381286621094,
"KL/mean": -110.26457214355469,
"KL/rejected_KL_mean": -153.8553466796875,
"KL/std": 87.53227233886719,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.0053495545871555805,
"fcm_dpo/delta": -0.08620619773864746,
"fcm_dpo/margin": 87.18153381347656,
"fcm_dpo/q_t": 0.39188152551651,
"grad_norm": 20.685396194458008,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.44135403633117676,
"logits/rejected": -0.4416983723640442,
"logps/chosen": -126.83819580078125,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -259.99578857421875,
"loss": 1.0522,
"margin_dpo/margin_mean": 87.18153381347656,
"margin_dpo/margin_std": 108.27259826660156,
"step": 152
},
{
"KL/chosen_KL_mean": -65.79019927978516,
"KL/mean": -103.00405883789062,
"KL/rejected_KL_mean": -140.2178955078125,
"KL/std": 96.10725402832031,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.005271708592772484,
"fcm_dpo/delta": 0.008975501172244549,
"fcm_dpo/margin": 74.42771911621094,
"fcm_dpo/q_t": 0.4141613245010376,
"grad_norm": 15.699677467346191,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.4330843687057495,
"logits/rejected": -0.4293479323387146,
"logps/chosen": -122.10546875,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -225.87374877929688,
"loss": 1.1194,
"margin_dpo/margin_mean": 74.42771911621094,
"margin_dpo/margin_std": 118.5946273803711,
"step": 153
},
{
"KL/chosen_KL_mean": -79.26509094238281,
"KL/mean": -114.19869995117188,
"KL/rejected_KL_mean": -149.13230895996094,
"KL/std": 93.6673812866211,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.005427438300102949,
"fcm_dpo/delta": 0.023521043360233307,
"fcm_dpo/margin": 69.86721801757812,
"fcm_dpo/q_t": 0.41615161299705505,
"grad_norm": 17.67850112915039,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.43425455689430237,
"logits/rejected": -0.4390127658843994,
"logps/chosen": -142.00765991210938,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -253.3765106201172,
"loss": 1.1462,
"margin_dpo/margin_mean": 69.86721801757812,
"margin_dpo/margin_std": 122.60856628417969,
"step": 154
},
{
"KL/chosen_KL_mean": -72.60445404052734,
"KL/mean": -106.42167663574219,
"KL/rejected_KL_mean": -140.2388916015625,
"KL/std": 85.55455017089844,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.005552366841584444,
"fcm_dpo/delta": 0.028694026172161102,
"fcm_dpo/margin": 67.63446044921875,
"fcm_dpo/q_t": 0.41562047600746155,
"grad_norm": 19.240365982055664,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.44728028774261475,
"logits/rejected": -0.4302216172218323,
"logps/chosen": -133.25762939453125,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -217.73110961914062,
"loss": 1.13,
"margin_dpo/margin_mean": 67.63445281982422,
"margin_dpo/margin_std": 108.91281127929688,
"step": 155
},
{
"KL/chosen_KL_mean": -99.16265869140625,
"KL/mean": -118.9976806640625,
"KL/rejected_KL_mean": -138.83270263671875,
"KL/std": 91.43487548828125,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.005871580448001623,
"fcm_dpo/delta": 0.06297972053289413,
"fcm_dpo/margin": 39.67005157470703,
"fcm_dpo/q_t": 0.44856178760528564,
"grad_norm": 27.07191276550293,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.3714250922203064,
"logits/rejected": -0.3661670684814453,
"logps/chosen": -168.654541015625,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -216.0019989013672,
"loss": 1.2847,
"margin_dpo/margin_mean": 39.67005157470703,
"margin_dpo/margin_std": 120.68568420410156,
"step": 156
},
{
"KL/chosen_KL_mean": -87.81330871582031,
"KL/mean": -132.32769775390625,
"KL/rejected_KL_mean": -176.84207153320312,
"KL/std": 99.45307159423828,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.005287863779813051,
"fcm_dpo/delta": -0.10068926960229874,
"fcm_dpo/margin": 89.02876281738281,
"fcm_dpo/q_t": 0.3861895203590393,
"grad_norm": 22.06880760192871,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.36405149102211,
"logits/rejected": -0.3723771572113037,
"logps/chosen": -149.18174743652344,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -284.4884338378906,
"loss": 1.0573,
"margin_dpo/margin_mean": 89.02876281738281,
"margin_dpo/margin_std": 111.94908142089844,
"step": 157
},
{
"KL/chosen_KL_mean": -89.31040954589844,
"KL/mean": -137.76190185546875,
"KL/rejected_KL_mean": -186.21340942382812,
"KL/std": 123.51625061035156,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.004927870351821184,
"fcm_dpo/delta": -0.10203598439693451,
"fcm_dpo/margin": 96.90299224853516,
"fcm_dpo/q_t": 0.3954607844352722,
"grad_norm": 16.532785415649414,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.3246377408504486,
"logits/rejected": -0.34248343110084534,
"logps/chosen": -146.92332458496094,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -299.9080810546875,
"loss": 1.0826,
"margin_dpo/margin_mean": 96.90299224853516,
"margin_dpo/margin_std": 150.00479125976562,
"step": 158
},
{
"KL/chosen_KL_mean": -102.0267333984375,
"KL/mean": -133.82208251953125,
"KL/rejected_KL_mean": -165.6173858642578,
"KL/std": 108.39896392822266,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.005198903381824493,
"fcm_dpo/delta": 0.07878997921943665,
"fcm_dpo/margin": 63.59064483642578,
"fcm_dpo/q_t": 0.42666539549827576,
"grad_norm": 25.30782127380371,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -0.35725462436676025,
"logits/rejected": -0.32486408948898315,
"logps/chosen": -183.5870819091797,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -254.51609802246094,
"loss": 1.1718,
"margin_dpo/margin_mean": 63.59064865112305,
"margin_dpo/margin_std": 117.3126220703125,
"step": 159
},
{
"KL/chosen_KL_mean": -103.35143280029297,
"KL/mean": -147.1392059326172,
"KL/rejected_KL_mean": -190.927001953125,
"KL/std": 114.80732727050781,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.0049042413011193275,
"fcm_dpo/delta": -0.04174552857875824,
"fcm_dpo/margin": 87.57557678222656,
"fcm_dpo/q_t": 0.40538012981414795,
"grad_norm": 21.559865951538086,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.3620595932006836,
"logits/rejected": -0.3560444116592407,
"logps/chosen": -169.08230590820312,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -288.14483642578125,
"loss": 1.1145,
"margin_dpo/margin_mean": 87.57556915283203,
"margin_dpo/margin_std": 143.64688110351562,
"step": 160
},
{
"KL/chosen_KL_mean": -87.88373565673828,
"KL/mean": -126.63963317871094,
"KL/rejected_KL_mean": -165.39553833007812,
"KL/std": 91.78167724609375,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.0050220671109855175,
"fcm_dpo/delta": 0.013326739892363548,
"fcm_dpo/margin": 77.51181030273438,
"fcm_dpo/q_t": 0.4130176603794098,
"grad_norm": 18.22276496887207,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.362446665763855,
"logits/rejected": -0.3627144396305084,
"logps/chosen": -140.32020568847656,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -248.82647705078125,
"loss": 1.1129,
"margin_dpo/margin_mean": 77.51181030273438,
"margin_dpo/margin_std": 115.73544311523438,
"step": 161
},
{
"KL/chosen_KL_mean": -86.13906860351562,
"KL/mean": -123.78965759277344,
"KL/rejected_KL_mean": -161.44024658203125,
"KL/std": 101.90530395507812,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.004698555450886488,
"fcm_dpo/delta": -0.07051464915275574,
"fcm_dpo/margin": 75.30118560791016,
"fcm_dpo/q_t": 0.4173450469970703,
"grad_norm": 18.306982040405273,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.4190049171447754,
"logits/rejected": -0.4046899676322937,
"logps/chosen": -148.74964904785156,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -250.83084106445312,
"loss": 1.1434,
"margin_dpo/margin_mean": 75.30118560791016,
"margin_dpo/margin_std": 122.22592163085938,
"step": 162
},
{
"KL/chosen_KL_mean": -95.6829833984375,
"KL/mean": -131.91665649414062,
"KL/rejected_KL_mean": -168.15032958984375,
"KL/std": 101.21453857421875,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.004903091117739677,
"fcm_dpo/delta": 0.054114192724227905,
"fcm_dpo/margin": 72.46734619140625,
"fcm_dpo/q_t": 0.4222571849822998,
"grad_norm": 17.79399871826172,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.3502505421638489,
"logits/rejected": -0.3333583474159241,
"logps/chosen": -150.70462036132812,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -243.5685577392578,
"loss": 1.1443,
"margin_dpo/margin_mean": 72.46734619140625,
"margin_dpo/margin_std": 121.69917297363281,
"step": 163
},
{
"KL/chosen_KL_mean": -88.13164520263672,
"KL/mean": -133.67013549804688,
"KL/rejected_KL_mean": -179.20860290527344,
"KL/std": 99.38763427734375,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.00465630367398262,
"fcm_dpo/delta": -0.03786290064454079,
"fcm_dpo/margin": 91.07695770263672,
"fcm_dpo/q_t": 0.4004104733467102,
"grad_norm": 21.530731201171875,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.34161853790283203,
"logits/rejected": -0.3289684057235718,
"logps/chosen": -143.77230834960938,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -258.87322998046875,
"loss": 1.0758,
"margin_dpo/margin_mean": 91.07695770263672,
"margin_dpo/margin_std": 114.45172119140625,
"step": 164
},
{
"KL/chosen_KL_mean": -95.0910873413086,
"KL/mean": -124.63126373291016,
"KL/rejected_KL_mean": -154.17144775390625,
"KL/std": 86.06841278076172,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.005135393235832453,
"fcm_dpo/delta": 0.1146107017993927,
"fcm_dpo/margin": 59.080352783203125,
"fcm_dpo/q_t": 0.434864342212677,
"grad_norm": 25.962055206298828,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.36488398909568787,
"logits/rejected": -0.34004294872283936,
"logps/chosen": -156.4017791748047,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -227.84205627441406,
"loss": 1.192,
"margin_dpo/margin_mean": 59.080360412597656,
"margin_dpo/margin_std": 115.70834350585938,
"step": 165
},
{
"KL/chosen_KL_mean": -85.34329223632812,
"KL/mean": -141.81805419921875,
"KL/rejected_KL_mean": -198.2928009033203,
"KL/std": 127.079345703125,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.004592292942106724,
"fcm_dpo/delta": -0.1705796718597412,
"fcm_dpo/margin": 112.94950866699219,
"fcm_dpo/q_t": 0.385653555393219,
"grad_norm": 15.64688777923584,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.29077380895614624,
"logits/rejected": -0.30136969685554504,
"logps/chosen": -136.326904296875,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -296.387939453125,
"loss": 1.0467,
"margin_dpo/margin_mean": 112.94950866699219,
"margin_dpo/margin_std": 160.54930114746094,
"step": 166
},
{
"KL/chosen_KL_mean": -87.4961166381836,
"KL/mean": -146.18600463867188,
"KL/rejected_KL_mean": -204.87586975097656,
"KL/std": 114.61824035644531,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.004137314390391111,
"fcm_dpo/delta": -0.11398793756961823,
"fcm_dpo/margin": 117.37975311279297,
"fcm_dpo/q_t": 0.3846902847290039,
"grad_norm": 18.566125869750977,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.31079137325286865,
"logits/rejected": -0.31497058272361755,
"logps/chosen": -137.9202117919922,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -300.90631103515625,
"loss": 1.0292,
"margin_dpo/margin_mean": 117.3797607421875,
"margin_dpo/margin_std": 134.6339874267578,
"step": 167
},
{
"KL/chosen_KL_mean": -97.19728088378906,
"KL/mean": -139.75576782226562,
"KL/rejected_KL_mean": -182.31427001953125,
"KL/std": 108.02088928222656,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.0041666412726044655,
"fcm_dpo/delta": 0.0552922822535038,
"fcm_dpo/margin": 85.11698913574219,
"fcm_dpo/q_t": 0.42072856426239014,
"grad_norm": 19.070110321044922,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.292841374874115,
"logits/rejected": -0.28525102138519287,
"logps/chosen": -146.6601104736328,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -257.6228332519531,
"loss": 1.1239,
"margin_dpo/margin_mean": 85.11698913574219,
"margin_dpo/margin_std": 122.18766021728516,
"step": 168
},
{
"KL/chosen_KL_mean": -102.30801391601562,
"KL/mean": -139.7085418701172,
"KL/rejected_KL_mean": -177.10906982421875,
"KL/std": 107.6697998046875,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.004094083793461323,
"fcm_dpo/delta": -0.03929879516363144,
"fcm_dpo/margin": 74.80106353759766,
"fcm_dpo/q_t": 0.43097662925720215,
"grad_norm": 22.41761016845703,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.3144596219062805,
"logits/rejected": -0.3004973530769348,
"logps/chosen": -162.1114501953125,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -260.454833984375,
"loss": 1.1915,
"margin_dpo/margin_mean": 74.80106353759766,
"margin_dpo/margin_std": 148.64874267578125,
"step": 169
},
{
"KL/chosen_KL_mean": -92.78736877441406,
"KL/mean": -136.38568115234375,
"KL/rejected_KL_mean": -179.98399353027344,
"KL/std": 105.04856872558594,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.00412098178640008,
"fcm_dpo/delta": 0.04728086292743683,
"fcm_dpo/margin": 87.19662475585938,
"fcm_dpo/q_t": 0.4182136654853821,
"grad_norm": 17.567453384399414,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.2699648141860962,
"logits/rejected": -0.2612574100494385,
"logps/chosen": -142.25914001464844,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -255.90133666992188,
"loss": 1.1277,
"margin_dpo/margin_mean": 87.19662475585938,
"margin_dpo/margin_std": 125.52247619628906,
"step": 170
},
{
"KL/chosen_KL_mean": -132.91552734375,
"KL/mean": -168.7076416015625,
"KL/rejected_KL_mean": -204.499755859375,
"KL/std": 122.58496856689453,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.004573632497340441,
"fcm_dpo/delta": 0.08680512011051178,
"fcm_dpo/margin": 71.584228515625,
"fcm_dpo/q_t": 0.4330134391784668,
"grad_norm": 25.66452980041504,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.2910054922103882,
"logits/rejected": -0.27732402086257935,
"logps/chosen": -217.41482543945312,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -313.8818359375,
"loss": 1.2,
"margin_dpo/margin_mean": 71.584228515625,
"margin_dpo/margin_std": 157.88197326660156,
"step": 171
},
{
"KL/chosen_KL_mean": -115.34175109863281,
"KL/mean": -152.7630615234375,
"KL/rejected_KL_mean": -190.18435668945312,
"KL/std": 117.2271957397461,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.004836169071495533,
"fcm_dpo/delta": 0.046509332954883575,
"fcm_dpo/margin": 74.84261322021484,
"fcm_dpo/q_t": 0.42060211300849915,
"grad_norm": 21.91267967224121,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.34988853335380554,
"logits/rejected": -0.33963823318481445,
"logps/chosen": -183.99566650390625,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -275.62103271484375,
"loss": 1.149,
"margin_dpo/margin_mean": 74.84262084960938,
"margin_dpo/margin_std": 131.61460876464844,
"step": 172
},
{
"KL/chosen_KL_mean": -105.72806549072266,
"KL/mean": -143.4074249267578,
"KL/rejected_KL_mean": -181.08677673339844,
"KL/std": 112.28417205810547,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.005077026784420013,
"fcm_dpo/delta": 0.019231334328651428,
"fcm_dpo/margin": 75.35870361328125,
"fcm_dpo/q_t": 0.4149942994117737,
"grad_norm": 22.99903678894043,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.34687381982803345,
"logits/rejected": -0.32024502754211426,
"logps/chosen": -168.77894592285156,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -259.77069091796875,
"loss": 1.1198,
"margin_dpo/margin_mean": 75.35870361328125,
"margin_dpo/margin_std": 116.21147155761719,
"step": 173
},
{
"KL/chosen_KL_mean": -97.80389404296875,
"KL/mean": -142.25250244140625,
"KL/rejected_KL_mean": -186.7010955810547,
"KL/std": 110.95504760742188,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.004968566354364157,
"fcm_dpo/delta": -0.05666285380721092,
"fcm_dpo/margin": 88.89718627929688,
"fcm_dpo/q_t": 0.4014216363430023,
"grad_norm": 25.833757400512695,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.28936031460762024,
"logits/rejected": -0.30766457319259644,
"logps/chosen": -151.1668701171875,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -288.6123046875,
"loss": 1.0988,
"margin_dpo/margin_mean": 88.8971939086914,
"margin_dpo/margin_std": 134.5579833984375,
"step": 174
},
{
"KL/chosen_KL_mean": -90.04649353027344,
"KL/mean": -149.2164306640625,
"KL/rejected_KL_mean": -208.3863525390625,
"KL/std": 116.98501586914062,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.004224025644361973,
"fcm_dpo/delta": -0.13619616627693176,
"fcm_dpo/margin": 118.33984375,
"fcm_dpo/q_t": 0.37867677211761475,
"grad_norm": 29.216033935546875,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.2830924093723297,
"logits/rejected": -0.2700659930706024,
"logps/chosen": -135.46426391601562,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -297.89215087890625,
"loss": 1.0097,
"margin_dpo/margin_mean": 118.33984375,
"margin_dpo/margin_std": 123.62623596191406,
"step": 175
},
{
"KL/chosen_KL_mean": -93.11599731445312,
"KL/mean": -144.95571899414062,
"KL/rejected_KL_mean": -196.79547119140625,
"KL/std": 118.29825592041016,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.004063988104462624,
"fcm_dpo/delta": -0.027029920369386673,
"fcm_dpo/margin": 103.6794662475586,
"fcm_dpo/q_t": 0.40472695231437683,
"grad_norm": 17.027528762817383,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.287243127822876,
"logits/rejected": -0.30266469717025757,
"logps/chosen": -143.56883239746094,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -292.35443115234375,
"loss": 1.0783,
"margin_dpo/margin_mean": 103.67945861816406,
"margin_dpo/margin_std": 136.32940673828125,
"step": 176
},
{
"KL/chosen_KL_mean": -108.14009094238281,
"KL/mean": -158.50189208984375,
"KL/rejected_KL_mean": -208.86370849609375,
"KL/std": 128.51873779296875,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.004049934446811676,
"fcm_dpo/delta": -0.011070270091295242,
"fcm_dpo/margin": 100.7236099243164,
"fcm_dpo/q_t": 0.4076334238052368,
"grad_norm": 22.69172477722168,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.3215820789337158,
"logits/rejected": -0.3144031763076782,
"logps/chosen": -169.35655212402344,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -304.75750732421875,
"loss": 1.0863,
"margin_dpo/margin_mean": 100.72361755371094,
"margin_dpo/margin_std": 134.19070434570312,
"step": 177
},
{
"KL/chosen_KL_mean": -118.08624267578125,
"KL/mean": -180.44430541992188,
"KL/rejected_KL_mean": -242.80238342285156,
"KL/std": 147.27687072753906,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.003680055495351553,
"fcm_dpo/delta": -0.0795358195900917,
"fcm_dpo/margin": 124.71614074707031,
"fcm_dpo/q_t": 0.3944624662399292,
"grad_norm": 22.354063034057617,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.2147635966539383,
"logits/rejected": -0.20747539401054382,
"logps/chosen": -176.35104370117188,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -348.167724609375,
"loss": 1.0579,
"margin_dpo/margin_mean": 124.71614074707031,
"margin_dpo/margin_std": 161.7968292236328,
"step": 178
},
{
"KL/chosen_KL_mean": -123.17328643798828,
"KL/mean": -166.720458984375,
"KL/rejected_KL_mean": -210.26763916015625,
"KL/std": 128.99139404296875,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.0038236351683735847,
"fcm_dpo/delta": 0.08045655488967896,
"fcm_dpo/margin": 87.09436798095703,
"fcm_dpo/q_t": 0.4275627136230469,
"grad_norm": 26.390382766723633,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.2743037939071655,
"logits/rejected": -0.2854520082473755,
"logps/chosen": -184.23159790039062,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -300.79547119140625,
"loss": 1.1747,
"margin_dpo/margin_mean": 87.09436798095703,
"margin_dpo/margin_std": 162.98179626464844,
"step": 179
},
{
"KL/chosen_KL_mean": -106.45706176757812,
"KL/mean": -164.29107666015625,
"KL/rejected_KL_mean": -222.1250762939453,
"KL/std": 113.41049194335938,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.003801894374191761,
"fcm_dpo/delta": -0.05152449384331703,
"fcm_dpo/margin": 115.66802978515625,
"fcm_dpo/q_t": 0.39676570892333984,
"grad_norm": 18.645387649536133,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.25623831152915955,
"logits/rejected": -0.25502607226371765,
"logps/chosen": -160.79977416992188,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -320.3369140625,
"loss": 1.0622,
"margin_dpo/margin_mean": 115.66802215576172,
"margin_dpo/margin_std": 143.18829345703125,
"step": 180
},
{
"KL/chosen_KL_mean": -91.30926513671875,
"KL/mean": -126.50997924804688,
"KL/rejected_KL_mean": -161.710693359375,
"KL/std": 105.37017822265625,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.004171198233962059,
"fcm_dpo/delta": 0.12288638204336166,
"fcm_dpo/margin": 70.40142059326172,
"fcm_dpo/q_t": 0.4360322952270508,
"grad_norm": 18.242246627807617,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.2822926938533783,
"logits/rejected": -0.25778597593307495,
"logps/chosen": -146.30972290039062,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -223.36685180664062,
"loss": 1.1884,
"margin_dpo/margin_mean": 70.40142059326172,
"margin_dpo/margin_std": 135.00912475585938,
"step": 181
},
{
"KL/chosen_KL_mean": -84.9980239868164,
"KL/mean": -147.97802734375,
"KL/rejected_KL_mean": -210.95803833007812,
"KL/std": 125.26875305175781,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.0038350564427673817,
"fcm_dpo/delta": -0.11499916762113571,
"fcm_dpo/margin": 125.96003723144531,
"fcm_dpo/q_t": 0.3841468095779419,
"grad_norm": 14.498435974121094,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.27837133407592773,
"logits/rejected": -0.28961560130119324,
"logps/chosen": -126.10587310791016,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -300.4796142578125,
"loss": 1.0402,
"margin_dpo/margin_mean": 125.96002960205078,
"margin_dpo/margin_std": 152.85032653808594,
"step": 182
},
{
"KL/chosen_KL_mean": -127.10066223144531,
"KL/mean": -160.03028869628906,
"KL/rejected_KL_mean": -192.95993041992188,
"KL/std": 105.72233581542969,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.0037473866250365973,
"fcm_dpo/delta": -0.014634971506893635,
"fcm_dpo/margin": 65.85924530029297,
"fcm_dpo/q_t": 0.4414650797843933,
"grad_norm": 16.12941551208496,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.2845485210418701,
"logits/rejected": -0.27767112851142883,
"logps/chosen": -184.62522888183594,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -268.9356689453125,
"loss": 1.2074,
"margin_dpo/margin_mean": 65.85924530029297,
"margin_dpo/margin_std": 116.302734375,
"step": 183
},
{
"KL/chosen_KL_mean": -108.40987396240234,
"KL/mean": -143.2193603515625,
"KL/rejected_KL_mean": -178.02880859375,
"KL/std": 99.74555206298828,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.0037929057143628597,
"fcm_dpo/delta": 0.024003401398658752,
"fcm_dpo/margin": 69.61894226074219,
"fcm_dpo/q_t": 0.4396555423736572,
"grad_norm": 15.583113670349121,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.3264748156070709,
"logits/rejected": -0.31739190220832825,
"logps/chosen": -166.954833984375,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -254.66287231445312,
"loss": 1.1946,
"margin_dpo/margin_mean": 69.61894226074219,
"margin_dpo/margin_std": 125.76559448242188,
"step": 184
},
{
"KL/chosen_KL_mean": -119.91249084472656,
"KL/mean": -147.1962890625,
"KL/rejected_KL_mean": -174.48007202148438,
"KL/std": 115.40476989746094,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.003838424803689122,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 54.567588806152344,
"fcm_dpo/q_t": 0.45364660024642944,
"grad_norm": 19.94534683227539,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.2822297513484955,
"logits/rejected": -0.2613981068134308,
"logps/chosen": -181.93833923339844,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -248.24266052246094,
"loss": 1.2533,
"margin_dpo/margin_mean": 54.567588806152344,
"margin_dpo/margin_std": 138.52407836914062,
"step": 185
},
{
"KL/chosen_KL_mean": -114.25096893310547,
"KL/mean": -166.89956665039062,
"KL/rejected_KL_mean": -219.54815673828125,
"KL/std": 117.91287231445312,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.003811160335317254,
"fcm_dpo/delta": -0.0017963321879506111,
"fcm_dpo/margin": 105.29721069335938,
"fcm_dpo/q_t": 0.40835633873939514,
"grad_norm": 19.27111053466797,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.31773799657821655,
"logits/rejected": -0.3034238815307617,
"logps/chosen": -183.60443115234375,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -307.62060546875,
"loss": 1.0898,
"margin_dpo/margin_mean": 105.29721069335938,
"margin_dpo/margin_std": 139.1805419921875,
"step": 186
},
{
"KL/chosen_KL_mean": -102.67378234863281,
"KL/mean": -148.59390258789062,
"KL/rejected_KL_mean": -194.5139923095703,
"KL/std": 112.72882080078125,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.004015129059553146,
"fcm_dpo/delta": 0.03561420738697052,
"fcm_dpo/margin": 91.84022521972656,
"fcm_dpo/q_t": 0.41552722454071045,
"grad_norm": 19.90249252319336,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.27692389488220215,
"logits/rejected": -0.27118462324142456,
"logps/chosen": -155.43023681640625,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -276.48309326171875,
"loss": 1.1093,
"margin_dpo/margin_mean": 91.84022521972656,
"margin_dpo/margin_std": 122.15457916259766,
"step": 187
},
{
"KL/chosen_KL_mean": -98.16572570800781,
"KL/mean": -153.54725646972656,
"KL/rejected_KL_mean": -208.92877197265625,
"KL/std": 125.24346923828125,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.003873111680150032,
"fcm_dpo/delta": -0.036867331713438034,
"fcm_dpo/margin": 110.76304626464844,
"fcm_dpo/q_t": 0.4013431966304779,
"grad_norm": 21.790515899658203,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.2571265697479248,
"logits/rejected": -0.26449519395828247,
"logps/chosen": -147.58120727539062,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -298.4692077636719,
"loss": 1.0701,
"margin_dpo/margin_mean": 110.76305389404297,
"margin_dpo/margin_std": 140.52169799804688,
"step": 188
},
{
"KL/chosen_KL_mean": -120.33238220214844,
"KL/mean": -165.42770385742188,
"KL/rejected_KL_mean": -210.52301025390625,
"KL/std": 131.658203125,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.003984817303717136,
"fcm_dpo/delta": 0.04916512593626976,
"fcm_dpo/margin": 90.19061279296875,
"fcm_dpo/q_t": 0.42296701669692993,
"grad_norm": 27.90737533569336,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.2483985424041748,
"logits/rejected": -0.23404812812805176,
"logps/chosen": -172.73133850097656,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -282.69036865234375,
"loss": 1.1524,
"margin_dpo/margin_mean": 90.19062042236328,
"margin_dpo/margin_std": 164.16030883789062,
"step": 189
},
{
"KL/chosen_KL_mean": -128.05345153808594,
"KL/mean": -182.98655700683594,
"KL/rejected_KL_mean": -237.91966247558594,
"KL/std": 141.72225952148438,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.004035481717437506,
"fcm_dpo/delta": -0.06281507015228271,
"fcm_dpo/margin": 109.86622619628906,
"fcm_dpo/q_t": 0.40009400248527527,
"grad_norm": 21.0288143157959,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.29694175720214844,
"logits/rejected": -0.2908221483230591,
"logps/chosen": -192.73651123046875,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -340.47021484375,
"loss": 1.1029,
"margin_dpo/margin_mean": 109.86621856689453,
"margin_dpo/margin_std": 163.8136444091797,
"step": 190
},
{
"KL/chosen_KL_mean": -120.60160827636719,
"KL/mean": -199.1689910888672,
"KL/rejected_KL_mean": -277.73638916015625,
"KL/std": 161.70599365234375,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.0032641002908349037,
"fcm_dpo/delta": -0.16109062731266022,
"fcm_dpo/margin": 157.13475036621094,
"fcm_dpo/q_t": 0.377569317817688,
"grad_norm": 17.576215744018555,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.2872377634048462,
"logits/rejected": -0.26138466596603394,
"logps/chosen": -189.26048278808594,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -387.8760681152344,
"loss": 1.0153,
"margin_dpo/margin_mean": 157.134765625,
"margin_dpo/margin_std": 182.57261657714844,
"step": 191
},
{
"KL/chosen_KL_mean": -150.72152709960938,
"KL/mean": -201.56427001953125,
"KL/rejected_KL_mean": -252.40701293945312,
"KL/std": 139.52310180664062,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.0034080445766448975,
"fcm_dpo/delta": 0.06307677179574966,
"fcm_dpo/margin": 101.68548583984375,
"fcm_dpo/q_t": 0.4241996109485626,
"grad_norm": 20.135229110717773,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.22673380374908447,
"logits/rejected": -0.2247309386730194,
"logps/chosen": -220.44842529296875,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -355.7283630371094,
"loss": 1.1507,
"margin_dpo/margin_mean": 101.68548583984375,
"margin_dpo/margin_std": 175.06744384765625,
"step": 192
},
{
"KL/chosen_KL_mean": -148.80224609375,
"KL/mean": -178.2123260498047,
"KL/rejected_KL_mean": -207.62240600585938,
"KL/std": 128.90753173828125,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.0035294066183269024,
"fcm_dpo/delta": 0.06064280495047569,
"fcm_dpo/margin": 58.82013702392578,
"fcm_dpo/q_t": 0.45174944400787354,
"grad_norm": 27.31705665588379,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.24892589449882507,
"logits/rejected": -0.23975345492362976,
"logps/chosen": -208.99273681640625,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -284.02996826171875,
"loss": 1.2674,
"margin_dpo/margin_mean": 58.82013702392578,
"margin_dpo/margin_std": 162.4898681640625,
"step": 193
},
{
"KL/chosen_KL_mean": -91.62898254394531,
"KL/mean": -139.33377075195312,
"KL/rejected_KL_mean": -187.03854370117188,
"KL/std": 104.66587829589844,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.003725461196154356,
"fcm_dpo/delta": 0.053960170596838,
"fcm_dpo/margin": 95.4095687866211,
"fcm_dpo/q_t": 0.4181906580924988,
"grad_norm": 16.410791397094727,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.19228887557983398,
"logits/rejected": -0.1776125431060791,
"logps/chosen": -129.4693603515625,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -247.72332763671875,
"loss": 1.1084,
"margin_dpo/margin_mean": 95.40956115722656,
"margin_dpo/margin_std": 115.09669494628906,
"step": 194
},
{
"KL/chosen_KL_mean": -141.21636962890625,
"KL/mean": -192.36294555664062,
"KL/rejected_KL_mean": -243.50949096679688,
"KL/std": 128.15513610839844,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.0038534309715032578,
"fcm_dpo/delta": 0.007251395843923092,
"fcm_dpo/margin": 102.29312133789062,
"fcm_dpo/q_t": 0.40964868664741516,
"grad_norm": 27.93120765686035,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.26708340644836426,
"logits/rejected": -0.285901814699173,
"logps/chosen": -196.10794067382812,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -340.28045654296875,
"loss": 1.0919,
"margin_dpo/margin_mean": 102.29312133789062,
"margin_dpo/margin_std": 133.5623779296875,
"step": 195
},
{
"KL/chosen_KL_mean": -111.13381958007812,
"KL/mean": -168.23362731933594,
"KL/rejected_KL_mean": -225.33343505859375,
"KL/std": 128.486083984375,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.0036738794296979904,
"fcm_dpo/delta": -0.02965060994029045,
"fcm_dpo/margin": 114.1996078491211,
"fcm_dpo/q_t": 0.40418341755867004,
"grad_norm": 19.077123641967773,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.2980375289916992,
"logits/rejected": -0.2835182845592499,
"logps/chosen": -164.37905883789062,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -301.3863830566406,
"loss": 1.0895,
"margin_dpo/margin_mean": 114.19961547851562,
"margin_dpo/margin_std": 154.56077575683594,
"step": 196
},
{
"KL/chosen_KL_mean": -110.27510833740234,
"KL/mean": -154.23858642578125,
"KL/rejected_KL_mean": -198.20204162597656,
"KL/std": 109.70982360839844,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.003983784466981888,
"fcm_dpo/delta": 0.05786813423037529,
"fcm_dpo/margin": 87.92694091796875,
"fcm_dpo/q_t": 0.4213848114013672,
"grad_norm": 21.03490447998047,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.28392934799194336,
"logits/rejected": -0.27791300415992737,
"logps/chosen": -170.6954345703125,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -275.41094970703125,
"loss": 1.125,
"margin_dpo/margin_mean": 87.92694091796875,
"margin_dpo/margin_std": 125.18641662597656,
"step": 197
},
{
"KL/chosen_KL_mean": -122.45361328125,
"KL/mean": -180.43850708007812,
"KL/rejected_KL_mean": -238.42340087890625,
"KL/std": 138.5358428955078,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.0038145678117871284,
"fcm_dpo/delta": -0.054569028317928314,
"fcm_dpo/margin": 115.96977996826172,
"fcm_dpo/q_t": 0.399750292301178,
"grad_norm": 20.83743667602539,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.32312026619911194,
"logits/rejected": -0.322989821434021,
"logps/chosen": -177.48980712890625,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -335.6666564941406,
"loss": 1.0714,
"margin_dpo/margin_mean": 115.96978759765625,
"margin_dpo/margin_std": 157.2657470703125,
"step": 198
},
{
"KL/chosen_KL_mean": -116.48646545410156,
"KL/mean": -172.57876586914062,
"KL/rejected_KL_mean": -228.67108154296875,
"KL/std": 122.16108703613281,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.0036303927190601826,
"fcm_dpo/delta": -0.012894738465547562,
"fcm_dpo/margin": 112.18461608886719,
"fcm_dpo/q_t": 0.4059009552001953,
"grad_norm": 20.86189079284668,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.3357727527618408,
"logits/rejected": -0.3183862566947937,
"logps/chosen": -173.31529235839844,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -313.31927490234375,
"loss": 1.094,
"margin_dpo/margin_mean": 112.18460845947266,
"margin_dpo/margin_std": 152.09129333496094,
"step": 199
},
{
"KL/chosen_KL_mean": -113.27946472167969,
"KL/mean": -171.24513244628906,
"KL/rejected_KL_mean": -229.2107696533203,
"KL/std": 133.42398071289062,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.0035990295000374317,
"fcm_dpo/delta": -0.02324402891099453,
"fcm_dpo/margin": 115.93130493164062,
"fcm_dpo/q_t": 0.40560150146484375,
"grad_norm": 18.164777755737305,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.2667478621006012,
"logits/rejected": -0.24850307404994965,
"logps/chosen": -166.34652709960938,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -309.8192138671875,
"loss": 1.0995,
"margin_dpo/margin_mean": 115.93130493164062,
"margin_dpo/margin_std": 171.93081665039062,
"step": 200
},
{
"KL/chosen_KL_mean": -123.16534423828125,
"KL/mean": -179.07032775878906,
"KL/rejected_KL_mean": -234.97531127929688,
"KL/std": 142.31027221679688,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.003661088179796934,
"fcm_dpo/delta": -0.012851493433117867,
"fcm_dpo/margin": 111.80996704101562,
"fcm_dpo/q_t": 0.40664464235305786,
"grad_norm": 20.639188766479492,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.37277939915657043,
"logits/rejected": -0.3691471219062805,
"logps/chosen": -198.56756591796875,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -349.78350830078125,
"loss": 1.1012,
"margin_dpo/margin_mean": 111.80996704101562,
"margin_dpo/margin_std": 161.63238525390625,
"step": 201
},
{
"KL/chosen_KL_mean": -127.29255676269531,
"KL/mean": -166.71469116210938,
"KL/rejected_KL_mean": -206.13681030273438,
"KL/std": 124.88301849365234,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.003907524980604649,
"fcm_dpo/delta": 0.10762214660644531,
"fcm_dpo/margin": 78.84428405761719,
"fcm_dpo/q_t": 0.4342569410800934,
"grad_norm": 23.870006561279297,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.28092265129089355,
"logits/rejected": -0.29571932554244995,
"logps/chosen": -177.3938751220703,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -293.1218566894531,
"loss": 1.1957,
"margin_dpo/margin_mean": 78.84428405761719,
"margin_dpo/margin_std": 163.82876586914062,
"step": 202
},
{
"KL/chosen_KL_mean": -127.91600036621094,
"KL/mean": -176.91217041015625,
"KL/rejected_KL_mean": -225.90835571289062,
"KL/std": 129.85752868652344,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.003999835811555386,
"fcm_dpo/delta": 0.009929049760103226,
"fcm_dpo/margin": 97.99237060546875,
"fcm_dpo/q_t": 0.4120560884475708,
"grad_norm": 20.524028778076172,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.31167030334472656,
"logits/rejected": -0.3005950450897217,
"logps/chosen": -188.52569580078125,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -311.8043212890625,
"loss": 1.1109,
"margin_dpo/margin_mean": 97.99237060546875,
"margin_dpo/margin_std": 146.369140625,
"step": 203
},
{
"KL/chosen_KL_mean": -140.44757080078125,
"KL/mean": -185.04940795898438,
"KL/rejected_KL_mean": -229.6512451171875,
"KL/std": 140.68167114257812,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.0041489386931061745,
"fcm_dpo/delta": 0.03660349175333977,
"fcm_dpo/margin": 89.20367431640625,
"fcm_dpo/q_t": 0.4208757281303406,
"grad_norm": 35.41379928588867,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.3780588209629059,
"logits/rejected": -0.33705025911331177,
"logps/chosen": -220.61251831054688,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -317.34716796875,
"loss": 1.1626,
"margin_dpo/margin_mean": 89.20367431640625,
"margin_dpo/margin_std": 168.600341796875,
"step": 204
},
{
"KL/chosen_KL_mean": -134.366455078125,
"KL/mean": -193.7669677734375,
"KL/rejected_KL_mean": -253.16749572753906,
"KL/std": 141.5926513671875,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.0037632733583450317,
"fcm_dpo/delta": -0.07647587358951569,
"fcm_dpo/margin": 118.80104064941406,
"fcm_dpo/q_t": 0.3929186463356018,
"grad_norm": 22.13498878479004,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.3193209767341614,
"logits/rejected": -0.2915324568748474,
"logps/chosen": -193.75119018554688,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -338.29254150390625,
"loss": 1.0786,
"margin_dpo/margin_mean": 118.80104064941406,
"margin_dpo/margin_std": 156.5557861328125,
"step": 205
},
{
"KL/chosen_KL_mean": -120.28572082519531,
"KL/mean": -186.2382354736328,
"KL/rejected_KL_mean": -252.1907501220703,
"KL/std": 128.577880859375,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.003537021577358246,
"fcm_dpo/delta": -0.0920427143573761,
"fcm_dpo/margin": 131.90501403808594,
"fcm_dpo/q_t": 0.3862435817718506,
"grad_norm": 23.741201400756836,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.2517659068107605,
"logits/rejected": -0.2520918548107147,
"logps/chosen": -167.25021362304688,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -351.14422607421875,
"loss": 1.0313,
"margin_dpo/margin_mean": 131.90501403808594,
"margin_dpo/margin_std": 139.59078979492188,
"step": 206
},
{
"KL/chosen_KL_mean": -114.99893188476562,
"KL/mean": -187.25506591796875,
"KL/rejected_KL_mean": -259.5111999511719,
"KL/std": 148.06365966796875,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.003286686260253191,
"fcm_dpo/delta": -0.09856441617012024,
"fcm_dpo/margin": 144.51226806640625,
"fcm_dpo/q_t": 0.38960766792297363,
"grad_norm": 21.21698760986328,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.35248827934265137,
"logits/rejected": -0.329440712928772,
"logps/chosen": -171.05519104003906,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -343.958984375,
"loss": 1.0401,
"margin_dpo/margin_mean": 144.51226806640625,
"margin_dpo/margin_std": 174.37086486816406,
"step": 207
},
{
"KL/chosen_KL_mean": -171.49325561523438,
"KL/mean": -230.3494110107422,
"KL/rejected_KL_mean": -289.20556640625,
"KL/std": 144.81240844726562,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.0032218745909631252,
"fcm_dpo/delta": 0.02522367238998413,
"fcm_dpo/margin": 117.71229553222656,
"fcm_dpo/q_t": 0.41357824206352234,
"grad_norm": 22.431468963623047,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.3535154461860657,
"logits/rejected": -0.3377763032913208,
"logps/chosen": -238.5608673095703,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -383.4924621582031,
"loss": 1.1181,
"margin_dpo/margin_mean": 117.71229553222656,
"margin_dpo/margin_std": 175.20639038085938,
"step": 208
},
{
"KL/chosen_KL_mean": -144.38653564453125,
"KL/mean": -197.31893920898438,
"KL/rejected_KL_mean": -250.25132751464844,
"KL/std": 131.18588256835938,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.003419704968109727,
"fcm_dpo/delta": 0.04580961540341377,
"fcm_dpo/margin": 105.86478424072266,
"fcm_dpo/q_t": 0.41855117678642273,
"grad_norm": 19.670215606689453,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.231040820479393,
"logits/rejected": -0.2151014506816864,
"logps/chosen": -200.5682373046875,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -331.1928405761719,
"loss": 1.1333,
"margin_dpo/margin_mean": 105.86478424072266,
"margin_dpo/margin_std": 164.6919708251953,
"step": 209
},
{
"KL/chosen_KL_mean": -127.3232192993164,
"KL/mean": -187.63894653320312,
"KL/rejected_KL_mean": -247.95468139648438,
"KL/std": 131.5801544189453,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.0034523247741162777,
"fcm_dpo/delta": -0.0221034474670887,
"fcm_dpo/margin": 120.6314697265625,
"fcm_dpo/q_t": 0.4037366807460785,
"grad_norm": 20.2919979095459,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.28856372833251953,
"logits/rejected": -0.2808060348033905,
"logps/chosen": -173.69503784179688,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -324.63629150390625,
"loss": 1.0825,
"margin_dpo/margin_mean": 120.6314697265625,
"margin_dpo/margin_std": 156.716552734375,
"step": 210
},
{
"KL/chosen_KL_mean": -186.89505004882812,
"KL/mean": -239.9453125,
"KL/rejected_KL_mean": -292.99560546875,
"KL/std": 155.1609649658203,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.003472290001809597,
"fcm_dpo/delta": 0.038492538034915924,
"fcm_dpo/margin": 106.10052490234375,
"fcm_dpo/q_t": 0.42122966051101685,
"grad_norm": 28.37523078918457,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.2967555522918701,
"logits/rejected": -0.2570844888687134,
"logps/chosen": -265.827392578125,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -379.81658935546875,
"loss": 1.1436,
"margin_dpo/margin_mean": 106.10053253173828,
"margin_dpo/margin_std": 179.58560180664062,
"step": 211
},
{
"KL/chosen_KL_mean": -154.07533264160156,
"KL/mean": -228.19821166992188,
"KL/rejected_KL_mean": -302.32110595703125,
"KL/std": 162.68435668945312,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.0031211567111313343,
"fcm_dpo/delta": -0.09513364732265472,
"fcm_dpo/margin": 148.24575805664062,
"fcm_dpo/q_t": 0.3897473216056824,
"grad_norm": 25.847610473632812,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.2827294170856476,
"logits/rejected": -0.2756040692329407,
"logps/chosen": -212.2723388671875,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -405.3789367675781,
"loss": 1.0549,
"margin_dpo/margin_mean": 148.24575805664062,
"margin_dpo/margin_std": 177.65496826171875,
"step": 212
},
{
"KL/chosen_KL_mean": -155.3287811279297,
"KL/mean": -228.24905395507812,
"KL/rejected_KL_mean": -301.16937255859375,
"KL/std": 149.9835205078125,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.0030004819855093956,
"fcm_dpo/delta": -0.05112471804022789,
"fcm_dpo/margin": 145.84056091308594,
"fcm_dpo/q_t": 0.3968704044818878,
"grad_norm": 19.53304672241211,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.25978952646255493,
"logits/rejected": -0.23081305623054504,
"logps/chosen": -222.84149169921875,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -395.08404541015625,
"loss": 1.061,
"margin_dpo/margin_mean": 145.84056091308594,
"margin_dpo/margin_std": 175.0486602783203,
"step": 213
},
{
"KL/chosen_KL_mean": -141.36532592773438,
"KL/mean": -206.33975219726562,
"KL/rejected_KL_mean": -271.31414794921875,
"KL/std": 151.19796752929688,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.0030389183666557074,
"fcm_dpo/delta": 0.006323062814772129,
"fcm_dpo/margin": 129.9488525390625,
"fcm_dpo/q_t": 0.41051051020622253,
"grad_norm": 20.851654052734375,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.2020268589258194,
"logits/rejected": -0.20587500929832458,
"logps/chosen": -182.97021484375,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -348.83160400390625,
"loss": 1.0964,
"margin_dpo/margin_mean": 129.94886779785156,
"margin_dpo/margin_std": 176.67694091796875,
"step": 214
},
{
"KL/chosen_KL_mean": -167.13980102539062,
"KL/mean": -235.6959228515625,
"KL/rejected_KL_mean": -304.2520446777344,
"KL/std": 154.10137939453125,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.0029814334120601416,
"fcm_dpo/delta": -0.011681612581014633,
"fcm_dpo/margin": 137.1122283935547,
"fcm_dpo/q_t": 0.4044470191001892,
"grad_norm": 21.97011947631836,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.2605951428413391,
"logits/rejected": -0.25532659888267517,
"logps/chosen": -220.4190673828125,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -394.2166748046875,
"loss": 1.0713,
"margin_dpo/margin_mean": 137.11224365234375,
"margin_dpo/margin_std": 158.41067504882812,
"step": 215
},
{
"KL/chosen_KL_mean": -169.673583984375,
"KL/mean": -237.0709228515625,
"KL/rejected_KL_mean": -304.46826171875,
"KL/std": 161.4061737060547,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.003034801222383976,
"fcm_dpo/delta": -0.012860134243965149,
"fcm_dpo/margin": 134.794677734375,
"fcm_dpo/q_t": 0.4068460166454315,
"grad_norm": 28.930782318115234,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.2749112844467163,
"logits/rejected": -0.260329008102417,
"logps/chosen": -218.56137084960938,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -381.66717529296875,
"loss": 1.1026,
"margin_dpo/margin_mean": 134.794677734375,
"margin_dpo/margin_std": 196.2810821533203,
"step": 216
},
{
"KL/chosen_KL_mean": -173.36903381347656,
"KL/mean": -254.24615478515625,
"KL/rejected_KL_mean": -335.123291015625,
"KL/std": 167.5054168701172,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.0027936077676713467,
"fcm_dpo/delta": -0.06825002282857895,
"fcm_dpo/margin": 161.75425720214844,
"fcm_dpo/q_t": 0.39398378133773804,
"grad_norm": 19.45440673828125,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.2623336613178253,
"logits/rejected": -0.2586092948913574,
"logps/chosen": -223.21434020996094,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -435.20159912109375,
"loss": 1.0481,
"margin_dpo/margin_mean": 161.75424194335938,
"margin_dpo/margin_std": 188.6632080078125,
"step": 217
},
{
"KL/chosen_KL_mean": -177.95050048828125,
"KL/mean": -239.9977264404297,
"KL/rejected_KL_mean": -302.0449523925781,
"KL/std": 164.52496337890625,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.0028708684258162975,
"fcm_dpo/delta": 0.05333707481622696,
"fcm_dpo/margin": 124.09444427490234,
"fcm_dpo/q_t": 0.42141902446746826,
"grad_norm": 22.139909744262695,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.296860933303833,
"logits/rejected": -0.29232728481292725,
"logps/chosen": -236.52719116210938,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -389.891357421875,
"loss": 1.1373,
"margin_dpo/margin_mean": 124.09444427490234,
"margin_dpo/margin_std": 196.5043487548828,
"step": 218
},
{
"KL/chosen_KL_mean": -184.52737426757812,
"KL/mean": -234.72857666015625,
"KL/rejected_KL_mean": -284.92974853515625,
"KL/std": 165.06085205078125,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.002932458883151412,
"fcm_dpo/delta": 0.0077366819605231285,
"fcm_dpo/margin": 100.40234375,
"fcm_dpo/q_t": 0.4327548146247864,
"grad_norm": 33.82648468017578,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.30966275930404663,
"logits/rejected": -0.3071323335170746,
"logps/chosen": -245.61123657226562,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -370.76019287109375,
"loss": 1.1977,
"margin_dpo/margin_mean": 100.40234375,
"margin_dpo/margin_std": 202.93426513671875,
"step": 219
},
{
"KL/chosen_KL_mean": -190.68328857421875,
"KL/mean": -234.95462036132812,
"KL/rejected_KL_mean": -279.2259216308594,
"KL/std": 134.0245361328125,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.00323110050521791,
"fcm_dpo/delta": 0.1334986686706543,
"fcm_dpo/margin": 88.54261779785156,
"fcm_dpo/q_t": 0.4363420009613037,
"grad_norm": 33.53060531616211,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.3398175835609436,
"logits/rejected": -0.31802213191986084,
"logps/chosen": -260.714599609375,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -366.91143798828125,
"loss": 1.1692,
"margin_dpo/margin_mean": 88.54261779785156,
"margin_dpo/margin_std": 137.38961791992188,
"step": 220
},
{
"KL/chosen_KL_mean": -137.20208740234375,
"KL/mean": -223.91976928710938,
"KL/rejected_KL_mean": -310.637451171875,
"KL/std": 157.56552124023438,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.003003270598128438,
"fcm_dpo/delta": -0.16315576434135437,
"fcm_dpo/margin": 173.43536376953125,
"fcm_dpo/q_t": 0.37127232551574707,
"grad_norm": 135.19117736816406,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.33175361156463623,
"logits/rejected": -0.3400239944458008,
"logps/chosen": -189.3568115234375,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -417.10516357421875,
"loss": 0.987,
"margin_dpo/margin_mean": 173.43536376953125,
"margin_dpo/margin_std": 158.20767211914062,
"step": 221
},
{
"KL/chosen_KL_mean": -147.8143768310547,
"KL/mean": -214.55850219726562,
"KL/rejected_KL_mean": -281.3026123046875,
"KL/std": 146.41705322265625,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.0029258220456540585,
"fcm_dpo/delta": 0.01042582280933857,
"fcm_dpo/margin": 133.4882354736328,
"fcm_dpo/q_t": 0.40958648920059204,
"grad_norm": 62.3775520324707,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.38752639293670654,
"logits/rejected": -0.3927309811115265,
"logps/chosen": -208.7854766845703,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -381.30377197265625,
"loss": 1.0883,
"margin_dpo/margin_mean": 133.4882354736328,
"margin_dpo/margin_std": 164.23587036132812,
"step": 222
},
{
"KL/chosen_KL_mean": -165.07833862304688,
"KL/mean": -222.50390625,
"KL/rejected_KL_mean": -279.92950439453125,
"KL/std": 146.90487670898438,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.0030144588090479374,
"fcm_dpo/delta": 0.06525313854217529,
"fcm_dpo/margin": 114.85115051269531,
"fcm_dpo/q_t": 0.4231482148170471,
"grad_norm": 22.007301330566406,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.2665255069732666,
"logits/rejected": -0.26349782943725586,
"logps/chosen": -217.71890258789062,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -362.7545166015625,
"loss": 1.1441,
"margin_dpo/margin_mean": 114.85115051269531,
"margin_dpo/margin_std": 187.14393615722656,
"step": 223
},
{
"KL/chosen_KL_mean": -144.54835510253906,
"KL/mean": -207.91268920898438,
"KL/rejected_KL_mean": -271.27703857421875,
"KL/std": 162.03009033203125,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.003018029499799013,
"fcm_dpo/delta": 0.017022643238306046,
"fcm_dpo/margin": 126.72867584228516,
"fcm_dpo/q_t": 0.41384878754615784,
"grad_norm": 22.61432456970215,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.29570257663726807,
"logits/rejected": -0.28287479281425476,
"logps/chosen": -193.14376831054688,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -348.39349365234375,
"loss": 1.1178,
"margin_dpo/margin_mean": 126.72867584228516,
"margin_dpo/margin_std": 185.35171508789062,
"step": 224
},
{
"KL/chosen_KL_mean": -163.48281860351562,
"KL/mean": -239.61199951171875,
"KL/rejected_KL_mean": -315.74114990234375,
"KL/std": 156.39309692382812,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.002961981575936079,
"fcm_dpo/delta": -0.066794753074646,
"fcm_dpo/margin": 152.25836181640625,
"fcm_dpo/q_t": 0.3953250050544739,
"grad_norm": 24.43362045288086,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.27601003646850586,
"logits/rejected": -0.26785707473754883,
"logps/chosen": -221.4832763671875,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -415.64410400390625,
"loss": 1.0628,
"margin_dpo/margin_mean": 152.25836181640625,
"margin_dpo/margin_std": 197.67453002929688,
"step": 225
},
{
"KL/chosen_KL_mean": -141.99343872070312,
"KL/mean": -200.15957641601562,
"KL/rejected_KL_mean": -258.32568359375,
"KL/std": 151.08013916015625,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.002992121037095785,
"fcm_dpo/delta": 0.06153475120663643,
"fcm_dpo/margin": 116.332275390625,
"fcm_dpo/q_t": 0.4230756461620331,
"grad_norm": 27.1488037109375,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.33648887276649475,
"logits/rejected": -0.3032319247722626,
"logps/chosen": -200.89224243164062,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -337.0134582519531,
"loss": 1.1404,
"margin_dpo/margin_mean": 116.332275390625,
"margin_dpo/margin_std": 179.59532165527344,
"step": 226
},
{
"KL/chosen_KL_mean": -164.3446807861328,
"KL/mean": -242.88470458984375,
"KL/rejected_KL_mean": -321.42474365234375,
"KL/std": 180.00967407226562,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.002881108084693551,
"fcm_dpo/delta": -0.07316943258047104,
"fcm_dpo/margin": 157.080078125,
"fcm_dpo/q_t": 0.3937509059906006,
"grad_norm": 23.208093643188477,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.31838715076446533,
"logits/rejected": -0.3055545687675476,
"logps/chosen": -223.41685485839844,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -420.8371276855469,
"loss": 1.0542,
"margin_dpo/margin_mean": 157.08009338378906,
"margin_dpo/margin_std": 193.092529296875,
"step": 227
},
{
"KL/chosen_KL_mean": -175.99685668945312,
"KL/mean": -229.3104248046875,
"KL/rejected_KL_mean": -282.62396240234375,
"KL/std": 151.4019775390625,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.003110603429377079,
"fcm_dpo/delta": 0.08030124008655548,
"fcm_dpo/margin": 106.62710571289062,
"fcm_dpo/q_t": 0.42634642124176025,
"grad_norm": 26.213809967041016,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.29573142528533936,
"logits/rejected": -0.28110069036483765,
"logps/chosen": -241.88815307617188,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -373.6727294921875,
"loss": 1.1646,
"margin_dpo/margin_mean": 106.62710571289062,
"margin_dpo/margin_std": 190.84280395507812,
"step": 228
},
{
"KL/chosen_KL_mean": -185.06878662109375,
"KL/mean": -240.96780395507812,
"KL/rejected_KL_mean": -296.8668212890625,
"KL/std": 172.71731567382812,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.003258654149249196,
"fcm_dpo/delta": 0.043283235281705856,
"fcm_dpo/margin": 111.79805755615234,
"fcm_dpo/q_t": 0.41782456636428833,
"grad_norm": 42.24066925048828,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.40128546953201294,
"logits/rejected": -0.3743479251861572,
"logps/chosen": -255.775146484375,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -381.39422607421875,
"loss": 1.1386,
"margin_dpo/margin_mean": 111.79806518554688,
"margin_dpo/margin_std": 184.9072265625,
"step": 229
},
{
"KL/chosen_KL_mean": -130.72586059570312,
"KL/mean": -217.06655883789062,
"KL/rejected_KL_mean": -303.4072265625,
"KL/std": 151.964111328125,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.0029333364218473434,
"fcm_dpo/delta": -0.14437267184257507,
"fcm_dpo/margin": 172.68138122558594,
"fcm_dpo/q_t": 0.37497806549072266,
"grad_norm": 67.53182983398438,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.2871912717819214,
"logits/rejected": -0.2930827736854553,
"logps/chosen": -170.0078582763672,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -389.0291442871094,
"loss": 0.9974,
"margin_dpo/margin_mean": 172.68138122558594,
"margin_dpo/margin_std": 165.72486877441406,
"step": 230
},
{
"KL/chosen_KL_mean": -154.43719482421875,
"KL/mean": -212.76937866210938,
"KL/rejected_KL_mean": -271.1015319824219,
"KL/std": 134.67486572265625,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.002986223204061389,
"fcm_dpo/delta": 0.06204049289226532,
"fcm_dpo/margin": 116.66436767578125,
"fcm_dpo/q_t": 0.42223888635635376,
"grad_norm": 29.774654388427734,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.36699360609054565,
"logits/rejected": -0.34106624126434326,
"logps/chosen": -217.713623046875,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -345.2254638671875,
"loss": 1.1221,
"margin_dpo/margin_mean": 116.66436004638672,
"margin_dpo/margin_std": 160.6752471923828,
"step": 231
},
{
"KL/chosen_KL_mean": -198.68173217773438,
"KL/mean": -247.72219848632812,
"KL/rejected_KL_mean": -296.7626953125,
"KL/std": 164.42752075195312,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.003230078611522913,
"fcm_dpo/delta": 0.09927871823310852,
"fcm_dpo/margin": 98.08092498779297,
"fcm_dpo/q_t": 0.43197596073150635,
"grad_norm": 22.91642189025879,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.3694447875022888,
"logits/rejected": -0.35070955753326416,
"logps/chosen": -269.43048095703125,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -380.73974609375,
"loss": 1.1624,
"margin_dpo/margin_mean": 98.08092498779297,
"margin_dpo/margin_std": 167.9994354248047,
"step": 232
},
{
"KL/chosen_KL_mean": -184.214599609375,
"KL/mean": -265.7254638671875,
"KL/rejected_KL_mean": -347.236328125,
"KL/std": 183.4271240234375,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.0030570742674171925,
"fcm_dpo/delta": -0.13046759366989136,
"fcm_dpo/margin": 163.021728515625,
"fcm_dpo/q_t": 0.38850700855255127,
"grad_norm": 26.11095428466797,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.33819979429244995,
"logits/rejected": -0.3396344780921936,
"logps/chosen": -239.0975341796875,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -454.7164001464844,
"loss": 1.0684,
"margin_dpo/margin_mean": 163.021728515625,
"margin_dpo/margin_std": 239.56597900390625,
"step": 233
},
{
"KL/chosen_KL_mean": -182.37313842773438,
"KL/mean": -264.9031982421875,
"KL/rejected_KL_mean": -347.43328857421875,
"KL/std": 161.71844482421875,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.0026757342275232077,
"fcm_dpo/delta": -0.06500078737735748,
"fcm_dpo/margin": 165.0601348876953,
"fcm_dpo/q_t": 0.3940809965133667,
"grad_norm": 23.45154571533203,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.3571465015411377,
"logits/rejected": -0.3816158175468445,
"logps/chosen": -226.4676513671875,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -447.4399108886719,
"loss": 1.0517,
"margin_dpo/margin_mean": 165.0601348876953,
"margin_dpo/margin_std": 179.3123779296875,
"step": 234
},
{
"KL/chosen_KL_mean": -224.09396362304688,
"KL/mean": -279.3703308105469,
"KL/rejected_KL_mean": -334.64666748046875,
"KL/std": 163.57676696777344,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.0029698603320866823,
"fcm_dpo/delta": 0.08258074522018433,
"fcm_dpo/margin": 110.55270385742188,
"fcm_dpo/q_t": 0.42682477831840515,
"grad_norm": 27.62843894958496,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.34939903020858765,
"logits/rejected": -0.33593958616256714,
"logps/chosen": -286.3318786621094,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -425.041748046875,
"loss": 1.1655,
"margin_dpo/margin_mean": 110.55270385742188,
"margin_dpo/margin_std": 197.34738159179688,
"step": 235
},
{
"KL/chosen_KL_mean": -168.29141235351562,
"KL/mean": -257.59796142578125,
"KL/rejected_KL_mean": -346.9044494628906,
"KL/std": 175.53897094726562,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.002734276931732893,
"fcm_dpo/delta": -0.11726871132850647,
"fcm_dpo/margin": 178.61306762695312,
"fcm_dpo/q_t": 0.3805997967720032,
"grad_norm": 28.633899688720703,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.33454635739326477,
"logits/rejected": -0.337698370218277,
"logps/chosen": -217.6327667236328,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -450.41607666015625,
"loss": 1.0055,
"margin_dpo/margin_mean": 178.61306762695312,
"margin_dpo/margin_std": 166.25515747070312,
"step": 236
},
{
"KL/chosen_KL_mean": -238.2225341796875,
"KL/mean": -303.3404541015625,
"KL/rejected_KL_mean": -368.45843505859375,
"KL/std": 182.74667358398438,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.0027634776197373867,
"fcm_dpo/delta": 0.04837137833237648,
"fcm_dpo/margin": 130.23587036132812,
"fcm_dpo/q_t": 0.4194791316986084,
"grad_norm": 28.578073501586914,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.3175830841064453,
"logits/rejected": -0.31506621837615967,
"logps/chosen": -292.3906555175781,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -463.23876953125,
"loss": 1.1321,
"margin_dpo/margin_mean": 130.23587036132812,
"margin_dpo/margin_std": 204.54656982421875,
"step": 237
},
{
"KL/chosen_KL_mean": -218.0069580078125,
"KL/mean": -284.595458984375,
"KL/rejected_KL_mean": -351.18402099609375,
"KL/std": 181.19268798828125,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.002782113617286086,
"fcm_dpo/delta": 0.034211453050374985,
"fcm_dpo/margin": 133.17703247070312,
"fcm_dpo/q_t": 0.41864013671875,
"grad_norm": 27.10376739501953,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.2829251289367676,
"logits/rejected": -0.29289430379867554,
"logps/chosen": -271.9801025390625,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -440.6019592285156,
"loss": 1.1244,
"margin_dpo/margin_mean": 133.17703247070312,
"margin_dpo/margin_std": 202.1953582763672,
"step": 238
},
{
"KL/chosen_KL_mean": -232.07142639160156,
"KL/mean": -307.0456848144531,
"KL/rejected_KL_mean": -382.0199279785156,
"KL/std": 173.7324981689453,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.002740685362368822,
"fcm_dpo/delta": -0.020037677139043808,
"fcm_dpo/margin": 149.948486328125,
"fcm_dpo/q_t": 0.4032841622829437,
"grad_norm": 26.712921142578125,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.308989942073822,
"logits/rejected": -0.3054299056529999,
"logps/chosen": -290.16925048828125,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -475.61285400390625,
"loss": 1.0861,
"margin_dpo/margin_mean": 149.948486328125,
"margin_dpo/margin_std": 189.98800659179688,
"step": 239
},
{
"KL/chosen_KL_mean": -229.48007202148438,
"KL/mean": -289.2586669921875,
"KL/rejected_KL_mean": -349.0372314453125,
"KL/std": 180.47412109375,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.0029091420583426952,
"fcm_dpo/delta": 0.06307174265384674,
"fcm_dpo/margin": 119.55715942382812,
"fcm_dpo/q_t": 0.4230736494064331,
"grad_norm": 33.61320114135742,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.33556127548217773,
"logits/rejected": -0.3180205523967743,
"logps/chosen": -290.09454345703125,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -423.15576171875,
"loss": 1.164,
"margin_dpo/margin_mean": 119.55715942382812,
"margin_dpo/margin_std": 214.35040283203125,
"step": 240
},
{
"KL/chosen_KL_mean": -187.94955444335938,
"KL/mean": -281.572998046875,
"KL/rejected_KL_mean": -375.19647216796875,
"KL/std": 189.39794921875,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.0027145131025463343,
"fcm_dpo/delta": -0.1450301706790924,
"fcm_dpo/margin": 187.2469024658203,
"fcm_dpo/q_t": 0.37965553998947144,
"grad_norm": 25.904090881347656,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.3710273504257202,
"logits/rejected": -0.35753804445266724,
"logps/chosen": -254.04063415527344,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -463.25738525390625,
"loss": 1.0206,
"margin_dpo/margin_mean": 187.24691772460938,
"margin_dpo/margin_std": 215.96273803710938,
"step": 241
},
{
"KL/chosen_KL_mean": -197.42364501953125,
"KL/mean": -263.04095458984375,
"KL/rejected_KL_mean": -328.6582336425781,
"KL/std": 158.09832763671875,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.002751118503510952,
"fcm_dpo/delta": 0.04367631673812866,
"fcm_dpo/margin": 131.2345428466797,
"fcm_dpo/q_t": 0.418499618768692,
"grad_norm": 23.890216827392578,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.3082933723926544,
"logits/rejected": -0.2922123968601227,
"logps/chosen": -265.2875671386719,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -412.0185546875,
"loss": 1.1207,
"margin_dpo/margin_mean": 131.2345428466797,
"margin_dpo/margin_std": 186.8716278076172,
"step": 242
},
{
"KL/chosen_KL_mean": -195.4427032470703,
"KL/mean": -275.4404296875,
"KL/rejected_KL_mean": -355.43817138671875,
"KL/std": 174.02914428710938,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.0026427144184708595,
"fcm_dpo/delta": -0.029283707961440086,
"fcm_dpo/margin": 159.99542236328125,
"fcm_dpo/q_t": 0.4009614586830139,
"grad_norm": 22.617460250854492,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.3353385329246521,
"logits/rejected": -0.31745070219039917,
"logps/chosen": -258.5269470214844,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -431.7738037109375,
"loss": 1.0667,
"margin_dpo/margin_mean": 159.99542236328125,
"margin_dpo/margin_std": 191.55514526367188,
"step": 243
},
{
"KL/chosen_KL_mean": -166.66122436523438,
"KL/mean": -250.8197021484375,
"KL/rejected_KL_mean": -334.9781799316406,
"KL/std": 169.25035095214844,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.002540828660130501,
"fcm_dpo/delta": -0.0361553393304348,
"fcm_dpo/margin": 168.3169403076172,
"fcm_dpo/q_t": 0.3987560272216797,
"grad_norm": 25.22274398803711,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.409039705991745,
"logits/rejected": -0.4093121290206909,
"logps/chosen": -227.80191040039062,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -429.8701171875,
"loss": 1.0496,
"margin_dpo/margin_mean": 168.3169403076172,
"margin_dpo/margin_std": 171.68531799316406,
"step": 244
},
{
"KL/chosen_KL_mean": -197.50634765625,
"KL/mean": -263.1775817871094,
"KL/rejected_KL_mean": -328.84881591796875,
"KL/std": 168.23464965820312,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.0024484535679221153,
"fcm_dpo/delta": -0.03805915266275406,
"fcm_dpo/margin": 131.34243774414062,
"fcm_dpo/q_t": 0.4226040244102478,
"grad_norm": 26.150102615356445,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.36390721797943115,
"logits/rejected": -0.351279079914093,
"logps/chosen": -264.76861572265625,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -416.4888916015625,
"loss": 1.1443,
"margin_dpo/margin_mean": 131.3424530029297,
"margin_dpo/margin_std": 187.38204956054688,
"step": 245
},
{
"KL/chosen_KL_mean": -202.00189208984375,
"KL/mean": -270.38726806640625,
"KL/rejected_KL_mean": -338.77264404296875,
"KL/std": 191.9010009765625,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.00253550847992301,
"fcm_dpo/delta": 0.0644979327917099,
"fcm_dpo/margin": 136.77073669433594,
"fcm_dpo/q_t": 0.42329055070877075,
"grad_norm": 24.134305953979492,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.394417405128479,
"logits/rejected": -0.3600098490715027,
"logps/chosen": -268.6988525390625,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -423.1189880371094,
"loss": 1.1347,
"margin_dpo/margin_mean": 136.770751953125,
"margin_dpo/margin_std": 207.49807739257812,
"step": 246
},
{
"KL/chosen_KL_mean": -252.14999389648438,
"KL/mean": -353.76324462890625,
"KL/rejected_KL_mean": -455.37652587890625,
"KL/std": 205.67657470703125,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.002377366181463003,
"fcm_dpo/delta": -0.11207044124603271,
"fcm_dpo/margin": 203.22650146484375,
"fcm_dpo/q_t": 0.38493090867996216,
"grad_norm": 25.502174377441406,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.320733904838562,
"logits/rejected": -0.31779003143310547,
"logps/chosen": -308.7553405761719,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -561.6697998046875,
"loss": 1.0303,
"margin_dpo/margin_mean": 203.22650146484375,
"margin_dpo/margin_std": 228.47640991210938,
"step": 247
},
{
"KL/chosen_KL_mean": -224.57611083984375,
"KL/mean": -309.53057861328125,
"KL/rejected_KL_mean": -394.4850158691406,
"KL/std": 169.4248046875,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.0023004484828561544,
"fcm_dpo/delta": 0.009408872574567795,
"fcm_dpo/margin": 169.908935546875,
"fcm_dpo/q_t": 0.4071027934551239,
"grad_norm": 27.232206344604492,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.2981037199497223,
"logits/rejected": -0.3177593946456909,
"logps/chosen": -268.61932373046875,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -486.3419189453125,
"loss": 1.0721,
"margin_dpo/margin_mean": 169.908935546875,
"margin_dpo/margin_std": 169.20985412597656,
"step": 248
},
{
"KL/chosen_KL_mean": -287.9716796875,
"KL/mean": -332.658203125,
"KL/rejected_KL_mean": -377.3447570800781,
"KL/std": 193.0209197998047,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.0025631722528487444,
"fcm_dpo/delta": 0.08395271003246307,
"fcm_dpo/margin": 89.3730697631836,
"fcm_dpo/q_t": 0.45033693313598633,
"grad_norm": 45.21783447265625,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.35276949405670166,
"logits/rejected": -0.33473479747772217,
"logps/chosen": -350.4140625,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -457.81280517578125,
"loss": 1.2555,
"margin_dpo/margin_mean": 89.37306213378906,
"margin_dpo/margin_std": 233.79147338867188,
"step": 249
},
{
"KL/chosen_KL_mean": -257.8434143066406,
"KL/mean": -340.539794921875,
"KL/rejected_KL_mean": -423.2361755371094,
"KL/std": 185.04319763183594,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.0025648935697972775,
"fcm_dpo/delta": -0.032477494329214096,
"fcm_dpo/margin": 165.39276123046875,
"fcm_dpo/q_t": 0.40042954683303833,
"grad_norm": 49.331077575683594,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.35356414318084717,
"logits/rejected": -0.31186503171920776,
"logps/chosen": -323.4801025390625,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -497.1080322265625,
"loss": 1.0711,
"margin_dpo/margin_mean": 165.39276123046875,
"margin_dpo/margin_std": 199.77915954589844,
"step": 250
},
{
"KL/chosen_KL_mean": -263.6090393066406,
"KL/mean": -316.90277099609375,
"KL/rejected_KL_mean": -370.1965026855469,
"KL/std": 189.91702270507812,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.002717065392062068,
"fcm_dpo/delta": 0.12958815693855286,
"fcm_dpo/margin": 106.58744812011719,
"fcm_dpo/q_t": 0.43745461106300354,
"grad_norm": 40.08333969116211,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.32426249980926514,
"logits/rejected": -0.2984544336795807,
"logps/chosen": -320.791748046875,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -447.85992431640625,
"loss": 1.1986,
"margin_dpo/margin_mean": 106.58744812011719,
"margin_dpo/margin_std": 211.80274963378906,
"step": 251
},
{
"KL/chosen_KL_mean": -247.48324584960938,
"KL/mean": -321.72723388671875,
"KL/rejected_KL_mean": -395.97125244140625,
"KL/std": 152.3659210205078,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.0027652715798467398,
"fcm_dpo/delta": -0.014062562957406044,
"fcm_dpo/margin": 148.48800659179688,
"fcm_dpo/q_t": 0.4011853337287903,
"grad_norm": 32.023529052734375,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.4131581485271454,
"logits/rejected": -0.37648361921310425,
"logps/chosen": -319.16888427734375,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -480.729248046875,
"loss": 1.06,
"margin_dpo/margin_mean": 148.48800659179688,
"margin_dpo/margin_std": 151.195068359375,
"step": 252
},
{
"KL/chosen_KL_mean": -203.74603271484375,
"KL/mean": -282.53802490234375,
"KL/rejected_KL_mean": -361.3300476074219,
"KL/std": 168.71078491210938,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.002772426698356867,
"fcm_dpo/delta": -0.049864716827869415,
"fcm_dpo/margin": 157.58399963378906,
"fcm_dpo/q_t": 0.3958263397216797,
"grad_norm": 24.330591201782227,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.42228442430496216,
"logits/rejected": -0.4147264063358307,
"logps/chosen": -272.8799743652344,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -460.0325927734375,
"loss": 1.0635,
"margin_dpo/margin_mean": 157.58401489257812,
"margin_dpo/margin_std": 181.4701690673828,
"step": 253
},
{
"KL/chosen_KL_mean": -186.63661193847656,
"KL/mean": -252.297119140625,
"KL/rejected_KL_mean": -317.9576416015625,
"KL/std": 167.77099609375,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.0027375142090022564,
"fcm_dpo/delta": 0.049500077962875366,
"fcm_dpo/margin": 131.32101440429688,
"fcm_dpo/q_t": 0.4207710027694702,
"grad_norm": 25.899538040161133,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.38440778851509094,
"logits/rejected": -0.37307465076446533,
"logps/chosen": -240.79161071777344,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -408.2652587890625,
"loss": 1.1333,
"margin_dpo/margin_mean": 131.32101440429688,
"margin_dpo/margin_std": 208.37350463867188,
"step": 254
},
{
"KL/chosen_KL_mean": -187.1531982421875,
"KL/mean": -254.55125427246094,
"KL/rejected_KL_mean": -321.9493103027344,
"KL/std": 147.3501739501953,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.002789452439174056,
"fcm_dpo/delta": 0.02846667543053627,
"fcm_dpo/margin": 134.7960968017578,
"fcm_dpo/q_t": 0.41337624192237854,
"grad_norm": 26.342166900634766,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.3332477807998657,
"logits/rejected": -0.3225042521953583,
"logps/chosen": -244.29486083984375,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -412.15789794921875,
"loss": 1.1121,
"margin_dpo/margin_mean": 134.7960968017578,
"margin_dpo/margin_std": 186.9753875732422,
"step": 255
},
{
"KL/chosen_KL_mean": -162.30020141601562,
"KL/mean": -233.31613159179688,
"KL/rejected_KL_mean": -304.33203125,
"KL/std": 154.13583374023438,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.0028459273744374514,
"fcm_dpo/delta": -0.0053253816440701485,
"fcm_dpo/margin": 142.03182983398438,
"fcm_dpo/q_t": 0.40578973293304443,
"grad_norm": 36.771114349365234,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.30460691452026367,
"logits/rejected": -0.3128143548965454,
"logps/chosen": -217.4636993408203,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -396.89495849609375,
"loss": 1.0809,
"margin_dpo/margin_mean": 142.03182983398438,
"margin_dpo/margin_std": 175.64019775390625,
"step": 256
},
{
"KL/chosen_KL_mean": -163.07763671875,
"KL/mean": -238.74008178710938,
"KL/rejected_KL_mean": -314.40252685546875,
"KL/std": 163.1609649658203,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.002718387870118022,
"fcm_dpo/delta": -0.019889619201421738,
"fcm_dpo/margin": 151.32489013671875,
"fcm_dpo/q_t": 0.4035347104072571,
"grad_norm": 23.515918731689453,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.35459989309310913,
"logits/rejected": -0.3430092930793762,
"logps/chosen": -212.5013427734375,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -393.9404296875,
"loss": 1.0789,
"margin_dpo/margin_mean": 151.32489013671875,
"margin_dpo/margin_std": 181.58273315429688,
"step": 257
},
{
"KL/chosen_KL_mean": -215.13340759277344,
"KL/mean": -289.7934875488281,
"KL/rejected_KL_mean": -364.45355224609375,
"KL/std": 176.51971435546875,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.002779233269393444,
"fcm_dpo/delta": -0.019095513969659805,
"fcm_dpo/margin": 149.32015991210938,
"fcm_dpo/q_t": 0.40277743339538574,
"grad_norm": 23.90601348876953,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.3055630028247833,
"logits/rejected": -0.3058106601238251,
"logps/chosen": -274.51751708984375,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -460.44366455078125,
"loss": 1.0945,
"margin_dpo/margin_mean": 149.32015991210938,
"margin_dpo/margin_std": 211.73117065429688,
"step": 258
},
{
"KL/chosen_KL_mean": -202.0823974609375,
"KL/mean": -263.96234130859375,
"KL/rejected_KL_mean": -325.84234619140625,
"KL/std": 168.348876953125,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.002821682021021843,
"fcm_dpo/delta": 0.06138894706964493,
"fcm_dpo/margin": 123.75993347167969,
"fcm_dpo/q_t": 0.4222329556941986,
"grad_norm": 20.800395965576172,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.35386550426483154,
"logits/rejected": -0.35250523686408997,
"logps/chosen": -254.91073608398438,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -415.03399658203125,
"loss": 1.1393,
"margin_dpo/margin_mean": 123.75993347167969,
"margin_dpo/margin_std": 192.9962921142578,
"step": 259
},
{
"KL/chosen_KL_mean": -198.77923583984375,
"KL/mean": -283.6119079589844,
"KL/rejected_KL_mean": -368.444580078125,
"KL/std": 172.5734405517578,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.0027726658154278994,
"fcm_dpo/delta": -0.09174390137195587,
"fcm_dpo/margin": 169.66534423828125,
"fcm_dpo/q_t": 0.3891771137714386,
"grad_norm": 29.0617733001709,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.33367860317230225,
"logits/rejected": -0.3492773771286011,
"logps/chosen": -246.1968994140625,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -463.53436279296875,
"loss": 1.0302,
"margin_dpo/margin_mean": 169.66534423828125,
"margin_dpo/margin_std": 185.7799530029297,
"step": 260
},
{
"KL/chosen_KL_mean": -203.46487426757812,
"KL/mean": -282.5809326171875,
"KL/rejected_KL_mean": -361.697021484375,
"KL/std": 188.90347290039062,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.0025986204855144024,
"fcm_dpo/delta": -0.014911869540810585,
"fcm_dpo/margin": 158.2321319580078,
"fcm_dpo/q_t": 0.4053801894187927,
"grad_norm": 20.706195831298828,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.30493831634521484,
"logits/rejected": -0.3092959225177765,
"logps/chosen": -256.4962463378906,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -450.21197509765625,
"loss": 1.0849,
"margin_dpo/margin_mean": 158.23214721679688,
"margin_dpo/margin_std": 208.61734008789062,
"step": 261
},
{
"KL/chosen_KL_mean": -237.848876953125,
"KL/mean": -303.23284912109375,
"KL/rejected_KL_mean": -368.6168212890625,
"KL/std": 171.96548461914062,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.0027585539501160383,
"fcm_dpo/delta": 0.045463643968105316,
"fcm_dpo/margin": 130.7679443359375,
"fcm_dpo/q_t": 0.4169616997241974,
"grad_norm": 26.5394344329834,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.2885762155056,
"logits/rejected": -0.26217561960220337,
"logps/chosen": -297.4690246582031,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -455.03533935546875,
"loss": 1.1097,
"margin_dpo/margin_mean": 130.7679443359375,
"margin_dpo/margin_std": 169.8227081298828,
"step": 262
},
{
"KL/chosen_KL_mean": -214.3430938720703,
"KL/mean": -308.7781982421875,
"KL/rejected_KL_mean": -403.21331787109375,
"KL/std": 202.47735595703125,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.002491295337677002,
"fcm_dpo/delta": -0.09768804162740707,
"fcm_dpo/margin": 188.87020874023438,
"fcm_dpo/q_t": 0.38718199729919434,
"grad_norm": 18.903581619262695,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.31606876850128174,
"logits/rejected": -0.2949512004852295,
"logps/chosen": -273.7640380859375,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -500.0705261230469,
"loss": 1.0366,
"margin_dpo/margin_mean": 188.87022399902344,
"margin_dpo/margin_std": 214.49734497070312,
"step": 263
},
{
"KL/chosen_KL_mean": -232.32882690429688,
"KL/mean": -313.197998046875,
"KL/rejected_KL_mean": -394.0671691894531,
"KL/std": 192.44293212890625,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.0024121187161654234,
"fcm_dpo/delta": 0.006785430014133453,
"fcm_dpo/margin": 161.7383270263672,
"fcm_dpo/q_t": 0.41011467576026917,
"grad_norm": 23.93086814880371,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.35104095935821533,
"logits/rejected": -0.3437178134918213,
"logps/chosen": -295.0509033203125,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -487.92340087890625,
"loss": 1.1065,
"margin_dpo/margin_mean": 161.7383270263672,
"margin_dpo/margin_std": 220.1507110595703,
"step": 264
},
{
"KL/chosen_KL_mean": -254.60040283203125,
"KL/mean": -330.61474609375,
"KL/rejected_KL_mean": -406.629150390625,
"KL/std": 214.48471069335938,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.0025378770660609007,
"fcm_dpo/delta": 0.017527392134070396,
"fcm_dpo/margin": 152.0287322998047,
"fcm_dpo/q_t": 0.4144975543022156,
"grad_norm": 22.09156036376953,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.2829780578613281,
"logits/rejected": -0.2669457197189331,
"logps/chosen": -316.5718688964844,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -494.6497497558594,
"loss": 1.1316,
"margin_dpo/margin_mean": 152.0287322998047,
"margin_dpo/margin_std": 254.72515869140625,
"step": 265
},
{
"KL/chosen_KL_mean": -259.6724853515625,
"KL/mean": -320.30584716796875,
"KL/rejected_KL_mean": -380.9391784667969,
"KL/std": 179.23141479492188,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.0027112066745758057,
"fcm_dpo/delta": 0.08521057665348053,
"fcm_dpo/margin": 121.26667022705078,
"fcm_dpo/q_t": 0.4275960922241211,
"grad_norm": 47.17190933227539,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.31882771849632263,
"logits/rejected": -0.2803471088409424,
"logps/chosen": -326.77215576171875,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -448.910400390625,
"loss": 1.1572,
"margin_dpo/margin_mean": 121.26667785644531,
"margin_dpo/margin_std": 205.98829650878906,
"step": 266
},
{
"KL/chosen_KL_mean": -229.09750366210938,
"KL/mean": -301.92169189453125,
"KL/rejected_KL_mean": -374.7459411621094,
"KL/std": 188.1318359375,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.002730957930907607,
"fcm_dpo/delta": 0.0015622973442077637,
"fcm_dpo/margin": 145.6484375,
"fcm_dpo/q_t": 0.4101436734199524,
"grad_norm": 41.06880187988281,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.3704749643802643,
"logits/rejected": -0.3451113700866699,
"logps/chosen": -298.0682373046875,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -464.91436767578125,
"loss": 1.1069,
"margin_dpo/margin_mean": 145.64842224121094,
"margin_dpo/margin_std": 215.05789184570312,
"step": 267
},
{
"KL/chosen_KL_mean": -242.15602111816406,
"KL/mean": -318.61297607421875,
"KL/rejected_KL_mean": -395.07000732421875,
"KL/std": 188.01754760742188,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.0026982720009982586,
"fcm_dpo/delta": -0.017409201711416245,
"fcm_dpo/margin": 152.9139404296875,
"fcm_dpo/q_t": 0.40618669986724854,
"grad_norm": 27.902847290039062,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.31142598390579224,
"logits/rejected": -0.31501471996307373,
"logps/chosen": -298.05633544921875,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -496.7176208496094,
"loss": 1.1023,
"margin_dpo/margin_mean": 152.9139404296875,
"margin_dpo/margin_std": 228.34695434570312,
"step": 268
},
{
"KL/chosen_KL_mean": -274.3188171386719,
"KL/mean": -364.34295654296875,
"KL/rejected_KL_mean": -454.36712646484375,
"KL/std": 197.2436981201172,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.0025738244876265526,
"fcm_dpo/delta": -0.0825297087430954,
"fcm_dpo/margin": 180.04827880859375,
"fcm_dpo/q_t": 0.39260607957839966,
"grad_norm": 29.512990951538086,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.3739697337150574,
"logits/rejected": -0.3583334684371948,
"logps/chosen": -344.3583984375,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -561.7164916992188,
"loss": 1.0732,
"margin_dpo/margin_mean": 180.04827880859375,
"margin_dpo/margin_std": 252.80072021484375,
"step": 269
},
{
"KL/chosen_KL_mean": -230.86578369140625,
"KL/mean": -299.2672119140625,
"KL/rejected_KL_mean": -367.66864013671875,
"KL/std": 178.17739868164062,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.0026433742605149746,
"fcm_dpo/delta": 0.0446162149310112,
"fcm_dpo/margin": 136.8028564453125,
"fcm_dpo/q_t": 0.41896626353263855,
"grad_norm": 47.853179931640625,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.4406346380710602,
"logits/rejected": -0.4400424361228943,
"logps/chosen": -300.3992614746094,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -477.5972900390625,
"loss": 1.1416,
"margin_dpo/margin_mean": 136.8028564453125,
"margin_dpo/margin_std": 227.7311248779297,
"step": 270
},
{
"KL/chosen_KL_mean": -227.30824279785156,
"KL/mean": -316.3374938964844,
"KL/rejected_KL_mean": -405.3667297363281,
"KL/std": 184.22903442382812,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.00247220229357481,
"fcm_dpo/delta": -0.054896239191293716,
"fcm_dpo/margin": 178.0584716796875,
"fcm_dpo/q_t": 0.3969544470310211,
"grad_norm": 26.639240264892578,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.3195192515850067,
"logits/rejected": -0.3056868612766266,
"logps/chosen": -284.07281494140625,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -497.88055419921875,
"loss": 1.0576,
"margin_dpo/margin_mean": 178.0584716796875,
"margin_dpo/margin_std": 211.52894592285156,
"step": 271
},
{
"KL/chosen_KL_mean": -217.37164306640625,
"KL/mean": -324.1234130859375,
"KL/rejected_KL_mean": -430.87518310546875,
"KL/std": 205.4072723388672,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.002309414092451334,
"fcm_dpo/delta": -0.12269287556409836,
"fcm_dpo/margin": 213.5035400390625,
"fcm_dpo/q_t": 0.3797604441642761,
"grad_norm": 40.22795486450195,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.2815614342689514,
"logits/rejected": -0.29494649171829224,
"logps/chosen": -266.8688049316406,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -536.41796875,
"loss": 1.0089,
"margin_dpo/margin_mean": 213.5035400390625,
"margin_dpo/margin_std": 207.53280639648438,
"step": 272
},
{
"KL/chosen_KL_mean": -249.883056640625,
"KL/mean": -348.0146789550781,
"KL/rejected_KL_mean": -446.14630126953125,
"KL/std": 197.6843719482422,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.002137008123099804,
"fcm_dpo/delta": -0.02540270984172821,
"fcm_dpo/margin": 196.2632293701172,
"fcm_dpo/q_t": 0.4012957215309143,
"grad_norm": 24.651897430419922,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.30211585760116577,
"logits/rejected": -0.28442516922950745,
"logps/chosen": -312.85845947265625,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -538.6448974609375,
"loss": 1.0672,
"margin_dpo/margin_mean": 196.2632293701172,
"margin_dpo/margin_std": 230.588134765625,
"step": 273
},
{
"KL/chosen_KL_mean": -293.39508056640625,
"KL/mean": -369.06951904296875,
"KL/rejected_KL_mean": -444.743896484375,
"KL/std": 191.70803833007812,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.002289203926920891,
"fcm_dpo/delta": 0.061743948608636856,
"fcm_dpo/margin": 151.3488311767578,
"fcm_dpo/q_t": 0.42134177684783936,
"grad_norm": 22.297304153442383,
"learning_rate": 3.75e-07,
"logits/chosen": -0.2628266215324402,
"logits/rejected": -0.24505800008773804,
"logps/chosen": -349.06280517578125,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -522.0770263671875,
"loss": 1.1298,
"margin_dpo/margin_mean": 151.3488311767578,
"margin_dpo/margin_std": 218.46170043945312,
"step": 274
},
{
"KL/chosen_KL_mean": -226.7274169921875,
"KL/mean": -308.6673889160156,
"KL/rejected_KL_mean": -390.60736083984375,
"KL/std": 187.83544921875,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.0023457312490791082,
"fcm_dpo/delta": 0.01810782589018345,
"fcm_dpo/margin": 163.87994384765625,
"fcm_dpo/q_t": 0.4110303521156311,
"grad_norm": 29.85940933227539,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.31675028800964355,
"logits/rejected": -0.32569241523742676,
"logps/chosen": -275.3221130371094,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -483.91107177734375,
"loss": 1.096,
"margin_dpo/margin_mean": 163.87994384765625,
"margin_dpo/margin_std": 209.12115478515625,
"step": 275
},
{
"KL/chosen_KL_mean": -254.89431762695312,
"KL/mean": -338.76141357421875,
"KL/rejected_KL_mean": -422.6285400390625,
"KL/std": 204.18536376953125,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.0023221762385219336,
"fcm_dpo/delta": 0.012708023190498352,
"fcm_dpo/margin": 167.73422241210938,
"fcm_dpo/q_t": 0.4127948582172394,
"grad_norm": 25.3048038482666,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.2963874936103821,
"logits/rejected": -0.2773512601852417,
"logps/chosen": -311.47174072265625,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -492.99420166015625,
"loss": 1.1126,
"margin_dpo/margin_mean": 167.73422241210938,
"margin_dpo/margin_std": 250.09458923339844,
"step": 276
},
{
"KL/chosen_KL_mean": -274.00732421875,
"KL/mean": -361.88043212890625,
"KL/rejected_KL_mean": -449.7535400390625,
"KL/std": 200.5372772216797,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.0023581404238939285,
"fcm_dpo/delta": -0.018989086151123047,
"fcm_dpo/margin": 175.74618530273438,
"fcm_dpo/q_t": 0.4043876826763153,
"grad_norm": 22.430313110351562,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.31193846464157104,
"logits/rejected": -0.2933298349380493,
"logps/chosen": -330.27886962890625,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -542.634765625,
"loss": 1.091,
"margin_dpo/margin_mean": 175.74620056152344,
"margin_dpo/margin_std": 240.4161376953125,
"step": 277
},
{
"KL/chosen_KL_mean": -238.85679626464844,
"KL/mean": -339.3966064453125,
"KL/rejected_KL_mean": -439.93634033203125,
"KL/std": 207.98587036132812,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.00218441104516387,
"fcm_dpo/delta": -0.052646711468696594,
"fcm_dpo/margin": 201.0795440673828,
"fcm_dpo/q_t": 0.3967970907688141,
"grad_norm": 22.02371597290039,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.30684125423431396,
"logits/rejected": -0.31390851736068726,
"logps/chosen": -291.7987365722656,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -531.18994140625,
"loss": 1.0559,
"margin_dpo/margin_mean": 201.0795440673828,
"margin_dpo/margin_std": 235.0767822265625,
"step": 278
},
{
"KL/chosen_KL_mean": -266.6314697265625,
"KL/mean": -369.5718994140625,
"KL/rejected_KL_mean": -472.5123291015625,
"KL/std": 229.59307861328125,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.0020728304516524076,
"fcm_dpo/delta": -0.038570303469896317,
"fcm_dpo/margin": 205.880859375,
"fcm_dpo/q_t": 0.4020066261291504,
"grad_norm": 23.30048179626465,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.25095367431640625,
"logits/rejected": -0.2509291172027588,
"logps/chosen": -315.2727966308594,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -560.36376953125,
"loss": 1.0833,
"margin_dpo/margin_mean": 205.880859375,
"margin_dpo/margin_std": 281.98443603515625,
"step": 279
},
{
"KL/chosen_KL_mean": -237.64895629882812,
"KL/mean": -337.5757751464844,
"KL/rejected_KL_mean": -437.50262451171875,
"KL/std": 190.99032592773438,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.0020760963670909405,
"fcm_dpo/delta": -0.018862612545490265,
"fcm_dpo/margin": 199.85365295410156,
"fcm_dpo/q_t": 0.40116894245147705,
"grad_norm": 22.5253963470459,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.3350566625595093,
"logits/rejected": -0.33742278814315796,
"logps/chosen": -296.4460754394531,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -536.1214599609375,
"loss": 1.0562,
"margin_dpo/margin_mean": 199.8536376953125,
"margin_dpo/margin_std": 203.86276245117188,
"step": 280
},
{
"KL/chosen_KL_mean": -224.87203979492188,
"KL/mean": -309.09222412109375,
"KL/rejected_KL_mean": -393.31243896484375,
"KL/std": 193.19955444335938,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.0021694411989301443,
"fcm_dpo/delta": 0.04002968966960907,
"fcm_dpo/margin": 168.44039916992188,
"fcm_dpo/q_t": 0.4139762818813324,
"grad_norm": 26.64156150817871,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.3864981532096863,
"logits/rejected": -0.3639860153198242,
"logps/chosen": -280.3605651855469,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -474.19500732421875,
"loss": 1.1042,
"margin_dpo/margin_mean": 168.4403839111328,
"margin_dpo/margin_std": 207.57632446289062,
"step": 281
},
{
"KL/chosen_KL_mean": -260.92059326171875,
"KL/mean": -328.2933349609375,
"KL/rejected_KL_mean": -395.66607666015625,
"KL/std": 199.9541015625,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.002180553739890456,
"fcm_dpo/delta": 0.025225864723324776,
"fcm_dpo/margin": 134.74557495117188,
"fcm_dpo/q_t": 0.43476957082748413,
"grad_norm": 23.243419647216797,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.39929962158203125,
"logits/rejected": -0.3811987638473511,
"logps/chosen": -333.99072265625,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -491.01708984375,
"loss": 1.1715,
"margin_dpo/margin_mean": 134.7455596923828,
"margin_dpo/margin_std": 234.26336669921875,
"step": 282
},
{
"KL/chosen_KL_mean": -273.17254638671875,
"KL/mean": -379.52557373046875,
"KL/rejected_KL_mean": -485.87860107421875,
"KL/std": 243.8074951171875,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.0021401268895715475,
"fcm_dpo/delta": -0.07182439416646957,
"fcm_dpo/margin": 212.7060546875,
"fcm_dpo/q_t": 0.3907451927661896,
"grad_norm": 35.38416290283203,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.3343729078769684,
"logits/rejected": -0.3363478183746338,
"logps/chosen": -335.07098388671875,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -582.8651123046875,
"loss": 1.0446,
"margin_dpo/margin_mean": 212.7060546875,
"margin_dpo/margin_std": 232.63815307617188,
"step": 283
},
{
"KL/chosen_KL_mean": -261.57415771484375,
"KL/mean": -360.19207763671875,
"KL/rejected_KL_mean": -458.80999755859375,
"KL/std": 222.4651336669922,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.0020390492863953114,
"fcm_dpo/delta": -0.0028852270916104317,
"fcm_dpo/margin": 197.23583984375,
"fcm_dpo/q_t": 0.4071544408798218,
"grad_norm": 24.07853126525879,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.37924909591674805,
"logits/rejected": -0.3620849549770355,
"logps/chosen": -320.00970458984375,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -552.279296875,
"loss": 1.0777,
"margin_dpo/margin_mean": 197.23580932617188,
"margin_dpo/margin_std": 236.99972534179688,
"step": 284
},
{
"KL/chosen_KL_mean": -298.85516357421875,
"KL/mean": -381.50390625,
"KL/rejected_KL_mean": -464.1526184082031,
"KL/std": 190.44850158691406,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.0021772286854684353,
"fcm_dpo/delta": 0.044019557535648346,
"fcm_dpo/margin": 165.29745483398438,
"fcm_dpo/q_t": 0.4151372015476227,
"grad_norm": 23.67416763305664,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.3634873628616333,
"logits/rejected": -0.348185658454895,
"logps/chosen": -365.08734130859375,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -563.2794189453125,
"loss": 1.1137,
"margin_dpo/margin_mean": 165.29745483398438,
"margin_dpo/margin_std": 213.57778930664062,
"step": 285
},
{
"KL/chosen_KL_mean": -292.51617431640625,
"KL/mean": -385.5911560058594,
"KL/rejected_KL_mean": -478.6661682128906,
"KL/std": 223.45005798339844,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.002157814335078001,
"fcm_dpo/delta": -0.0024737920612096786,
"fcm_dpo/margin": 186.14996337890625,
"fcm_dpo/q_t": 0.40657156705856323,
"grad_norm": 23.773475646972656,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.3397786319255829,
"logits/rejected": -0.32064247131347656,
"logps/chosen": -365.4671936035156,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -567.254638671875,
"loss": 1.0899,
"margin_dpo/margin_mean": 186.14996337890625,
"margin_dpo/margin_std": 243.01104736328125,
"step": 286
},
{
"KL/chosen_KL_mean": -262.6454162597656,
"KL/mean": -341.39129638671875,
"KL/rejected_KL_mean": -420.13714599609375,
"KL/std": 194.3848114013672,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.0022398338187485933,
"fcm_dpo/delta": 0.05684395506978035,
"fcm_dpo/margin": 157.49172973632812,
"fcm_dpo/q_t": 0.4202299118041992,
"grad_norm": 20.517465591430664,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.30254948139190674,
"logits/rejected": -0.27677813172340393,
"logps/chosen": -324.18658447265625,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -497.833251953125,
"loss": 1.1189,
"margin_dpo/margin_mean": 157.49172973632812,
"margin_dpo/margin_std": 216.00836181640625,
"step": 287
},
{
"KL/chosen_KL_mean": -242.85244750976562,
"KL/mean": -340.43890380859375,
"KL/rejected_KL_mean": -438.025390625,
"KL/std": 184.95721435546875,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.002138757612556219,
"fcm_dpo/delta": -0.028658173978328705,
"fcm_dpo/margin": 195.1729278564453,
"fcm_dpo/q_t": 0.3991842269897461,
"grad_norm": 20.475711822509766,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.35998380184173584,
"logits/rejected": -0.34525611996650696,
"logps/chosen": -299.513671875,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -525.361083984375,
"loss": 1.0586,
"margin_dpo/margin_mean": 195.17294311523438,
"margin_dpo/margin_std": 196.0745391845703,
"step": 288
},
{
"KL/chosen_KL_mean": -236.07003784179688,
"KL/mean": -339.13861083984375,
"KL/rejected_KL_mean": -442.2071533203125,
"KL/std": 204.4755859375,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.002140854485332966,
"fcm_dpo/delta": -0.05301383137702942,
"fcm_dpo/margin": 206.13711547851562,
"fcm_dpo/q_t": 0.3952919840812683,
"grad_norm": 35.90175247192383,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.28917020559310913,
"logits/rejected": -0.29095658659935,
"logps/chosen": -281.3004455566406,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -529.849853515625,
"loss": 1.0471,
"margin_dpo/margin_mean": 206.13711547851562,
"margin_dpo/margin_std": 221.66661071777344,
"step": 289
},
{
"KL/chosen_KL_mean": -259.9752502441406,
"KL/mean": -367.92950439453125,
"KL/rejected_KL_mean": -475.8837890625,
"KL/std": 215.7952880859375,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.0020013581961393356,
"fcm_dpo/delta": -0.04185499623417854,
"fcm_dpo/margin": 215.90850830078125,
"fcm_dpo/q_t": 0.39859092235565186,
"grad_norm": 23.12173843383789,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.29490506649017334,
"logits/rejected": -0.3174816370010376,
"logps/chosen": -315.4467468261719,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -592.5923461914062,
"loss": 1.0603,
"margin_dpo/margin_mean": 215.9084930419922,
"margin_dpo/margin_std": 255.39891052246094,
"step": 290
},
{
"KL/chosen_KL_mean": -236.2718505859375,
"KL/mean": -337.82806396484375,
"KL/rejected_KL_mean": -439.3843078613281,
"KL/std": 192.71876525878906,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.0020202212035655975,
"fcm_dpo/delta": -0.014722079038619995,
"fcm_dpo/margin": 203.11245727539062,
"fcm_dpo/q_t": 0.4027057886123657,
"grad_norm": 20.603652954101562,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.3010188043117523,
"logits/rejected": -0.29001563787460327,
"logps/chosen": -299.5322265625,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -528.681396484375,
"loss": 1.0707,
"margin_dpo/margin_mean": 203.11245727539062,
"margin_dpo/margin_std": 228.27931213378906,
"step": 291
},
{
"KL/chosen_KL_mean": -271.1993408203125,
"KL/mean": -373.78936767578125,
"KL/rejected_KL_mean": -476.37933349609375,
"KL/std": 224.70799255371094,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.0019872216507792473,
"fcm_dpo/delta": -0.010810526087880135,
"fcm_dpo/margin": 205.17999267578125,
"fcm_dpo/q_t": 0.40419653058052063,
"grad_norm": 20.72088050842285,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.3157344460487366,
"logits/rejected": -0.3041796088218689,
"logps/chosen": -325.1178894042969,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -566.3406982421875,
"loss": 1.0851,
"margin_dpo/margin_mean": 205.17999267578125,
"margin_dpo/margin_std": 263.08953857421875,
"step": 292
},
{
"KL/chosen_KL_mean": -287.68072509765625,
"KL/mean": -370.7479248046875,
"KL/rejected_KL_mean": -453.81512451171875,
"KL/std": 234.54473876953125,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.0020656294655054808,
"fcm_dpo/delta": 0.06754275411367416,
"fcm_dpo/margin": 166.13436889648438,
"fcm_dpo/q_t": 0.42421412467956543,
"grad_norm": 27.3696346282959,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.332236647605896,
"logits/rejected": -0.3292074203491211,
"logps/chosen": -348.0567626953125,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -531.6675415039062,
"loss": 1.144,
"margin_dpo/margin_mean": 166.13436889648438,
"margin_dpo/margin_std": 270.0863037109375,
"step": 293
},
{
"KL/chosen_KL_mean": -266.4648742675781,
"KL/mean": -360.0727844238281,
"KL/rejected_KL_mean": -453.68072509765625,
"KL/std": 205.4690704345703,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.002078534569591284,
"fcm_dpo/delta": 0.012987792491912842,
"fcm_dpo/margin": 187.21585083007812,
"fcm_dpo/q_t": 0.41207996010780334,
"grad_norm": 24.852489471435547,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.3020135164260864,
"logits/rejected": -0.30931025743484497,
"logps/chosen": -314.5523986816406,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -535.5776977539062,
"loss": 1.102,
"margin_dpo/margin_mean": 187.21585083007812,
"margin_dpo/margin_std": 258.83837890625,
"step": 294
},
{
"KL/chosen_KL_mean": -315.635009765625,
"KL/mean": -417.57843017578125,
"KL/rejected_KL_mean": -519.52197265625,
"KL/std": 232.22276306152344,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.0020754141733050346,
"fcm_dpo/delta": -0.029316715896129608,
"fcm_dpo/margin": 203.88693237304688,
"fcm_dpo/q_t": 0.40314337611198425,
"grad_norm": 19.910776138305664,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.31562861800193787,
"logits/rejected": -0.3209300637245178,
"logps/chosen": -365.5596618652344,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -606.978271484375,
"loss": 1.0858,
"margin_dpo/margin_mean": 203.88693237304688,
"margin_dpo/margin_std": 279.4998779296875,
"step": 295
},
{
"KL/chosen_KL_mean": -389.09619140625,
"KL/mean": -459.28369140625,
"KL/rejected_KL_mean": -529.4712524414062,
"KL/std": 224.94338989257812,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.00205283984541893,
"fcm_dpo/delta": 0.0012772375484928489,
"fcm_dpo/margin": 140.37503051757812,
"fcm_dpo/q_t": 0.4331241846084595,
"grad_norm": 30.800323486328125,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.3613106906414032,
"logits/rejected": -0.3486577272415161,
"logps/chosen": -454.5874328613281,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -622.560302734375,
"loss": 1.2004,
"margin_dpo/margin_mean": 140.37501525878906,
"margin_dpo/margin_std": 281.86309814453125,
"step": 296
},
{
"KL/chosen_KL_mean": -337.59783935546875,
"KL/mean": -417.08062744140625,
"KL/rejected_KL_mean": -496.5633850097656,
"KL/std": 210.40191650390625,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.002149302512407303,
"fcm_dpo/delta": 0.07061058282852173,
"fcm_dpo/margin": 158.96554565429688,
"fcm_dpo/q_t": 0.4227258563041687,
"grad_norm": 33.55876541137695,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.368080735206604,
"logits/rejected": -0.3742336928844452,
"logps/chosen": -394.0747985839844,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -591.701904296875,
"loss": 1.1254,
"margin_dpo/margin_mean": 158.96554565429688,
"margin_dpo/margin_std": 216.52734375,
"step": 297
},
{
"KL/chosen_KL_mean": -349.87359619140625,
"KL/mean": -458.6307373046875,
"KL/rejected_KL_mean": -567.387939453125,
"KL/std": 274.2647399902344,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.002113178838044405,
"fcm_dpo/delta": -0.0772165060043335,
"fcm_dpo/margin": 217.51431274414062,
"fcm_dpo/q_t": 0.3965296745300293,
"grad_norm": 24.389272689819336,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.401616632938385,
"logits/rejected": -0.4193766713142395,
"logps/chosen": -417.19873046875,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -684.050048828125,
"loss": 1.0813,
"margin_dpo/margin_mean": 217.51431274414062,
"margin_dpo/margin_std": 321.95660400390625,
"step": 298
},
{
"KL/chosen_KL_mean": -289.93463134765625,
"KL/mean": -381.6058044433594,
"KL/rejected_KL_mean": -473.2769470214844,
"KL/std": 202.38690185546875,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.002111277775838971,
"fcm_dpo/delta": 0.012855540961027145,
"fcm_dpo/margin": 183.3423614501953,
"fcm_dpo/q_t": 0.4118254780769348,
"grad_norm": 35.68212890625,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.35824018716812134,
"logits/rejected": -0.36311280727386475,
"logps/chosen": -338.896728515625,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -557.605224609375,
"loss": 1.104,
"margin_dpo/margin_mean": 183.34234619140625,
"margin_dpo/margin_std": 251.83197021484375,
"step": 299
},
{
"KL/chosen_KL_mean": -350.1387939453125,
"KL/mean": -458.59375,
"KL/rejected_KL_mean": -567.0487060546875,
"KL/std": 237.21231079101562,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.0020199865102767944,
"fcm_dpo/delta": -0.048955146223306656,
"fcm_dpo/margin": 216.90985107421875,
"fcm_dpo/q_t": 0.39751458168029785,
"grad_norm": 26.843948364257812,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.3743543028831482,
"logits/rejected": -0.3817921280860901,
"logps/chosen": -409.2125244140625,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -663.01513671875,
"loss": 1.0789,
"margin_dpo/margin_mean": 216.90985107421875,
"margin_dpo/margin_std": 290.77459716796875,
"step": 300
},
{
"KL/chosen_KL_mean": -294.7914733886719,
"KL/mean": -403.30487060546875,
"KL/rejected_KL_mean": -511.8182373046875,
"KL/std": 218.48614501953125,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.0019110521534457803,
"fcm_dpo/delta": -0.019859077408909798,
"fcm_dpo/margin": 217.02674865722656,
"fcm_dpo/q_t": 0.40373334288597107,
"grad_norm": 25.57866096496582,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.3827190399169922,
"logits/rejected": -0.38337594270706177,
"logps/chosen": -352.04083251953125,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -604.1717529296875,
"loss": 1.0724,
"margin_dpo/margin_mean": 217.02676391601562,
"margin_dpo/margin_std": 262.17144775390625,
"step": 301
},
{
"KL/chosen_KL_mean": -263.9697570800781,
"KL/mean": -353.9146728515625,
"KL/rejected_KL_mean": -443.85955810546875,
"KL/std": 204.00518798828125,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.0019708811305463314,
"fcm_dpo/delta": 0.054747726768255234,
"fcm_dpo/margin": 179.88980102539062,
"fcm_dpo/q_t": 0.4198175072669983,
"grad_norm": 19.74114227294922,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.3587857484817505,
"logits/rejected": -0.36537155508995056,
"logps/chosen": -315.167724609375,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -541.0859375,
"loss": 1.1197,
"margin_dpo/margin_mean": 179.88980102539062,
"margin_dpo/margin_std": 243.09542846679688,
"step": 302
},
{
"KL/chosen_KL_mean": -275.6668701171875,
"KL/mean": -358.65966796875,
"KL/rejected_KL_mean": -441.6524658203125,
"KL/std": 205.9825439453125,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.002106656786054373,
"fcm_dpo/delta": 0.061119839549064636,
"fcm_dpo/margin": 165.985595703125,
"fcm_dpo/q_t": 0.42138999700546265,
"grad_norm": 28.86655044555664,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.3849058747291565,
"logits/rejected": -0.3747110366821289,
"logps/chosen": -342.38079833984375,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -528.597900390625,
"loss": 1.1338,
"margin_dpo/margin_mean": 165.985595703125,
"margin_dpo/margin_std": 249.7038116455078,
"step": 303
},
{
"KL/chosen_KL_mean": -257.2261962890625,
"KL/mean": -351.8838806152344,
"KL/rejected_KL_mean": -446.54156494140625,
"KL/std": 196.91989135742188,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.0021874920930713415,
"fcm_dpo/delta": -0.019974112510681152,
"fcm_dpo/margin": 189.31533813476562,
"fcm_dpo/q_t": 0.40017253160476685,
"grad_norm": 45.90519332885742,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.41976654529571533,
"logits/rejected": -0.4154810905456543,
"logps/chosen": -329.1769104003906,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -537.0135498046875,
"loss": 1.0521,
"margin_dpo/margin_mean": 189.31533813476562,
"margin_dpo/margin_std": 171.4813232421875,
"step": 304
},
{
"KL/chosen_KL_mean": -267.2148132324219,
"KL/mean": -351.92645263671875,
"KL/rejected_KL_mean": -436.63812255859375,
"KL/std": 221.58193969726562,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.0021368793677538633,
"fcm_dpo/delta": 0.04446440190076828,
"fcm_dpo/margin": 169.42330932617188,
"fcm_dpo/q_t": 0.4194003641605377,
"grad_norm": 24.6429500579834,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.431082159280777,
"logits/rejected": -0.417194128036499,
"logps/chosen": -334.0100402832031,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -529.3927001953125,
"loss": 1.1257,
"margin_dpo/margin_mean": 169.42330932617188,
"margin_dpo/margin_std": 248.18634033203125,
"step": 305
},
{
"KL/chosen_KL_mean": -270.4483947753906,
"KL/mean": -353.85845947265625,
"KL/rejected_KL_mean": -437.2685241699219,
"KL/std": 196.28871154785156,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.002273739315569401,
"fcm_dpo/delta": 0.025212926790118217,
"fcm_dpo/margin": 166.82009887695312,
"fcm_dpo/q_t": 0.41293442249298096,
"grad_norm": 20.455917358398438,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.4016483426094055,
"logits/rejected": -0.37884002923965454,
"logps/chosen": -340.1322937011719,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -522.427734375,
"loss": 1.0949,
"margin_dpo/margin_mean": 166.82009887695312,
"margin_dpo/margin_std": 209.34255981445312,
"step": 306
},
{
"KL/chosen_KL_mean": -245.0524139404297,
"KL/mean": -331.04681396484375,
"KL/rejected_KL_mean": -417.041259765625,
"KL/std": 183.830810546875,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.0022681187838315964,
"fcm_dpo/delta": 0.011811528354883194,
"fcm_dpo/margin": 171.98887634277344,
"fcm_dpo/q_t": 0.4100651741027832,
"grad_norm": 28.9613037109375,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.4207409620285034,
"logits/rejected": -0.40590375661849976,
"logps/chosen": -315.21783447265625,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -504.0135803222656,
"loss": 1.096,
"margin_dpo/margin_mean": 171.98886108398438,
"margin_dpo/margin_std": 226.1441650390625,
"step": 307
},
{
"KL/chosen_KL_mean": -252.00428771972656,
"KL/mean": -349.07568359375,
"KL/rejected_KL_mean": -446.1470642089844,
"KL/std": 207.59048461914062,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.002229331061244011,
"fcm_dpo/delta": -0.04198146238923073,
"fcm_dpo/margin": 194.14276123046875,
"fcm_dpo/q_t": 0.3967881202697754,
"grad_norm": 28.756671905517578,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.4297657012939453,
"logits/rejected": -0.4254001975059509,
"logps/chosen": -307.249267578125,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -525.5193481445312,
"loss": 1.0511,
"margin_dpo/margin_mean": 194.14276123046875,
"margin_dpo/margin_std": 207.66583251953125,
"step": 308
},
{
"KL/chosen_KL_mean": -257.0223388671875,
"KL/mean": -352.66180419921875,
"KL/rejected_KL_mean": -448.30126953125,
"KL/std": 222.42701721191406,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.0022140173241496086,
"fcm_dpo/delta": -0.03186354413628578,
"fcm_dpo/margin": 191.2789306640625,
"fcm_dpo/q_t": 0.3998178541660309,
"grad_norm": 29.72759437561035,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.3393629789352417,
"logits/rejected": -0.33330753445625305,
"logps/chosen": -305.98138427734375,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -530.6419677734375,
"loss": 1.0754,
"margin_dpo/margin_mean": 191.2789306640625,
"margin_dpo/margin_std": 235.2989959716797,
"step": 309
},
{
"KL/chosen_KL_mean": -304.2635498046875,
"KL/mean": -393.31304931640625,
"KL/rejected_KL_mean": -482.362548828125,
"KL/std": 200.93919372558594,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.0021948455832898617,
"fcm_dpo/delta": 0.009528759866952896,
"fcm_dpo/margin": 178.0990447998047,
"fcm_dpo/q_t": 0.4077993631362915,
"grad_norm": 36.597747802734375,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.4227510690689087,
"logits/rejected": -0.41301751136779785,
"logps/chosen": -367.00531005859375,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -562.2928466796875,
"loss": 1.0787,
"margin_dpo/margin_mean": 178.09906005859375,
"margin_dpo/margin_std": 195.08804321289062,
"step": 310
},
{
"KL/chosen_KL_mean": -307.46826171875,
"KL/mean": -412.06182861328125,
"KL/rejected_KL_mean": -516.6553955078125,
"KL/std": 252.58099365234375,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.0020815101452171803,
"fcm_dpo/delta": -0.045531198382377625,
"fcm_dpo/margin": 209.18707275390625,
"fcm_dpo/q_t": 0.39924418926239014,
"grad_norm": 25.931556701660156,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.4020995497703552,
"logits/rejected": -0.3801601529121399,
"logps/chosen": -360.4962463378906,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -594.0935668945312,
"loss": 1.0693,
"margin_dpo/margin_mean": 209.18707275390625,
"margin_dpo/margin_std": 267.8011474609375,
"step": 311
},
{
"KL/chosen_KL_mean": -301.262451171875,
"KL/mean": -398.7344970703125,
"KL/rejected_KL_mean": -496.20654296875,
"KL/std": 227.84207153320312,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.002090971916913986,
"fcm_dpo/delta": -0.011386025696992874,
"fcm_dpo/margin": 194.94412231445312,
"fcm_dpo/q_t": 0.40784695744514465,
"grad_norm": 25.662092208862305,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -0.40964388847351074,
"logits/rejected": -0.40318238735198975,
"logps/chosen": -350.6546630859375,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -571.9993896484375,
"loss": 1.0933,
"margin_dpo/margin_mean": 194.94412231445312,
"margin_dpo/margin_std": 271.61602783203125,
"step": 312
},
{
"KL/chosen_KL_mean": -282.75341796875,
"KL/mean": -380.6422119140625,
"KL/rejected_KL_mean": -478.531005859375,
"KL/std": 236.21067810058594,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.002096590120345354,
"fcm_dpo/delta": -0.019848383963108063,
"fcm_dpo/margin": 195.77755737304688,
"fcm_dpo/q_t": 0.4035242199897766,
"grad_norm": 27.727643966674805,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -0.4416351914405823,
"logits/rejected": -0.43936672806739807,
"logps/chosen": -332.90618896484375,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -564.9371948242188,
"loss": 1.0933,
"margin_dpo/margin_mean": 195.77755737304688,
"margin_dpo/margin_std": 253.7050018310547,
"step": 313
},
{
"KL/chosen_KL_mean": -304.5816345214844,
"KL/mean": -394.99072265625,
"KL/rejected_KL_mean": -485.39984130859375,
"KL/std": 222.5576934814453,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.0020326506346464157,
"fcm_dpo/delta": 0.03956056758761406,
"fcm_dpo/margin": 180.8181915283203,
"fcm_dpo/q_t": 0.41847917437553406,
"grad_norm": 20.639629364013672,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.476603627204895,
"logits/rejected": -0.4820059835910797,
"logps/chosen": -361.8192138671875,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -582.996337890625,
"loss": 1.1336,
"margin_dpo/margin_mean": 180.81817626953125,
"margin_dpo/margin_std": 289.74945068359375,
"step": 314
},
{
"KL/chosen_KL_mean": -279.29681396484375,
"KL/mean": -364.7213134765625,
"KL/rejected_KL_mean": -450.1457824707031,
"KL/std": 205.07464599609375,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.00213609728962183,
"fcm_dpo/delta": 0.04297637194395065,
"fcm_dpo/margin": 170.8489990234375,
"fcm_dpo/q_t": 0.41723307967185974,
"grad_norm": 21.063323974609375,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.3772663474082947,
"logits/rejected": -0.3812904357910156,
"logps/chosen": -328.36639404296875,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -535.82666015625,
"loss": 1.1108,
"margin_dpo/margin_mean": 170.8489990234375,
"margin_dpo/margin_std": 229.18209838867188,
"step": 315
},
{
"KL/chosen_KL_mean": -275.72784423828125,
"KL/mean": -397.86114501953125,
"KL/rejected_KL_mean": -519.994384765625,
"KL/std": 247.28073120117188,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.001942518400028348,
"fcm_dpo/delta": -0.10697119683027267,
"fcm_dpo/margin": 244.26651000976562,
"fcm_dpo/q_t": 0.3864254951477051,
"grad_norm": 23.46164321899414,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.42778435349464417,
"logits/rejected": -0.43224072456359863,
"logps/chosen": -329.9886169433594,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -621.27587890625,
"loss": 1.0309,
"margin_dpo/margin_mean": 244.2665252685547,
"margin_dpo/margin_std": 268.9966125488281,
"step": 316
},
{
"KL/chosen_KL_mean": -286.0876159667969,
"KL/mean": -423.0874938964844,
"KL/rejected_KL_mean": -560.0873413085938,
"KL/std": 227.30641174316406,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.0018125809729099274,
"fcm_dpo/delta": -0.1276942938566208,
"fcm_dpo/margin": 273.9997253417969,
"fcm_dpo/q_t": 0.3785746097564697,
"grad_norm": 27.784400939941406,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.4147995114326477,
"logits/rejected": -0.4096861481666565,
"logps/chosen": -342.18182373046875,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -660.786376953125,
"loss": 1.0049,
"margin_dpo/margin_mean": 273.99969482421875,
"margin_dpo/margin_std": 257.4659423828125,
"step": 317
},
{
"KL/chosen_KL_mean": -317.5200500488281,
"KL/mean": -415.6494140625,
"KL/rejected_KL_mean": -513.7787475585938,
"KL/std": 229.84683227539062,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.0018144804053008556,
"fcm_dpo/delta": 0.05106057599186897,
"fcm_dpo/margin": 196.25872802734375,
"fcm_dpo/q_t": 0.41813182830810547,
"grad_norm": 26.109601974487305,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.4953124225139618,
"logits/rejected": -0.46893250942230225,
"logps/chosen": -382.1657409667969,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -596.5430297851562,
"loss": 1.1096,
"margin_dpo/margin_mean": 196.25872802734375,
"margin_dpo/margin_std": 247.5173797607422,
"step": 318
},
{
"KL/chosen_KL_mean": -285.78082275390625,
"KL/mean": -403.2270812988281,
"KL/rejected_KL_mean": -520.67333984375,
"KL/std": 257.381591796875,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.0017810399876907468,
"fcm_dpo/delta": -0.023179180920124054,
"fcm_dpo/margin": 234.89251708984375,
"fcm_dpo/q_t": 0.40157684683799744,
"grad_norm": 22.635221481323242,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.38388240337371826,
"logits/rejected": -0.396739661693573,
"logps/chosen": -335.16455078125,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -634.579833984375,
"loss": 1.0691,
"margin_dpo/margin_mean": 234.8925323486328,
"margin_dpo/margin_std": 279.86474609375,
"step": 319
},
{
"KL/chosen_KL_mean": -290.6177978515625,
"KL/mean": -416.1560974121094,
"KL/rejected_KL_mean": -541.6944580078125,
"KL/std": 259.626220703125,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.0016947748372331262,
"fcm_dpo/delta": -0.03422039747238159,
"fcm_dpo/margin": 251.07666015625,
"fcm_dpo/q_t": 0.39976027607917786,
"grad_norm": 21.635658264160156,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.44464847445487976,
"logits/rejected": -0.4517730474472046,
"logps/chosen": -350.1226806640625,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -639.361572265625,
"loss": 1.0582,
"margin_dpo/margin_mean": 251.07666015625,
"margin_dpo/margin_std": 280.9095458984375,
"step": 320
},
{
"KL/chosen_KL_mean": -359.953369140625,
"KL/mean": -468.767333984375,
"KL/rejected_KL_mean": -577.5813598632812,
"KL/std": 275.1668701171875,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.0017093562055379152,
"fcm_dpo/delta": 0.03284335136413574,
"fcm_dpo/margin": 217.62799072265625,
"fcm_dpo/q_t": 0.41511040925979614,
"grad_norm": 19.775318145751953,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -0.3967721462249756,
"logits/rejected": -0.38455623388290405,
"logps/chosen": -421.5020446777344,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -669.222412109375,
"loss": 1.1103,
"margin_dpo/margin_mean": 217.62796020507812,
"margin_dpo/margin_std": 292.2899169921875,
"step": 321
},
{
"KL/chosen_KL_mean": -302.359130859375,
"KL/mean": -423.87506103515625,
"KL/rejected_KL_mean": -545.3909912109375,
"KL/std": 239.97723388671875,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.001710197888314724,
"fcm_dpo/delta": -0.02115067094564438,
"fcm_dpo/margin": 243.0318603515625,
"fcm_dpo/q_t": 0.4011186957359314,
"grad_norm": 20.75707244873047,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -0.381791889667511,
"logits/rejected": -0.3810097575187683,
"logps/chosen": -359.649169921875,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -641.1409912109375,
"loss": 1.0584,
"margin_dpo/margin_mean": 243.0318603515625,
"margin_dpo/margin_std": 252.65171813964844,
"step": 322
},
{
"KL/chosen_KL_mean": -329.60552978515625,
"KL/mean": -433.67523193359375,
"KL/rejected_KL_mean": -537.7449340820312,
"KL/std": 240.89913940429688,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.0017628297209739685,
"fcm_dpo/delta": 0.04052872955799103,
"fcm_dpo/margin": 208.13937377929688,
"fcm_dpo/q_t": 0.4153671860694885,
"grad_norm": 26.74566078186035,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -0.4394986629486084,
"logits/rejected": -0.4391285479068756,
"logps/chosen": -380.8395080566406,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -612.806884765625,
"loss": 1.1044,
"margin_dpo/margin_mean": 208.13937377929688,
"margin_dpo/margin_std": 257.1087951660156,
"step": 323
},
{
"KL/chosen_KL_mean": -379.00439453125,
"KL/mean": -478.7786865234375,
"KL/rejected_KL_mean": -578.552978515625,
"KL/std": 264.73980712890625,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.0018952081445604563,
"fcm_dpo/delta": 0.019777312874794006,
"fcm_dpo/margin": 199.5486297607422,
"fcm_dpo/q_t": 0.415118932723999,
"grad_norm": 40.34514236450195,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -0.446133553981781,
"logits/rejected": -0.43301481008529663,
"logps/chosen": -444.1395568847656,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -665.030517578125,
"loss": 1.1304,
"margin_dpo/margin_mean": 199.5486297607422,
"margin_dpo/margin_std": 308.2591247558594,
"step": 324
},
{
"KL/chosen_KL_mean": -287.88055419921875,
"KL/mean": -370.6061096191406,
"KL/rejected_KL_mean": -453.3316650390625,
"KL/std": 218.6322021484375,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.001969131175428629,
"fcm_dpo/delta": 0.08780458569526672,
"fcm_dpo/margin": 165.45111083984375,
"fcm_dpo/q_t": 0.4262668192386627,
"grad_norm": 25.783239364624023,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.46015477180480957,
"logits/rejected": -0.4445416331291199,
"logps/chosen": -344.09613037109375,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -523.4176025390625,
"loss": 1.1354,
"margin_dpo/margin_mean": 165.45111083984375,
"margin_dpo/margin_std": 224.27264404296875,
"step": 325
},
{
"KL/chosen_KL_mean": -307.934814453125,
"KL/mean": -389.87255859375,
"KL/rejected_KL_mean": -471.8103332519531,
"KL/std": 201.8898162841797,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.0020997067913413048,
"fcm_dpo/delta": 0.06763522326946259,
"fcm_dpo/margin": 163.87550354003906,
"fcm_dpo/q_t": 0.4207178056240082,
"grad_norm": 62.081993103027344,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.4982990622520447,
"logits/rejected": -0.4790222644805908,
"logps/chosen": -380.6597595214844,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -551.6571044921875,
"loss": 1.1107,
"margin_dpo/margin_mean": 163.87550354003906,
"margin_dpo/margin_std": 188.82302856445312,
"step": 326
},
{
"KL/chosen_KL_mean": -277.514404296875,
"KL/mean": -394.7227783203125,
"KL/rejected_KL_mean": -511.93115234375,
"KL/std": 218.5492706298828,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.0019716641400009394,
"fcm_dpo/delta": -0.08498930931091309,
"fcm_dpo/margin": 234.416748046875,
"fcm_dpo/q_t": 0.38751423358917236,
"grad_norm": 51.83660888671875,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.45873162150382996,
"logits/rejected": -0.4484935998916626,
"logps/chosen": -346.6488037109375,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -623.8649291992188,
"loss": 1.0338,
"margin_dpo/margin_mean": 234.41673278808594,
"margin_dpo/margin_std": 248.63563537597656,
"step": 327
},
{
"KL/chosen_KL_mean": -291.4725341796875,
"KL/mean": -397.5861511230469,
"KL/rejected_KL_mean": -503.69976806640625,
"KL/std": 236.2239990234375,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.0019511812133714557,
"fcm_dpo/delta": -0.01778026856482029,
"fcm_dpo/margin": 212.22723388671875,
"fcm_dpo/q_t": 0.4037235677242279,
"grad_norm": 34.96001052856445,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.4395790100097656,
"logits/rejected": -0.4473843276500702,
"logps/chosen": -351.15972900390625,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -594.5548095703125,
"loss": 1.0754,
"margin_dpo/margin_mean": 212.22726440429688,
"margin_dpo/margin_std": 262.13189697265625,
"step": 328
},
{
"KL/chosen_KL_mean": -334.9793395996094,
"KL/mean": -451.97674560546875,
"KL/rejected_KL_mean": -568.97412109375,
"KL/std": 270.3482666015625,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.0018274824833497405,
"fcm_dpo/delta": -0.03972601890563965,
"fcm_dpo/margin": 233.9947509765625,
"fcm_dpo/q_t": 0.3984408974647522,
"grad_norm": 25.084033966064453,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -0.4626386761665344,
"logits/rejected": -0.46101057529449463,
"logps/chosen": -400.22552490234375,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -669.671875,
"loss": 1.0719,
"margin_dpo/margin_mean": 233.9947509765625,
"margin_dpo/margin_std": 285.23626708984375,
"step": 329
},
{
"KL/chosen_KL_mean": -292.37579345703125,
"KL/mean": -412.8188171386719,
"KL/rejected_KL_mean": -533.2619018554688,
"KL/std": 249.09326171875,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.0017980177653953433,
"fcm_dpo/delta": -0.04239234700798988,
"fcm_dpo/margin": 240.88613891601562,
"fcm_dpo/q_t": 0.3976569175720215,
"grad_norm": 20.89375877380371,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -0.4702164828777313,
"logits/rejected": -0.46967270970344543,
"logps/chosen": -339.3741149902344,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -620.1387329101562,
"loss": 1.0555,
"margin_dpo/margin_mean": 240.88613891601562,
"margin_dpo/margin_std": 273.5047607421875,
"step": 330
},
{
"KL/chosen_KL_mean": -330.68896484375,
"KL/mean": -452.95233154296875,
"KL/rejected_KL_mean": -575.2156982421875,
"KL/std": 218.610107421875,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.0017413150053471327,
"fcm_dpo/delta": -0.032744720578193665,
"fcm_dpo/margin": 244.52676391601562,
"fcm_dpo/q_t": 0.3974621891975403,
"grad_norm": 26.476423263549805,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -0.49178826808929443,
"logits/rejected": -0.4904711842536926,
"logps/chosen": -381.21319580078125,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -664.231201171875,
"loss": 1.0385,
"margin_dpo/margin_mean": 244.52676391601562,
"margin_dpo/margin_std": 216.18482971191406,
"step": 331
},
{
"KL/chosen_KL_mean": -348.40509033203125,
"KL/mean": -431.3126220703125,
"KL/rejected_KL_mean": -514.2201538085938,
"KL/std": 222.2275390625,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.0018814262002706528,
"fcm_dpo/delta": 0.10219782590866089,
"fcm_dpo/margin": 165.81512451171875,
"fcm_dpo/q_t": 0.43078944087028503,
"grad_norm": 26.434301376342773,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -0.5475494861602783,
"logits/rejected": -0.5299344062805176,
"logps/chosen": -397.5853576660156,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -590.705322265625,
"loss": 1.152,
"margin_dpo/margin_mean": 165.81512451171875,
"margin_dpo/margin_std": 254.50332641601562,
"step": 332
},
{
"KL/chosen_KL_mean": -377.22314453125,
"KL/mean": -493.54986572265625,
"KL/rejected_KL_mean": -609.8765869140625,
"KL/std": 277.18450927734375,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.0017979848198592663,
"fcm_dpo/delta": -0.030853308737277985,
"fcm_dpo/margin": 232.65338134765625,
"fcm_dpo/q_t": 0.40362921357154846,
"grad_norm": 32.03265380859375,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -0.5083990097045898,
"logits/rejected": -0.5068196654319763,
"logps/chosen": -440.9789123535156,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -704.920654296875,
"loss": 1.1067,
"margin_dpo/margin_mean": 232.6533660888672,
"margin_dpo/margin_std": 344.5853271484375,
"step": 333
},
{
"KL/chosen_KL_mean": -350.3871765136719,
"KL/mean": -450.59454345703125,
"KL/rejected_KL_mean": -550.802001953125,
"KL/std": 298.42266845703125,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.0018730323063209653,
"fcm_dpo/delta": 0.030153676867485046,
"fcm_dpo/margin": 200.414794921875,
"fcm_dpo/q_t": 0.41732466220855713,
"grad_norm": 27.673667907714844,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.4616066813468933,
"logits/rejected": -0.45988792181015015,
"logps/chosen": -417.366943359375,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -646.118896484375,
"loss": 1.1399,
"margin_dpo/margin_mean": 200.41477966308594,
"margin_dpo/margin_std": 338.0478515625,
"step": 334
},
{
"KL/chosen_KL_mean": -360.7254333496094,
"KL/mean": -446.093017578125,
"KL/rejected_KL_mean": -531.4605712890625,
"KL/std": 273.3331298828125,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.00204592477530241,
"fcm_dpo/delta": 0.05596485733985901,
"fcm_dpo/margin": 170.73513793945312,
"fcm_dpo/q_t": 0.42263150215148926,
"grad_norm": 35.19724655151367,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -0.569267749786377,
"logits/rejected": -0.5530425310134888,
"logps/chosen": -423.2679138183594,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -619.0782470703125,
"loss": 1.15,
"margin_dpo/margin_mean": 170.73513793945312,
"margin_dpo/margin_std": 286.39031982421875,
"step": 335
},
{
"KL/chosen_KL_mean": -362.0695495605469,
"KL/mean": -486.9052429199219,
"KL/rejected_KL_mean": -611.740966796875,
"KL/std": 306.3258972167969,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.0019088383996859193,
"fcm_dpo/delta": -0.10008023679256439,
"fcm_dpo/margin": 249.67137145996094,
"fcm_dpo/q_t": 0.3910997807979584,
"grad_norm": 27.048595428466797,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -0.4783538281917572,
"logits/rejected": -0.4791324734687805,
"logps/chosen": -416.6007080078125,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -716.1451416015625,
"loss": 1.078,
"margin_dpo/margin_mean": 249.67135620117188,
"margin_dpo/margin_std": 367.23291015625,
"step": 336
},
{
"KL/chosen_KL_mean": -319.1661376953125,
"KL/mean": -445.8056335449219,
"KL/rejected_KL_mean": -572.445068359375,
"KL/std": 243.7745361328125,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.0017321293707937002,
"fcm_dpo/delta": -0.051446449011564255,
"fcm_dpo/margin": 253.27894592285156,
"fcm_dpo/q_t": 0.39632317423820496,
"grad_norm": 19.951826095581055,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -0.5001170635223389,
"logits/rejected": -0.4860590696334839,
"logps/chosen": -384.29486083984375,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -674.172119140625,
"loss": 1.0528,
"margin_dpo/margin_mean": 253.27896118164062,
"margin_dpo/margin_std": 288.4793701171875,
"step": 337
},
{
"KL/chosen_KL_mean": -283.015869140625,
"KL/mean": -405.06072998046875,
"KL/rejected_KL_mean": -527.1055908203125,
"KL/std": 232.23532104492188,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.0016998077044263482,
"fcm_dpo/delta": -0.01908147893846035,
"fcm_dpo/margin": 244.0897216796875,
"fcm_dpo/q_t": 0.40122056007385254,
"grad_norm": 33.17329025268555,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -0.5003715753555298,
"logits/rejected": -0.49724671244621277,
"logps/chosen": -341.4385986328125,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -616.1741333007812,
"loss": 1.0533,
"margin_dpo/margin_mean": 244.0897216796875,
"margin_dpo/margin_std": 240.3944854736328,
"step": 338
},
{
"KL/chosen_KL_mean": -306.18310546875,
"KL/mean": -420.02911376953125,
"KL/rejected_KL_mean": -533.8751220703125,
"KL/std": 236.42514038085938,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.0017625135369598866,
"fcm_dpo/delta": -0.008848987519741058,
"fcm_dpo/margin": 227.69198608398438,
"fcm_dpo/q_t": 0.40451204776763916,
"grad_norm": 25.832143783569336,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -0.5664355754852295,
"logits/rejected": -0.5560377836227417,
"logps/chosen": -366.17840576171875,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -637.7860107421875,
"loss": 1.077,
"margin_dpo/margin_mean": 227.69198608398438,
"margin_dpo/margin_std": 242.7012481689453,
"step": 339
},
{
"KL/chosen_KL_mean": -306.2630920410156,
"KL/mean": -406.93267822265625,
"KL/rejected_KL_mean": -507.6022644042969,
"KL/std": 228.15411376953125,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.001739348517730832,
"fcm_dpo/delta": 0.06040128692984581,
"fcm_dpo/margin": 201.33917236328125,
"fcm_dpo/q_t": 0.4204002022743225,
"grad_norm": 19.13730812072754,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -0.46875977516174316,
"logits/rejected": -0.4506962299346924,
"logps/chosen": -359.09332275390625,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -580.70947265625,
"loss": 1.1223,
"margin_dpo/margin_mean": 201.33917236328125,
"margin_dpo/margin_std": 275.57354736328125,
"step": 340
},
{
"KL/chosen_KL_mean": -299.728515625,
"KL/mean": -421.93115234375,
"KL/rejected_KL_mean": -544.1337890625,
"KL/std": 255.9670867919922,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.001722757937386632,
"fcm_dpo/delta": -0.028678692877292633,
"fcm_dpo/margin": 244.40525817871094,
"fcm_dpo/q_t": 0.40045589208602905,
"grad_norm": 22.329763412475586,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.4424135684967041,
"logits/rejected": -0.4432716369628906,
"logps/chosen": -347.6282958984375,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -645.9436645507812,
"loss": 1.0604,
"margin_dpo/margin_mean": 244.40525817871094,
"margin_dpo/margin_std": 266.0626525878906,
"step": 341
},
{
"KL/chosen_KL_mean": -297.13800048828125,
"KL/mean": -400.4307861328125,
"KL/rejected_KL_mean": -503.7236022949219,
"KL/std": 236.86354064941406,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.0018168420065194368,
"fcm_dpo/delta": 0.027337031438946724,
"fcm_dpo/margin": 206.58563232421875,
"fcm_dpo/q_t": 0.41394251585006714,
"grad_norm": 22.068641662597656,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.49012503027915955,
"logits/rejected": -0.4645516276359558,
"logps/chosen": -369.1346435546875,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -596.313232421875,
"loss": 1.109,
"margin_dpo/margin_mean": 206.5856170654297,
"margin_dpo/margin_std": 283.00140380859375,
"step": 342
},
{
"KL/chosen_KL_mean": -280.635986328125,
"KL/mean": -405.8623046875,
"KL/rejected_KL_mean": -531.088623046875,
"KL/std": 239.86712646484375,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.001725044334307313,
"fcm_dpo/delta": -0.0413195863366127,
"fcm_dpo/margin": 250.45265197753906,
"fcm_dpo/q_t": 0.39542150497436523,
"grad_norm": 20.26138687133789,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.5200803875923157,
"logits/rejected": -0.5258131623268127,
"logps/chosen": -335.0416259765625,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -642.1300048828125,
"loss": 1.0338,
"margin_dpo/margin_mean": 250.45265197753906,
"margin_dpo/margin_std": 214.74595642089844,
"step": 343
},
{
"KL/chosen_KL_mean": -282.39642333984375,
"KL/mean": -391.85748291015625,
"KL/rejected_KL_mean": -501.31854248046875,
"KL/std": 262.4002685546875,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.0017953403294086456,
"fcm_dpo/delta": -0.00011947751045227051,
"fcm_dpo/margin": 218.922119140625,
"fcm_dpo/q_t": 0.40945079922676086,
"grad_norm": 18.697933197021484,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.5233709216117859,
"logits/rejected": -0.5257829427719116,
"logps/chosen": -336.361083984375,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -591.94189453125,
"loss": 1.0912,
"margin_dpo/margin_mean": 218.922119140625,
"margin_dpo/margin_std": 262.0262756347656,
"step": 344
},
{
"KL/chosen_KL_mean": -341.17315673828125,
"KL/mean": -463.88018798828125,
"KL/rejected_KL_mean": -586.5872802734375,
"KL/std": 246.993408203125,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.0017087084706872702,
"fcm_dpo/delta": -0.024968912824988365,
"fcm_dpo/margin": 245.41409301757812,
"fcm_dpo/q_t": 0.4007132053375244,
"grad_norm": 23.478904724121094,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.5109938383102417,
"logits/rejected": -0.5099775791168213,
"logps/chosen": -402.85882568359375,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -686.0776977539062,
"loss": 1.0622,
"margin_dpo/margin_mean": 245.41409301757812,
"margin_dpo/margin_std": 272.0403137207031,
"step": 345
},
{
"KL/chosen_KL_mean": -347.7568359375,
"KL/mean": -463.9387512207031,
"KL/rejected_KL_mean": -580.12060546875,
"KL/std": 252.9781494140625,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.001682568108662963,
"fcm_dpo/delta": 0.011223375797271729,
"fcm_dpo/margin": 232.36383056640625,
"fcm_dpo/q_t": 0.40780335664749146,
"grad_norm": 23.162200927734375,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -0.49171721935272217,
"logits/rejected": -0.4840758442878723,
"logps/chosen": -403.0130920410156,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -657.5359497070312,
"loss": 1.0805,
"margin_dpo/margin_mean": 232.36383056640625,
"margin_dpo/margin_std": 264.86688232421875,
"step": 346
},
{
"KL/chosen_KL_mean": -359.0545654296875,
"KL/mean": -472.8544616699219,
"KL/rejected_KL_mean": -586.6543579101562,
"KL/std": 266.8034973144531,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.001710890093818307,
"fcm_dpo/delta": 0.013139501214027405,
"fcm_dpo/margin": 227.59982299804688,
"fcm_dpo/q_t": 0.4103218615055084,
"grad_norm": 22.08174705505371,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -0.5109871625900269,
"logits/rejected": -0.5056133270263672,
"logps/chosen": -416.62078857421875,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -679.0094604492188,
"loss": 1.095,
"margin_dpo/margin_mean": 227.59979248046875,
"margin_dpo/margin_std": 297.329833984375,
"step": 347
},
{
"KL/chosen_KL_mean": -317.605712890625,
"KL/mean": -419.5076904296875,
"KL/rejected_KL_mean": -521.40966796875,
"KL/std": 239.0012969970703,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.0017384930979460478,
"fcm_dpo/delta": 0.0538782924413681,
"fcm_dpo/margin": 203.80392456054688,
"fcm_dpo/q_t": 0.41967394948005676,
"grad_norm": 22.87740707397461,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -0.5410622358322144,
"logits/rejected": -0.5414994359016418,
"logps/chosen": -373.9234313964844,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -610.5479736328125,
"loss": 1.122,
"margin_dpo/margin_mean": 203.80392456054688,
"margin_dpo/margin_std": 276.4021301269531,
"step": 348
},
{
"KL/chosen_KL_mean": -315.38214111328125,
"KL/mean": -426.52508544921875,
"KL/rejected_KL_mean": -537.6680297851562,
"KL/std": 259.76336669921875,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.0018351702019572258,
"fcm_dpo/delta": -0.011926159262657166,
"fcm_dpo/margin": 222.285888671875,
"fcm_dpo/q_t": 0.407881498336792,
"grad_norm": 21.936599731445312,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -0.5372369289398193,
"logits/rejected": -0.5374791622161865,
"logps/chosen": -373.40765380859375,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -635.1732177734375,
"loss": 1.1019,
"margin_dpo/margin_mean": 222.28587341308594,
"margin_dpo/margin_std": 322.0589904785156,
"step": 349
},
{
"KL/chosen_KL_mean": -336.73089599609375,
"KL/mean": -447.26806640625,
"KL/rejected_KL_mean": -557.8052978515625,
"KL/std": 256.64642333984375,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.001765610184520483,
"fcm_dpo/delta": 0.010335829108953476,
"fcm_dpo/margin": 221.074462890625,
"fcm_dpo/q_t": 0.40952080488204956,
"grad_norm": 30.984949111938477,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -0.5725632905960083,
"logits/rejected": -0.5793622136116028,
"logps/chosen": -401.0613708496094,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -647.677001953125,
"loss": 1.1081,
"margin_dpo/margin_mean": 221.074462890625,
"margin_dpo/margin_std": 314.63995361328125,
"step": 350
},
{
"KL/chosen_KL_mean": -301.34967041015625,
"KL/mean": -436.94683837890625,
"KL/rejected_KL_mean": -572.5440673828125,
"KL/std": 292.54278564453125,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.0016822493635118008,
"fcm_dpo/delta": -0.07549858838319778,
"fcm_dpo/margin": 271.1943664550781,
"fcm_dpo/q_t": 0.3937299847602844,
"grad_norm": 28.201416015625,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -0.5415765047073364,
"logits/rejected": -0.5444436073303223,
"logps/chosen": -362.02178955078125,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -674.1094970703125,
"loss": 1.0575,
"margin_dpo/margin_mean": 271.19439697265625,
"margin_dpo/margin_std": 342.4002685546875,
"step": 351
},
{
"KL/chosen_KL_mean": -353.643798828125,
"KL/mean": -438.07720947265625,
"KL/rejected_KL_mean": -522.5106811523438,
"KL/std": 269.5029602050781,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.0018008626066148281,
"fcm_dpo/delta": 0.11381582170724869,
"fcm_dpo/margin": 168.86691284179688,
"fcm_dpo/q_t": 0.4331480860710144,
"grad_norm": 61.400474548339844,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -0.5702152252197266,
"logits/rejected": -0.5352950096130371,
"logps/chosen": -424.58721923828125,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -599.1526489257812,
"loss": 1.1724,
"margin_dpo/margin_mean": 168.86691284179688,
"margin_dpo/margin_std": 291.0610656738281,
"step": 352
},
{
"KL/chosen_KL_mean": -321.31304931640625,
"KL/mean": -428.07708740234375,
"KL/rejected_KL_mean": -534.8411865234375,
"KL/std": 256.8054504394531,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.0018330267630517483,
"fcm_dpo/delta": 0.006989631801843643,
"fcm_dpo/margin": 213.528076171875,
"fcm_dpo/q_t": 0.4106243848800659,
"grad_norm": 29.105669021606445,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -0.6194638609886169,
"logits/rejected": -0.604763925075531,
"logps/chosen": -379.7083740234375,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -615.1766967773438,
"loss": 1.104,
"margin_dpo/margin_mean": 213.528076171875,
"margin_dpo/margin_std": 291.9771728515625,
"step": 353
},
{
"KL/chosen_KL_mean": -276.51214599609375,
"KL/mean": -391.6829833984375,
"KL/rejected_KL_mean": -506.8538513183594,
"KL/std": 234.94125366210938,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.0018385781440883875,
"fcm_dpo/delta": -0.030174316838383675,
"fcm_dpo/margin": 230.3416748046875,
"fcm_dpo/q_t": 0.39935770630836487,
"grad_norm": 32.30131149291992,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -0.6025009155273438,
"logits/rejected": -0.5897366404533386,
"logps/chosen": -336.3151550292969,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -595.611328125,
"loss": 1.0512,
"margin_dpo/margin_mean": 230.3416748046875,
"margin_dpo/margin_std": 236.0069580078125,
"step": 354
},
{
"KL/chosen_KL_mean": -267.94769287109375,
"KL/mean": -392.4739685058594,
"KL/rejected_KL_mean": -517.000244140625,
"KL/std": 249.90625,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.0017431924352422357,
"fcm_dpo/delta": -0.045911140739917755,
"fcm_dpo/margin": 249.0526123046875,
"fcm_dpo/q_t": 0.3952483534812927,
"grad_norm": 41.20962905883789,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -0.6518046855926514,
"logits/rejected": -0.6433999538421631,
"logps/chosen": -322.076171875,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -599.4063110351562,
"loss": 1.0418,
"margin_dpo/margin_mean": 249.05262756347656,
"margin_dpo/margin_std": 242.5153045654297,
"step": 355
},
{
"KL/chosen_KL_mean": -348.6526184082031,
"KL/mean": -410.7809753417969,
"KL/rejected_KL_mean": -472.9093322753906,
"KL/std": 255.02890014648438,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.0017826826078817248,
"fcm_dpo/delta": 0.03428832069039345,
"fcm_dpo/margin": 124.2567367553711,
"fcm_dpo/q_t": 0.45051053166389465,
"grad_norm": 29.759109497070312,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -0.5975973010063171,
"logits/rejected": -0.57161945104599,
"logps/chosen": -413.326416015625,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -548.80859375,
"loss": 1.2561,
"margin_dpo/margin_mean": 124.25672912597656,
"margin_dpo/margin_std": 320.7979736328125,
"step": 356
},
{
"KL/chosen_KL_mean": -309.07611083984375,
"KL/mean": -413.7547607421875,
"KL/rejected_KL_mean": -518.4334106445312,
"KL/std": 256.1674499511719,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.0018109192606061697,
"fcm_dpo/delta": 0.02454628050327301,
"fcm_dpo/margin": 209.35728454589844,
"fcm_dpo/q_t": 0.41207998991012573,
"grad_norm": 25.09831428527832,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -0.5878846645355225,
"logits/rejected": -0.5737650990486145,
"logps/chosen": -361.80194091796875,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -605.2745361328125,
"loss": 1.0991,
"margin_dpo/margin_mean": 209.3572998046875,
"margin_dpo/margin_std": 263.3780517578125,
"step": 357
},
{
"KL/chosen_KL_mean": -287.3702697753906,
"KL/mean": -386.8647155761719,
"KL/rejected_KL_mean": -486.3591613769531,
"KL/std": 241.16976928710938,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.0018941482994705439,
"fcm_dpo/delta": 0.0284799225628376,
"fcm_dpo/margin": 198.98886108398438,
"fcm_dpo/q_t": 0.4145725965499878,
"grad_norm": 26.890031814575195,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.5883272886276245,
"logits/rejected": -0.5693084001541138,
"logps/chosen": -350.5757141113281,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -574.732421875,
"loss": 1.1049,
"margin_dpo/margin_mean": 198.98886108398438,
"margin_dpo/margin_std": 268.44134521484375,
"step": 358
},
{
"KL/chosen_KL_mean": -328.99090576171875,
"KL/mean": -444.7538146972656,
"KL/rejected_KL_mean": -560.5167236328125,
"KL/std": 247.1792755126953,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.0018442978616803885,
"fcm_dpo/delta": -0.0352255180478096,
"fcm_dpo/margin": 231.5258331298828,
"fcm_dpo/q_t": 0.4013112187385559,
"grad_norm": 33.82530212402344,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -0.5302337408065796,
"logits/rejected": -0.5221052765846252,
"logps/chosen": -385.3611145019531,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -642.6904907226562,
"loss": 1.0792,
"margin_dpo/margin_mean": 231.5258331298828,
"margin_dpo/margin_std": 305.66802978515625,
"step": 359
},
{
"KL/chosen_KL_mean": -329.69427490234375,
"KL/mean": -418.23699951171875,
"KL/rejected_KL_mean": -506.77972412109375,
"KL/std": 206.728271484375,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.0019492888823151588,
"fcm_dpo/delta": 0.0655849277973175,
"fcm_dpo/margin": 177.08547973632812,
"fcm_dpo/q_t": 0.420721173286438,
"grad_norm": 44.412681579589844,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -0.5013865828514099,
"logits/rejected": -0.48618659377098083,
"logps/chosen": -381.1546630859375,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -576.61865234375,
"loss": 1.1116,
"margin_dpo/margin_mean": 177.08547973632812,
"margin_dpo/margin_std": 211.73977661132812,
"step": 360
},
{
"KL/chosen_KL_mean": -342.9272766113281,
"KL/mean": -429.4185791015625,
"KL/rejected_KL_mean": -515.9098510742188,
"KL/std": 240.47056579589844,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.0020752213895320892,
"fcm_dpo/delta": 0.047938957810401917,
"fcm_dpo/margin": 172.98260498046875,
"fcm_dpo/q_t": 0.41855740547180176,
"grad_norm": 35.09357833862305,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -0.5015807747840881,
"logits/rejected": -0.49445807933807373,
"logps/chosen": -396.79681396484375,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -606.6791381835938,
"loss": 1.1306,
"margin_dpo/margin_mean": 172.9825897216797,
"margin_dpo/margin_std": 263.9880676269531,
"step": 361
},
{
"KL/chosen_KL_mean": -294.02984619140625,
"KL/mean": -431.71832275390625,
"KL/rejected_KL_mean": -569.40673828125,
"KL/std": 248.303955078125,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.0018198469188064337,
"fcm_dpo/delta": -0.14192384481430054,
"fcm_dpo/margin": 275.376953125,
"fcm_dpo/q_t": 0.37714123725891113,
"grad_norm": 30.883852005004883,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -0.5016046166419983,
"logits/rejected": -0.5126606225967407,
"logps/chosen": -352.66888427734375,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -674.9887084960938,
"loss": 0.9979,
"margin_dpo/margin_mean": 275.376953125,
"margin_dpo/margin_std": 260.5057373046875,
"step": 362
},
{
"KL/chosen_KL_mean": -272.70916748046875,
"KL/mean": -416.3315124511719,
"KL/rejected_KL_mean": -559.953857421875,
"KL/std": 247.9210205078125,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.0016754900570958853,
"fcm_dpo/delta": -0.10649609565734863,
"fcm_dpo/margin": 287.24468994140625,
"fcm_dpo/q_t": 0.3827672302722931,
"grad_norm": 27.67137908935547,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -0.4572371542453766,
"logits/rejected": -0.4535822868347168,
"logps/chosen": -317.26751708984375,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -634.6488037109375,
"loss": 1.0097,
"margin_dpo/margin_mean": 287.24468994140625,
"margin_dpo/margin_std": 268.51934814453125,
"step": 363
},
{
"KL/chosen_KL_mean": -304.93585205078125,
"KL/mean": -410.62762451171875,
"KL/rejected_KL_mean": -516.3193969726562,
"KL/std": 243.23239135742188,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.0017072297632694244,
"fcm_dpo/delta": 0.043868713080883026,
"fcm_dpo/margin": 211.38351440429688,
"fcm_dpo/q_t": 0.4163602590560913,
"grad_norm": 26.54780387878418,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.4880487322807312,
"logits/rejected": -0.49908387660980225,
"logps/chosen": -353.8304443359375,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -607.7151489257812,
"loss": 1.1142,
"margin_dpo/margin_mean": 211.38351440429688,
"margin_dpo/margin_std": 280.0267028808594,
"step": 364
},
{
"KL/chosen_KL_mean": -315.26373291015625,
"KL/mean": -426.0964050292969,
"KL/rejected_KL_mean": -536.9290771484375,
"KL/std": 257.34033203125,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.0017189306672662497,
"fcm_dpo/delta": 0.0233314111828804,
"fcm_dpo/margin": 221.66537475585938,
"fcm_dpo/q_t": 0.41273248195648193,
"grad_norm": 22.404428482055664,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -0.5078902244567871,
"logits/rejected": -0.5193514227867126,
"logps/chosen": -366.7564697265625,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -629.6307373046875,
"loss": 1.0905,
"margin_dpo/margin_mean": 221.66537475585938,
"margin_dpo/margin_std": 264.7396240234375,
"step": 365
},
{
"KL/chosen_KL_mean": -298.6568603515625,
"KL/mean": -417.4523010253906,
"KL/rejected_KL_mean": -536.2477416992188,
"KL/std": 257.2602233886719,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.0016834220150485635,
"fcm_dpo/delta": -0.002375733107328415,
"fcm_dpo/margin": 237.59083557128906,
"fcm_dpo/q_t": 0.4071364402770996,
"grad_norm": 26.219554901123047,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -0.49354439973831177,
"logits/rejected": -0.507148027420044,
"logps/chosen": -343.37744140625,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -619.55810546875,
"loss": 1.0866,
"margin_dpo/margin_mean": 237.59083557128906,
"margin_dpo/margin_std": 297.007080078125,
"step": 366
},
{
"KL/chosen_KL_mean": -325.56622314453125,
"KL/mean": -423.55853271484375,
"KL/rejected_KL_mean": -521.5508422851562,
"KL/std": 235.90292358398438,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.001811149064451456,
"fcm_dpo/delta": 0.053424857556819916,
"fcm_dpo/margin": 195.98464965820312,
"fcm_dpo/q_t": 0.41807931661605835,
"grad_norm": 25.911218643188477,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -0.5182558298110962,
"logits/rejected": -0.5040343999862671,
"logps/chosen": -383.97161865234375,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -598.3021850585938,
"loss": 1.1129,
"margin_dpo/margin_mean": 195.98464965820312,
"margin_dpo/margin_std": 253.58770751953125,
"step": 367
},
{
"KL/chosen_KL_mean": -320.83441162109375,
"KL/mean": -478.84674072265625,
"KL/rejected_KL_mean": -636.8590087890625,
"KL/std": 274.4027099609375,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.001620815135538578,
"fcm_dpo/delta": -0.1518523395061493,
"fcm_dpo/margin": 316.0246276855469,
"fcm_dpo/q_t": 0.3704509735107422,
"grad_norm": 30.158796310424805,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -0.5701684951782227,
"logits/rejected": -0.5831023454666138,
"logps/chosen": -365.28692626953125,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -735.414306640625,
"loss": 0.9731,
"margin_dpo/margin_mean": 316.02459716796875,
"margin_dpo/margin_std": 235.96087646484375,
"step": 368
},
{
"KL/chosen_KL_mean": -417.2987976074219,
"KL/mean": -500.9288330078125,
"KL/rejected_KL_mean": -584.5588989257812,
"KL/std": 269.81658935546875,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.0015922598540782928,
"fcm_dpo/delta": 0.041051216423511505,
"fcm_dpo/margin": 167.26007080078125,
"fcm_dpo/q_t": 0.4384276270866394,
"grad_norm": 24.34749984741211,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -0.5948071479797363,
"logits/rejected": -0.575736403465271,
"logps/chosen": -488.6802978515625,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -675.854736328125,
"loss": 1.1934,
"margin_dpo/margin_mean": 167.26007080078125,
"margin_dpo/margin_std": 302.8238525390625,
"step": 369
},
{
"KL/chosen_KL_mean": -449.61785888671875,
"KL/mean": -520.163330078125,
"KL/rejected_KL_mean": -590.7088623046875,
"KL/std": 294.0730895996094,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.0016860844334587455,
"fcm_dpo/delta": 0.07256323844194412,
"fcm_dpo/margin": 141.0910186767578,
"fcm_dpo/q_t": 0.4467887282371521,
"grad_norm": 34.77388000488281,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -0.6763471364974976,
"logits/rejected": -0.6677573323249817,
"logps/chosen": -521.225341796875,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -687.9686279296875,
"loss": 1.2396,
"margin_dpo/margin_mean": 141.0910186767578,
"margin_dpo/margin_std": 338.9228515625,
"step": 370
},
{
"KL/chosen_KL_mean": -435.55035400390625,
"KL/mean": -551.7735595703125,
"KL/rejected_KL_mean": -667.9967041015625,
"KL/std": 304.8518371582031,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.0017521032132208347,
"fcm_dpo/delta": -0.009448290802538395,
"fcm_dpo/margin": 232.4463348388672,
"fcm_dpo/q_t": 0.40719670057296753,
"grad_norm": 27.79642105102539,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -0.7335249185562134,
"logits/rejected": -0.720642626285553,
"logps/chosen": -504.96484375,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -767.1689453125,
"loss": 1.099,
"margin_dpo/margin_mean": 232.4463348388672,
"margin_dpo/margin_std": 332.423095703125,
"step": 371
},
{
"KL/chosen_KL_mean": -432.0025939941406,
"KL/mean": -568.457763671875,
"KL/rejected_KL_mean": -704.9129638671875,
"KL/std": 349.4569396972656,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.001637202687561512,
"fcm_dpo/delta": -0.061564333736896515,
"fcm_dpo/margin": 272.91033935546875,
"fcm_dpo/q_t": 0.3999387323856354,
"grad_norm": 26.669132232666016,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -0.7301241755485535,
"logits/rejected": -0.7251428365707397,
"logps/chosen": -493.8205871582031,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -783.452392578125,
"loss": 1.0802,
"margin_dpo/margin_mean": 272.9103088378906,
"margin_dpo/margin_std": 398.3546142578125,
"step": 372
},
{
"KL/chosen_KL_mean": -448.3118896484375,
"KL/mean": -581.019775390625,
"KL/rejected_KL_mean": -713.7276000976562,
"KL/std": 327.57122802734375,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.0016204612329602242,
"fcm_dpo/delta": -0.039967313408851624,
"fcm_dpo/margin": 265.41571044921875,
"fcm_dpo/q_t": 0.39898842573165894,
"grad_norm": 32.85871505737305,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -0.7037230730056763,
"logits/rejected": -0.6890777349472046,
"logps/chosen": -512.529052734375,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -799.687255859375,
"loss": 1.0727,
"margin_dpo/margin_mean": 265.41571044921875,
"margin_dpo/margin_std": 323.6532897949219,
"step": 373
},
{
"KL/chosen_KL_mean": -407.01507568359375,
"KL/mean": -524.6700439453125,
"KL/rejected_KL_mean": -642.324951171875,
"KL/std": 347.67022705078125,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.0015636862954124808,
"fcm_dpo/delta": 0.03692319244146347,
"fcm_dpo/margin": 235.3098602294922,
"fcm_dpo/q_t": 0.42001497745513916,
"grad_norm": 27.749187469482422,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -0.7614662647247314,
"logits/rejected": -0.7477431297302246,
"logps/chosen": -477.6652526855469,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -735.965087890625,
"loss": 1.1424,
"margin_dpo/margin_mean": 235.3098602294922,
"margin_dpo/margin_std": 396.4349060058594,
"step": 374
},
{
"KL/chosen_KL_mean": -424.40948486328125,
"KL/mean": -535.830078125,
"KL/rejected_KL_mean": -647.2506713867188,
"KL/std": 291.63531494140625,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.0016217189840972424,
"fcm_dpo/delta": 0.04367804527282715,
"fcm_dpo/margin": 222.84117126464844,
"fcm_dpo/q_t": 0.418157160282135,
"grad_norm": 41.795082092285156,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -0.6727270483970642,
"logits/rejected": -0.6715774536132812,
"logps/chosen": -484.4897155761719,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -736.18896484375,
"loss": 1.1357,
"margin_dpo/margin_mean": 222.84115600585938,
"margin_dpo/margin_std": 338.76904296875,
"step": 375
},
{
"KL/chosen_KL_mean": -414.8541259765625,
"KL/mean": -552.9400634765625,
"KL/rejected_KL_mean": -691.0260009765625,
"KL/std": 303.83843994140625,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.0016104718670248985,
"fcm_dpo/delta": -0.058481134474277496,
"fcm_dpo/margin": 276.171875,
"fcm_dpo/q_t": 0.39490610361099243,
"grad_norm": 31.177749633789062,
"learning_rate": 2.5e-07,
"logits/chosen": -0.7422023415565491,
"logits/rejected": -0.7344397306442261,
"logps/chosen": -477.5144348144531,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -796.5526123046875,
"loss": 1.0616,
"margin_dpo/margin_mean": 276.171875,
"margin_dpo/margin_std": 342.929443359375,
"step": 376
},
{
"KL/chosen_KL_mean": -424.4516906738281,
"KL/mean": -556.843994140625,
"KL/rejected_KL_mean": -689.2363891601562,
"KL/std": 344.01544189453125,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.0015855635283514857,
"fcm_dpo/delta": -0.02529209665954113,
"fcm_dpo/margin": 264.78466796875,
"fcm_dpo/q_t": 0.40445077419281006,
"grad_norm": 25.540964126586914,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -0.6894611120223999,
"logits/rejected": -0.6936970949172974,
"logps/chosen": -478.930419921875,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -787.939697265625,
"loss": 1.0868,
"margin_dpo/margin_mean": 264.78466796875,
"margin_dpo/margin_std": 363.934326171875,
"step": 377
},
{
"KL/chosen_KL_mean": -386.81268310546875,
"KL/mean": -535.3828735351562,
"KL/rejected_KL_mean": -683.9530029296875,
"KL/std": 294.50201416015625,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.001496739685535431,
"fcm_dpo/delta": -0.057502150535583496,
"fcm_dpo/margin": 297.14031982421875,
"fcm_dpo/q_t": 0.3941471576690674,
"grad_norm": 38.976097106933594,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -0.7334337830543518,
"logits/rejected": -0.7559584379196167,
"logps/chosen": -431.8332214355469,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -772.0,
"loss": 1.0487,
"margin_dpo/margin_mean": 297.14031982421875,
"margin_dpo/margin_std": 334.1382141113281,
"step": 378
},
{
"KL/chosen_KL_mean": -417.34716796875,
"KL/mean": -559.3892211914062,
"KL/rejected_KL_mean": -701.4312744140625,
"KL/std": 313.11883544921875,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.001429541502147913,
"fcm_dpo/delta": -0.010058283805847168,
"fcm_dpo/margin": 284.0840759277344,
"fcm_dpo/q_t": 0.4058372974395752,
"grad_norm": 29.72431182861328,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -0.6593658328056335,
"logits/rejected": -0.6784616112709045,
"logps/chosen": -470.52923583984375,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -815.7314453125,
"loss": 1.0836,
"margin_dpo/margin_mean": 284.0840759277344,
"margin_dpo/margin_std": 356.0497131347656,
"step": 379
},
{
"KL/chosen_KL_mean": -403.3964538574219,
"KL/mean": -557.42529296875,
"KL/rejected_KL_mean": -711.4541625976562,
"KL/std": 354.645751953125,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.0014169125352054834,
"fcm_dpo/delta": -0.04655863344669342,
"fcm_dpo/margin": 308.05767822265625,
"fcm_dpo/q_t": 0.39721158146858215,
"grad_norm": 28.100101470947266,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -0.632209300994873,
"logits/rejected": -0.664146900177002,
"logps/chosen": -454.7494812011719,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -815.6458740234375,
"loss": 1.0674,
"margin_dpo/margin_mean": 308.05767822265625,
"margin_dpo/margin_std": 388.6807861328125,
"step": 380
},
{
"KL/chosen_KL_mean": -407.2333068847656,
"KL/mean": -519.652587890625,
"KL/rejected_KL_mean": -632.0718994140625,
"KL/std": 297.33251953125,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.0014676698483526707,
"fcm_dpo/delta": 0.08423132449388504,
"fcm_dpo/margin": 224.83859252929688,
"fcm_dpo/q_t": 0.4268224537372589,
"grad_norm": 25.78670883178711,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -0.6134881973266602,
"logits/rejected": -0.5925121307373047,
"logps/chosen": -465.036376953125,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -711.291259765625,
"loss": 1.1633,
"margin_dpo/margin_mean": 224.83859252929688,
"margin_dpo/margin_std": 391.60101318359375,
"step": 381
},
{
"KL/chosen_KL_mean": -392.236572265625,
"KL/mean": -526.9730834960938,
"KL/rejected_KL_mean": -661.7095947265625,
"KL/std": 286.292724609375,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.0015065961051732302,
"fcm_dpo/delta": -0.007509663701057434,
"fcm_dpo/margin": 269.4730529785156,
"fcm_dpo/q_t": 0.40533730387687683,
"grad_norm": 28.11603546142578,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -0.6188876628875732,
"logits/rejected": -0.6265856027603149,
"logps/chosen": -458.2568359375,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -772.4197387695312,
"loss": 1.0722,
"margin_dpo/margin_mean": 269.4730224609375,
"margin_dpo/margin_std": 310.5892333984375,
"step": 382
},
{
"KL/chosen_KL_mean": -413.4278869628906,
"KL/mean": -549.6930541992188,
"KL/rejected_KL_mean": -685.958251953125,
"KL/std": 322.25799560546875,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.0014835065230727196,
"fcm_dpo/delta": -0.005721503868699074,
"fcm_dpo/margin": 272.53033447265625,
"fcm_dpo/q_t": 0.4068153500556946,
"grad_norm": 22.861156463623047,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -0.5609561204910278,
"logits/rejected": -0.5730615854263306,
"logps/chosen": -463.81939697265625,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -779.674072265625,
"loss": 1.1028,
"margin_dpo/margin_mean": 272.53033447265625,
"margin_dpo/margin_std": 395.47784423828125,
"step": 383
},
{
"KL/chosen_KL_mean": -410.1112060546875,
"KL/mean": -516.384033203125,
"KL/rejected_KL_mean": -622.6568603515625,
"KL/std": 280.2078857421875,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.001605054596439004,
"fcm_dpo/delta": 0.06686155498027802,
"fcm_dpo/margin": 212.545654296875,
"fcm_dpo/q_t": 0.42119306325912476,
"grad_norm": 24.116987228393555,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -0.5835964679718018,
"logits/rejected": -0.5749599933624268,
"logps/chosen": -462.15728759765625,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -708.417724609375,
"loss": 1.1212,
"margin_dpo/margin_mean": 212.54566955566406,
"margin_dpo/margin_std": 272.0644226074219,
"step": 384
},
{
"KL/chosen_KL_mean": -378.491455078125,
"KL/mean": -509.3629150390625,
"KL/rejected_KL_mean": -640.234375,
"KL/std": 248.0293426513672,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.0015787326265126467,
"fcm_dpo/delta": -0.016653146594762802,
"fcm_dpo/margin": 261.7428894042969,
"fcm_dpo/q_t": 0.4017704725265503,
"grad_norm": 38.663368225097656,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -0.6341022253036499,
"logits/rejected": -0.6096962690353394,
"logps/chosen": -444.04364013671875,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -718.062255859375,
"loss": 1.0631,
"margin_dpo/margin_mean": 261.7428894042969,
"margin_dpo/margin_std": 283.84356689453125,
"step": 385
},
{
"KL/chosen_KL_mean": -397.4541015625,
"KL/mean": -538.3778686523438,
"KL/rejected_KL_mean": -679.3016357421875,
"KL/std": 300.32208251953125,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.0015297075733542442,
"fcm_dpo/delta": -0.039631377905607224,
"fcm_dpo/margin": 281.8475341796875,
"fcm_dpo/q_t": 0.39953434467315674,
"grad_norm": 25.86907196044922,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -0.5929363965988159,
"logits/rejected": -0.5882759094238281,
"logps/chosen": -455.67596435546875,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -771.6290283203125,
"loss": 1.0695,
"margin_dpo/margin_mean": 281.8475341796875,
"margin_dpo/margin_std": 355.7461242675781,
"step": 386
},
{
"KL/chosen_KL_mean": -450.69427490234375,
"KL/mean": -571.585693359375,
"KL/rejected_KL_mean": -692.4771728515625,
"KL/std": 309.3702392578125,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.0015234483871608973,
"fcm_dpo/delta": 0.03800199180841446,
"fcm_dpo/margin": 241.78292846679688,
"fcm_dpo/q_t": 0.4150587320327759,
"grad_norm": 32.03988265991211,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -0.6444403529167175,
"logits/rejected": -0.6319328546524048,
"logps/chosen": -517.1137084960938,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -784.6463623046875,
"loss": 1.1086,
"margin_dpo/margin_mean": 241.78292846679688,
"margin_dpo/margin_std": 314.6268310546875,
"step": 387
},
{
"KL/chosen_KL_mean": -483.2527770996094,
"KL/mean": -631.25,
"KL/rejected_KL_mean": -779.247314453125,
"KL/std": 369.490234375,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.001472370233386755,
"fcm_dpo/delta": -0.05027584731578827,
"fcm_dpo/margin": 295.9945068359375,
"fcm_dpo/q_t": 0.40026527643203735,
"grad_norm": 68.39986419677734,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -0.6584256887435913,
"logits/rejected": -0.6686611175537109,
"logps/chosen": -533.3822021484375,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -883.6803588867188,
"loss": 1.0768,
"margin_dpo/margin_mean": 295.9945068359375,
"margin_dpo/margin_std": 401.4984130859375,
"step": 388
},
{
"KL/chosen_KL_mean": -505.9647521972656,
"KL/mean": -641.0068359375,
"KL/rejected_KL_mean": -776.0489501953125,
"KL/std": 349.13677978515625,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.001461312174797058,
"fcm_dpo/delta": 0.00432373583316803,
"fcm_dpo/margin": 270.084228515625,
"fcm_dpo/q_t": 0.4113960266113281,
"grad_norm": 51.183631896972656,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -0.6953232288360596,
"logits/rejected": -0.696498453617096,
"logps/chosen": -563.871337890625,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -853.9635009765625,
"loss": 1.1163,
"margin_dpo/margin_mean": 270.084228515625,
"margin_dpo/margin_std": 410.4221496582031,
"step": 389
},
{
"KL/chosen_KL_mean": -545.4517211914062,
"KL/mean": -687.8853149414062,
"KL/rejected_KL_mean": -830.3189086914062,
"KL/std": 405.5350646972656,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.0014416101621463895,
"fcm_dpo/delta": -0.01683669537305832,
"fcm_dpo/margin": 284.8671875,
"fcm_dpo/q_t": 0.4116922616958618,
"grad_norm": 30.454208374023438,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -0.614460825920105,
"logits/rejected": -0.6173591017723083,
"logps/chosen": -594.6776123046875,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -915.8471069335938,
"loss": 1.127,
"margin_dpo/margin_mean": 284.8671875,
"margin_dpo/margin_std": 498.7476806640625,
"step": 390
},
{
"KL/chosen_KL_mean": -546.4203491210938,
"KL/mean": -634.4364013671875,
"KL/rejected_KL_mean": -722.452392578125,
"KL/std": 375.4483337402344,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.0016221825499087572,
"fcm_dpo/delta": 0.13364286720752716,
"fcm_dpo/margin": 176.031982421875,
"fcm_dpo/q_t": 0.4394233822822571,
"grad_norm": 55.282936096191406,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -0.6284788846969604,
"logits/rejected": -0.6224967241287231,
"logps/chosen": -610.75,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -809.1905517578125,
"loss": 1.2094,
"margin_dpo/margin_mean": 176.03196716308594,
"margin_dpo/margin_std": 371.1688537597656,
"step": 391
},
{
"KL/chosen_KL_mean": -451.90289306640625,
"KL/mean": -614.0924072265625,
"KL/rejected_KL_mean": -776.2818603515625,
"KL/std": 364.5159912109375,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.001549735781736672,
"fcm_dpo/delta": -0.13615985214710236,
"fcm_dpo/margin": 324.3789367675781,
"fcm_dpo/q_t": 0.3801257610321045,
"grad_norm": 34.18647766113281,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -0.6204630136489868,
"logits/rejected": -0.6229850053787231,
"logps/chosen": -505.4068908691406,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -878.627685546875,
"loss": 1.0167,
"margin_dpo/margin_mean": 324.3789367675781,
"margin_dpo/margin_std": 355.14715576171875,
"step": 392
},
{
"KL/chosen_KL_mean": -472.4049072265625,
"KL/mean": -620.7923583984375,
"KL/rejected_KL_mean": -769.1798095703125,
"KL/std": 418.1456604003906,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.0014102270361036062,
"fcm_dpo/delta": -0.026089845225214958,
"fcm_dpo/margin": 296.77490234375,
"fcm_dpo/q_t": 0.4100596308708191,
"grad_norm": 26.125350952148438,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -0.6747171878814697,
"logits/rejected": -0.6873846054077148,
"logps/chosen": -518.8788452148438,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -841.148681640625,
"loss": 1.1204,
"margin_dpo/margin_mean": 296.77490234375,
"margin_dpo/margin_std": 503.2057189941406,
"step": 393
},
{
"KL/chosen_KL_mean": -548.73779296875,
"KL/mean": -690.5399169921875,
"KL/rejected_KL_mean": -832.342041015625,
"KL/std": 433.2088623046875,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.0014103709254413843,
"fcm_dpo/delta": -0.0008256174623966217,
"fcm_dpo/margin": 283.604248046875,
"fcm_dpo/q_t": 0.4093853831291199,
"grad_norm": 22.54509162902832,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -0.64982008934021,
"logits/rejected": -0.654583215713501,
"logps/chosen": -601.6493530273438,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -923.1646728515625,
"loss": 1.1035,
"margin_dpo/margin_mean": 283.604248046875,
"margin_dpo/margin_std": 410.2900695800781,
"step": 394
},
{
"KL/chosen_KL_mean": -520.23828125,
"KL/mean": -669.6795654296875,
"KL/rejected_KL_mean": -819.120849609375,
"KL/std": 406.344482421875,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.0013870133552700281,
"fcm_dpo/delta": -0.019791439175605774,
"fcm_dpo/margin": 298.882568359375,
"fcm_dpo/q_t": 0.40929269790649414,
"grad_norm": 33.52304458618164,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -0.7437783479690552,
"logits/rejected": -0.7509876489639282,
"logps/chosen": -582.784423828125,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -902.9034423828125,
"loss": 1.1252,
"margin_dpo/margin_mean": 298.882568359375,
"margin_dpo/margin_std": 504.57586669921875,
"step": 395
},
{
"KL/chosen_KL_mean": -525.498779296875,
"KL/mean": -663.1034545898438,
"KL/rejected_KL_mean": -800.7081298828125,
"KL/std": 373.2579650878906,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.0014105092268437147,
"fcm_dpo/delta": 0.014646814204752445,
"fcm_dpo/margin": 275.20941162109375,
"fcm_dpo/q_t": 0.4090290665626526,
"grad_norm": 26.613161087036133,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.7049363851547241,
"logits/rejected": -0.6887962818145752,
"logps/chosen": -594.4947509765625,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -889.3547973632812,
"loss": 1.0943,
"margin_dpo/margin_mean": 275.20941162109375,
"margin_dpo/margin_std": 351.0611572265625,
"step": 396
},
{
"KL/chosen_KL_mean": -484.30328369140625,
"KL/mean": -674.1336669921875,
"KL/rejected_KL_mean": -863.964111328125,
"KL/std": 388.2706298828125,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.0012838148977607489,
"fcm_dpo/delta": -0.11897562444210052,
"fcm_dpo/margin": 379.66082763671875,
"fcm_dpo/q_t": 0.3814941942691803,
"grad_norm": 36.56934356689453,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -0.7270249128341675,
"logits/rejected": -0.7500828504562378,
"logps/chosen": -545.5804443359375,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -967.0802001953125,
"loss": 1.0163,
"margin_dpo/margin_mean": 379.66082763671875,
"margin_dpo/margin_std": 395.0962829589844,
"step": 397
},
{
"KL/chosen_KL_mean": -559.9149169921875,
"KL/mean": -713.6265869140625,
"KL/rejected_KL_mean": -867.3382568359375,
"KL/std": 434.58392333984375,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.0012684231624007225,
"fcm_dpo/delta": 0.012464390136301517,
"fcm_dpo/margin": 307.4233093261719,
"fcm_dpo/q_t": 0.41516533493995667,
"grad_norm": 29.09773063659668,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.7229080200195312,
"logits/rejected": -0.7291736006736755,
"logps/chosen": -628.06640625,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -975.86181640625,
"loss": 1.1261,
"margin_dpo/margin_mean": 307.42333984375,
"margin_dpo/margin_std": 505.4172058105469,
"step": 398
},
{
"KL/chosen_KL_mean": -482.8316650390625,
"KL/mean": -642.2623901367188,
"KL/rejected_KL_mean": -801.6930541992188,
"KL/std": 400.37786865234375,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.0012626744573935866,
"fcm_dpo/delta": -0.003726685419678688,
"fcm_dpo/margin": 318.8613586425781,
"fcm_dpo/q_t": 0.4093470573425293,
"grad_norm": 36.91798400878906,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -0.7428172826766968,
"logits/rejected": -0.7134918570518494,
"logps/chosen": -543.7214965820312,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -879.6586303710938,
"loss": 1.1144,
"margin_dpo/margin_mean": 318.8613586425781,
"margin_dpo/margin_std": 496.3259582519531,
"step": 399
},
{
"KL/chosen_KL_mean": -457.9895324707031,
"KL/mean": -660.6600341796875,
"KL/rejected_KL_mean": -863.33056640625,
"KL/std": 368.4857177734375,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.0011767219984903932,
"fcm_dpo/delta": -0.10189603269100189,
"fcm_dpo/margin": 405.341064453125,
"fcm_dpo/q_t": 0.38432997465133667,
"grad_norm": 36.20450973510742,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.7186334133148193,
"logits/rejected": -0.7250533103942871,
"logps/chosen": -521.6331176757812,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -968.583251953125,
"loss": 1.0222,
"margin_dpo/margin_mean": 405.341064453125,
"margin_dpo/margin_std": 422.13092041015625,
"step": 400
},
{
"KL/chosen_KL_mean": -539.432861328125,
"KL/mean": -647.3896484375,
"KL/rejected_KL_mean": -755.346435546875,
"KL/std": 385.900146484375,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.0012087201466783881,
"fcm_dpo/delta": 0.04972143471240997,
"fcm_dpo/margin": 215.91360473632812,
"fcm_dpo/q_t": 0.44173485040664673,
"grad_norm": 31.359392166137695,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -0.7117223739624023,
"logits/rejected": -0.7047185301780701,
"logps/chosen": -596.5958862304688,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -839.138916015625,
"loss": 1.2256,
"margin_dpo/margin_mean": 215.9136199951172,
"margin_dpo/margin_std": 492.71820068359375,
"step": 401
},
{
"KL/chosen_KL_mean": -420.78167724609375,
"KL/mean": -644.1975708007812,
"KL/rejected_KL_mean": -867.613525390625,
"KL/std": 450.4141845703125,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.0010973564349114895,
"fcm_dpo/delta": -0.12137363106012344,
"fcm_dpo/margin": 446.831787109375,
"fcm_dpo/q_t": 0.3830028772354126,
"grad_norm": 26.827964782714844,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -0.7724575996398926,
"logits/rejected": -0.7748699188232422,
"logps/chosen": -471.52203369140625,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -948.6595458984375,
"loss": 1.0181,
"margin_dpo/margin_mean": 446.831787109375,
"margin_dpo/margin_std": 491.77197265625,
"step": 402
},
{
"KL/chosen_KL_mean": -427.5725402832031,
"KL/mean": -592.4838256835938,
"KL/rejected_KL_mean": -757.39501953125,
"KL/std": 362.35382080078125,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.0011073750210925937,
"fcm_dpo/delta": 0.04235755652189255,
"fcm_dpo/margin": 329.82257080078125,
"fcm_dpo/q_t": 0.41572099924087524,
"grad_norm": 28.50166893005371,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -0.7852243185043335,
"logits/rejected": -0.7927581071853638,
"logps/chosen": -474.7198486328125,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -834.6616821289062,
"loss": 1.1015,
"margin_dpo/margin_mean": 329.82257080078125,
"margin_dpo/margin_std": 395.7626037597656,
"step": 403
},
{
"KL/chosen_KL_mean": -516.7240600585938,
"KL/mean": -690.0592041015625,
"KL/rejected_KL_mean": -863.3944091796875,
"KL/std": 423.0172119140625,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.0011222802568227053,
"fcm_dpo/delta": 0.01353571005165577,
"fcm_dpo/margin": 346.67041015625,
"fcm_dpo/q_t": 0.4125150740146637,
"grad_norm": 25.302902221679688,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -0.7894694805145264,
"logits/rejected": -0.7903016805648804,
"logps/chosen": -564.5993041992188,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -940.5494384765625,
"loss": 1.1134,
"margin_dpo/margin_mean": 346.67041015625,
"margin_dpo/margin_std": 523.8775024414062,
"step": 404
},
{
"KL/chosen_KL_mean": -592.7296142578125,
"KL/mean": -713.805419921875,
"KL/rejected_KL_mean": -834.8812255859375,
"KL/std": 422.7801513671875,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.001129148993641138,
"fcm_dpo/delta": -0.005241929553449154,
"fcm_dpo/margin": 242.15170288085938,
"fcm_dpo/q_t": 0.4377569258213043,
"grad_norm": 39.06339645385742,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -0.783086895942688,
"logits/rejected": -0.7755570411682129,
"logps/chosen": -657.8924560546875,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -922.0680541992188,
"loss": 1.2157,
"margin_dpo/margin_mean": 242.15170288085938,
"margin_dpo/margin_std": 514.3509521484375,
"step": 405
},
{
"KL/chosen_KL_mean": -550.7774658203125,
"KL/mean": -743.4031982421875,
"KL/rejected_KL_mean": -936.0289306640625,
"KL/std": 466.4752197265625,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.0011199053842574358,
"fcm_dpo/delta": -0.04166682809591293,
"fcm_dpo/margin": 385.25140380859375,
"fcm_dpo/q_t": 0.40454065799713135,
"grad_norm": 29.93971061706543,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -0.8605983257293701,
"logits/rejected": -0.8947657346725464,
"logps/chosen": -600.518310546875,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -1028.1075439453125,
"loss": 1.0915,
"margin_dpo/margin_mean": 385.25140380859375,
"margin_dpo/margin_std": 562.6111450195312,
"step": 406
},
{
"KL/chosen_KL_mean": -557.0032348632812,
"KL/mean": -658.51025390625,
"KL/rejected_KL_mean": -760.0172729492188,
"KL/std": 408.6435852050781,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.0011084176367148757,
"fcm_dpo/delta": 0.05087389424443245,
"fcm_dpo/margin": 203.0139923095703,
"fcm_dpo/q_t": 0.44716814160346985,
"grad_norm": 29.988346099853516,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -0.9323893785476685,
"logits/rejected": -0.8998297452926636,
"logps/chosen": -613.333984375,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -837.5293579101562,
"loss": 1.2531,
"margin_dpo/margin_mean": 203.01400756835938,
"margin_dpo/margin_std": 529.3558959960938,
"step": 407
},
{
"KL/chosen_KL_mean": -562.3118896484375,
"KL/mean": -679.2642822265625,
"KL/rejected_KL_mean": -796.2166748046875,
"KL/std": 361.14996337890625,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.0012600787449628115,
"fcm_dpo/delta": 0.12069714069366455,
"fcm_dpo/margin": 233.90478515625,
"fcm_dpo/q_t": 0.43428733944892883,
"grad_norm": 34.71678161621094,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -0.9175188541412354,
"logits/rejected": -0.8953307867050171,
"logps/chosen": -632.1011962890625,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -886.3135986328125,
"loss": 1.1787,
"margin_dpo/margin_mean": 233.90478515625,
"margin_dpo/margin_std": 410.2579345703125,
"step": 408
},
{
"KL/chosen_KL_mean": -531.5665893554688,
"KL/mean": -659.5404663085938,
"KL/rejected_KL_mean": -787.514404296875,
"KL/std": 434.8625183105469,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.0013779137516394258,
"fcm_dpo/delta": 0.05021357536315918,
"fcm_dpo/margin": 255.94775390625,
"fcm_dpo/q_t": 0.41811615228652954,
"grad_norm": 53.07970428466797,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -0.801446795463562,
"logits/rejected": -0.78272545337677,
"logps/chosen": -598.884033203125,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -872.4186401367188,
"loss": 1.1736,
"margin_dpo/margin_mean": 255.94773864746094,
"margin_dpo/margin_std": 472.74395751953125,
"step": 409
},
{
"KL/chosen_KL_mean": -491.54156494140625,
"KL/mean": -645.4672241210938,
"KL/rejected_KL_mean": -799.392822265625,
"KL/std": 392.8507385253906,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.0013317305129021406,
"fcm_dpo/delta": -0.012640236876904964,
"fcm_dpo/margin": 307.85125732421875,
"fcm_dpo/q_t": 0.40540868043899536,
"grad_norm": 28.094219207763672,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -0.8844398260116577,
"logits/rejected": -0.8990808725357056,
"logps/chosen": -543.0069580078125,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -882.591796875,
"loss": 1.0982,
"margin_dpo/margin_mean": 307.85125732421875,
"margin_dpo/margin_std": 440.26861572265625,
"step": 410
},
{
"KL/chosen_KL_mean": -492.3099670410156,
"KL/mean": -623.0086059570312,
"KL/rejected_KL_mean": -753.7071533203125,
"KL/std": 405.5526123046875,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.0012510113883763552,
"fcm_dpo/delta": -0.062365103513002396,
"fcm_dpo/margin": 261.3972473144531,
"fcm_dpo/q_t": 0.42639321088790894,
"grad_norm": 39.17562484741211,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -0.9218274354934692,
"logits/rejected": -0.9327045679092407,
"logps/chosen": -544.6172485351562,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -834.4021606445312,
"loss": 1.1733,
"margin_dpo/margin_mean": 261.397216796875,
"margin_dpo/margin_std": 473.8624267578125,
"step": 411
},
{
"KL/chosen_KL_mean": -472.92578125,
"KL/mean": -619.5107421875,
"KL/rejected_KL_mean": -766.095703125,
"KL/std": 371.1781005859375,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.0012766936561092734,
"fcm_dpo/delta": 0.031684231013059616,
"fcm_dpo/margin": 293.1698913574219,
"fcm_dpo/q_t": 0.41487789154052734,
"grad_norm": 37.587581634521484,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -0.9481087327003479,
"logits/rejected": -0.9869192838668823,
"logps/chosen": -526.0699462890625,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -866.156494140625,
"loss": 1.1127,
"margin_dpo/margin_mean": 293.1698913574219,
"margin_dpo/margin_std": 415.78509521484375,
"step": 412
},
{
"KL/chosen_KL_mean": -546.9744873046875,
"KL/mean": -695.7308349609375,
"KL/rejected_KL_mean": -844.4873046875,
"KL/std": 423.4039306640625,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.0013015582226216793,
"fcm_dpo/delta": 0.0158353503793478,
"fcm_dpo/margin": 297.5128173828125,
"fcm_dpo/q_t": 0.4121004343032837,
"grad_norm": 36.7993049621582,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -0.8297325968742371,
"logits/rejected": -0.8475293517112732,
"logps/chosen": -608.556396484375,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -943.960693359375,
"loss": 1.1252,
"margin_dpo/margin_mean": 297.5128173828125,
"margin_dpo/margin_std": 477.887451171875,
"step": 413
},
{
"KL/chosen_KL_mean": -513.6051025390625,
"KL/mean": -674.848876953125,
"KL/rejected_KL_mean": -836.0927124023438,
"KL/std": 387.4964294433594,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.0013245465233922005,
"fcm_dpo/delta": -0.03918169438838959,
"fcm_dpo/margin": 322.4875793457031,
"fcm_dpo/q_t": 0.3994929790496826,
"grad_norm": 43.984615325927734,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -0.7619544267654419,
"logits/rejected": -0.7554280757904053,
"logps/chosen": -560.236572265625,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -923.7392578125,
"loss": 1.096,
"margin_dpo/margin_mean": 322.4875793457031,
"margin_dpo/margin_std": 455.6772766113281,
"step": 414
},
{
"KL/chosen_KL_mean": -519.7416381835938,
"KL/mean": -640.4639892578125,
"KL/rejected_KL_mean": -761.186279296875,
"KL/std": 358.64996337890625,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.001341789960861206,
"fcm_dpo/delta": 0.09107168018817902,
"fcm_dpo/margin": 241.4446258544922,
"fcm_dpo/q_t": 0.42783957719802856,
"grad_norm": 31.809572219848633,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -0.7860684990882874,
"logits/rejected": -0.7803636789321899,
"logps/chosen": -598.3599853515625,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -861.663818359375,
"loss": 1.1536,
"margin_dpo/margin_mean": 241.44464111328125,
"margin_dpo/margin_std": 387.5479431152344,
"step": 415
},
{
"KL/chosen_KL_mean": -462.289306640625,
"KL/mean": -652.3590087890625,
"KL/rejected_KL_mean": -842.4287109375,
"KL/std": 418.4091796875,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.0012784367427229881,
"fcm_dpo/delta": -0.11329878866672516,
"fcm_dpo/margin": 380.139404296875,
"fcm_dpo/q_t": 0.38521865010261536,
"grad_norm": 30.71705436706543,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -0.7888989448547363,
"logits/rejected": -0.80275559425354,
"logps/chosen": -520.5684204101562,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -932.9974365234375,
"loss": 1.0321,
"margin_dpo/margin_mean": 380.1394348144531,
"margin_dpo/margin_std": 444.5953369140625,
"step": 416
},
{
"KL/chosen_KL_mean": -447.36212158203125,
"KL/mean": -610.0339965820312,
"KL/rejected_KL_mean": -772.7059326171875,
"KL/std": 384.17840576171875,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.0012431307695806026,
"fcm_dpo/delta": -0.006137235090136528,
"fcm_dpo/margin": 325.34375,
"fcm_dpo/q_t": 0.4053758382797241,
"grad_norm": 39.485050201416016,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -0.7083637714385986,
"logits/rejected": -0.6805850863456726,
"logps/chosen": -497.56085205078125,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -840.8577880859375,
"loss": 1.0748,
"margin_dpo/margin_mean": 325.3437805175781,
"margin_dpo/margin_std": 378.35235595703125,
"step": 417
},
{
"KL/chosen_KL_mean": -498.91168212890625,
"KL/mean": -664.226318359375,
"KL/rejected_KL_mean": -829.5408935546875,
"KL/std": 416.49688720703125,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.0012598626781255007,
"fcm_dpo/delta": -0.02940238267183304,
"fcm_dpo/margin": 330.62921142578125,
"fcm_dpo/q_t": 0.4053995609283447,
"grad_norm": 24.680252075195312,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -0.796850323677063,
"logits/rejected": -0.7809560894966125,
"logps/chosen": -580.89013671875,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -934.2323608398438,
"loss": 1.1034,
"margin_dpo/margin_mean": 330.62921142578125,
"margin_dpo/margin_std": 468.53973388671875,
"step": 418
},
{
"KL/chosen_KL_mean": -468.39617919921875,
"KL/mean": -656.1996459960938,
"KL/rejected_KL_mean": -844.0031127929688,
"KL/std": 375.4820556640625,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.0011481782421469688,
"fcm_dpo/delta": -0.040392979979515076,
"fcm_dpo/margin": 375.6069641113281,
"fcm_dpo/q_t": 0.39766836166381836,
"grad_norm": 35.54838562011719,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -0.6798853278160095,
"logits/rejected": -0.6791231632232666,
"logps/chosen": -521.3448486328125,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -935.586181640625,
"loss": 1.0594,
"margin_dpo/margin_mean": 375.60693359375,
"margin_dpo/margin_std": 432.56622314453125,
"step": 419
},
{
"KL/chosen_KL_mean": -585.70361328125,
"KL/mean": -711.439208984375,
"KL/rejected_KL_mean": -837.1748657226562,
"KL/std": 429.8891296386719,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.0012487138155847788,
"fcm_dpo/delta": 0.09940655529499054,
"fcm_dpo/margin": 251.47128295898438,
"fcm_dpo/q_t": 0.4328416585922241,
"grad_norm": 44.155242919921875,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -0.810115396976471,
"logits/rejected": -0.7681174278259277,
"logps/chosen": -663.4735107421875,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -906.4947509765625,
"loss": 1.1992,
"margin_dpo/margin_mean": 251.47128295898438,
"margin_dpo/margin_std": 534.525390625,
"step": 420
},
{
"KL/chosen_KL_mean": -498.656005859375,
"KL/mean": -636.3494873046875,
"KL/rejected_KL_mean": -774.04296875,
"KL/std": 402.7722473144531,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.0013234158977866173,
"fcm_dpo/delta": 0.041565168648958206,
"fcm_dpo/margin": 275.386962890625,
"fcm_dpo/q_t": 0.41690126061439514,
"grad_norm": 27.106782913208008,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -0.7847892045974731,
"logits/rejected": -0.7707439661026001,
"logps/chosen": -552.421875,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -863.324462890625,
"loss": 1.1328,
"margin_dpo/margin_mean": 275.3869934082031,
"margin_dpo/margin_std": 421.31298828125,
"step": 421
},
{
"KL/chosen_KL_mean": -588.316162109375,
"KL/mean": -753.931640625,
"KL/rejected_KL_mean": -919.5469970703125,
"KL/std": 450.30499267578125,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.0012808962492272258,
"fcm_dpo/delta": -0.031088268384337425,
"fcm_dpo/margin": 331.2308349609375,
"fcm_dpo/q_t": 0.4033207297325134,
"grad_norm": 26.31610107421875,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -0.8320428133010864,
"logits/rejected": -0.8418641090393066,
"logps/chosen": -656.949951171875,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -1007.4105224609375,
"loss": 1.1125,
"margin_dpo/margin_mean": 331.2308349609375,
"margin_dpo/margin_std": 535.4093627929688,
"step": 422
},
{
"KL/chosen_KL_mean": -558.1864624023438,
"KL/mean": -731.43212890625,
"KL/rejected_KL_mean": -904.677734375,
"KL/std": 429.63372802734375,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.0012374825309962034,
"fcm_dpo/delta": -0.0368797592818737,
"fcm_dpo/margin": 346.49127197265625,
"fcm_dpo/q_t": 0.40145501494407654,
"grad_norm": 33.875404357910156,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -0.8104654550552368,
"logits/rejected": -0.827370822429657,
"logps/chosen": -613.1767578125,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -990.9842529296875,
"loss": 1.0925,
"margin_dpo/margin_mean": 346.4912414550781,
"margin_dpo/margin_std": 503.84454345703125,
"step": 423
},
{
"KL/chosen_KL_mean": -510.28045654296875,
"KL/mean": -682.5518188476562,
"KL/rejected_KL_mean": -854.8231201171875,
"KL/std": 413.15972900390625,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.001186060020700097,
"fcm_dpo/delta": -0.01438332349061966,
"fcm_dpo/margin": 344.54266357421875,
"fcm_dpo/q_t": 0.40733110904693604,
"grad_norm": 37.09081268310547,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -0.7975935935974121,
"logits/rejected": -0.7788383960723877,
"logps/chosen": -566.2923583984375,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -921.3020629882812,
"loss": 1.0954,
"margin_dpo/margin_mean": 344.54266357421875,
"margin_dpo/margin_std": 480.84130859375,
"step": 424
},
{
"KL/chosen_KL_mean": -609.7572021484375,
"KL/mean": -721.0950927734375,
"KL/rejected_KL_mean": -832.43310546875,
"KL/std": 407.82891845703125,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.0012294601183384657,
"fcm_dpo/delta": 0.027459844946861267,
"fcm_dpo/margin": 222.6759490966797,
"fcm_dpo/q_t": 0.4368723928928375,
"grad_norm": 42.740379333496094,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -0.7805401086807251,
"logits/rejected": -0.7881333827972412,
"logps/chosen": -656.626220703125,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -928.3585815429688,
"loss": 1.2167,
"margin_dpo/margin_mean": 222.67593383789062,
"margin_dpo/margin_std": 486.15093994140625,
"step": 425
},
{
"KL/chosen_KL_mean": -552.8094482421875,
"KL/mean": -687.5247192382812,
"KL/rejected_KL_mean": -822.239990234375,
"KL/std": 365.3626403808594,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.0013338683638721704,
"fcm_dpo/delta": 0.04180013760924339,
"fcm_dpo/margin": 269.4305419921875,
"fcm_dpo/q_t": 0.4174906015396118,
"grad_norm": 31.232389450073242,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -0.7621093988418579,
"logits/rejected": -0.7303955554962158,
"logps/chosen": -629.3930053710938,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -903.506591796875,
"loss": 1.1272,
"margin_dpo/margin_mean": 269.4305419921875,
"margin_dpo/margin_std": 378.654052734375,
"step": 426
},
{
"KL/chosen_KL_mean": -467.79608154296875,
"KL/mean": -581.63623046875,
"KL/rejected_KL_mean": -695.476318359375,
"KL/std": 333.85369873046875,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.001394979190081358,
"fcm_dpo/delta": 0.09802726656198502,
"fcm_dpo/margin": 227.68026733398438,
"fcm_dpo/q_t": 0.4301951825618744,
"grad_norm": 31.7608642578125,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -0.8063141703605652,
"logits/rejected": -0.78669673204422,
"logps/chosen": -532.6499633789062,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -774.0423583984375,
"loss": 1.1608,
"margin_dpo/margin_mean": 227.68026733398438,
"margin_dpo/margin_std": 382.787353515625,
"step": 427
},
{
"KL/chosen_KL_mean": -608.790771484375,
"KL/mean": -786.853515625,
"KL/rejected_KL_mean": -964.916259765625,
"KL/std": 468.04583740234375,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.0013052403228357434,
"fcm_dpo/delta": -0.090778648853302,
"fcm_dpo/margin": 356.1253967285156,
"fcm_dpo/q_t": 0.3941190838813782,
"grad_norm": 28.149211883544922,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -0.7752005457878113,
"logits/rejected": -0.7893061637878418,
"logps/chosen": -671.427490234375,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -1068.197998046875,
"loss": 1.088,
"margin_dpo/margin_mean": 356.1253967285156,
"margin_dpo/margin_std": 543.524169921875,
"step": 428
},
{
"KL/chosen_KL_mean": -629.3082275390625,
"KL/mean": -756.3204345703125,
"KL/rejected_KL_mean": -883.3326416015625,
"KL/std": 410.94122314453125,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.0012956894934177399,
"fcm_dpo/delta": -0.020538704469799995,
"fcm_dpo/margin": 254.02444458007812,
"fcm_dpo/q_t": 0.4243999719619751,
"grad_norm": 43.50310516357422,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -0.7827075719833374,
"logits/rejected": -0.7630304098129272,
"logps/chosen": -710.542236328125,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -975.1275634765625,
"loss": 1.1856,
"margin_dpo/margin_mean": 254.02444458007812,
"margin_dpo/margin_std": 494.28680419921875,
"step": 429
},
{
"KL/chosen_KL_mean": -546.2984619140625,
"KL/mean": -644.2332763671875,
"KL/rejected_KL_mean": -742.1680297851562,
"KL/std": 393.195556640625,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.0013126166304573417,
"fcm_dpo/delta": 0.04607279598712921,
"fcm_dpo/margin": 195.86962890625,
"fcm_dpo/q_t": 0.44172602891921997,
"grad_norm": 39.36235427856445,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.7814037799835205,
"logits/rejected": -0.7843170166015625,
"logps/chosen": -607.21875,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -846.5908203125,
"loss": 1.2239,
"margin_dpo/margin_mean": 195.86962890625,
"margin_dpo/margin_std": 441.08013916015625,
"step": 430
},
{
"KL/chosen_KL_mean": -474.2396240234375,
"KL/mean": -651.661376953125,
"KL/rejected_KL_mean": -829.0831298828125,
"KL/std": 401.1722412109375,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.0012775419745594263,
"fcm_dpo/delta": -0.06899101287126541,
"fcm_dpo/margin": 354.843505859375,
"fcm_dpo/q_t": 0.39390021562576294,
"grad_norm": 31.863265991210938,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -0.7518160343170166,
"logits/rejected": -0.7592271566390991,
"logps/chosen": -531.58837890625,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -921.9234008789062,
"loss": 1.067,
"margin_dpo/margin_mean": 354.8435363769531,
"margin_dpo/margin_std": 455.1669921875,
"step": 431
},
{
"KL/chosen_KL_mean": -427.3382263183594,
"KL/mean": -617.7256469726562,
"KL/rejected_KL_mean": -808.113037109375,
"KL/std": 403.57916259765625,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.001215646741911769,
"fcm_dpo/delta": -0.08303441107273102,
"fcm_dpo/margin": 380.77484130859375,
"fcm_dpo/q_t": 0.39162373542785645,
"grad_norm": 47.37468719482422,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -0.6925067901611328,
"logits/rejected": -0.685772180557251,
"logps/chosen": -498.41302490234375,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -906.692626953125,
"loss": 1.0385,
"margin_dpo/margin_mean": 380.77484130859375,
"margin_dpo/margin_std": 422.8524169921875,
"step": 432
},
{
"KL/chosen_KL_mean": -561.0573120117188,
"KL/mean": -678.9639282226562,
"KL/rejected_KL_mean": -796.8705444335938,
"KL/std": 433.2787780761719,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.0012703599641099572,
"fcm_dpo/delta": 0.11619932949542999,
"fcm_dpo/margin": 235.8131561279297,
"fcm_dpo/q_t": 0.4302235245704651,
"grad_norm": 37.460628509521484,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -0.7900607585906982,
"logits/rejected": -0.7896082401275635,
"logps/chosen": -619.3305053710938,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -892.8214111328125,
"loss": 1.2116,
"margin_dpo/margin_mean": 235.81317138671875,
"margin_dpo/margin_std": 522.4442749023438,
"step": 433
},
{
"KL/chosen_KL_mean": -496.65863037109375,
"KL/mean": -640.20361328125,
"KL/rejected_KL_mean": -783.7486572265625,
"KL/std": 404.4720458984375,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.0013023980427533388,
"fcm_dpo/delta": 0.0312434583902359,
"fcm_dpo/margin": 287.09002685546875,
"fcm_dpo/q_t": 0.4213806688785553,
"grad_norm": 25.82297706604004,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -0.8235654830932617,
"logits/rejected": -0.8260775208473206,
"logps/chosen": -558.63232421875,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -862.2472534179688,
"loss": 1.1396,
"margin_dpo/margin_mean": 287.09002685546875,
"margin_dpo/margin_std": 501.02880859375,
"step": 434
},
{
"KL/chosen_KL_mean": -480.15570068359375,
"KL/mean": -635.9765625,
"KL/rejected_KL_mean": -791.79736328125,
"KL/std": 393.2747802734375,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.0013056218158453703,
"fcm_dpo/delta": -0.01002051867544651,
"fcm_dpo/margin": 311.6417236328125,
"fcm_dpo/q_t": 0.4069361090660095,
"grad_norm": 45.785640716552734,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -0.8830646276473999,
"logits/rejected": -0.8846416473388672,
"logps/chosen": -531.65771484375,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -879.3642578125,
"loss": 1.0991,
"margin_dpo/margin_mean": 311.6416931152344,
"margin_dpo/margin_std": 440.7518615722656,
"step": 435
},
{
"KL/chosen_KL_mean": -499.36517333984375,
"KL/mean": -633.2975463867188,
"KL/rejected_KL_mean": -767.22998046875,
"KL/std": 352.81036376953125,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.0013422095216810703,
"fcm_dpo/delta": 0.048316676169633865,
"fcm_dpo/margin": 267.86474609375,
"fcm_dpo/q_t": 0.4203526973724365,
"grad_norm": 30.017845153808594,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -0.732751727104187,
"logits/rejected": -0.6973142623901367,
"logps/chosen": -570.7689208984375,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -849.9577026367188,
"loss": 1.1375,
"margin_dpo/margin_mean": 267.86474609375,
"margin_dpo/margin_std": 426.91168212890625,
"step": 436
},
{
"KL/chosen_KL_mean": -500.2353515625,
"KL/mean": -638.3201293945312,
"KL/rejected_KL_mean": -776.4049072265625,
"KL/std": 347.3167724609375,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.0014004592085257173,
"fcm_dpo/delta": 0.01640990376472473,
"fcm_dpo/margin": 276.16949462890625,
"fcm_dpo/q_t": 0.4110352396965027,
"grad_norm": 28.889192581176758,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -0.8302721977233887,
"logits/rejected": -0.814739465713501,
"logps/chosen": -564.9796142578125,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -858.4484252929688,
"loss": 1.1092,
"margin_dpo/margin_mean": 276.16949462890625,
"margin_dpo/margin_std": 396.4515686035156,
"step": 437
},
{
"KL/chosen_KL_mean": -522.9442749023438,
"KL/mean": -677.5450439453125,
"KL/rejected_KL_mean": -832.145751953125,
"KL/std": 379.323974609375,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.001349923899397254,
"fcm_dpo/delta": -0.025827843695878983,
"fcm_dpo/margin": 309.201416015625,
"fcm_dpo/q_t": 0.40191444754600525,
"grad_norm": 42.13845443725586,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -0.8437032699584961,
"logits/rejected": -0.8374447226524353,
"logps/chosen": -581.9629516601562,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -915.2225341796875,
"loss": 1.0822,
"margin_dpo/margin_mean": 309.2014465332031,
"margin_dpo/margin_std": 397.62255859375,
"step": 438
},
{
"KL/chosen_KL_mean": -549.39306640625,
"KL/mean": -681.3974609375,
"KL/rejected_KL_mean": -813.4017944335938,
"KL/std": 387.31488037109375,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.001252303714863956,
"fcm_dpo/delta": -0.09410447627305984,
"fcm_dpo/margin": 264.0087890625,
"fcm_dpo/q_t": 0.4208075702190399,
"grad_norm": 36.43398666381836,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -0.8629584908485413,
"logits/rejected": -0.8729925155639648,
"logps/chosen": -603.1771240234375,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -897.3872680664062,
"loss": 1.1705,
"margin_dpo/margin_mean": 264.0087890625,
"margin_dpo/margin_std": 464.46630859375,
"step": 439
},
{
"KL/chosen_KL_mean": -529.5724487304688,
"KL/mean": -701.4462890625,
"KL/rejected_KL_mean": -873.320068359375,
"KL/std": 479.51513671875,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.0012315923813730478,
"fcm_dpo/delta": -0.029587693512439728,
"fcm_dpo/margin": 343.74761962890625,
"fcm_dpo/q_t": 0.4066435694694519,
"grad_norm": 53.8433837890625,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -0.8866897821426392,
"logits/rejected": -0.8979896903038025,
"logps/chosen": -608.13916015625,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -969.81787109375,
"loss": 1.085,
"margin_dpo/margin_mean": 343.74761962890625,
"margin_dpo/margin_std": 490.02239990234375,
"step": 440
},
{
"KL/chosen_KL_mean": -619.644775390625,
"KL/mean": -767.532470703125,
"KL/rejected_KL_mean": -915.4201049804688,
"KL/std": 493.6292419433594,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.0012763803824782372,
"fcm_dpo/delta": 0.022389892488718033,
"fcm_dpo/margin": 295.7752685546875,
"fcm_dpo/q_t": 0.4217098355293274,
"grad_norm": 33.3763313293457,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -0.8680696487426758,
"logits/rejected": -0.8612392544746399,
"logps/chosen": -680.46923828125,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -1011.890869140625,
"loss": 1.1805,
"margin_dpo/margin_mean": 295.7752990722656,
"margin_dpo/margin_std": 602.29541015625,
"step": 441
},
{
"KL/chosen_KL_mean": -522.046875,
"KL/mean": -706.1878662109375,
"KL/rejected_KL_mean": -890.3289184570312,
"KL/std": 459.5678405761719,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.001231772475875914,
"fcm_dpo/delta": -0.07431311160326004,
"fcm_dpo/margin": 368.2820129394531,
"fcm_dpo/q_t": 0.3975549638271332,
"grad_norm": 34.67390823364258,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -0.8087929487228394,
"logits/rejected": -0.8081426620483398,
"logps/chosen": -569.05810546875,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -966.8681640625,
"loss": 1.089,
"margin_dpo/margin_mean": 368.28204345703125,
"margin_dpo/margin_std": 529.7601928710938,
"step": 442
},
{
"KL/chosen_KL_mean": -569.135009765625,
"KL/mean": -666.7360229492188,
"KL/rejected_KL_mean": -764.3370361328125,
"KL/std": 426.0760498046875,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.0012039494467899203,
"fcm_dpo/delta": 0.04213346913456917,
"fcm_dpo/margin": 195.20201110839844,
"fcm_dpo/q_t": 0.44704896211624146,
"grad_norm": 43.334136962890625,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -0.8095992803573608,
"logits/rejected": -0.7834180593490601,
"logps/chosen": -640.4080810546875,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -851.0169677734375,
"loss": 1.2562,
"margin_dpo/margin_mean": 195.20199584960938,
"margin_dpo/margin_std": 502.8472900390625,
"step": 443
},
{
"KL/chosen_KL_mean": -557.4791259765625,
"KL/mean": -734.4202270507812,
"KL/rejected_KL_mean": -911.361328125,
"KL/std": 455.6096496582031,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.001166836591437459,
"fcm_dpo/delta": -0.01813267171382904,
"fcm_dpo/margin": 353.8822021484375,
"fcm_dpo/q_t": 0.4072955846786499,
"grad_norm": 32.24861145019531,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -0.7987985610961914,
"logits/rejected": -0.8144040703773499,
"logps/chosen": -614.6928100585938,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -1008.6162109375,
"loss": 1.099,
"margin_dpo/margin_mean": 353.8822021484375,
"margin_dpo/margin_std": 520.219970703125,
"step": 444
},
{
"KL/chosen_KL_mean": -508.13671875,
"KL/mean": -659.2096557617188,
"KL/rejected_KL_mean": -810.2826538085938,
"KL/std": 361.33935546875,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.0012313367333263159,
"fcm_dpo/delta": 0.032493121922016144,
"fcm_dpo/margin": 302.14593505859375,
"fcm_dpo/q_t": 0.4164279103279114,
"grad_norm": 38.02067565917969,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -0.7978518009185791,
"logits/rejected": -0.7957039475440979,
"logps/chosen": -575.4365234375,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -902.96533203125,
"loss": 1.1248,
"margin_dpo/margin_mean": 302.14593505859375,
"margin_dpo/margin_std": 454.81597900390625,
"step": 445
},
{
"KL/chosen_KL_mean": -450.0630798339844,
"KL/mean": -626.6373291015625,
"KL/rejected_KL_mean": -803.211669921875,
"KL/std": 392.5597229003906,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.001226876862347126,
"fcm_dpo/delta": -0.046956416219472885,
"fcm_dpo/margin": 353.14862060546875,
"fcm_dpo/q_t": 0.3958050608634949,
"grad_norm": 34.05545425415039,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -0.8068540692329407,
"logits/rejected": -0.8256067037582397,
"logps/chosen": -509.16156005859375,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -904.475830078125,
"loss": 1.0592,
"margin_dpo/margin_mean": 353.14862060546875,
"margin_dpo/margin_std": 380.9633483886719,
"step": 446
},
{
"KL/chosen_KL_mean": -495.2170715332031,
"KL/mean": -681.0345458984375,
"KL/rejected_KL_mean": -866.8519897460938,
"KL/std": 464.1243591308594,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.0011278802994638681,
"fcm_dpo/delta": -0.025529345497488976,
"fcm_dpo/margin": 371.6348876953125,
"fcm_dpo/q_t": 0.4053482413291931,
"grad_norm": 30.737730026245117,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -0.9250874519348145,
"logits/rejected": -0.9507585763931274,
"logps/chosen": -551.2924194335938,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -965.5467529296875,
"loss": 1.086,
"margin_dpo/margin_mean": 371.6348876953125,
"margin_dpo/margin_std": 520.7388916015625,
"step": 447
},
{
"KL/chosen_KL_mean": -539.2561645507812,
"KL/mean": -716.0037841796875,
"KL/rejected_KL_mean": -892.7515258789062,
"KL/std": 413.3590087890625,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.0011455777566879988,
"fcm_dpo/delta": -0.006920462474226952,
"fcm_dpo/margin": 353.4953308105469,
"fcm_dpo/q_t": 0.4064434766769409,
"grad_norm": 37.22543716430664,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -0.8032995462417603,
"logits/rejected": -0.8209420442581177,
"logps/chosen": -599.260009765625,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -995.0161743164062,
"loss": 1.083,
"margin_dpo/margin_mean": 353.4953308105469,
"margin_dpo/margin_std": 442.79266357421875,
"step": 448
},
{
"KL/chosen_KL_mean": -577.392333984375,
"KL/mean": -767.4705810546875,
"KL/rejected_KL_mean": -957.548828125,
"KL/std": 516.8084716796875,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.001104456139728427,
"fcm_dpo/delta": -0.025262048467993736,
"fcm_dpo/margin": 380.15655517578125,
"fcm_dpo/q_t": 0.40797027945518494,
"grad_norm": 27.65226173400879,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -0.8589591979980469,
"logits/rejected": -0.8559813499450684,
"logps/chosen": -644.9189453125,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -1046.145751953125,
"loss": 1.1076,
"margin_dpo/margin_mean": 380.15655517578125,
"margin_dpo/margin_std": 612.1381225585938,
"step": 449
},
{
"KL/chosen_KL_mean": -496.8646240234375,
"KL/mean": -713.1530151367188,
"KL/rejected_KL_mean": -929.4413452148438,
"KL/std": 456.3707275390625,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.00104955630376935,
"fcm_dpo/delta": -0.06971128284931183,
"fcm_dpo/margin": 432.5767822265625,
"fcm_dpo/q_t": 0.3907149136066437,
"grad_norm": 37.34757614135742,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -0.9782512187957764,
"logits/rejected": -1.0043165683746338,
"logps/chosen": -553.9727172851562,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -1032.1962890625,
"loss": 1.0356,
"margin_dpo/margin_mean": 432.5767822265625,
"margin_dpo/margin_std": 448.83770751953125,
"step": 450
},
{
"KL/chosen_KL_mean": -742.9453125,
"KL/mean": -891.0797119140625,
"KL/rejected_KL_mean": -1039.214111328125,
"KL/std": 587.5035400390625,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.0011060864198952913,
"fcm_dpo/delta": 0.08447873592376709,
"fcm_dpo/margin": 296.268798828125,
"fcm_dpo/q_t": 0.42436882853507996,
"grad_norm": 39.329833984375,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -1.0949430465698242,
"logits/rejected": -1.0765621662139893,
"logps/chosen": -801.4141845703125,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -1112.1435546875,
"loss": 1.2346,
"margin_dpo/margin_mean": 296.268798828125,
"margin_dpo/margin_std": 738.4291381835938,
"step": 451
},
{
"KL/chosen_KL_mean": -594.1732177734375,
"KL/mean": -763.3720703125,
"KL/rejected_KL_mean": -932.5709228515625,
"KL/std": 460.3730773925781,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.0011289503891021013,
"fcm_dpo/delta": 0.02220895327627659,
"fcm_dpo/margin": 338.39764404296875,
"fcm_dpo/q_t": 0.41150838136672974,
"grad_norm": 33.76683044433594,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -0.9474809169769287,
"logits/rejected": -0.9722226858139038,
"logps/chosen": -644.5637817382812,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -1030.34228515625,
"loss": 1.1039,
"margin_dpo/margin_mean": 338.39764404296875,
"margin_dpo/margin_std": 456.95947265625,
"step": 452
},
{
"KL/chosen_KL_mean": -649.19287109375,
"KL/mean": -851.3839721679688,
"KL/rejected_KL_mean": -1053.5750732421875,
"KL/std": 513.1878051757812,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.0010724079329520464,
"fcm_dpo/delta": -0.04788634181022644,
"fcm_dpo/margin": 404.3822021484375,
"fcm_dpo/q_t": 0.40426748991012573,
"grad_norm": 26.855628967285156,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -1.0186734199523926,
"logits/rejected": -1.0134930610656738,
"logps/chosen": -706.90771484375,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -1135.782470703125,
"loss": 1.1022,
"margin_dpo/margin_mean": 404.3821716308594,
"margin_dpo/margin_std": 631.357421875,
"step": 453
},
{
"KL/chosen_KL_mean": -824.8762817382812,
"KL/mean": -1020.529052734375,
"KL/rejected_KL_mean": -1216.181640625,
"KL/std": 584.076171875,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.0010508534032851458,
"fcm_dpo/delta": -0.01683557778596878,
"fcm_dpo/margin": 391.305419921875,
"fcm_dpo/q_t": 0.4103407859802246,
"grad_norm": 36.570125579833984,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -1.0660200119018555,
"logits/rejected": -1.0628576278686523,
"logps/chosen": -885.8218994140625,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -1301.1324462890625,
"loss": 1.1279,
"margin_dpo/margin_mean": 391.305419921875,
"margin_dpo/margin_std": 684.9031982421875,
"step": 454
},
{
"KL/chosen_KL_mean": -688.5186157226562,
"KL/mean": -917.4297485351562,
"KL/rejected_KL_mean": -1146.3408203125,
"KL/std": 531.5499877929688,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.0010322749149054289,
"fcm_dpo/delta": -0.09659485518932343,
"fcm_dpo/margin": 457.82232666015625,
"fcm_dpo/q_t": 0.38912808895111084,
"grad_norm": 33.90567398071289,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -0.9941831827163696,
"logits/rejected": -1.0336955785751343,
"logps/chosen": -733.4053344726562,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -1261.642333984375,
"loss": 1.063,
"margin_dpo/margin_mean": 457.8222961425781,
"margin_dpo/margin_std": 598.6575927734375,
"step": 455
},
{
"KL/chosen_KL_mean": -691.5862426757812,
"KL/mean": -909.7722778320312,
"KL/rejected_KL_mean": -1127.958251953125,
"KL/std": 566.892578125,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.0009644461097195745,
"fcm_dpo/delta": -0.02674005925655365,
"fcm_dpo/margin": 436.37200927734375,
"fcm_dpo/q_t": 0.40306729078292847,
"grad_norm": 33.307159423828125,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -0.9692044258117676,
"logits/rejected": -0.9966904520988464,
"logps/chosen": -748.623046875,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -1233.1761474609375,
"loss": 1.0658,
"margin_dpo/margin_mean": 436.37200927734375,
"margin_dpo/margin_std": 519.3140258789062,
"step": 456
},
{
"KL/chosen_KL_mean": -584.7559204101562,
"KL/mean": -828.9456787109375,
"KL/rejected_KL_mean": -1073.135498046875,
"KL/std": 520.8341064453125,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.0009004294406622648,
"fcm_dpo/delta": -0.0521702915430069,
"fcm_dpo/margin": 488.3794860839844,
"fcm_dpo/q_t": 0.39493173360824585,
"grad_norm": 42.788997650146484,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -0.982511043548584,
"logits/rejected": -1.0036410093307495,
"logps/chosen": -638.9984130859375,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -1158.2449951171875,
"loss": 1.0442,
"margin_dpo/margin_mean": 488.3795166015625,
"margin_dpo/margin_std": 512.117919921875,
"step": 457
},
{
"KL/chosen_KL_mean": -680.8214721679688,
"KL/mean": -844.1943359375,
"KL/rejected_KL_mean": -1007.5672607421875,
"KL/std": 468.6507568359375,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.0009023591410368681,
"fcm_dpo/delta": 0.005208446644246578,
"fcm_dpo/margin": 326.74578857421875,
"fcm_dpo/q_t": 0.4328933656215668,
"grad_norm": 29.520099639892578,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -0.9825581312179565,
"logits/rejected": -0.9830326437950134,
"logps/chosen": -736.2303466796875,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -1105.25048828125,
"loss": 1.1728,
"margin_dpo/margin_mean": 326.74578857421875,
"margin_dpo/margin_std": 542.132080078125,
"step": 458
},
{
"KL/chosen_KL_mean": -676.8546752929688,
"KL/mean": -901.5927734375,
"KL/rejected_KL_mean": -1126.330810546875,
"KL/std": 550.5717163085938,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.0008877612417563796,
"fcm_dpo/delta": -1.3709068298339844e-06,
"fcm_dpo/margin": 449.47607421875,
"fcm_dpo/q_t": 0.40948015451431274,
"grad_norm": 22.407264709472656,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -0.9684402942657471,
"logits/rejected": -1.0020538568496704,
"logps/chosen": -723.4121704101562,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -1212.4993896484375,
"loss": 1.1013,
"margin_dpo/margin_mean": 449.47607421875,
"margin_dpo/margin_std": 635.2473754882812,
"step": 459
},
{
"KL/chosen_KL_mean": -719.9354248046875,
"KL/mean": -945.6378173828125,
"KL/rejected_KL_mean": -1171.3402099609375,
"KL/std": 503.3649597167969,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.00090052280575037,
"fcm_dpo/delta": -0.008194062858819962,
"fcm_dpo/margin": 451.40478515625,
"fcm_dpo/q_t": 0.40608513355255127,
"grad_norm": 31.070011138916016,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -0.9713107943534851,
"logits/rejected": -1.0081372261047363,
"logps/chosen": -771.5703125,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -1275.45947265625,
"loss": 1.0757,
"margin_dpo/margin_mean": 451.40478515625,
"margin_dpo/margin_std": 542.3765869140625,
"step": 460
},
{
"KL/chosen_KL_mean": -740.08251953125,
"KL/mean": -919.7174072265625,
"KL/rejected_KL_mean": -1099.352294921875,
"KL/std": 511.79473876953125,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.0009528920636512339,
"fcm_dpo/delta": 0.06786449253559113,
"fcm_dpo/margin": 359.2697448730469,
"fcm_dpo/q_t": 0.4245432913303375,
"grad_norm": 25.247751235961914,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -1.029995083808899,
"logits/rejected": -1.0514973402023315,
"logps/chosen": -795.2644653320312,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -1185.8291015625,
"loss": 1.1423,
"margin_dpo/margin_mean": 359.269775390625,
"margin_dpo/margin_std": 578.6834716796875,
"step": 461
},
{
"KL/chosen_KL_mean": -744.9185791015625,
"KL/mean": -907.572265625,
"KL/rejected_KL_mean": -1070.2259521484375,
"KL/std": 544.3043212890625,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.0010181250981986523,
"fcm_dpo/delta": 0.0823717713356018,
"fcm_dpo/margin": 325.307373046875,
"fcm_dpo/q_t": 0.4292236864566803,
"grad_norm": 35.24434280395508,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -1.0601680278778076,
"logits/rejected": -1.0431756973266602,
"logps/chosen": -814.8466796875,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -1149.067138671875,
"loss": 1.1657,
"margin_dpo/margin_mean": 325.307373046875,
"margin_dpo/margin_std": 593.1353759765625,
"step": 462
},
{
"KL/chosen_KL_mean": -733.4375,
"KL/mean": -934.1990966796875,
"KL/rejected_KL_mean": -1134.960693359375,
"KL/std": 516.4095458984375,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.001036065979860723,
"fcm_dpo/delta": -0.020512185990810394,
"fcm_dpo/margin": 401.523193359375,
"fcm_dpo/q_t": 0.4055131673812866,
"grad_norm": 23.178422927856445,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -0.9877623319625854,
"logits/rejected": -1.0045299530029297,
"logps/chosen": -788.7119140625,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -1223.9857177734375,
"loss": 1.0915,
"margin_dpo/margin_mean": 401.523193359375,
"margin_dpo/margin_std": 563.6845092773438,
"step": 463
},
{
"KL/chosen_KL_mean": -697.1048583984375,
"KL/mean": -991.504150390625,
"KL/rejected_KL_mean": -1285.9033203125,
"KL/std": 571.4072265625,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.0008614182588644326,
"fcm_dpo/delta": -0.16394121944904327,
"fcm_dpo/margin": 588.7985229492188,
"fcm_dpo/q_t": 0.37307262420654297,
"grad_norm": 46.0228157043457,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -0.9791674017906189,
"logits/rejected": -1.0607080459594727,
"logps/chosen": -748.0171508789062,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -1388.3927001953125,
"loss": 1.002,
"margin_dpo/margin_mean": 588.7985229492188,
"margin_dpo/margin_std": 583.0584716796875,
"step": 464
},
{
"KL/chosen_KL_mean": -694.8887939453125,
"KL/mean": -952.684814453125,
"KL/rejected_KL_mean": -1210.4808349609375,
"KL/std": 611.1224365234375,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.0008492013439536095,
"fcm_dpo/delta": -0.048867009580135345,
"fcm_dpo/margin": 515.5921020507812,
"fcm_dpo/q_t": 0.39614564180374146,
"grad_norm": 30.500961303710938,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -0.9008214473724365,
"logits/rejected": -0.9379734396934509,
"logps/chosen": -755.005615234375,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -1324.4268798828125,
"loss": 1.0666,
"margin_dpo/margin_mean": 515.5921020507812,
"margin_dpo/margin_std": 642.5521240234375,
"step": 465
},
{
"KL/chosen_KL_mean": -780.0445556640625,
"KL/mean": -987.330322265625,
"KL/rejected_KL_mean": -1194.6162109375,
"KL/std": 571.94091796875,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.0008426806889474392,
"fcm_dpo/delta": 0.06057006120681763,
"fcm_dpo/margin": 414.57159423828125,
"fcm_dpo/q_t": 0.4206228256225586,
"grad_norm": 33.371273040771484,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -1.0010290145874023,
"logits/rejected": -1.007284164428711,
"logps/chosen": -832.9654541015625,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -1284.931640625,
"loss": 1.1329,
"margin_dpo/margin_mean": 414.57159423828125,
"margin_dpo/margin_std": 603.785400390625,
"step": 466
},
{
"KL/chosen_KL_mean": -1015.46484375,
"KL/mean": -1223.3994140625,
"KL/rejected_KL_mean": -1431.3341064453125,
"KL/std": 710.5194091796875,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.0008342203218489885,
"fcm_dpo/delta": -0.047805603593587875,
"fcm_dpo/margin": 415.86932373046875,
"fcm_dpo/q_t": 0.42023301124572754,
"grad_norm": 45.32796859741211,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -1.1425104141235352,
"logits/rejected": -1.14357328414917,
"logps/chosen": -1094.1806640625,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -1534.1943359375,
"loss": 1.1881,
"margin_dpo/margin_mean": 415.86932373046875,
"margin_dpo/margin_std": 850.1127319335938,
"step": 467
},
{
"KL/chosen_KL_mean": -783.6282348632812,
"KL/mean": -1058.208984375,
"KL/rejected_KL_mean": -1332.789794921875,
"KL/std": 650.537109375,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.0008028654847294092,
"fcm_dpo/delta": -0.052505575120449066,
"fcm_dpo/margin": 549.16162109375,
"fcm_dpo/q_t": 0.40023478865623474,
"grad_norm": 24.539749145507812,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -1.0484925508499146,
"logits/rejected": -1.0707437992095947,
"logps/chosen": -853.5635986328125,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -1433.818603515625,
"loss": 1.074,
"margin_dpo/margin_mean": 549.16162109375,
"margin_dpo/margin_std": 754.311279296875,
"step": 468
},
{
"KL/chosen_KL_mean": -840.949951171875,
"KL/mean": -1125.788330078125,
"KL/rejected_KL_mean": -1410.626953125,
"KL/std": 636.927001953125,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.0007483696099370718,
"fcm_dpo/delta": -0.038286350667476654,
"fcm_dpo/margin": 569.6768798828125,
"fcm_dpo/q_t": 0.3994740843772888,
"grad_norm": 37.96442413330078,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -1.0542805194854736,
"logits/rejected": -1.0895564556121826,
"logps/chosen": -909.0746459960938,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -1515.413330078125,
"loss": 1.0689,
"margin_dpo/margin_mean": 569.6769409179688,
"margin_dpo/margin_std": 686.1154174804688,
"step": 469
},
{
"KL/chosen_KL_mean": -708.578125,
"KL/mean": -937.5423583984375,
"KL/rejected_KL_mean": -1166.5064697265625,
"KL/std": 602.3682861328125,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.0007999057415872812,
"fcm_dpo/delta": 0.038524776697158813,
"fcm_dpo/margin": 457.9284362792969,
"fcm_dpo/q_t": 0.41636377573013306,
"grad_norm": 24.361658096313477,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -0.9929207563400269,
"logits/rejected": -1.0160140991210938,
"logps/chosen": -752.3699951171875,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -1249.2093505859375,
"loss": 1.1274,
"margin_dpo/margin_mean": 457.9284362792969,
"margin_dpo/margin_std": 697.048095703125,
"step": 470
},
{
"KL/chosen_KL_mean": -843.971923828125,
"KL/mean": -1043.7379150390625,
"KL/rejected_KL_mean": -1243.5037841796875,
"KL/std": 644.9224853515625,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.0008243155316449702,
"fcm_dpo/delta": 0.0843021422624588,
"fcm_dpo/margin": 399.5318603515625,
"fcm_dpo/q_t": 0.42968451976776123,
"grad_norm": 39.733333587646484,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -1.1035377979278564,
"logits/rejected": -1.1254429817199707,
"logps/chosen": -907.3114624023438,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -1327.1142578125,
"loss": 1.1649,
"margin_dpo/margin_mean": 399.5318603515625,
"margin_dpo/margin_std": 707.281982421875,
"step": 471
},
{
"KL/chosen_KL_mean": -862.677734375,
"KL/mean": -1102.48583984375,
"KL/rejected_KL_mean": -1342.2938232421875,
"KL/std": 753.7301025390625,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.0008416681084781885,
"fcm_dpo/delta": -0.006307002156972885,
"fcm_dpo/margin": 479.61614990234375,
"fcm_dpo/q_t": 0.411409854888916,
"grad_norm": 30.461442947387695,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -1.062950849533081,
"logits/rejected": -1.0488746166229248,
"logps/chosen": -946.3438110351562,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -1459.5030517578125,
"loss": 1.1356,
"margin_dpo/margin_mean": 479.61614990234375,
"margin_dpo/margin_std": 826.3499755859375,
"step": 472
},
{
"KL/chosen_KL_mean": -1003.8850708007812,
"KL/mean": -1114.1983642578125,
"KL/rejected_KL_mean": -1224.5115966796875,
"KL/std": 738.931640625,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0008559616981074214,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 220.62660217285156,
"fcm_dpo/q_t": 0.4583190083503723,
"grad_norm": 101.26480102539062,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -1.0762083530426025,
"logits/rejected": -1.0582425594329834,
"logps/chosen": -1067.382080078125,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -1305.658203125,
"loss": 1.3516,
"margin_dpo/margin_mean": 220.62660217285156,
"margin_dpo/margin_std": 942.107421875,
"step": 473
},
{
"KL/chosen_KL_mean": -769.4073486328125,
"KL/mean": -986.923583984375,
"KL/rejected_KL_mean": -1204.43994140625,
"KL/std": 693.4749755859375,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.00081443547969684,
"fcm_dpo/delta": -0.10206399112939835,
"fcm_dpo/margin": 435.03271484375,
"fcm_dpo/q_t": 0.4185516834259033,
"grad_norm": 32.762020111083984,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -1.048344612121582,
"logits/rejected": -1.0651113986968994,
"logps/chosen": -822.019287109375,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -1294.5203857421875,
"loss": 1.1724,
"margin_dpo/margin_mean": 435.03271484375,
"margin_dpo/margin_std": 791.048583984375,
"step": 474
},
{
"KL/chosen_KL_mean": -539.2615356445312,
"KL/mean": -790.7316284179688,
"KL/rejected_KL_mean": -1042.20166015625,
"KL/std": 493.97027587890625,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.0007748671341687441,
"fcm_dpo/delta": 0.01266252901405096,
"fcm_dpo/margin": 502.940185546875,
"fcm_dpo/q_t": 0.40863102674484253,
"grad_norm": 52.324066162109375,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -1.0229177474975586,
"logits/rejected": -1.0607411861419678,
"logps/chosen": -581.7567138671875,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -1132.2646484375,
"loss": 1.0766,
"margin_dpo/margin_mean": 502.940185546875,
"margin_dpo/margin_std": 544.89208984375,
"step": 475
},
{
"KL/chosen_KL_mean": -634.4459228515625,
"KL/mean": -844.801025390625,
"KL/rejected_KL_mean": -1055.1561279296875,
"KL/std": 564.7354736328125,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.0008217949070967734,
"fcm_dpo/delta": 0.06553815305233002,
"fcm_dpo/margin": 420.71014404296875,
"fcm_dpo/q_t": 0.4223518669605255,
"grad_norm": 62.65557098388672,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -0.9976848363876343,
"logits/rejected": -1.0245535373687744,
"logps/chosen": -677.3953247070312,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -1128.866455078125,
"loss": 1.1221,
"margin_dpo/margin_mean": 420.71014404296875,
"margin_dpo/margin_std": 576.537353515625,
"step": 476
},
{
"KL/chosen_KL_mean": -674.2581787109375,
"KL/mean": -865.6505126953125,
"KL/rejected_KL_mean": -1057.0428466796875,
"KL/std": 557.9041748046875,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.0008936204831115901,
"fcm_dpo/delta": 0.06722961366176605,
"fcm_dpo/margin": 382.7846984863281,
"fcm_dpo/q_t": 0.42294949293136597,
"grad_norm": 28.50010108947754,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -1.0419853925704956,
"logits/rejected": -1.0101861953735352,
"logps/chosen": -745.03076171875,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -1133.18017578125,
"loss": 1.1533,
"margin_dpo/margin_mean": 382.78466796875,
"margin_dpo/margin_std": 655.9279174804688,
"step": 477
},
{
"KL/chosen_KL_mean": -557.695068359375,
"KL/mean": -778.8765869140625,
"KL/rejected_KL_mean": -1000.0580444335938,
"KL/std": 482.09033203125,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.0009058309369720519,
"fcm_dpo/delta": -0.0017963144928216934,
"fcm_dpo/margin": 442.363037109375,
"fcm_dpo/q_t": 0.407336950302124,
"grad_norm": 43.02243423461914,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.946250319480896,
"logits/rejected": -0.9788520336151123,
"logps/chosen": -599.1356201171875,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -1085.4200439453125,
"loss": 1.0866,
"margin_dpo/margin_mean": 442.363037109375,
"margin_dpo/margin_std": 562.7466430664062,
"step": 478
},
{
"KL/chosen_KL_mean": -741.406982421875,
"KL/mean": -950.5042114257812,
"KL/rejected_KL_mean": -1159.601318359375,
"KL/std": 631.4393920898438,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.0009333643829450011,
"fcm_dpo/delta": 0.004260525107383728,
"fcm_dpo/margin": 418.19439697265625,
"fcm_dpo/q_t": 0.4147453308105469,
"grad_norm": 30.091028213500977,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -0.9998948574066162,
"logits/rejected": -1.0395790338516235,
"logps/chosen": -795.31494140625,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -1254.7177734375,
"loss": 1.1436,
"margin_dpo/margin_mean": 418.19439697265625,
"margin_dpo/margin_std": 732.4306640625,
"step": 479
},
{
"KL/chosen_KL_mean": -865.813720703125,
"KL/mean": -1064.1876220703125,
"KL/rejected_KL_mean": -1262.5615234375,
"KL/std": 623.685791015625,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.0009484676993452013,
"fcm_dpo/delta": 0.020787112414836884,
"fcm_dpo/margin": 396.74774169921875,
"fcm_dpo/q_t": 0.41742902994155884,
"grad_norm": 50.18581008911133,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -1.0877306461334229,
"logits/rejected": -1.0981130599975586,
"logps/chosen": -924.4964599609375,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -1345.493896484375,
"loss": 1.1513,
"margin_dpo/margin_mean": 396.74774169921875,
"margin_dpo/margin_std": 687.515869140625,
"step": 480
},
{
"KL/chosen_KL_mean": -735.0712890625,
"KL/mean": -1004.9774169921875,
"KL/rejected_KL_mean": -1274.883544921875,
"KL/std": 640.9407958984375,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.000862010580021888,
"fcm_dpo/delta": -0.08503904193639755,
"fcm_dpo/margin": 539.812255859375,
"fcm_dpo/q_t": 0.391613632440567,
"grad_norm": 32.593658447265625,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -1.1026833057403564,
"logits/rejected": -1.142751693725586,
"logps/chosen": -790.0355224609375,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -1367.303955078125,
"loss": 1.0549,
"margin_dpo/margin_mean": 539.812255859375,
"margin_dpo/margin_std": 687.2750854492188,
"step": 481
},
{
"KL/chosen_KL_mean": -830.9714965820312,
"KL/mean": -957.885009765625,
"KL/rejected_KL_mean": -1084.7984619140625,
"KL/std": 625.652099609375,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.0008747933316044509,
"fcm_dpo/delta": 0.0788145363330841,
"fcm_dpo/margin": 253.82705688476562,
"fcm_dpo/q_t": 0.4496998190879822,
"grad_norm": 46.61983871459961,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -0.9891968369483948,
"logits/rejected": -0.9823375940322876,
"logps/chosen": -898.5249633789062,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -1172.387939453125,
"loss": 1.2716,
"margin_dpo/margin_mean": 253.82705688476562,
"margin_dpo/margin_std": 734.9395751953125,
"step": 482
},
{
"KL/chosen_KL_mean": -761.9283447265625,
"KL/mean": -1030.1768798828125,
"KL/rejected_KL_mean": -1298.425537109375,
"KL/std": 638.6688232421875,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.0008516391972079873,
"fcm_dpo/delta": -0.07522018998861313,
"fcm_dpo/margin": 536.4971923828125,
"fcm_dpo/q_t": 0.3947756588459015,
"grad_norm": 29.71031379699707,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -1.0698552131652832,
"logits/rejected": -1.0944595336914062,
"logps/chosen": -825.2581787109375,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -1394.21240234375,
"loss": 1.0606,
"margin_dpo/margin_mean": 536.4971313476562,
"margin_dpo/margin_std": 694.40087890625,
"step": 483
},
{
"KL/chosen_KL_mean": -655.5699462890625,
"KL/mean": -913.00341796875,
"KL/rejected_KL_mean": -1170.43701171875,
"KL/std": 608.8349609375,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.000842659268528223,
"fcm_dpo/delta": -0.0462777316570282,
"fcm_dpo/margin": 514.8670043945312,
"fcm_dpo/q_t": 0.3998936414718628,
"grad_norm": 36.89812469482422,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -1.0860092639923096,
"logits/rejected": -1.119147539138794,
"logps/chosen": -714.7080688476562,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -1254.808349609375,
"loss": 1.0775,
"margin_dpo/margin_mean": 514.8670043945312,
"margin_dpo/margin_std": 665.43310546875,
"step": 484
},
{
"KL/chosen_KL_mean": -662.37353515625,
"KL/mean": -922.7737426757812,
"KL/rejected_KL_mean": -1183.174072265625,
"KL/std": 650.9013671875,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.0007992424070835114,
"fcm_dpo/delta": -0.020606661215424538,
"fcm_dpo/margin": 520.8004760742188,
"fcm_dpo/q_t": 0.4069255590438843,
"grad_norm": 32.7896842956543,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -0.9968384504318237,
"logits/rejected": -1.0324972867965698,
"logps/chosen": -721.2230834960938,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -1286.5380859375,
"loss": 1.0967,
"margin_dpo/margin_mean": 520.8004760742188,
"margin_dpo/margin_std": 764.5574951171875,
"step": 485
},
{
"KL/chosen_KL_mean": -828.6250610351562,
"KL/mean": -1101.12109375,
"KL/rejected_KL_mean": -1373.6173095703125,
"KL/std": 724.00244140625,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.0007718969136476517,
"fcm_dpo/delta": -0.02629101648926735,
"fcm_dpo/margin": 544.9921875,
"fcm_dpo/q_t": 0.4044193923473358,
"grad_norm": 39.17950439453125,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -1.1160706281661987,
"logits/rejected": -1.1396567821502686,
"logps/chosen": -883.884765625,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -1465.756591796875,
"loss": 1.1214,
"margin_dpo/margin_mean": 544.9921875,
"margin_dpo/margin_std": 907.802001953125,
"step": 486
},
{
"KL/chosen_KL_mean": -896.8312377929688,
"KL/mean": -1101.290771484375,
"KL/rejected_KL_mean": -1305.750244140625,
"KL/std": 672.8538818359375,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.0008255624561570585,
"fcm_dpo/delta": 0.07200143486261368,
"fcm_dpo/margin": 408.9190673828125,
"fcm_dpo/q_t": 0.42319005727767944,
"grad_norm": 29.267309188842773,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -1.1223185062408447,
"logits/rejected": -1.135122537612915,
"logps/chosen": -949.89453125,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -1398.1690673828125,
"loss": 1.1616,
"margin_dpo/margin_mean": 408.9190673828125,
"margin_dpo/margin_std": 730.7452392578125,
"step": 487
},
{
"KL/chosen_KL_mean": -746.6181640625,
"KL/mean": -975.8692626953125,
"KL/rejected_KL_mean": -1205.120361328125,
"KL/std": 613.75439453125,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.0008516233647242188,
"fcm_dpo/delta": 0.0090005062520504,
"fcm_dpo/margin": 458.5020751953125,
"fcm_dpo/q_t": 0.41139358282089233,
"grad_norm": 33.18574523925781,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -1.0628390312194824,
"logits/rejected": -1.0790064334869385,
"logps/chosen": -798.8463745117188,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -1289.1268310546875,
"loss": 1.1022,
"margin_dpo/margin_mean": 458.5020751953125,
"margin_dpo/margin_std": 630.7826538085938,
"step": 488
},
{
"KL/chosen_KL_mean": -688.994873046875,
"KL/mean": -898.1849365234375,
"KL/rejected_KL_mean": -1107.375,
"KL/std": 607.5086669921875,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.0008741497294977307,
"fcm_dpo/delta": 0.03951010853052139,
"fcm_dpo/margin": 418.3800964355469,
"fcm_dpo/q_t": 0.41885343194007874,
"grad_norm": 23.560508728027344,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -1.0819485187530518,
"logits/rejected": -1.084855318069458,
"logps/chosen": -744.9844970703125,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -1186.7730712890625,
"loss": 1.1243,
"margin_dpo/margin_mean": 418.380126953125,
"margin_dpo/margin_std": 633.3609619140625,
"step": 489
},
{
"KL/chosen_KL_mean": -841.709716796875,
"KL/mean": -1069.041015625,
"KL/rejected_KL_mean": -1296.372314453125,
"KL/std": 748.8676147460938,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.0008761119097471237,
"fcm_dpo/delta": 0.0016551017761230469,
"fcm_dpo/margin": 454.66265869140625,
"fcm_dpo/q_t": 0.41499996185302734,
"grad_norm": 53.128875732421875,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -1.144291639328003,
"logits/rejected": -1.196203589439392,
"logps/chosen": -894.0761108398438,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -1406.7813720703125,
"loss": 1.1641,
"margin_dpo/margin_mean": 454.6626281738281,
"margin_dpo/margin_std": 890.287353515625,
"step": 490
},
{
"KL/chosen_KL_mean": -801.8388061523438,
"KL/mean": -956.7105102539062,
"KL/rejected_KL_mean": -1111.5821533203125,
"KL/std": 636.75390625,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.0009558784076943994,
"fcm_dpo/delta": 0.12077458947896957,
"fcm_dpo/margin": 309.7433776855469,
"fcm_dpo/q_t": 0.4361428916454315,
"grad_norm": 39.206363677978516,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -1.1435798406600952,
"logits/rejected": -1.1361366510391235,
"logps/chosen": -861.955078125,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -1184.85498046875,
"loss": 1.1942,
"margin_dpo/margin_mean": 309.7433776855469,
"margin_dpo/margin_std": 618.770751953125,
"step": 491
},
{
"KL/chosen_KL_mean": -781.2764892578125,
"KL/mean": -925.2930908203125,
"KL/rejected_KL_mean": -1069.3095703125,
"KL/std": 593.3115844726562,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.0010589244775474072,
"fcm_dpo/delta": 0.11252547055482864,
"fcm_dpo/margin": 288.0330810546875,
"fcm_dpo/q_t": 0.43613219261169434,
"grad_norm": 65.94483184814453,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -1.2232658863067627,
"logits/rejected": -1.1998727321624756,
"logps/chosen": -835.270751953125,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -1141.96923828125,
"loss": 1.2344,
"margin_dpo/margin_mean": 288.0330810546875,
"margin_dpo/margin_std": 703.6218872070312,
"step": 492
},
{
"KL/chosen_KL_mean": -823.78759765625,
"KL/mean": -995.8330078125,
"KL/rejected_KL_mean": -1167.87841796875,
"KL/std": 663.9137573242188,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.001078893314115703,
"fcm_dpo/delta": 0.029794402420520782,
"fcm_dpo/margin": 344.0907897949219,
"fcm_dpo/q_t": 0.41683828830718994,
"grad_norm": 46.201839447021484,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -1.2597391605377197,
"logits/rejected": -1.2192950248718262,
"logps/chosen": -899.2848510742188,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -1255.201416015625,
"loss": 1.1584,
"margin_dpo/margin_mean": 344.0908203125,
"margin_dpo/margin_std": 617.4366455078125,
"step": 493
},
{
"KL/chosen_KL_mean": -648.2301025390625,
"KL/mean": -912.5694580078125,
"KL/rejected_KL_mean": -1176.9088134765625,
"KL/std": 621.4219360351562,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.000986246857792139,
"fcm_dpo/delta": -0.16815879940986633,
"fcm_dpo/margin": 528.6787109375,
"fcm_dpo/q_t": 0.37546783685684204,
"grad_norm": 32.91933822631836,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -1.0481430292129517,
"logits/rejected": -1.0907659530639648,
"logps/chosen": -689.58935546875,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -1263.000244140625,
"loss": 1.0206,
"margin_dpo/margin_mean": 528.6787109375,
"margin_dpo/margin_std": 635.6686401367188,
"step": 494
},
{
"KL/chosen_KL_mean": -706.6998291015625,
"KL/mean": -920.7889404296875,
"KL/rejected_KL_mean": -1134.8779296875,
"KL/std": 593.68115234375,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.0009533504489809275,
"fcm_dpo/delta": -0.010413305833935738,
"fcm_dpo/margin": 428.17816162109375,
"fcm_dpo/q_t": 0.4118821620941162,
"grad_norm": 38.663597106933594,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -1.2045929431915283,
"logits/rejected": -1.22074294090271,
"logps/chosen": -770.23486328125,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -1226.302490234375,
"loss": 1.1255,
"margin_dpo/margin_mean": 428.17816162109375,
"margin_dpo/margin_std": 734.76904296875,
"step": 495
},
{
"KL/chosen_KL_mean": -847.32470703125,
"KL/mean": -956.1488037109375,
"KL/rejected_KL_mean": -1064.972900390625,
"KL/std": 484.2099609375,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.0009861316066235304,
"fcm_dpo/delta": 0.08399142324924469,
"fcm_dpo/margin": 217.6483154296875,
"fcm_dpo/q_t": 0.45106980204582214,
"grad_norm": 73.58963012695312,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -1.1229890584945679,
"logits/rejected": -1.0876094102859497,
"logps/chosen": -919.9166259765625,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -1149.30224609375,
"loss": 1.2516,
"margin_dpo/margin_mean": 217.6483154296875,
"margin_dpo/margin_std": 544.780517578125,
"step": 496
},
{
"KL/chosen_KL_mean": -713.8504028320312,
"KL/mean": -832.6170654296875,
"KL/rejected_KL_mean": -951.3837890625,
"KL/std": 529.62255859375,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.0010455565061420202,
"fcm_dpo/delta": 0.017400572076439857,
"fcm_dpo/margin": 237.53341674804688,
"fcm_dpo/q_t": 0.44449126720428467,
"grad_norm": 23.886857986450195,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -1.10499107837677,
"logits/rejected": -1.0940041542053223,
"logps/chosen": -772.4443359375,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -1027.672119140625,
"loss": 1.2264,
"margin_dpo/margin_mean": 237.53341674804688,
"margin_dpo/margin_std": 521.065673828125,
"step": 497
},
{
"KL/chosen_KL_mean": -697.0528564453125,
"KL/mean": -902.389404296875,
"KL/rejected_KL_mean": -1107.725830078125,
"KL/std": 593.9395751953125,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.0010199977550655603,
"fcm_dpo/delta": -0.024301331490278244,
"fcm_dpo/margin": 410.67303466796875,
"fcm_dpo/q_t": 0.40479618310928345,
"grad_norm": 30.96971321105957,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -1.1298916339874268,
"logits/rejected": -1.1229459047317505,
"logps/chosen": -768.258544921875,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -1191.6839599609375,
"loss": 1.1151,
"margin_dpo/margin_mean": 410.67303466796875,
"margin_dpo/margin_std": 663.222900390625,
"step": 498
},
{
"KL/chosen_KL_mean": -577.2625732421875,
"KL/mean": -813.7861328125,
"KL/rejected_KL_mean": -1050.3096923828125,
"KL/std": 576.3260498046875,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.0009620316559448838,
"fcm_dpo/delta": -0.0720800906419754,
"fcm_dpo/margin": 473.047119140625,
"fcm_dpo/q_t": 0.3953206241130829,
"grad_norm": 36.52189254760742,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -1.118801474571228,
"logits/rejected": -1.1649246215820312,
"logps/chosen": -628.5177001953125,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -1151.388427734375,
"loss": 1.0697,
"margin_dpo/margin_mean": 473.047119140625,
"margin_dpo/margin_std": 653.6705322265625,
"step": 499
},
{
"KL/chosen_KL_mean": -667.104248046875,
"KL/mean": -863.0795288085938,
"KL/rejected_KL_mean": -1059.054931640625,
"KL/std": 504.5117492675781,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.000964233884587884,
"fcm_dpo/delta": 0.027243074029684067,
"fcm_dpo/margin": 391.9505615234375,
"fcm_dpo/q_t": 0.4149992763996124,
"grad_norm": 31.563304901123047,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -1.1256649494171143,
"logits/rejected": -1.1642621755599976,
"logps/chosen": -724.1317138671875,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -1152.989013671875,
"loss": 1.1294,
"margin_dpo/margin_mean": 391.9505615234375,
"margin_dpo/margin_std": 622.8431396484375,
"step": 500
},
{
"KL/chosen_KL_mean": -576.75732421875,
"KL/mean": -784.8577880859375,
"KL/rejected_KL_mean": -992.958251953125,
"KL/std": 533.4993286132812,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.0009827818721532822,
"fcm_dpo/delta": -0.012303851544857025,
"fcm_dpo/margin": 416.20098876953125,
"fcm_dpo/q_t": 0.40838170051574707,
"grad_norm": 32.86457061767578,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -1.0381156206130981,
"logits/rejected": -1.0312669277191162,
"logps/chosen": -631.1168212890625,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -1073.114990234375,
"loss": 1.1167,
"margin_dpo/margin_mean": 416.2009582519531,
"margin_dpo/margin_std": 665.16796875,
"step": 501
},
{
"KL/chosen_KL_mean": -536.04345703125,
"KL/mean": -745.6707763671875,
"KL/rejected_KL_mean": -955.2979736328125,
"KL/std": 505.134765625,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.0009646775433793664,
"fcm_dpo/delta": -0.00565625075250864,
"fcm_dpo/margin": 419.25445556640625,
"fcm_dpo/q_t": 0.40891292691230774,
"grad_norm": 21.100860595703125,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -1.0066035985946655,
"logits/rejected": -0.9983298182487488,
"logps/chosen": -603.64404296875,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -1038.246826171875,
"loss": 1.0891,
"margin_dpo/margin_mean": 419.25445556640625,
"margin_dpo/margin_std": 562.9284057617188,
"step": 502
},
{
"KL/chosen_KL_mean": -536.14404296875,
"KL/mean": -733.31298828125,
"KL/rejected_KL_mean": -930.4820556640625,
"KL/std": 481.45635986328125,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.0009646597900427878,
"fcm_dpo/delta": 0.023929888382554054,
"fcm_dpo/margin": 394.33795166015625,
"fcm_dpo/q_t": 0.4147661328315735,
"grad_norm": 21.984729766845703,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -0.999343752861023,
"logits/rejected": -0.9944263696670532,
"logps/chosen": -591.222412109375,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -1012.9874877929688,
"loss": 1.1043,
"margin_dpo/margin_mean": 394.33795166015625,
"margin_dpo/margin_std": 542.9268798828125,
"step": 503
},
{
"KL/chosen_KL_mean": -596.7254028320312,
"KL/mean": -843.4456787109375,
"KL/rejected_KL_mean": -1090.1658935546875,
"KL/std": 561.0550537109375,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.0009342943667434156,
"fcm_dpo/delta": -0.07896675169467926,
"fcm_dpo/margin": 493.44061279296875,
"fcm_dpo/q_t": 0.39245522022247314,
"grad_norm": 32.341758728027344,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -1.0672714710235596,
"logits/rejected": -1.1046226024627686,
"logps/chosen": -656.691162109375,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -1193.927978515625,
"loss": 1.058,
"margin_dpo/margin_mean": 493.4405517578125,
"margin_dpo/margin_std": 614.1143188476562,
"step": 504
},
{
"KL/chosen_KL_mean": -736.76806640625,
"KL/mean": -838.422607421875,
"KL/rejected_KL_mean": -940.0771484375,
"KL/std": 535.1500244140625,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.0009070243104360998,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 203.3091583251953,
"fcm_dpo/q_t": 0.45628899335861206,
"grad_norm": 33.11906814575195,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -1.0850073099136353,
"logits/rejected": -1.0549143552780151,
"logps/chosen": -812.9229736328125,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -1028.6624755859375,
"loss": 1.2666,
"margin_dpo/margin_mean": 203.30914306640625,
"margin_dpo/margin_std": 520.629638671875,
"step": 505
},
{
"KL/chosen_KL_mean": -579.7349853515625,
"KL/mean": -793.5402221679688,
"KL/rejected_KL_mean": -1007.345458984375,
"KL/std": 549.4849853515625,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.0009010919602587819,
"fcm_dpo/delta": 0.016882240772247314,
"fcm_dpo/margin": 427.61041259765625,
"fcm_dpo/q_t": 0.4115670919418335,
"grad_norm": 39.05805587768555,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -1.088112235069275,
"logits/rejected": -1.1115524768829346,
"logps/chosen": -628.695556640625,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -1085.760498046875,
"loss": 1.1101,
"margin_dpo/margin_mean": 427.61041259765625,
"margin_dpo/margin_std": 606.4409790039062,
"step": 506
},
{
"KL/chosen_KL_mean": -689.96875,
"KL/mean": -874.892578125,
"KL/rejected_KL_mean": -1059.81640625,
"KL/std": 653.38037109375,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.0009622873039916158,
"fcm_dpo/delta": 0.053409378975629807,
"fcm_dpo/margin": 369.84759521484375,
"fcm_dpo/q_t": 0.42470353841781616,
"grad_norm": 33.041019439697266,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -1.1047194004058838,
"logits/rejected": -1.1137211322784424,
"logps/chosen": -756.770263671875,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -1155.1893310546875,
"loss": 1.1655,
"margin_dpo/margin_mean": 369.84759521484375,
"margin_dpo/margin_std": 710.9828491210938,
"step": 507
},
{
"KL/chosen_KL_mean": -782.4303588867188,
"KL/mean": -958.6895751953125,
"KL/rejected_KL_mean": -1134.94873046875,
"KL/std": 643.3538818359375,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.001016983762383461,
"fcm_dpo/delta": 0.04988085478544235,
"fcm_dpo/margin": 352.5183410644531,
"fcm_dpo/q_t": 0.4246112108230591,
"grad_norm": 35.355621337890625,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -1.1216602325439453,
"logits/rejected": -1.1300599575042725,
"logps/chosen": -853.7338256835938,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -1230.576171875,
"loss": 1.1984,
"margin_dpo/margin_mean": 352.51837158203125,
"margin_dpo/margin_std": 768.9783935546875,
"step": 508
},
{
"KL/chosen_KL_mean": -592.2100830078125,
"KL/mean": -759.6229248046875,
"KL/rejected_KL_mean": -927.03564453125,
"KL/std": 474.44573974609375,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.0010790289379656315,
"fcm_dpo/delta": 0.04428262263536453,
"fcm_dpo/margin": 334.8254699707031,
"fcm_dpo/q_t": 0.4207102656364441,
"grad_norm": 41.111671447753906,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -0.9959691166877747,
"logits/rejected": -0.9935662150382996,
"logps/chosen": -656.029052734375,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -1010.2921142578125,
"loss": 1.1492,
"margin_dpo/margin_mean": 334.8254699707031,
"margin_dpo/margin_std": 584.1224975585938,
"step": 509
},
{
"KL/chosen_KL_mean": -746.1397705078125,
"KL/mean": -961.1923828125,
"KL/rejected_KL_mean": -1176.244873046875,
"KL/std": 542.093017578125,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.0010361545719206333,
"fcm_dpo/delta": -0.05862744525074959,
"fcm_dpo/margin": 430.10516357421875,
"fcm_dpo/q_t": 0.394826740026474,
"grad_norm": 30.757688522338867,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -1.0323694944381714,
"logits/rejected": -1.0630911588668823,
"logps/chosen": -798.0181884765625,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -1279.0101318359375,
"loss": 1.0551,
"margin_dpo/margin_mean": 430.10516357421875,
"margin_dpo/margin_std": 506.75775146484375,
"step": 510
},
{
"KL/chosen_KL_mean": -670.8756103515625,
"KL/mean": -856.5685424804688,
"KL/rejected_KL_mean": -1042.261474609375,
"KL/std": 570.9805908203125,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.001025435863994062,
"fcm_dpo/delta": 0.02368216961622238,
"fcm_dpo/margin": 371.38592529296875,
"fcm_dpo/q_t": 0.415951669216156,
"grad_norm": 40.323726654052734,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -1.109389305114746,
"logits/rejected": -1.1236944198608398,
"logps/chosen": -731.1136474609375,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -1135.1181640625,
"loss": 1.155,
"margin_dpo/margin_mean": 371.38592529296875,
"margin_dpo/margin_std": 696.55078125,
"step": 511
},
{
"KL/chosen_KL_mean": -533.606201171875,
"KL/mean": -791.3221435546875,
"KL/rejected_KL_mean": -1049.0382080078125,
"KL/std": 548.933349609375,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.000933335511945188,
"fcm_dpo/delta": -0.11121343821287155,
"fcm_dpo/margin": 515.4320068359375,
"fcm_dpo/q_t": 0.385158896446228,
"grad_norm": 59.88717269897461,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -0.9930525422096252,
"logits/rejected": -1.025518536567688,
"logps/chosen": -588.51171875,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -1130.9140625,
"loss": 1.0262,
"margin_dpo/margin_mean": 515.4320068359375,
"margin_dpo/margin_std": 572.5833740234375,
"step": 512
},
{
"KL/chosen_KL_mean": -714.98828125,
"KL/mean": -890.2576904296875,
"KL/rejected_KL_mean": -1065.5272216796875,
"KL/std": 549.4346923828125,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.0008974907686933875,
"fcm_dpo/delta": -0.028862863779067993,
"fcm_dpo/margin": 350.5388488769531,
"fcm_dpo/q_t": 0.4272831082344055,
"grad_norm": 52.60966491699219,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -1.1262786388397217,
"logits/rejected": -1.1136434078216553,
"logps/chosen": -779.90478515625,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -1141.589599609375,
"loss": 1.1783,
"margin_dpo/margin_mean": 350.5388488769531,
"margin_dpo/margin_std": 643.74267578125,
"step": 513
},
{
"KL/chosen_KL_mean": -711.4935302734375,
"KL/mean": -886.0343017578125,
"KL/rejected_KL_mean": -1060.5751953125,
"KL/std": 575.5819091796875,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.0009602649370208383,
"fcm_dpo/delta": 0.07626126706600189,
"fcm_dpo/margin": 349.08160400390625,
"fcm_dpo/q_t": 0.4296727478504181,
"grad_norm": 45.58797073364258,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -1.0165659189224243,
"logits/rejected": -1.0113214254379272,
"logps/chosen": -785.72314453125,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -1139.520751953125,
"loss": 1.1706,
"margin_dpo/margin_mean": 349.08160400390625,
"margin_dpo/margin_std": 668.0552978515625,
"step": 514
},
{
"KL/chosen_KL_mean": -587.929443359375,
"KL/mean": -739.9951171875,
"KL/rejected_KL_mean": -892.060791015625,
"KL/std": 471.86102294921875,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.0010444549843668938,
"fcm_dpo/delta": 0.09740975499153137,
"fcm_dpo/margin": 304.1312255859375,
"fcm_dpo/q_t": 0.42857980728149414,
"grad_norm": 32.755496978759766,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -0.9877028465270996,
"logits/rejected": -1.01090669631958,
"logps/chosen": -638.3310546875,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -979.1585083007812,
"loss": 1.1743,
"margin_dpo/margin_mean": 304.1312561035156,
"margin_dpo/margin_std": 567.49853515625,
"step": 515
},
{
"KL/chosen_KL_mean": -636.4197998046875,
"KL/mean": -818.3901977539062,
"KL/rejected_KL_mean": -1000.3605346679688,
"KL/std": 521.9057006835938,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.001094970153644681,
"fcm_dpo/delta": -0.001060720533132553,
"fcm_dpo/margin": 363.9407958984375,
"fcm_dpo/q_t": 0.40709108114242554,
"grad_norm": 48.295448303222656,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -1.0736280679702759,
"logits/rejected": -1.1098659038543701,
"logps/chosen": -701.0693359375,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -1112.0830078125,
"loss": 1.1144,
"margin_dpo/margin_mean": 363.9407958984375,
"margin_dpo/margin_std": 547.8302001953125,
"step": 516
},
{
"KL/chosen_KL_mean": -603.1934814453125,
"KL/mean": -841.0816650390625,
"KL/rejected_KL_mean": -1078.9698486328125,
"KL/std": 555.549560546875,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.0009808378526940942,
"fcm_dpo/delta": -0.0903298482298851,
"fcm_dpo/margin": 475.7763671875,
"fcm_dpo/q_t": 0.39226794242858887,
"grad_norm": 38.17343521118164,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -1.0043656826019287,
"logits/rejected": -1.0225131511688232,
"logps/chosen": -664.1070556640625,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -1168.052978515625,
"loss": 1.0563,
"margin_dpo/margin_mean": 475.7763671875,
"margin_dpo/margin_std": 629.1284790039062,
"step": 517
},
{
"KL/chosen_KL_mean": -587.3173828125,
"KL/mean": -774.2802734375,
"KL/rejected_KL_mean": -961.2432250976562,
"KL/std": 448.082275390625,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.000986230792477727,
"fcm_dpo/delta": 0.03777404874563217,
"fcm_dpo/margin": 373.92578125,
"fcm_dpo/q_t": 0.4149107336997986,
"grad_norm": 56.80478286743164,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -1.0099992752075195,
"logits/rejected": -1.0327041149139404,
"logps/chosen": -644.7733154296875,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -1046.555908203125,
"loss": 1.111,
"margin_dpo/margin_mean": 373.92578125,
"margin_dpo/margin_std": 499.00146484375,
"step": 518
},
{
"KL/chosen_KL_mean": -576.0409545898438,
"KL/mean": -715.2539672851562,
"KL/rejected_KL_mean": -854.4669189453125,
"KL/std": 430.7130432128906,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.0011134599335491657,
"fcm_dpo/delta": 0.10244397819042206,
"fcm_dpo/margin": 278.426025390625,
"fcm_dpo/q_t": 0.4303070902824402,
"grad_norm": 32.800045013427734,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -0.9758340716362,
"logits/rejected": -0.9646449089050293,
"logps/chosen": -650.104248046875,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -958.9111328125,
"loss": 1.1866,
"margin_dpo/margin_mean": 278.426025390625,
"margin_dpo/margin_std": 533.133544921875,
"step": 519
},
{
"KL/chosen_KL_mean": -643.1387939453125,
"KL/mean": -824.5906372070312,
"KL/rejected_KL_mean": -1006.0426025390625,
"KL/std": 498.93865966796875,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.0011077395174652338,
"fcm_dpo/delta": -0.002978809177875519,
"fcm_dpo/margin": 362.90374755859375,
"fcm_dpo/q_t": 0.4119797348976135,
"grad_norm": 35.526668548583984,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -0.9478990435600281,
"logits/rejected": -0.9614932537078857,
"logps/chosen": -713.4385986328125,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -1106.02392578125,
"loss": 1.1183,
"margin_dpo/margin_mean": 362.90374755859375,
"margin_dpo/margin_std": 579.5814208984375,
"step": 520
},
{
"KL/chosen_KL_mean": -554.781005859375,
"KL/mean": -760.6822509765625,
"KL/rejected_KL_mean": -966.5836181640625,
"KL/std": 510.9144287109375,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.001102518173865974,
"fcm_dpo/delta": -0.07349302619695663,
"fcm_dpo/margin": 411.80267333984375,
"fcm_dpo/q_t": 0.394964337348938,
"grad_norm": 34.72807312011719,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -1.0138747692108154,
"logits/rejected": -1.0302083492279053,
"logps/chosen": -612.9238891601562,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -1049.8642578125,
"loss": 1.0735,
"margin_dpo/margin_mean": 411.80267333984375,
"margin_dpo/margin_std": 541.2650146484375,
"step": 521
},
{
"KL/chosen_KL_mean": -658.7055053710938,
"KL/mean": -834.223388671875,
"KL/rejected_KL_mean": -1009.7413330078125,
"KL/std": 540.191162109375,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.0010687486501410604,
"fcm_dpo/delta": 0.030154742300510406,
"fcm_dpo/margin": 351.0357666015625,
"fcm_dpo/q_t": 0.4162691831588745,
"grad_norm": 32.86531448364258,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -1.0368986129760742,
"logits/rejected": -1.046702265739441,
"logps/chosen": -705.253173828125,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -1075.755126953125,
"loss": 1.1409,
"margin_dpo/margin_mean": 351.0357666015625,
"margin_dpo/margin_std": 594.4010009765625,
"step": 522
},
{
"KL/chosen_KL_mean": -706.0521240234375,
"KL/mean": -906.0491333007812,
"KL/rejected_KL_mean": -1106.046142578125,
"KL/std": 648.95263671875,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.0010503309313207865,
"fcm_dpo/delta": -0.025494202971458435,
"fcm_dpo/margin": 399.9941101074219,
"fcm_dpo/q_t": 0.4063454568386078,
"grad_norm": 37.535797119140625,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -1.0674784183502197,
"logits/rejected": -1.0807902812957764,
"logps/chosen": -767.8216552734375,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -1189.8076171875,
"loss": 1.1229,
"margin_dpo/margin_mean": 399.9941101074219,
"margin_dpo/margin_std": 670.1151733398438,
"step": 523
},
{
"KL/chosen_KL_mean": -687.168212890625,
"KL/mean": -907.5570068359375,
"KL/rejected_KL_mean": -1127.94580078125,
"KL/std": 601.67529296875,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.0009792076889425516,
"fcm_dpo/delta": -0.04597056284546852,
"fcm_dpo/margin": 440.77752685546875,
"fcm_dpo/q_t": 0.3970939517021179,
"grad_norm": 35.97694396972656,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -1.1234776973724365,
"logits/rejected": -1.125186562538147,
"logps/chosen": -765.240234375,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -1209.247802734375,
"loss": 1.0894,
"margin_dpo/margin_mean": 440.77752685546875,
"margin_dpo/margin_std": 615.4475708007812,
"step": 524
},
{
"KL/chosen_KL_mean": -658.93115234375,
"KL/mean": -915.503173828125,
"KL/rejected_KL_mean": -1172.0750732421875,
"KL/std": 587.5389404296875,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.0009053426329046488,
"fcm_dpo/delta": -0.09118733555078506,
"fcm_dpo/margin": 513.14404296875,
"fcm_dpo/q_t": 0.3911314010620117,
"grad_norm": 35.581817626953125,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -1.0382120609283447,
"logits/rejected": -1.0887622833251953,
"logps/chosen": -709.7589721679688,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -1272.1280517578125,
"loss": 1.0522,
"margin_dpo/margin_mean": 513.1439819335938,
"margin_dpo/margin_std": 641.33203125,
"step": 525
},
{
"KL/chosen_KL_mean": -707.2711791992188,
"KL/mean": -966.6031494140625,
"KL/rejected_KL_mean": -1225.93505859375,
"KL/std": 583.5233154296875,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.0008811709121800959,
"fcm_dpo/delta": -0.07432842254638672,
"fcm_dpo/margin": 518.663818359375,
"fcm_dpo/q_t": 0.3920350670814514,
"grad_norm": 27.579317092895508,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -1.0754790306091309,
"logits/rejected": -1.084367036819458,
"logps/chosen": -770.4384155273438,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -1312.244384765625,
"loss": 1.0666,
"margin_dpo/margin_mean": 518.663818359375,
"margin_dpo/margin_std": 673.5228271484375,
"step": 526
},
{
"KL/chosen_KL_mean": -777.7960815429688,
"KL/mean": -947.6534423828125,
"KL/rejected_KL_mean": -1117.5108642578125,
"KL/std": 600.279296875,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.000911904382519424,
"fcm_dpo/delta": 0.10690906643867493,
"fcm_dpo/margin": 339.7147521972656,
"fcm_dpo/q_t": 0.4307979345321655,
"grad_norm": 33.39142990112305,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -1.33701753616333,
"logits/rejected": -1.3142709732055664,
"logps/chosen": -848.9248046875,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -1195.853515625,
"loss": 1.154,
"margin_dpo/margin_mean": 339.71478271484375,
"margin_dpo/margin_std": 517.4578857421875,
"step": 527
},
{
"KL/chosen_KL_mean": -758.856689453125,
"KL/mean": -977.1895751953125,
"KL/rejected_KL_mean": -1195.5224609375,
"KL/std": 594.4635620117188,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.0009343413985334337,
"fcm_dpo/delta": -0.010038571432232857,
"fcm_dpo/margin": 436.66583251953125,
"fcm_dpo/q_t": 0.4094654321670532,
"grad_norm": 30.87571144104004,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -1.206683874130249,
"logits/rejected": -1.1996240615844727,
"logps/chosen": -826.9461059570312,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -1289.4324951171875,
"loss": 1.1199,
"margin_dpo/margin_mean": 436.66583251953125,
"margin_dpo/margin_std": 710.944580078125,
"step": 528
},
{
"KL/chosen_KL_mean": -954.2504272460938,
"KL/mean": -1106.777099609375,
"KL/rejected_KL_mean": -1259.303955078125,
"KL/std": 612.4547119140625,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.0009363851859234273,
"fcm_dpo/delta": 0.0063527897000312805,
"fcm_dpo/margin": 305.05340576171875,
"fcm_dpo/q_t": 0.43478527665138245,
"grad_norm": 38.029884338378906,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -1.2603811025619507,
"logits/rejected": -1.278627872467041,
"logps/chosen": -1009.825439453125,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -1348.512939453125,
"loss": 1.2423,
"margin_dpo/margin_mean": 305.0534362792969,
"margin_dpo/margin_std": 756.2225341796875,
"step": 529
},
{
"KL/chosen_KL_mean": -787.18115234375,
"KL/mean": -1030.659912109375,
"KL/rejected_KL_mean": -1274.138671875,
"KL/std": 670.89990234375,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.00090701476437971,
"fcm_dpo/delta": -0.053333625197410583,
"fcm_dpo/margin": 486.95751953125,
"fcm_dpo/q_t": 0.40362924337387085,
"grad_norm": 35.42485427856445,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -1.1458381414413452,
"logits/rejected": -1.1499226093292236,
"logps/chosen": -834.7825927734375,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -1361.4232177734375,
"loss": 1.1316,
"margin_dpo/margin_mean": 486.95745849609375,
"margin_dpo/margin_std": 884.4109497070312,
"step": 530
},
{
"KL/chosen_KL_mean": -894.1881713867188,
"KL/mean": -1081.9078369140625,
"KL/rejected_KL_mean": -1269.6273193359375,
"KL/std": 714.610595703125,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.0009350181790068746,
"fcm_dpo/delta": 0.05843368172645569,
"fcm_dpo/margin": 375.4392395019531,
"fcm_dpo/q_t": 0.42438310384750366,
"grad_norm": 46.751792907714844,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -1.278576374053955,
"logits/rejected": -1.2574585676193237,
"logps/chosen": -949.4342041015625,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -1340.2333984375,
"loss": 1.1589,
"margin_dpo/margin_mean": 375.439208984375,
"margin_dpo/margin_std": 681.22314453125,
"step": 531
},
{
"KL/chosen_KL_mean": -914.6450805664062,
"KL/mean": -1124.872314453125,
"KL/rejected_KL_mean": -1335.0997314453125,
"KL/std": 674.3599853515625,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.0009111958206631243,
"fcm_dpo/delta": 0.013179205358028412,
"fcm_dpo/margin": 420.45458984375,
"fcm_dpo/q_t": 0.4144301414489746,
"grad_norm": 74.0493392944336,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -1.2184380292892456,
"logits/rejected": -1.1921205520629883,
"logps/chosen": -984.93115234375,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -1421.6910400390625,
"loss": 1.184,
"margin_dpo/margin_mean": 420.4546203613281,
"margin_dpo/margin_std": 841.721435546875,
"step": 532
},
{
"KL/chosen_KL_mean": -811.30615234375,
"KL/mean": -966.7421875,
"KL/rejected_KL_mean": -1122.1781005859375,
"KL/std": 577.7342529296875,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.0009565654909238219,
"fcm_dpo/delta": 0.0060767000541090965,
"fcm_dpo/margin": 310.87200927734375,
"fcm_dpo/q_t": 0.4336027204990387,
"grad_norm": 33.39649200439453,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -1.299651861190796,
"logits/rejected": -1.2833778858184814,
"logps/chosen": -876.1612548828125,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -1198.76611328125,
"loss": 1.2121,
"margin_dpo/margin_mean": 310.87200927734375,
"margin_dpo/margin_std": 690.0610961914062,
"step": 533
},
{
"KL/chosen_KL_mean": -830.42724609375,
"KL/mean": -1037.4508056640625,
"KL/rejected_KL_mean": -1244.474365234375,
"KL/std": 619.809814453125,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.0009335308568552136,
"fcm_dpo/delta": 0.01120779663324356,
"fcm_dpo/margin": 414.0472412109375,
"fcm_dpo/q_t": 0.41367307305336,
"grad_norm": 32.04544448852539,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -1.2206509113311768,
"logits/rejected": -1.2227022647857666,
"logps/chosen": -890.546630859375,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -1323.017822265625,
"loss": 1.1307,
"margin_dpo/margin_mean": 414.0472412109375,
"margin_dpo/margin_std": 661.77587890625,
"step": 534
},
{
"KL/chosen_KL_mean": -699.33740234375,
"KL/mean": -953.7452392578125,
"KL/rejected_KL_mean": -1208.1529541015625,
"KL/std": 633.1478271484375,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.0009318743832409382,
"fcm_dpo/delta": -0.09808388352394104,
"fcm_dpo/margin": 508.8156433105469,
"fcm_dpo/q_t": 0.390717089176178,
"grad_norm": 24.964298248291016,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -1.1217422485351562,
"logits/rejected": -1.1360340118408203,
"logps/chosen": -753.6676635742188,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -1304.460693359375,
"loss": 1.0588,
"margin_dpo/margin_mean": 508.81561279296875,
"margin_dpo/margin_std": 653.384521484375,
"step": 535
},
{
"KL/chosen_KL_mean": -622.41259765625,
"KL/mean": -909.5772705078125,
"KL/rejected_KL_mean": -1196.741943359375,
"KL/std": 661.69384765625,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.0008232737891376019,
"fcm_dpo/delta": -0.09523215144872665,
"fcm_dpo/margin": 574.329345703125,
"fcm_dpo/q_t": 0.3940427899360657,
"grad_norm": 29.166486740112305,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -1.0844101905822754,
"logits/rejected": -1.1293668746948242,
"logps/chosen": -669.4931030273438,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -1285.83984375,
"loss": 1.0587,
"margin_dpo/margin_mean": 574.329345703125,
"margin_dpo/margin_std": 803.4403076171875,
"step": 536
},
{
"KL/chosen_KL_mean": -635.1380615234375,
"KL/mean": -826.725830078125,
"KL/rejected_KL_mean": -1018.3135375976562,
"KL/std": 565.7158203125,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.0007962306845001876,
"fcm_dpo/delta": -0.008790962398052216,
"fcm_dpo/margin": 383.1754150390625,
"fcm_dpo/q_t": 0.42851394414901733,
"grad_norm": 35.423980712890625,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -1.1369528770446777,
"logits/rejected": -1.1194478273391724,
"logps/chosen": -692.8856201171875,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -1088.751953125,
"loss": 1.1758,
"margin_dpo/margin_mean": 383.1754150390625,
"margin_dpo/margin_std": 672.041015625,
"step": 537
},
{
"KL/chosen_KL_mean": -784.5943603515625,
"KL/mean": -1004.52880859375,
"KL/rejected_KL_mean": -1224.46337890625,
"KL/std": 607.7210693359375,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.0008164436439983547,
"fcm_dpo/delta": 0.049948450177907944,
"fcm_dpo/margin": 439.8689270019531,
"fcm_dpo/q_t": 0.4188556671142578,
"grad_norm": 27.3602237701416,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -1.1543678045272827,
"logits/rejected": -1.1441106796264648,
"logps/chosen": -851.0103149414062,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -1308.69140625,
"loss": 1.1229,
"margin_dpo/margin_mean": 439.86895751953125,
"margin_dpo/margin_std": 635.7586669921875,
"step": 538
},
{
"KL/chosen_KL_mean": -652.3861083984375,
"KL/mean": -859.1839599609375,
"KL/rejected_KL_mean": -1065.981689453125,
"KL/std": 471.9359130859375,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.0008725939551368356,
"fcm_dpo/delta": 0.04645565152168274,
"fcm_dpo/margin": 413.5956115722656,
"fcm_dpo/q_t": 0.417635977268219,
"grad_norm": 29.32832145690918,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -1.157194972038269,
"logits/rejected": -1.1582703590393066,
"logps/chosen": -710.8789672851562,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -1157.835693359375,
"loss": 1.1091,
"margin_dpo/margin_mean": 413.5955810546875,
"margin_dpo/margin_std": 535.5076904296875,
"step": 539
},
{
"KL/chosen_KL_mean": -629.462890625,
"KL/mean": -898.05859375,
"KL/rejected_KL_mean": -1166.654296875,
"KL/std": 595.0738525390625,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.0008491129847243428,
"fcm_dpo/delta": -0.07360121607780457,
"fcm_dpo/margin": 537.1914672851562,
"fcm_dpo/q_t": 0.396462619304657,
"grad_norm": 36.619747161865234,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -1.031808614730835,
"logits/rejected": -1.0707483291625977,
"logps/chosen": -692.9454345703125,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -1283.0843505859375,
"loss": 1.0689,
"margin_dpo/margin_mean": 537.1914672851562,
"margin_dpo/margin_std": 714.2697143554688,
"step": 540
},
{
"KL/chosen_KL_mean": -754.865966796875,
"KL/mean": -900.2635498046875,
"KL/rejected_KL_mean": -1045.6612548828125,
"KL/std": 539.58740234375,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.000835648737847805,
"fcm_dpo/delta": 0.02992171235382557,
"fcm_dpo/margin": 290.7952880859375,
"fcm_dpo/q_t": 0.44497525691986084,
"grad_norm": 49.2709846496582,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -1.191582202911377,
"logits/rejected": -1.1900990009307861,
"logps/chosen": -833.1463012695312,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -1148.934814453125,
"loss": 1.2274,
"margin_dpo/margin_mean": 290.79522705078125,
"margin_dpo/margin_std": 635.66015625,
"step": 541
},
{
"KL/chosen_KL_mean": -641.0167236328125,
"KL/mean": -899.6398315429688,
"KL/rejected_KL_mean": -1158.262939453125,
"KL/std": 611.9912109375,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.0007914283778518438,
"fcm_dpo/delta": -0.019995450973510742,
"fcm_dpo/margin": 517.24609375,
"fcm_dpo/q_t": 0.40515321493148804,
"grad_norm": 45.66024398803711,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -1.1752982139587402,
"logits/rejected": -1.219357967376709,
"logps/chosen": -698.501708984375,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -1254.738037109375,
"loss": 1.0996,
"margin_dpo/margin_mean": 517.24609375,
"margin_dpo/margin_std": 717.5678100585938,
"step": 542
},
{
"KL/chosen_KL_mean": -710.8384399414062,
"KL/mean": -893.802001953125,
"KL/rejected_KL_mean": -1076.7655029296875,
"KL/std": 660.750244140625,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.0008738016476854682,
"fcm_dpo/delta": 0.09592510759830475,
"fcm_dpo/margin": 365.9271240234375,
"fcm_dpo/q_t": 0.4370731711387634,
"grad_norm": 29.643529891967773,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -1.101868748664856,
"logits/rejected": -1.1417419910430908,
"logps/chosen": -771.4559326171875,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -1182.364501953125,
"loss": 1.2076,
"margin_dpo/margin_mean": 365.9271545410156,
"margin_dpo/margin_std": 836.8877563476562,
"step": 543
},
{
"KL/chosen_KL_mean": -727.2119140625,
"KL/mean": -949.6517333984375,
"KL/rejected_KL_mean": -1172.091552734375,
"KL/std": 569.668701171875,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.0008942891145125031,
"fcm_dpo/delta": 0.0022725816816091537,
"fcm_dpo/margin": 444.879638671875,
"fcm_dpo/q_t": 0.4095289707183838,
"grad_norm": 34.229854583740234,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -1.1131432056427002,
"logits/rejected": -1.14817476272583,
"logps/chosen": -786.8541870117188,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -1273.046142578125,
"loss": 1.1092,
"margin_dpo/margin_mean": 444.879638671875,
"margin_dpo/margin_std": 663.5440673828125,
"step": 544
},
{
"KL/chosen_KL_mean": -659.4088745117188,
"KL/mean": -919.2174072265625,
"KL/rejected_KL_mean": -1179.02587890625,
"KL/std": 598.9300537109375,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.000862646906170994,
"fcm_dpo/delta": -0.06230539083480835,
"fcm_dpo/margin": 519.6170043945312,
"fcm_dpo/q_t": 0.3970434069633484,
"grad_norm": 32.03169631958008,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -0.9890373349189758,
"logits/rejected": -1.033557653427124,
"logps/chosen": -727.0574951171875,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -1274.933837890625,
"loss": 1.0686,
"margin_dpo/margin_mean": 519.6170654296875,
"margin_dpo/margin_std": 691.0440673828125,
"step": 545
},
{
"KL/chosen_KL_mean": -640.8040771484375,
"KL/mean": -804.4512939453125,
"KL/rejected_KL_mean": -968.0985717773438,
"KL/std": 497.0710144042969,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.0009246976114809513,
"fcm_dpo/delta": 0.11454042792320251,
"fcm_dpo/margin": 327.29449462890625,
"fcm_dpo/q_t": 0.4316098690032959,
"grad_norm": 30.14081573486328,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -1.0944793224334717,
"logits/rejected": -1.0822169780731201,
"logps/chosen": -691.5482788085938,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -1049.96484375,
"loss": 1.1688,
"margin_dpo/margin_mean": 327.2945251464844,
"margin_dpo/margin_std": 540.7611694335938,
"step": 546
},
{
"KL/chosen_KL_mean": -660.4383544921875,
"KL/mean": -884.519287109375,
"KL/rejected_KL_mean": -1108.60009765625,
"KL/std": 564.5079345703125,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.0009412175277248025,
"fcm_dpo/delta": -0.02773449756205082,
"fcm_dpo/margin": 448.1617431640625,
"fcm_dpo/q_t": 0.4049096703529358,
"grad_norm": 41.60638427734375,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -1.058458685874939,
"logits/rejected": -1.0802452564239502,
"logps/chosen": -734.1260986328125,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -1199.3614501953125,
"loss": 1.0946,
"margin_dpo/margin_mean": 448.1617431640625,
"margin_dpo/margin_std": 651.8272094726562,
"step": 547
},
{
"KL/chosen_KL_mean": -706.6986694335938,
"KL/mean": -914.0684814453125,
"KL/rejected_KL_mean": -1121.4383544921875,
"KL/std": 620.1270141601562,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.0009347201557829976,
"fcm_dpo/delta": 0.015313264913856983,
"fcm_dpo/margin": 414.7396240234375,
"fcm_dpo/q_t": 0.41696617007255554,
"grad_norm": 34.767852783203125,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -1.1285653114318848,
"logits/rejected": -1.1398406028747559,
"logps/chosen": -771.9450073242188,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -1215.556396484375,
"loss": 1.1259,
"margin_dpo/margin_mean": 414.7396240234375,
"margin_dpo/margin_std": 696.6866455078125,
"step": 548
},
{
"KL/chosen_KL_mean": -716.9879760742188,
"KL/mean": -874.5885009765625,
"KL/rejected_KL_mean": -1032.1890869140625,
"KL/std": 489.6221008300781,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.0009415035601705313,
"fcm_dpo/delta": 0.00250604678876698,
"fcm_dpo/margin": 315.2010803222656,
"fcm_dpo/q_t": 0.4317595362663269,
"grad_norm": 44.2504768371582,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -1.176891803741455,
"logits/rejected": -1.1708965301513672,
"logps/chosen": -766.2003173828125,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -1106.099365234375,
"loss": 1.188,
"margin_dpo/margin_mean": 315.2010803222656,
"margin_dpo/margin_std": 595.5227661132812,
"step": 549
},
{
"KL/chosen_KL_mean": -721.7799682617188,
"KL/mean": -916.480224609375,
"KL/rejected_KL_mean": -1111.1806640625,
"KL/std": 571.6375732421875,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.000960228149779141,
"fcm_dpo/delta": 0.03213486075401306,
"fcm_dpo/margin": 389.4005126953125,
"fcm_dpo/q_t": 0.4163832664489746,
"grad_norm": 41.49302673339844,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -1.0517723560333252,
"logits/rejected": -1.0827746391296387,
"logps/chosen": -778.5869140625,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -1206.306396484375,
"loss": 1.1212,
"margin_dpo/margin_mean": 389.4005126953125,
"margin_dpo/margin_std": 595.0445556640625,
"step": 550
},
{
"KL/chosen_KL_mean": -613.4744262695312,
"KL/mean": -925.0634765625,
"KL/rejected_KL_mean": -1236.652587890625,
"KL/std": 624.64697265625,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.0008410347509197891,
"fcm_dpo/delta": -0.17439556121826172,
"fcm_dpo/margin": 623.1781005859375,
"fcm_dpo/q_t": 0.3711840510368347,
"grad_norm": 51.76773452758789,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -1.0433658361434937,
"logits/rejected": -1.1170668601989746,
"logps/chosen": -672.5807495117188,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -1348.3253173828125,
"loss": 0.9923,
"margin_dpo/margin_mean": 623.1781005859375,
"margin_dpo/margin_std": 630.8492431640625,
"step": 551
},
{
"KL/chosen_KL_mean": -602.475341796875,
"KL/mean": -958.9727783203125,
"KL/rejected_KL_mean": -1315.47021484375,
"KL/std": 719.78564453125,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.0006975846481509507,
"fcm_dpo/delta": -0.14975111186504364,
"fcm_dpo/margin": 712.994873046875,
"fcm_dpo/q_t": 0.3810081481933594,
"grad_norm": 42.0149040222168,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -1.1741893291473389,
"logits/rejected": -1.2385377883911133,
"logps/chosen": -664.8298950195312,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -1420.0322265625,
"loss": 1.0282,
"margin_dpo/margin_mean": 712.994873046875,
"margin_dpo/margin_std": 833.3380737304688,
"step": 552
},
{
"KL/chosen_KL_mean": -699.2514038085938,
"KL/mean": -884.60302734375,
"KL/rejected_KL_mean": -1069.95458984375,
"KL/std": 591.6678466796875,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.0007735902327112854,
"fcm_dpo/delta": 0.1325617879629135,
"fcm_dpo/margin": 370.7031555175781,
"fcm_dpo/q_t": 0.43872201442718506,
"grad_norm": 23.199739456176758,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -1.1220920085906982,
"logits/rejected": -1.1197218894958496,
"logps/chosen": -767.51025390625,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -1168.0517578125,
"loss": 1.1789,
"margin_dpo/margin_mean": 370.70318603515625,
"margin_dpo/margin_std": 653.1337890625,
"step": 553
},
{
"KL/chosen_KL_mean": -751.51806640625,
"KL/mean": -983.1180419921875,
"KL/rejected_KL_mean": -1214.7178955078125,
"KL/std": 659.7369995117188,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.0008373720338568091,
"fcm_dpo/delta": 0.009692121297121048,
"fcm_dpo/margin": 463.19989013671875,
"fcm_dpo/q_t": 0.4117772877216339,
"grad_norm": 118.59552001953125,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -1.087932825088501,
"logits/rejected": -1.0606003999710083,
"logps/chosen": -819.4658203125,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -1304.5006103515625,
"loss": 1.1619,
"margin_dpo/margin_mean": 463.1998596191406,
"margin_dpo/margin_std": 863.3409423828125,
"step": 554
},
{
"KL/chosen_KL_mean": -739.2120971679688,
"KL/mean": -1043.806396484375,
"KL/rejected_KL_mean": -1348.400634765625,
"KL/std": 680.260009765625,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.0007673618383705616,
"fcm_dpo/delta": -0.08767017722129822,
"fcm_dpo/margin": 609.1884155273438,
"fcm_dpo/q_t": 0.3946698307991028,
"grad_norm": 30.83856773376465,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -1.1824922561645508,
"logits/rejected": -1.2582507133483887,
"logps/chosen": -792.5426025390625,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -1456.8798828125,
"loss": 1.0729,
"margin_dpo/margin_mean": 609.1884765625,
"margin_dpo/margin_std": 872.76953125,
"step": 555
},
{
"KL/chosen_KL_mean": -732.7947998046875,
"KL/mean": -903.1065673828125,
"KL/rejected_KL_mean": -1073.418212890625,
"KL/std": 530.9589233398438,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.0008233854314312339,
"fcm_dpo/delta": 0.13905900716781616,
"fcm_dpo/margin": 340.62347412109375,
"fcm_dpo/q_t": 0.4381856620311737,
"grad_norm": 29.910762786865234,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -1.185271143913269,
"logits/rejected": -1.1860347986221313,
"logps/chosen": -791.4393310546875,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -1174.7587890625,
"loss": 1.1731,
"margin_dpo/margin_mean": 340.6235046386719,
"margin_dpo/margin_std": 539.8701171875,
"step": 556
},
{
"KL/chosen_KL_mean": -786.4280395507812,
"KL/mean": -1055.865234375,
"KL/rejected_KL_mean": -1325.302490234375,
"KL/std": 711.47607421875,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.0008084324654191732,
"fcm_dpo/delta": -0.04845455288887024,
"fcm_dpo/margin": 538.87451171875,
"fcm_dpo/q_t": 0.4022066295146942,
"grad_norm": 46.43569564819336,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -1.2151281833648682,
"logits/rejected": -1.2664202451705933,
"logps/chosen": -854.2686767578125,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -1435.2421875,
"loss": 1.0994,
"margin_dpo/margin_mean": 538.8744506835938,
"margin_dpo/margin_std": 822.3883056640625,
"step": 557
},
{
"KL/chosen_KL_mean": -752.0896606445312,
"KL/mean": -1117.599853515625,
"KL/rejected_KL_mean": -1483.10986328125,
"KL/std": 693.58544921875,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.0007167826406657696,
"fcm_dpo/delta": -0.16944512724876404,
"fcm_dpo/margin": 731.0201416015625,
"fcm_dpo/q_t": 0.3706546425819397,
"grad_norm": 24.203054428100586,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -1.274876594543457,
"logits/rejected": -1.3060567378997803,
"logps/chosen": -814.4578857421875,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -1585.2708740234375,
"loss": 1.0044,
"margin_dpo/margin_mean": 731.0202026367188,
"margin_dpo/margin_std": 783.4100341796875,
"step": 558
},
{
"KL/chosen_KL_mean": -840.5665893554688,
"KL/mean": -1157.062744140625,
"KL/rejected_KL_mean": -1473.558837890625,
"KL/std": 744.0479736328125,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.0006758394883945584,
"fcm_dpo/delta": -0.03540560603141785,
"fcm_dpo/margin": 632.9921264648438,
"fcm_dpo/q_t": 0.401652991771698,
"grad_norm": 23.325172424316406,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -1.4109092950820923,
"logits/rejected": -1.432593822479248,
"logps/chosen": -901.3189697265625,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -1567.0009765625,
"loss": 1.0774,
"margin_dpo/margin_mean": 632.9921875,
"margin_dpo/margin_std": 839.2745361328125,
"step": 559
},
{
"KL/chosen_KL_mean": -801.013916015625,
"KL/mean": -1016.2203369140625,
"KL/rejected_KL_mean": -1231.4267578125,
"KL/std": 670.739990234375,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.0006765261059626937,
"fcm_dpo/delta": 0.019914967939257622,
"fcm_dpo/margin": 430.41290283203125,
"fcm_dpo/q_t": 0.43432098627090454,
"grad_norm": 40.84651184082031,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -1.0937294960021973,
"logits/rejected": -1.0813250541687012,
"logps/chosen": -859.11767578125,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -1311.41796875,
"loss": 1.1813,
"margin_dpo/margin_mean": 430.41290283203125,
"margin_dpo/margin_std": 772.7431640625,
"step": 560
},
{
"KL/chosen_KL_mean": -897.7193603515625,
"KL/mean": -1091.67431640625,
"KL/rejected_KL_mean": -1285.6295166015625,
"KL/std": 642.45654296875,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.0007529509020969272,
"fcm_dpo/delta": 0.1230059415102005,
"fcm_dpo/margin": 387.91015625,
"fcm_dpo/q_t": 0.435150682926178,
"grad_norm": 30.059293746948242,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -1.2934693098068237,
"logits/rejected": -1.3272886276245117,
"logps/chosen": -948.6322021484375,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -1384.6981201171875,
"loss": 1.1972,
"margin_dpo/margin_mean": 387.91015625,
"margin_dpo/margin_std": 791.5865478515625,
"step": 561
},
{
"KL/chosen_KL_mean": -864.7420654296875,
"KL/mean": -1108.170654296875,
"KL/rejected_KL_mean": -1351.5992431640625,
"KL/std": 697.1679077148438,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.0007649194449186325,
"fcm_dpo/delta": 0.03210524842143059,
"fcm_dpo/margin": 486.857177734375,
"fcm_dpo/q_t": 0.416918009519577,
"grad_norm": 28.984006881713867,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -1.2221224308013916,
"logits/rejected": -1.2382316589355469,
"logps/chosen": -924.2064819335938,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -1448.141845703125,
"loss": 1.1243,
"margin_dpo/margin_mean": 486.857177734375,
"margin_dpo/margin_std": 740.4078369140625,
"step": 562
},
{
"KL/chosen_KL_mean": -973.072021484375,
"KL/mean": -1153.509033203125,
"KL/rejected_KL_mean": -1333.9459228515625,
"KL/std": 739.7679443359375,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.0007791572716087103,
"fcm_dpo/delta": -0.02793397195637226,
"fcm_dpo/margin": 360.8739013671875,
"fcm_dpo/q_t": 0.4309625029563904,
"grad_norm": 40.373382568359375,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -1.2764613628387451,
"logits/rejected": -1.2854523658752441,
"logps/chosen": -1033.68017578125,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -1428.513671875,
"loss": 1.2446,
"margin_dpo/margin_mean": 360.8739013671875,
"margin_dpo/margin_std": 878.1210327148438,
"step": 563
},
{
"KL/chosen_KL_mean": -857.3223266601562,
"KL/mean": -1105.55712890625,
"KL/rejected_KL_mean": -1353.7919921875,
"KL/std": 703.2030029296875,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.0007832963019609451,
"fcm_dpo/delta": 0.013243109919130802,
"fcm_dpo/margin": 496.4696960449219,
"fcm_dpo/q_t": 0.4130350947380066,
"grad_norm": 32.131954193115234,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -1.2859337329864502,
"logits/rejected": -1.2894582748413086,
"logps/chosen": -931.5906982421875,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -1447.0301513671875,
"loss": 1.1066,
"margin_dpo/margin_mean": 496.4696960449219,
"margin_dpo/margin_std": 708.5772094726562,
"step": 564
},
{
"KL/chosen_KL_mean": -940.7181396484375,
"KL/mean": -1180.5489501953125,
"KL/rejected_KL_mean": -1420.3797607421875,
"KL/std": 811.2318115234375,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.0007922237273305655,
"fcm_dpo/delta": 0.024702370166778564,
"fcm_dpo/margin": 479.6616516113281,
"fcm_dpo/q_t": 0.4200623631477356,
"grad_norm": 52.22495651245117,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -1.1902871131896973,
"logits/rejected": -1.1874666213989258,
"logps/chosen": -1009.738037109375,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -1506.15869140625,
"loss": 1.1803,
"margin_dpo/margin_mean": 479.6616516113281,
"margin_dpo/margin_std": 994.3869018554688,
"step": 565
},
{
"KL/chosen_KL_mean": -863.586669921875,
"KL/mean": -1147.6942138671875,
"KL/rejected_KL_mean": -1431.8017578125,
"KL/std": 777.2615966796875,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.0007855127332732081,
"fcm_dpo/delta": -0.061380840837955475,
"fcm_dpo/margin": 568.2152099609375,
"fcm_dpo/q_t": 0.3984350264072418,
"grad_norm": 28.156330108642578,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -1.2587931156158447,
"logits/rejected": -1.2928824424743652,
"logps/chosen": -930.1319580078125,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -1535.671142578125,
"loss": 1.1124,
"margin_dpo/margin_mean": 568.2152099609375,
"margin_dpo/margin_std": 901.3643188476562,
"step": 566
},
{
"KL/chosen_KL_mean": -786.615478515625,
"KL/mean": -1003.1062622070312,
"KL/rejected_KL_mean": -1219.5970458984375,
"KL/std": 541.6770629882812,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.000789005309343338,
"fcm_dpo/delta": 0.07048732787370682,
"fcm_dpo/margin": 432.98162841796875,
"fcm_dpo/q_t": 0.4219757318496704,
"grad_norm": 25.511581420898438,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -1.3670188188552856,
"logits/rejected": -1.3844826221466064,
"logps/chosen": -839.4737548828125,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -1304.968017578125,
"loss": 1.1217,
"margin_dpo/margin_mean": 432.98162841796875,
"margin_dpo/margin_std": 570.1590576171875,
"step": 567
},
{
"KL/chosen_KL_mean": -747.6436767578125,
"KL/mean": -1037.877197265625,
"KL/rejected_KL_mean": -1328.11083984375,
"KL/std": 627.2274780273438,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.0007679754635319114,
"fcm_dpo/delta": -0.05931420996785164,
"fcm_dpo/margin": 580.4671630859375,
"fcm_dpo/q_t": 0.39421334862709045,
"grad_norm": 34.71445846557617,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -1.1856874227523804,
"logits/rejected": -1.243179202079773,
"logps/chosen": -792.8360595703125,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -1417.2032470703125,
"loss": 1.0511,
"margin_dpo/margin_mean": 580.4671630859375,
"margin_dpo/margin_std": 675.10009765625,
"step": 568
},
{
"KL/chosen_KL_mean": -887.8179321289062,
"KL/mean": -1117.02783203125,
"KL/rejected_KL_mean": -1346.2376708984375,
"KL/std": 795.90283203125,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.0008052626508288085,
"fcm_dpo/delta": 0.030735157430171967,
"fcm_dpo/margin": 458.41961669921875,
"fcm_dpo/q_t": 0.41605180501937866,
"grad_norm": 35.24816131591797,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -1.2379553318023682,
"logits/rejected": -1.2469182014465332,
"logps/chosen": -951.5385131835938,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -1425.3408203125,
"loss": 1.1615,
"margin_dpo/margin_mean": 458.41961669921875,
"margin_dpo/margin_std": 826.8089599609375,
"step": 569
},
{
"KL/chosen_KL_mean": -801.3278198242188,
"KL/mean": -1011.711669921875,
"KL/rejected_KL_mean": -1222.095458984375,
"KL/std": 619.7181396484375,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.0008181704906746745,
"fcm_dpo/delta": 0.06755281984806061,
"fcm_dpo/margin": 420.76763916015625,
"fcm_dpo/q_t": 0.42422983050346375,
"grad_norm": 28.688831329345703,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -1.3244519233703613,
"logits/rejected": -1.3194363117218018,
"logps/chosen": -862.9423828125,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -1304.2373046875,
"loss": 1.1451,
"margin_dpo/margin_mean": 420.76763916015625,
"margin_dpo/margin_std": 679.84033203125,
"step": 570
},
{
"KL/chosen_KL_mean": -844.9622802734375,
"KL/mean": -1082.0240478515625,
"KL/rejected_KL_mean": -1319.085693359375,
"KL/std": 643.911865234375,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.0008347969851456583,
"fcm_dpo/delta": 0.005073328502476215,
"fcm_dpo/margin": 474.1234130859375,
"fcm_dpo/q_t": 0.412005752325058,
"grad_norm": 48.486839294433594,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -1.1971943378448486,
"logits/rejected": -1.2182611227035522,
"logps/chosen": -898.016357421875,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -1410.422607421875,
"loss": 1.1393,
"margin_dpo/margin_mean": 474.1234436035156,
"margin_dpo/margin_std": 829.1712646484375,
"step": 571
},
{
"KL/chosen_KL_mean": -877.3575439453125,
"KL/mean": -1135.824951171875,
"KL/rejected_KL_mean": -1394.29248046875,
"KL/std": 772.8182373046875,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.000814650789834559,
"fcm_dpo/delta": -0.027804527431726456,
"fcm_dpo/margin": 516.9349365234375,
"fcm_dpo/q_t": 0.40952882170677185,
"grad_norm": 26.61905288696289,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -1.2630120515823364,
"logits/rejected": -1.3064543008804321,
"logps/chosen": -925.8168334960938,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -1477.849609375,
"loss": 1.1352,
"margin_dpo/margin_mean": 516.9349365234375,
"margin_dpo/margin_std": 933.8126220703125,
"step": 572
},
{
"KL/chosen_KL_mean": -831.7431640625,
"KL/mean": -1071.42333984375,
"KL/rejected_KL_mean": -1311.103515625,
"KL/std": 665.356201171875,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.0008302472997456789,
"fcm_dpo/delta": 0.0017357263714075089,
"fcm_dpo/margin": 479.3603515625,
"fcm_dpo/q_t": 0.4101864695549011,
"grad_norm": 33.60465621948242,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -1.2730963230133057,
"logits/rejected": -1.296579360961914,
"logps/chosen": -883.36572265625,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -1396.428466796875,
"loss": 1.0996,
"margin_dpo/margin_mean": 479.3603515625,
"margin_dpo/margin_std": 673.96435546875,
"step": 573
},
{
"KL/chosen_KL_mean": -761.8274536132812,
"KL/mean": -1024.19091796875,
"KL/rejected_KL_mean": -1286.554443359375,
"KL/std": 611.08935546875,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.0008096909150481224,
"fcm_dpo/delta": -0.03171788156032562,
"fcm_dpo/margin": 524.7269287109375,
"fcm_dpo/q_t": 0.40161073207855225,
"grad_norm": 34.691165924072266,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -1.2997612953186035,
"logits/rejected": -1.3460713624954224,
"logps/chosen": -812.8719482421875,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -1379.36083984375,
"loss": 1.0834,
"margin_dpo/margin_mean": 524.7269287109375,
"margin_dpo/margin_std": 706.32373046875,
"step": 574
},
{
"KL/chosen_KL_mean": -815.347412109375,
"KL/mean": -1056.3719482421875,
"KL/rejected_KL_mean": -1297.396484375,
"KL/std": 656.7642211914062,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.0007790890522301197,
"fcm_dpo/delta": 0.024538405239582062,
"fcm_dpo/margin": 482.0490417480469,
"fcm_dpo/q_t": 0.4154299199581146,
"grad_norm": 46.73399353027344,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -1.071021318435669,
"logits/rejected": -1.0894410610198975,
"logps/chosen": -887.1375732421875,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -1392.78271484375,
"loss": 1.1529,
"margin_dpo/margin_mean": 482.049072265625,
"margin_dpo/margin_std": 828.638671875,
"step": 575
},
{
"KL/chosen_KL_mean": -783.9774780273438,
"KL/mean": -965.3140869140625,
"KL/rejected_KL_mean": -1146.650634765625,
"KL/std": 556.8798828125,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.000867416150867939,
"fcm_dpo/delta": 0.10194016993045807,
"fcm_dpo/margin": 362.67315673828125,
"fcm_dpo/q_t": 0.4307815134525299,
"grad_norm": 33.000823974609375,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -1.2250196933746338,
"logits/rejected": -1.2645316123962402,
"logps/chosen": -838.240478515625,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -1247.405029296875,
"loss": 1.1724,
"margin_dpo/margin_mean": 362.67315673828125,
"margin_dpo/margin_std": 646.2176513671875,
"step": 576
},
{
"KL/chosen_KL_mean": -681.5286254882812,
"KL/mean": -917.073974609375,
"KL/rejected_KL_mean": -1152.619384765625,
"KL/std": 618.7384033203125,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.0008785222889855504,
"fcm_dpo/delta": -0.01841025985777378,
"fcm_dpo/margin": 471.0908203125,
"fcm_dpo/q_t": 0.4073464274406433,
"grad_norm": 27.403722763061523,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -1.1902132034301758,
"logits/rejected": -1.181257724761963,
"logps/chosen": -743.4383544921875,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -1236.6900634765625,
"loss": 1.1109,
"margin_dpo/margin_mean": 471.0908203125,
"margin_dpo/margin_std": 701.18603515625,
"step": 577
},
{
"KL/chosen_KL_mean": -655.4379272460938,
"KL/mean": -906.4620361328125,
"KL/rejected_KL_mean": -1157.486083984375,
"KL/std": 596.8307495117188,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.000839193002320826,
"fcm_dpo/delta": -0.03209435194730759,
"fcm_dpo/margin": 502.04815673828125,
"fcm_dpo/q_t": 0.4018974304199219,
"grad_norm": 33.629188537597656,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -1.0385322570800781,
"logits/rejected": -1.082366943359375,
"logps/chosen": -704.7015991210938,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -1240.92236328125,
"loss": 1.0797,
"margin_dpo/margin_mean": 502.04815673828125,
"margin_dpo/margin_std": 644.6083984375,
"step": 578
},
{
"KL/chosen_KL_mean": -802.5963134765625,
"KL/mean": -953.2222900390625,
"KL/rejected_KL_mean": -1103.8482666015625,
"KL/std": 631.9645385742188,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.0009417862165719271,
"fcm_dpo/delta": 0.13616657257080078,
"fcm_dpo/margin": 301.25201416015625,
"fcm_dpo/q_t": 0.43976956605911255,
"grad_norm": 40.763694763183594,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -1.1537394523620605,
"logits/rejected": -1.164426326751709,
"logps/chosen": -861.482177734375,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -1198.635986328125,
"loss": 1.2241,
"margin_dpo/margin_mean": 301.25201416015625,
"margin_dpo/margin_std": 703.5513916015625,
"step": 579
},
{
"KL/chosen_KL_mean": -557.6210327148438,
"KL/mean": -839.1734619140625,
"KL/rejected_KL_mean": -1120.725830078125,
"KL/std": 694.3538208007812,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.0008712954586371779,
"fcm_dpo/delta": -0.1274590790271759,
"fcm_dpo/margin": 563.1048583984375,
"fcm_dpo/q_t": 0.38948333263397217,
"grad_norm": 29.91592788696289,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -0.9779269695281982,
"logits/rejected": -1.0172150135040283,
"logps/chosen": -606.327880859375,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -1202.4842529296875,
"loss": 1.0591,
"margin_dpo/margin_mean": 563.1048583984375,
"margin_dpo/margin_std": 814.8396606445312,
"step": 580
},
{
"KL/chosen_KL_mean": -770.3370361328125,
"KL/mean": -978.4288330078125,
"KL/rejected_KL_mean": -1186.520751953125,
"KL/std": 666.4249267578125,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.0008886848227120936,
"fcm_dpo/delta": 0.03695109486579895,
"fcm_dpo/margin": 416.1837158203125,
"fcm_dpo/q_t": 0.41858458518981934,
"grad_norm": 36.696754455566406,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -1.2336806058883667,
"logits/rejected": -1.262026071548462,
"logps/chosen": -832.559326171875,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -1278.25634765625,
"loss": 1.1399,
"margin_dpo/margin_mean": 416.1837158203125,
"margin_dpo/margin_std": 706.4039306640625,
"step": 581
},
{
"KL/chosen_KL_mean": -691.3490600585938,
"KL/mean": -915.9049072265625,
"KL/rejected_KL_mean": -1140.460693359375,
"KL/std": 554.6382446289062,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.000924235675483942,
"fcm_dpo/delta": -0.028372012078762054,
"fcm_dpo/margin": 449.111572265625,
"fcm_dpo/q_t": 0.40326303243637085,
"grad_norm": 34.956790924072266,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -1.1378693580627441,
"logits/rejected": -1.1712710857391357,
"logps/chosen": -749.5777587890625,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -1250.5302734375,
"loss": 1.1015,
"margin_dpo/margin_mean": 449.111572265625,
"margin_dpo/margin_std": 610.8257446289062,
"step": 582
},
{
"KL/chosen_KL_mean": -759.1519775390625,
"KL/mean": -915.6939697265625,
"KL/rejected_KL_mean": -1072.2359619140625,
"KL/std": 620.579833984375,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.0008432507747784257,
"fcm_dpo/delta": -0.03345673531293869,
"fcm_dpo/margin": 313.083984375,
"fcm_dpo/q_t": 0.43994593620300293,
"grad_norm": 49.73270034790039,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -1.1442230939865112,
"logits/rejected": -1.1517397165298462,
"logps/chosen": -816.0148315429688,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -1160.639892578125,
"loss": 1.2397,
"margin_dpo/margin_mean": 313.083984375,
"margin_dpo/margin_std": 745.907470703125,
"step": 583
},
{
"KL/chosen_KL_mean": -676.06298828125,
"KL/mean": -953.474853515625,
"KL/rejected_KL_mean": -1230.88671875,
"KL/std": 634.57861328125,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.0008079511462710798,
"fcm_dpo/delta": -0.06212177127599716,
"fcm_dpo/margin": 554.8237915039062,
"fcm_dpo/q_t": 0.3955593705177307,
"grad_norm": 27.940731048583984,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -1.0924582481384277,
"logits/rejected": -1.1197701692581177,
"logps/chosen": -732.9636840820312,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -1328.5228271484375,
"loss": 1.0545,
"margin_dpo/margin_mean": 554.8238525390625,
"margin_dpo/margin_std": 655.2168579101562,
"step": 584
},
{
"KL/chosen_KL_mean": -754.1314697265625,
"KL/mean": -1025.2532958984375,
"KL/rejected_KL_mean": -1296.3751220703125,
"KL/std": 712.186279296875,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.0007676214445382357,
"fcm_dpo/delta": -0.022018061950802803,
"fcm_dpo/margin": 542.24365234375,
"fcm_dpo/q_t": 0.40494245290756226,
"grad_norm": 26.670785903930664,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -1.1742076873779297,
"logits/rejected": -1.2190337181091309,
"logps/chosen": -815.10546875,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -1380.544677734375,
"loss": 1.1091,
"margin_dpo/margin_mean": 542.24365234375,
"margin_dpo/margin_std": 845.0147094726562,
"step": 585
},
{
"KL/chosen_KL_mean": -803.51416015625,
"KL/mean": -982.947021484375,
"KL/rejected_KL_mean": -1162.3798828125,
"KL/std": 643.473876953125,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.0008414636831730604,
"fcm_dpo/delta": 0.11576494574546814,
"fcm_dpo/margin": 358.8658447265625,
"fcm_dpo/q_t": 0.43414974212646484,
"grad_norm": 35.62641525268555,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -1.1895267963409424,
"logits/rejected": -1.2259807586669922,
"logps/chosen": -860.3697509765625,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -1254.1826171875,
"loss": 1.1841,
"margin_dpo/margin_mean": 358.8658447265625,
"margin_dpo/margin_std": 677.7850952148438,
"step": 586
},
{
"KL/chosen_KL_mean": -560.0857543945312,
"KL/mean": -783.292724609375,
"KL/rejected_KL_mean": -1006.4996337890625,
"KL/std": 627.01611328125,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.0008853634353727102,
"fcm_dpo/delta": 0.005062144249677658,
"fcm_dpo/margin": 446.41387939453125,
"fcm_dpo/q_t": 0.4096928834915161,
"grad_norm": 31.3030948638916,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -1.2153189182281494,
"logits/rejected": -1.2506670951843262,
"logps/chosen": -604.77734375,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -1089.12353515625,
"loss": 1.1015,
"margin_dpo/margin_mean": 446.41387939453125,
"margin_dpo/margin_std": 625.9844970703125,
"step": 587
},
{
"KL/chosen_KL_mean": -758.8787841796875,
"KL/mean": -942.8692626953125,
"KL/rejected_KL_mean": -1126.859619140625,
"KL/std": 569.2108154296875,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.0009099560556933284,
"fcm_dpo/delta": 0.07821156829595566,
"fcm_dpo/margin": 367.9808349609375,
"fcm_dpo/q_t": 0.42506054043769836,
"grad_norm": 27.26927947998047,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -1.142134666442871,
"logits/rejected": -1.1726654767990112,
"logps/chosen": -809.1737670898438,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -1234.2294921875,
"loss": 1.1514,
"margin_dpo/margin_mean": 367.9808349609375,
"margin_dpo/margin_std": 589.927734375,
"step": 588
},
{
"KL/chosen_KL_mean": -769.4779663085938,
"KL/mean": -1022.2039794921875,
"KL/rejected_KL_mean": -1274.929931640625,
"KL/std": 642.8394775390625,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.0008996998658403754,
"fcm_dpo/delta": -0.07083002477884293,
"fcm_dpo/margin": 505.4520263671875,
"fcm_dpo/q_t": 0.39429712295532227,
"grad_norm": 30.192138671875,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -1.128014087677002,
"logits/rejected": -1.1533284187316895,
"logps/chosen": -829.4078369140625,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -1386.5853271484375,
"loss": 1.0682,
"margin_dpo/margin_mean": 505.4520263671875,
"margin_dpo/margin_std": 678.6458740234375,
"step": 589
},
{
"KL/chosen_KL_mean": -634.3226928710938,
"KL/mean": -899.4765625,
"KL/rejected_KL_mean": -1164.63037109375,
"KL/std": 646.88037109375,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.000832444173283875,
"fcm_dpo/delta": -0.05546820908784866,
"fcm_dpo/margin": 530.3076782226562,
"fcm_dpo/q_t": 0.3984396457672119,
"grad_norm": 26.239267349243164,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -1.1118061542510986,
"logits/rejected": -1.1564977169036865,
"logps/chosen": -690.1324462890625,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -1270.693115234375,
"loss": 1.0645,
"margin_dpo/margin_mean": 530.3077392578125,
"margin_dpo/margin_std": 680.8991088867188,
"step": 590
},
{
"KL/chosen_KL_mean": -681.802490234375,
"KL/mean": -927.5146484375,
"KL/rejected_KL_mean": -1173.226806640625,
"KL/std": 607.2724609375,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.0008302570786327124,
"fcm_dpo/delta": -0.010073492303490639,
"fcm_dpo/margin": 491.42431640625,
"fcm_dpo/q_t": 0.40566641092300415,
"grad_norm": 36.62338638305664,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -1.1887190341949463,
"logits/rejected": -1.203287124633789,
"logps/chosen": -738.0430908203125,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -1257.012939453125,
"loss": 1.0977,
"margin_dpo/margin_mean": 491.42431640625,
"margin_dpo/margin_std": 697.486572265625,
"step": 591
},
{
"KL/chosen_KL_mean": -740.6405639648438,
"KL/mean": -975.1524658203125,
"KL/rejected_KL_mean": -1209.6644287109375,
"KL/std": 639.179931640625,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.0008298831526190042,
"fcm_dpo/delta": 0.013343624770641327,
"fcm_dpo/margin": 469.0238037109375,
"fcm_dpo/q_t": 0.413235068321228,
"grad_norm": 32.85460662841797,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -1.1930391788482666,
"logits/rejected": -1.2116713523864746,
"logps/chosen": -788.580810546875,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -1285.397216796875,
"loss": 1.152,
"margin_dpo/margin_mean": 469.0238342285156,
"margin_dpo/margin_std": 840.769775390625,
"step": 592
},
{
"KL/chosen_KL_mean": -722.489013671875,
"KL/mean": -891.135986328125,
"KL/rejected_KL_mean": -1059.782958984375,
"KL/std": 666.1737060546875,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.0008565220050513744,
"fcm_dpo/delta": 0.02374492585659027,
"fcm_dpo/margin": 337.2939147949219,
"fcm_dpo/q_t": 0.4361189305782318,
"grad_norm": 36.292320251464844,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -1.0026201009750366,
"logits/rejected": -1.000308871269226,
"logps/chosen": -771.1798095703125,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -1120.69091796875,
"loss": 1.2122,
"margin_dpo/margin_mean": 337.2939147949219,
"margin_dpo/margin_std": 735.4708862304688,
"step": 593
},
{
"KL/chosen_KL_mean": -694.244384765625,
"KL/mean": -903.6416015625,
"KL/rejected_KL_mean": -1113.038818359375,
"KL/std": 638.9234619140625,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.000834466889500618,
"fcm_dpo/delta": -0.05287259444594383,
"fcm_dpo/margin": 418.79449462890625,
"fcm_dpo/q_t": 0.41835910081863403,
"grad_norm": 43.15362548828125,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -1.1837817430496216,
"logits/rejected": -1.1828114986419678,
"logps/chosen": -749.1792602539062,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -1199.1385498046875,
"loss": 1.1504,
"margin_dpo/margin_mean": 418.7945556640625,
"margin_dpo/margin_std": 674.802001953125,
"step": 594
},
{
"KL/chosen_KL_mean": -639.7455444335938,
"KL/mean": -877.6624755859375,
"KL/rejected_KL_mean": -1115.579345703125,
"KL/std": 593.7979736328125,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.0008135140524245799,
"fcm_dpo/delta": 0.015700072050094604,
"fcm_dpo/margin": 475.833740234375,
"fcm_dpo/q_t": 0.4137133061885834,
"grad_norm": 40.35921859741211,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -1.1239649057388306,
"logits/rejected": -1.1565492153167725,
"logps/chosen": -689.1659545898438,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -1196.20654296875,
"loss": 1.1059,
"margin_dpo/margin_mean": 475.83380126953125,
"margin_dpo/margin_std": 678.7783203125,
"step": 595
},
{
"KL/chosen_KL_mean": -760.5462646484375,
"KL/mean": -910.779052734375,
"KL/rejected_KL_mean": -1061.011962890625,
"KL/std": 600.0560913085938,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.0009213665034621954,
"fcm_dpo/delta": 0.14197216928005219,
"fcm_dpo/margin": 300.4656982421875,
"fcm_dpo/q_t": 0.4435434341430664,
"grad_norm": 95.65324401855469,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -1.1321377754211426,
"logits/rejected": -1.1026864051818848,
"logps/chosen": -820.3379516601562,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -1141.10302734375,
"loss": 1.2344,
"margin_dpo/margin_mean": 300.4656677246094,
"margin_dpo/margin_std": 739.6461181640625,
"step": 596
},
{
"KL/chosen_KL_mean": -720.8272705078125,
"KL/mean": -1009.4996337890625,
"KL/rejected_KL_mean": -1298.1719970703125,
"KL/std": 710.2843017578125,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.0008209669031202793,
"fcm_dpo/delta": -0.12117333710193634,
"fcm_dpo/margin": 577.3447265625,
"fcm_dpo/q_t": 0.389828622341156,
"grad_norm": 26.715089797973633,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -1.0798090696334839,
"logits/rejected": -1.139901876449585,
"logps/chosen": -778.0880126953125,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -1398.86572265625,
"loss": 1.0726,
"margin_dpo/margin_mean": 577.3447265625,
"margin_dpo/margin_std": 803.3443603515625,
"step": 597
},
{
"KL/chosen_KL_mean": -685.9241943359375,
"KL/mean": -916.6578369140625,
"KL/rejected_KL_mean": -1147.391357421875,
"KL/std": 597.9251098632812,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.0008235409623011947,
"fcm_dpo/delta": 0.01904139667749405,
"fcm_dpo/margin": 461.46710205078125,
"fcm_dpo/q_t": 0.4120750427246094,
"grad_norm": 30.65525245666504,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -1.1381169557571411,
"logits/rejected": -1.1627564430236816,
"logps/chosen": -738.4427490234375,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -1236.835205078125,
"loss": 1.1215,
"margin_dpo/margin_mean": 461.4671630859375,
"margin_dpo/margin_std": 674.5496826171875,
"step": 598
},
{
"KL/chosen_KL_mean": -730.5025634765625,
"KL/mean": -930.897216796875,
"KL/rejected_KL_mean": -1131.2918701171875,
"KL/std": 586.238037109375,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.0008123376173898578,
"fcm_dpo/delta": -0.05563164874911308,
"fcm_dpo/margin": 400.7892761230469,
"fcm_dpo/q_t": 0.4195353388786316,
"grad_norm": 32.0465202331543,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -1.0951640605926514,
"logits/rejected": -1.108811616897583,
"logps/chosen": -780.3052978515625,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -1214.2703857421875,
"loss": 1.1353,
"margin_dpo/margin_mean": 400.789306640625,
"margin_dpo/margin_std": 522.7140502929688,
"step": 599
},
{
"KL/chosen_KL_mean": -804.6961669921875,
"KL/mean": -995.6710205078125,
"KL/rejected_KL_mean": -1186.645751953125,
"KL/std": 597.6373291015625,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.0008582416921854019,
"fcm_dpo/delta": 0.08676525950431824,
"fcm_dpo/margin": 381.94964599609375,
"fcm_dpo/q_t": 0.4284188151359558,
"grad_norm": 34.12265396118164,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -1.236138939857483,
"logits/rejected": -1.2366291284561157,
"logps/chosen": -871.131103515625,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -1272.102294921875,
"loss": 1.1788,
"margin_dpo/margin_mean": 381.94964599609375,
"margin_dpo/margin_std": 733.8538818359375,
"step": 600
},
{
"KL/chosen_KL_mean": -790.10302734375,
"KL/mean": -1049.4549560546875,
"KL/rejected_KL_mean": -1308.806884765625,
"KL/std": 661.8978271484375,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.000848105177283287,
"fcm_dpo/delta": -0.05170179903507233,
"fcm_dpo/margin": 518.703857421875,
"fcm_dpo/q_t": 0.39643436670303345,
"grad_norm": 31.908023834228516,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -1.1642279624938965,
"logits/rejected": -1.2025851011276245,
"logps/chosen": -849.2366943359375,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -1403.497802734375,
"loss": 1.0642,
"margin_dpo/margin_mean": 518.703857421875,
"margin_dpo/margin_std": 650.58935546875,
"step": 601
},
{
"KL/chosen_KL_mean": -551.074951171875,
"KL/mean": -840.6611938476562,
"KL/rejected_KL_mean": -1130.24755859375,
"KL/std": 586.427490234375,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.0008106931345537305,
"fcm_dpo/delta": -0.0917593240737915,
"fcm_dpo/margin": 579.1724853515625,
"fcm_dpo/q_t": 0.38940227031707764,
"grad_norm": 74.04801177978516,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -1.1794953346252441,
"logits/rejected": -1.2283287048339844,
"logps/chosen": -599.66845703125,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -1217.916015625,
"loss": 1.0376,
"margin_dpo/margin_mean": 579.1725463867188,
"margin_dpo/margin_std": 647.2698974609375,
"step": 602
},
{
"KL/chosen_KL_mean": -703.7813720703125,
"KL/mean": -975.3844604492188,
"KL/rejected_KL_mean": -1246.987548828125,
"KL/std": 681.866943359375,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.0007574381306767464,
"fcm_dpo/delta": -0.014486655592918396,
"fcm_dpo/margin": 543.206298828125,
"fcm_dpo/q_t": 0.4078108072280884,
"grad_norm": 36.255306243896484,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -1.1725047826766968,
"logits/rejected": -1.1789551973342896,
"logps/chosen": -774.1959838867188,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -1347.313232421875,
"loss": 1.0949,
"margin_dpo/margin_mean": 543.206298828125,
"margin_dpo/margin_std": 783.991943359375,
"step": 603
},
{
"KL/chosen_KL_mean": -677.0938110351562,
"KL/mean": -970.4219360351562,
"KL/rejected_KL_mean": -1263.75,
"KL/std": 666.1785888671875,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.0007347924984060228,
"fcm_dpo/delta": -0.03963564336299896,
"fcm_dpo/margin": 586.65625,
"fcm_dpo/q_t": 0.40320873260498047,
"grad_norm": 34.325931549072266,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -1.1817028522491455,
"logits/rejected": -1.2359271049499512,
"logps/chosen": -723.5518798828125,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -1355.6044921875,
"loss": 1.0899,
"margin_dpo/margin_mean": 586.65625,
"margin_dpo/margin_std": 854.323974609375,
"step": 604
},
{
"KL/chosen_KL_mean": -748.796875,
"KL/mean": -951.0224609375,
"KL/rejected_KL_mean": -1153.248046875,
"KL/std": 612.2020263671875,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.000774614978581667,
"fcm_dpo/delta": 0.10340875387191772,
"fcm_dpo/margin": 404.4511413574219,
"fcm_dpo/q_t": 0.4324561655521393,
"grad_norm": 47.217437744140625,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -1.118881344795227,
"logits/rejected": -1.1395517587661743,
"logps/chosen": -815.0462646484375,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -1255.552978515625,
"loss": 1.176,
"margin_dpo/margin_mean": 404.451171875,
"margin_dpo/margin_std": 743.8991088867188,
"step": 605
},
{
"KL/chosen_KL_mean": -717.6261596679688,
"KL/mean": -960.13623046875,
"KL/rejected_KL_mean": -1202.646240234375,
"KL/std": 601.002685546875,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.0008218929287977517,
"fcm_dpo/delta": 0.00010516494512557983,
"fcm_dpo/margin": 485.0201721191406,
"fcm_dpo/q_t": 0.40895628929138184,
"grad_norm": 25.46479606628418,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -1.141870141029358,
"logits/rejected": -1.163278579711914,
"logps/chosen": -772.4452514648438,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -1301.017822265625,
"loss": 1.0914,
"margin_dpo/margin_mean": 485.02020263671875,
"margin_dpo/margin_std": 640.8707275390625,
"step": 606
},
{
"KL/chosen_KL_mean": -767.1453247070312,
"KL/mean": -966.7313232421875,
"KL/rejected_KL_mean": -1166.3173828125,
"KL/std": 642.6415405273438,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.0008614829275757074,
"fcm_dpo/delta": 0.06497818231582642,
"fcm_dpo/margin": 399.1719970703125,
"fcm_dpo/q_t": 0.42393672466278076,
"grad_norm": 33.80935287475586,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -1.1142241954803467,
"logits/rejected": -1.104911208152771,
"logps/chosen": -825.2293701171875,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -1246.094482421875,
"loss": 1.159,
"margin_dpo/margin_mean": 399.1720275878906,
"margin_dpo/margin_std": 708.9419555664062,
"step": 607
},
{
"KL/chosen_KL_mean": -686.8588256835938,
"KL/mean": -919.4632568359375,
"KL/rejected_KL_mean": -1152.0677490234375,
"KL/std": 572.4481811523438,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.0008695480646565557,
"fcm_dpo/delta": -0.006582053378224373,
"fcm_dpo/margin": 465.2089538574219,
"fcm_dpo/q_t": 0.40590620040893555,
"grad_norm": 34.56772232055664,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -1.1690213680267334,
"logits/rejected": -1.1923552751541138,
"logps/chosen": -744.3096923828125,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -1246.841064453125,
"loss": 1.0818,
"margin_dpo/margin_mean": 465.208984375,
"margin_dpo/margin_std": 576.3578491210938,
"step": 608
},
{
"KL/chosen_KL_mean": -712.329345703125,
"KL/mean": -1004.8988037109375,
"KL/rejected_KL_mean": -1297.46826171875,
"KL/std": 753.2168579101562,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.000799752539023757,
"fcm_dpo/delta": -0.0890444964170456,
"fcm_dpo/margin": 585.1388549804688,
"fcm_dpo/q_t": 0.39761674404144287,
"grad_norm": 28.98295783996582,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -1.2440345287322998,
"logits/rejected": -1.281120777130127,
"logps/chosen": -771.134765625,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -1386.2841796875,
"loss": 1.082,
"margin_dpo/margin_mean": 585.1388549804688,
"margin_dpo/margin_std": 920.4691162109375,
"step": 609
},
{
"KL/chosen_KL_mean": -670.919189453125,
"KL/mean": -864.196044921875,
"KL/rejected_KL_mean": -1057.472900390625,
"KL/std": 553.8067626953125,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.0008423338877037168,
"fcm_dpo/delta": 0.08745961636304855,
"fcm_dpo/margin": 386.55364990234375,
"fcm_dpo/q_t": 0.42681533098220825,
"grad_norm": 36.83287811279297,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -1.1055896282196045,
"logits/rejected": -1.0923852920532227,
"logps/chosen": -736.6142578125,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -1140.878173828125,
"loss": 1.1577,
"margin_dpo/margin_mean": 386.55364990234375,
"margin_dpo/margin_std": 650.5970458984375,
"step": 610
},
{
"KL/chosen_KL_mean": -696.1116943359375,
"KL/mean": -999.7044677734375,
"KL/rejected_KL_mean": -1303.29736328125,
"KL/std": 736.2969970703125,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.0007899197516962886,
"fcm_dpo/delta": -0.10505129396915436,
"fcm_dpo/margin": 607.1857299804688,
"fcm_dpo/q_t": 0.39069780707359314,
"grad_norm": 37.02203369140625,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -1.1951907873153687,
"logits/rejected": -1.256514310836792,
"logps/chosen": -748.7111206054688,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -1389.62841796875,
"loss": 1.0611,
"margin_dpo/margin_mean": 607.1856689453125,
"margin_dpo/margin_std": 833.6317138671875,
"step": 611
},
{
"KL/chosen_KL_mean": -774.832275390625,
"KL/mean": -1000.640625,
"KL/rejected_KL_mean": -1226.448974609375,
"KL/std": 606.4058837890625,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.0008069322793744504,
"fcm_dpo/delta": 0.04110585153102875,
"fcm_dpo/margin": 451.61676025390625,
"fcm_dpo/q_t": 0.417686402797699,
"grad_norm": 29.89786148071289,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -1.1885337829589844,
"logits/rejected": -1.1959373950958252,
"logps/chosen": -834.156005859375,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -1314.761474609375,
"loss": 1.1247,
"margin_dpo/margin_mean": 451.61676025390625,
"margin_dpo/margin_std": 676.6371459960938,
"step": 612
},
{
"KL/chosen_KL_mean": -677.6046142578125,
"KL/mean": -974.4893188476562,
"KL/rejected_KL_mean": -1271.3740234375,
"KL/std": 672.0018310546875,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.0007531556766480207,
"fcm_dpo/delta": -0.0629609227180481,
"fcm_dpo/margin": 593.7694702148438,
"fcm_dpo/q_t": 0.39546507596969604,
"grad_norm": 26.967117309570312,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -1.1772477626800537,
"logits/rejected": -1.2423419952392578,
"logps/chosen": -737.3345336914062,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -1376.481689453125,
"loss": 1.0554,
"margin_dpo/margin_mean": 593.7694702148438,
"margin_dpo/margin_std": 724.92333984375,
"step": 613
},
{
"KL/chosen_KL_mean": -767.2695922851562,
"KL/mean": -1050.36181640625,
"KL/rejected_KL_mean": -1333.453857421875,
"KL/std": 694.145751953125,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.0007397818844765425,
"fcm_dpo/delta": -0.0238279327750206,
"fcm_dpo/margin": 566.184326171875,
"fcm_dpo/q_t": 0.4034821391105652,
"grad_norm": 31.130779266357422,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -1.1655349731445312,
"logits/rejected": -1.2575234174728394,
"logps/chosen": -820.2085571289062,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -1438.13330078125,
"loss": 1.0995,
"margin_dpo/margin_mean": 566.184326171875,
"margin_dpo/margin_std": 844.2413330078125,
"step": 614
},
{
"KL/chosen_KL_mean": -710.9827270507812,
"KL/mean": -932.9111328125,
"KL/rejected_KL_mean": -1154.839599609375,
"KL/std": 673.3746337890625,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.0007818863959982991,
"fcm_dpo/delta": 0.06144233047962189,
"fcm_dpo/margin": 443.85687255859375,
"fcm_dpo/q_t": 0.4220801591873169,
"grad_norm": 32.169227600097656,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -1.1130198240280151,
"logits/rejected": -1.1419811248779297,
"logps/chosen": -776.800048828125,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -1250.01708984375,
"loss": 1.1437,
"margin_dpo/margin_mean": 443.85687255859375,
"margin_dpo/margin_std": 725.8150634765625,
"step": 615
},
{
"KL/chosen_KL_mean": -881.0988159179688,
"KL/mean": -1073.1583251953125,
"KL/rejected_KL_mean": -1265.2177734375,
"KL/std": 594.451171875,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.0008393003372475505,
"fcm_dpo/delta": 0.09190573543310165,
"fcm_dpo/margin": 384.11895751953125,
"fcm_dpo/q_t": 0.428872287273407,
"grad_norm": 44.38339614868164,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -1.2580994367599487,
"logits/rejected": -1.2420811653137207,
"logps/chosen": -946.231689453125,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -1339.9183349609375,
"loss": 1.1633,
"margin_dpo/margin_mean": 384.11895751953125,
"margin_dpo/margin_std": 659.7879638671875,
"step": 616
},
{
"KL/chosen_KL_mean": -825.0336303710938,
"KL/mean": -983.9627685546875,
"KL/rejected_KL_mean": -1142.891845703125,
"KL/std": 582.34765625,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.0009453271050006151,
"fcm_dpo/delta": 0.11414283514022827,
"fcm_dpo/margin": 317.8581848144531,
"fcm_dpo/q_t": 0.4370740056037903,
"grad_norm": 73.6228256225586,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -1.1125876903533936,
"logits/rejected": -1.0867927074432373,
"logps/chosen": -888.0391845703125,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -1207.1259765625,
"loss": 1.2115,
"margin_dpo/margin_mean": 317.85821533203125,
"margin_dpo/margin_std": 713.1170654296875,
"step": 617
},
{
"KL/chosen_KL_mean": -846.281494140625,
"KL/mean": -1157.50244140625,
"KL/rejected_KL_mean": -1468.7235107421875,
"KL/std": 774.007080078125,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.0008609164506196976,
"fcm_dpo/delta": -0.18371938169002533,
"fcm_dpo/margin": 622.4420776367188,
"fcm_dpo/q_t": 0.38035058975219727,
"grad_norm": 32.81733322143555,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -1.1643829345703125,
"logits/rejected": -1.221153736114502,
"logps/chosen": -913.3828125,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -1560.876953125,
"loss": 1.087,
"margin_dpo/margin_mean": 622.4420776367188,
"margin_dpo/margin_std": 1005.3324584960938,
"step": 618
},
{
"KL/chosen_KL_mean": -804.8983154296875,
"KL/mean": -1043.05419921875,
"KL/rejected_KL_mean": -1281.2099609375,
"KL/std": 693.5120239257812,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.000811479170806706,
"fcm_dpo/delta": 0.0165516696870327,
"fcm_dpo/margin": 476.311767578125,
"fcm_dpo/q_t": 0.4198111295700073,
"grad_norm": 43.509002685546875,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -1.1278049945831299,
"logits/rejected": -1.1508402824401855,
"logps/chosen": -860.87646484375,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -1374.3953857421875,
"loss": 1.1626,
"margin_dpo/margin_mean": 476.311767578125,
"margin_dpo/margin_std": 932.0376586914062,
"step": 619
},
{
"KL/chosen_KL_mean": -741.0186157226562,
"KL/mean": -960.3197021484375,
"KL/rejected_KL_mean": -1179.6207275390625,
"KL/std": 604.758544921875,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.000822564004920423,
"fcm_dpo/delta": 0.046854112297296524,
"fcm_dpo/margin": 438.6020812988281,
"fcm_dpo/q_t": 0.4194413125514984,
"grad_norm": 36.871070861816406,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -1.2204086780548096,
"logits/rejected": -1.2356070280075073,
"logps/chosen": -800.8161010742188,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -1258.031494140625,
"loss": 1.131,
"margin_dpo/margin_mean": 438.6020812988281,
"margin_dpo/margin_std": 668.6788940429688,
"step": 620
},
{
"KL/chosen_KL_mean": -726.4075317382812,
"KL/mean": -1026.935546875,
"KL/rejected_KL_mean": -1327.463623046875,
"KL/std": 694.7574462890625,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.0008126673055812716,
"fcm_dpo/delta": -0.11893101036548615,
"fcm_dpo/margin": 601.0560302734375,
"fcm_dpo/q_t": 0.38432037830352783,
"grad_norm": 41.782691955566406,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -1.2633055448532104,
"logits/rejected": -1.3201026916503906,
"logps/chosen": -780.34130859375,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -1415.8330078125,
"loss": 1.0593,
"margin_dpo/margin_mean": 601.0560302734375,
"margin_dpo/margin_std": 780.2889404296875,
"step": 621
},
{
"KL/chosen_KL_mean": -730.9711303710938,
"KL/mean": -956.7696533203125,
"KL/rejected_KL_mean": -1182.568115234375,
"KL/std": 568.827392578125,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.0007760204607620835,
"fcm_dpo/delta": 0.0598348043859005,
"fcm_dpo/margin": 451.59698486328125,
"fcm_dpo/q_t": 0.4228161573410034,
"grad_norm": 30.10220718383789,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -1.0861146450042725,
"logits/rejected": -1.0753684043884277,
"logps/chosen": -791.2569580078125,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -1268.0869140625,
"loss": 1.1376,
"margin_dpo/margin_mean": 451.5970153808594,
"margin_dpo/margin_std": 714.3831787109375,
"step": 622
},
{
"KL/chosen_KL_mean": -812.854736328125,
"KL/mean": -974.1890258789062,
"KL/rejected_KL_mean": -1135.523193359375,
"KL/std": 589.9398193359375,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.000887159607373178,
"fcm_dpo/delta": 0.13209515810012817,
"fcm_dpo/margin": 322.6684875488281,
"fcm_dpo/q_t": 0.440715491771698,
"grad_norm": 47.28063201904297,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -1.2864371538162231,
"logits/rejected": -1.2883734703063965,
"logps/chosen": -877.01171875,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -1220.606201171875,
"loss": 1.2157,
"margin_dpo/margin_mean": 322.6684875488281,
"margin_dpo/margin_std": 727.4967041015625,
"step": 623
},
{
"KL/chosen_KL_mean": -803.52783203125,
"KL/mean": -1085.2625732421875,
"KL/rejected_KL_mean": -1366.997314453125,
"KL/std": 636.1299438476562,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.0008517538080923259,
"fcm_dpo/delta": -0.10489484667778015,
"fcm_dpo/margin": 563.469482421875,
"fcm_dpo/q_t": 0.3851096034049988,
"grad_norm": 33.36922836303711,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -1.2476253509521484,
"logits/rejected": -1.259102463722229,
"logps/chosen": -875.446533203125,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -1464.12939453125,
"loss": 1.0463,
"margin_dpo/margin_mean": 563.469482421875,
"margin_dpo/margin_std": 700.0657958984375,
"step": 624
},
{
"KL/chosen_KL_mean": -754.678466796875,
"KL/mean": -1054.7698974609375,
"KL/rejected_KL_mean": -1354.8614501953125,
"KL/std": 678.5252685546875,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.000783138326369226,
"fcm_dpo/delta": -0.09115005284547806,
"fcm_dpo/margin": 600.1829833984375,
"fcm_dpo/q_t": 0.38854146003723145,
"grad_norm": 52.66489791870117,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -1.1888365745544434,
"logits/rejected": -1.228683352470398,
"logps/chosen": -813.0205078125,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -1440.9517822265625,
"loss": 1.0299,
"margin_dpo/margin_mean": 600.1829833984375,
"margin_dpo/margin_std": 645.32763671875,
"step": 625
},
{
"KL/chosen_KL_mean": -938.5303344726562,
"KL/mean": -1111.8682861328125,
"KL/rejected_KL_mean": -1285.206298828125,
"KL/std": 750.4013671875,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.0008260097820311785,
"fcm_dpo/delta": 0.13311442732810974,
"fcm_dpo/margin": 346.6759033203125,
"fcm_dpo/q_t": 0.4367343485355377,
"grad_norm": 28.725139617919922,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -1.1261563301086426,
"logits/rejected": -1.1264156103134155,
"logps/chosen": -1013.6429443359375,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -1384.39501953125,
"loss": 1.2138,
"margin_dpo/margin_mean": 346.6759033203125,
"margin_dpo/margin_std": 732.3070068359375,
"step": 626
},
{
"KL/chosen_KL_mean": -613.0308837890625,
"KL/mean": -944.4686279296875,
"KL/rejected_KL_mean": -1275.90625,
"KL/std": 782.295166015625,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.000785567332059145,
"fcm_dpo/delta": -0.16194023191928864,
"fcm_dpo/margin": 662.8754272460938,
"fcm_dpo/q_t": 0.38459348678588867,
"grad_norm": 26.760295867919922,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -1.0580464601516724,
"logits/rejected": -1.1465545892715454,
"logps/chosen": -660.7740478515625,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -1382.6607666015625,
"loss": 1.0457,
"margin_dpo/margin_mean": 662.8753662109375,
"margin_dpo/margin_std": 917.1488647460938,
"step": 627
},
{
"KL/chosen_KL_mean": -848.233642578125,
"KL/mean": -1102.293212890625,
"KL/rejected_KL_mean": -1356.35302734375,
"KL/std": 702.3494873046875,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.0007409874233417213,
"fcm_dpo/delta": 0.02891882136464119,
"fcm_dpo/margin": 508.11920166015625,
"fcm_dpo/q_t": 0.41614317893981934,
"grad_norm": 28.3284854888916,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -1.2321488857269287,
"logits/rejected": -1.258644700050354,
"logps/chosen": -908.4166259765625,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -1457.90771484375,
"loss": 1.1272,
"margin_dpo/margin_mean": 508.1192626953125,
"margin_dpo/margin_std": 800.9361572265625,
"step": 628
},
{
"KL/chosen_KL_mean": -841.8361206054688,
"KL/mean": -1071.118408203125,
"KL/rejected_KL_mean": -1300.4007568359375,
"KL/std": 656.1351318359375,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.0007858018507249653,
"fcm_dpo/delta": 0.04754742980003357,
"fcm_dpo/margin": 458.56463623046875,
"fcm_dpo/q_t": 0.4175514578819275,
"grad_norm": 37.84564208984375,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -1.2372636795043945,
"logits/rejected": -1.2568615674972534,
"logps/chosen": -906.0496826171875,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -1392.054443359375,
"loss": 1.1255,
"margin_dpo/margin_mean": 458.56463623046875,
"margin_dpo/margin_std": 679.4099731445312,
"step": 629
},
{
"KL/chosen_KL_mean": -726.173583984375,
"KL/mean": -865.2925415039062,
"KL/rejected_KL_mean": -1004.4114990234375,
"KL/std": 653.0287475585938,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.0008041888941079378,
"fcm_dpo/delta": 0.038379114121198654,
"fcm_dpo/margin": 278.2379455566406,
"fcm_dpo/q_t": 0.45435744524002075,
"grad_norm": 49.139530181884766,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -1.1940525770187378,
"logits/rejected": -1.1697330474853516,
"logps/chosen": -785.464599609375,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -1088.009765625,
"loss": 1.2713,
"margin_dpo/margin_mean": 278.2379455566406,
"margin_dpo/margin_std": 808.236572265625,
"step": 630
},
{
"KL/chosen_KL_mean": -864.2023315429688,
"KL/mean": -1085.380126953125,
"KL/rejected_KL_mean": -1306.557861328125,
"KL/std": 646.6876831054688,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.000846027978695929,
"fcm_dpo/delta": 0.031014172360301018,
"fcm_dpo/margin": 442.35552978515625,
"fcm_dpo/q_t": 0.4147540330886841,
"grad_norm": 34.697113037109375,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -1.3160864114761353,
"logits/rejected": -1.3359469175338745,
"logps/chosen": -923.6559448242188,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -1387.509521484375,
"loss": 1.1439,
"margin_dpo/margin_mean": 442.35552978515625,
"margin_dpo/margin_std": 760.668212890625,
"step": 631
},
{
"KL/chosen_KL_mean": -795.5738525390625,
"KL/mean": -1007.8973388671875,
"KL/rejected_KL_mean": -1220.220947265625,
"KL/std": 619.8748779296875,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.0008544891024939716,
"fcm_dpo/delta": 0.044195935130119324,
"fcm_dpo/margin": 424.64697265625,
"fcm_dpo/q_t": 0.4180447459220886,
"grad_norm": 51.36729049682617,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -1.279064416885376,
"logits/rejected": -1.281667947769165,
"logps/chosen": -856.9254150390625,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -1306.381103515625,
"loss": 1.1631,
"margin_dpo/margin_mean": 424.64697265625,
"margin_dpo/margin_std": 784.182373046875,
"step": 632
},
{
"KL/chosen_KL_mean": -906.9592895507812,
"KL/mean": -1059.207275390625,
"KL/rejected_KL_mean": -1211.455322265625,
"KL/std": 601.04248046875,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.000900345272384584,
"fcm_dpo/delta": 0.03711273893713951,
"fcm_dpo/margin": 304.4959716796875,
"fcm_dpo/q_t": 0.4383799433708191,
"grad_norm": 50.936614990234375,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -1.30977463722229,
"logits/rejected": -1.322211503982544,
"logps/chosen": -964.2374267578125,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -1303.039306640625,
"loss": 1.2248,
"margin_dpo/margin_mean": 304.4960021972656,
"margin_dpo/margin_std": 702.047119140625,
"step": 633
},
{
"KL/chosen_KL_mean": -957.939208984375,
"KL/mean": -1103.0045166015625,
"KL/rejected_KL_mean": -1248.06982421875,
"KL/std": 700.0567626953125,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.0010044092778116465,
"fcm_dpo/delta": 0.12742158770561218,
"fcm_dpo/margin": 290.1304931640625,
"fcm_dpo/q_t": 0.4410281479358673,
"grad_norm": 48.39108657836914,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -1.1118695735931396,
"logits/rejected": -1.1093440055847168,
"logps/chosen": -1024.5582275390625,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -1355.1954345703125,
"loss": 1.2305,
"margin_dpo/margin_mean": 290.1304931640625,
"margin_dpo/margin_std": 708.994873046875,
"step": 634
},
{
"KL/chosen_KL_mean": -850.6639404296875,
"KL/mean": -1053.29736328125,
"KL/rejected_KL_mean": -1255.930908203125,
"KL/std": 765.2185668945312,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.001036031637340784,
"fcm_dpo/delta": -0.02556215599179268,
"fcm_dpo/margin": 405.26690673828125,
"fcm_dpo/q_t": 0.41270193457603455,
"grad_norm": 61.66108703613281,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -1.2386977672576904,
"logits/rejected": -1.2290127277374268,
"logps/chosen": -924.6194458007812,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -1338.431396484375,
"loss": 1.1466,
"margin_dpo/margin_mean": 405.2669372558594,
"margin_dpo/margin_std": 769.2952880859375,
"step": 635
},
{
"KL/chosen_KL_mean": -771.644775390625,
"KL/mean": -1004.4685668945312,
"KL/rejected_KL_mean": -1237.29248046875,
"KL/std": 704.637451171875,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.0009967228397727013,
"fcm_dpo/delta": -0.08795134723186493,
"fcm_dpo/margin": 465.6476135253906,
"fcm_dpo/q_t": 0.39764294028282166,
"grad_norm": 32.073509216308594,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -1.1573783159255981,
"logits/rejected": -1.1642378568649292,
"logps/chosen": -831.273681640625,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -1319.271240234375,
"loss": 1.1023,
"margin_dpo/margin_mean": 465.6475830078125,
"margin_dpo/margin_std": 754.404541015625,
"step": 636
},
{
"KL/chosen_KL_mean": -774.3438110351562,
"KL/mean": -1051.230712890625,
"KL/rejected_KL_mean": -1328.117431640625,
"KL/std": 712.5130615234375,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.0008392821764573455,
"fcm_dpo/delta": -0.09347308427095413,
"fcm_dpo/margin": 553.773681640625,
"fcm_dpo/q_t": 0.39158618450164795,
"grad_norm": 31.033771514892578,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -1.2771704196929932,
"logits/rejected": -1.3423259258270264,
"logps/chosen": -823.9964599609375,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -1426.5225830078125,
"loss": 1.0572,
"margin_dpo/margin_mean": 553.773681640625,
"margin_dpo/margin_std": 714.4674072265625,
"step": 637
},
{
"KL/chosen_KL_mean": -740.0574951171875,
"KL/mean": -960.1834716796875,
"KL/rejected_KL_mean": -1180.3095703125,
"KL/std": 647.9360961914062,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.0008902228437364101,
"fcm_dpo/delta": -0.0007760748267173767,
"fcm_dpo/margin": 440.2519836425781,
"fcm_dpo/q_t": 0.40659117698669434,
"grad_norm": 46.73960494995117,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -1.0999634265899658,
"logits/rejected": -1.0819281339645386,
"logps/chosen": -798.2141723632812,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -1259.6109619140625,
"loss": 1.1511,
"margin_dpo/margin_mean": 440.2519836425781,
"margin_dpo/margin_std": 770.6236572265625,
"step": 638
},
{
"KL/chosen_KL_mean": -1026.832275390625,
"KL/mean": -1113.654052734375,
"KL/rejected_KL_mean": -1200.4757080078125,
"KL/std": 620.957275390625,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.0008462556288577616,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 173.64344787597656,
"fcm_dpo/q_t": 0.46698197722435,
"grad_norm": 103.9713363647461,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -1.081902265548706,
"logits/rejected": -1.0169289112091064,
"logps/chosen": -1099.155517578125,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -1274.7506103515625,
"loss": 1.3501,
"margin_dpo/margin_mean": 173.64346313476562,
"margin_dpo/margin_std": 787.2015380859375,
"step": 639
},
{
"KL/chosen_KL_mean": -761.8798828125,
"KL/mean": -1064.907958984375,
"KL/rejected_KL_mean": -1367.93603515625,
"KL/std": 741.346923828125,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.000793866696767509,
"fcm_dpo/delta": -0.10620252788066864,
"fcm_dpo/margin": 606.05615234375,
"fcm_dpo/q_t": 0.3925554156303406,
"grad_norm": 52.245452880859375,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -1.0945394039154053,
"logits/rejected": -1.133887767791748,
"logps/chosen": -818.0142822265625,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -1476.5361328125,
"loss": 1.0622,
"margin_dpo/margin_mean": 606.05615234375,
"margin_dpo/margin_std": 859.9293212890625,
"step": 640
},
{
"KL/chosen_KL_mean": -864.1361083984375,
"KL/mean": -1076.710693359375,
"KL/rejected_KL_mean": -1289.285400390625,
"KL/std": 632.7630615234375,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.0008008034783415496,
"fcm_dpo/delta": 0.0718461126089096,
"fcm_dpo/margin": 425.14935302734375,
"fcm_dpo/q_t": 0.4279997944831848,
"grad_norm": 27.010753631591797,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -1.1861107349395752,
"logits/rejected": -1.2011582851409912,
"logps/chosen": -929.1329345703125,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -1376.2777099609375,
"loss": 1.1776,
"margin_dpo/margin_mean": 425.14935302734375,
"margin_dpo/margin_std": 845.0499877929688,
"step": 641
},
{
"KL/chosen_KL_mean": -837.578125,
"KL/mean": -1093.5718994140625,
"KL/rejected_KL_mean": -1349.565673828125,
"KL/std": 812.8326416015625,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.0008224358316510916,
"fcm_dpo/delta": -0.02903025597333908,
"fcm_dpo/margin": 511.987548828125,
"fcm_dpo/q_t": 0.412765234708786,
"grad_norm": 36.307071685791016,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -1.1126117706298828,
"logits/rejected": -1.1542222499847412,
"logps/chosen": -903.267333984375,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -1459.8076171875,
"loss": 1.152,
"margin_dpo/margin_mean": 511.987548828125,
"margin_dpo/margin_std": 1003.8565673828125,
"step": 642
},
{
"KL/chosen_KL_mean": -778.371826171875,
"KL/mean": -960.838623046875,
"KL/rejected_KL_mean": -1143.305419921875,
"KL/std": 605.3773193359375,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.0007829120149835944,
"fcm_dpo/delta": -0.014414433389902115,
"fcm_dpo/margin": 364.93359375,
"fcm_dpo/q_t": 0.43285301327705383,
"grad_norm": 33.376853942871094,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -1.2616610527038574,
"logits/rejected": -1.2592954635620117,
"logps/chosen": -830.32177734375,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -1230.7738037109375,
"loss": 1.1815,
"margin_dpo/margin_mean": 364.93359375,
"margin_dpo/margin_std": 620.4666748046875,
"step": 643
},
{
"KL/chosen_KL_mean": -716.4739990234375,
"KL/mean": -920.228515625,
"KL/rejected_KL_mean": -1123.983154296875,
"KL/std": 688.679931640625,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.0008282763883471489,
"fcm_dpo/delta": 0.07501886785030365,
"fcm_dpo/margin": 407.50909423828125,
"fcm_dpo/q_t": 0.4266040325164795,
"grad_norm": 32.15571594238281,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -1.1580393314361572,
"logits/rejected": -1.1756101846694946,
"logps/chosen": -775.49169921875,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -1211.1197509765625,
"loss": 1.1794,
"margin_dpo/margin_mean": 407.50909423828125,
"margin_dpo/margin_std": 801.0850830078125,
"step": 644
},
{
"KL/chosen_KL_mean": -772.6549072265625,
"KL/mean": -870.29248046875,
"KL/rejected_KL_mean": -967.9300537109375,
"KL/std": 544.7769165039062,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.0009023309103213251,
"fcm_dpo/delta": 0.06694209575653076,
"fcm_dpo/margin": 195.2750701904297,
"fcm_dpo/q_t": 0.4632098078727722,
"grad_norm": 85.04663848876953,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -1.126030683517456,
"logits/rejected": -1.115586280822754,
"logps/chosen": -828.5309448242188,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -1065.7108154296875,
"loss": 1.3318,
"margin_dpo/margin_mean": 195.2751007080078,
"margin_dpo/margin_std": 740.9451904296875,
"step": 645
},
{
"KL/chosen_KL_mean": -735.0755615234375,
"KL/mean": -908.2972412109375,
"KL/rejected_KL_mean": -1081.5189208984375,
"KL/std": 522.7762451171875,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.0009436524705961347,
"fcm_dpo/delta": 0.08735965192317963,
"fcm_dpo/margin": 346.443359375,
"fcm_dpo/q_t": 0.4290798604488373,
"grad_norm": 62.62091064453125,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -1.1851344108581543,
"logits/rejected": -1.1642956733703613,
"logps/chosen": -796.351318359375,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -1159.024658203125,
"loss": 1.1762,
"margin_dpo/margin_mean": 346.4433288574219,
"margin_dpo/margin_std": 640.25537109375,
"step": 646
},
{
"KL/chosen_KL_mean": -630.638671875,
"KL/mean": -835.8479614257812,
"KL/rejected_KL_mean": -1041.0572509765625,
"KL/std": 575.6964111328125,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.0009774916106835008,
"fcm_dpo/delta": -0.0016585299745202065,
"fcm_dpo/margin": 410.41851806640625,
"fcm_dpo/q_t": 0.40817171335220337,
"grad_norm": 27.86429786682129,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -1.0603611469268799,
"logits/rejected": -1.0982961654663086,
"logps/chosen": -685.4910888671875,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -1134.57666015625,
"loss": 1.1114,
"margin_dpo/margin_mean": 410.41851806640625,
"margin_dpo/margin_std": 619.114990234375,
"step": 647
},
{
"KL/chosen_KL_mean": -673.4736328125,
"KL/mean": -996.9141845703125,
"KL/rejected_KL_mean": -1320.354736328125,
"KL/std": 748.5928955078125,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.0008425582782365382,
"fcm_dpo/delta": -0.20228338241577148,
"fcm_dpo/margin": 646.8811645507812,
"fcm_dpo/q_t": 0.3778739869594574,
"grad_norm": 32.02235412597656,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -1.1987550258636475,
"logits/rejected": -1.2712297439575195,
"logps/chosen": -727.6451416015625,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -1419.067626953125,
"loss": 1.0347,
"margin_dpo/margin_mean": 646.8811645507812,
"margin_dpo/margin_std": 918.9876708984375,
"step": 648
},
{
"KL/chosen_KL_mean": -728.5607299804688,
"KL/mean": -868.1865234375,
"KL/rejected_KL_mean": -1007.812255859375,
"KL/std": 557.9706420898438,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.0008311655838042498,
"fcm_dpo/delta": 0.06771388649940491,
"fcm_dpo/margin": 279.2515869140625,
"fcm_dpo/q_t": 0.4502527713775635,
"grad_norm": 32.22618865966797,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -1.224090814590454,
"logits/rejected": -1.2157777547836304,
"logps/chosen": -791.0410766601562,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -1087.8895263671875,
"loss": 1.2478,
"margin_dpo/margin_mean": 279.2515869140625,
"margin_dpo/margin_std": 673.363525390625,
"step": 649
},
{
"KL/chosen_KL_mean": -760.45458984375,
"KL/mean": -985.4943237304688,
"KL/rejected_KL_mean": -1210.533935546875,
"KL/std": 681.9244995117188,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.0008620332810096443,
"fcm_dpo/delta": 0.014763827435672283,
"fcm_dpo/margin": 450.07940673828125,
"fcm_dpo/q_t": 0.4147789478302002,
"grad_norm": 35.338233947753906,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -1.1790560483932495,
"logits/rejected": -1.2069194316864014,
"logps/chosen": -816.5474243164062,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -1308.798828125,
"loss": 1.1394,
"margin_dpo/margin_mean": 450.0793762207031,
"margin_dpo/margin_std": 782.4627685546875,
"step": 650
},
{
"KL/chosen_KL_mean": -495.2056884765625,
"KL/mean": -796.1655883789062,
"KL/rejected_KL_mean": -1097.12548828125,
"KL/std": 634.3084716796875,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.0007995986961759627,
"fcm_dpo/delta": -0.10846766829490662,
"fcm_dpo/margin": 601.9198608398438,
"fcm_dpo/q_t": 0.3852821886539459,
"grad_norm": 45.97235107421875,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -1.143014669418335,
"logits/rejected": -1.2076648473739624,
"logps/chosen": -538.631103515625,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -1197.08349609375,
"loss": 1.023,
"margin_dpo/margin_mean": 601.9197998046875,
"margin_dpo/margin_std": 651.2605590820312,
"step": 651
},
{
"KL/chosen_KL_mean": -655.0028076171875,
"KL/mean": -886.6578979492188,
"KL/rejected_KL_mean": -1118.31298828125,
"KL/std": 625.3679809570312,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.0008092697244137526,
"fcm_dpo/delta": 0.029820626601576805,
"fcm_dpo/margin": 463.31011962890625,
"fcm_dpo/q_t": 0.41610437631607056,
"grad_norm": 35.829437255859375,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -1.1463913917541504,
"logits/rejected": -1.1780339479446411,
"logps/chosen": -717.5796508789062,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -1230.080810546875,
"loss": 1.1258,
"margin_dpo/margin_mean": 463.31011962890625,
"margin_dpo/margin_std": 727.44580078125,
"step": 652
},
{
"KL/chosen_KL_mean": -785.6455078125,
"KL/mean": -1017.7234497070312,
"KL/rejected_KL_mean": -1249.8013916015625,
"KL/std": 702.5745849609375,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.0008226313511840999,
"fcm_dpo/delta": 0.022288989275693893,
"fcm_dpo/margin": 464.1558837890625,
"fcm_dpo/q_t": 0.41595858335494995,
"grad_norm": 36.48124313354492,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -1.2419447898864746,
"logits/rejected": -1.2747020721435547,
"logps/chosen": -846.7584838867188,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -1353.051025390625,
"loss": 1.123,
"margin_dpo/margin_mean": 464.1558837890625,
"margin_dpo/margin_std": 734.2294311523438,
"step": 653
},
{
"KL/chosen_KL_mean": -720.1655883789062,
"KL/mean": -930.6776733398438,
"KL/rejected_KL_mean": -1141.189697265625,
"KL/std": 579.1441650390625,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.0008555342210456729,
"fcm_dpo/delta": 0.04842513054609299,
"fcm_dpo/margin": 421.0242004394531,
"fcm_dpo/q_t": 0.4207690358161926,
"grad_norm": 29.795970916748047,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -1.1985931396484375,
"logits/rejected": -1.2237248420715332,
"logps/chosen": -781.8937377929688,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -1239.963623046875,
"loss": 1.1322,
"margin_dpo/margin_mean": 421.0242004394531,
"margin_dpo/margin_std": 659.9177856445312,
"step": 654
},
{
"KL/chosen_KL_mean": -680.0551147460938,
"KL/mean": -888.6227416992188,
"KL/rejected_KL_mean": -1097.1903076171875,
"KL/std": 573.6949462890625,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.000887269270606339,
"fcm_dpo/delta": 0.036719754338264465,
"fcm_dpo/margin": 417.13525390625,
"fcm_dpo/q_t": 0.4181985855102539,
"grad_norm": 32.07929229736328,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -1.17026948928833,
"logits/rejected": -1.2217252254486084,
"logps/chosen": -729.6319580078125,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -1195.482177734375,
"loss": 1.1344,
"margin_dpo/margin_mean": 417.1352233886719,
"margin_dpo/margin_std": 680.7271728515625,
"step": 655
},
{
"KL/chosen_KL_mean": -690.5496826171875,
"KL/mean": -1050.8857421875,
"KL/rejected_KL_mean": -1411.2216796875,
"KL/std": 771.2440185546875,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.0007734264945611358,
"fcm_dpo/delta": -0.2174438238143921,
"fcm_dpo/margin": 720.6719970703125,
"fcm_dpo/q_t": 0.3665843605995178,
"grad_norm": 77.32756805419922,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -1.065166711807251,
"logits/rejected": -1.151442289352417,
"logps/chosen": -743.09912109375,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -1524.896484375,
"loss": 0.9841,
"margin_dpo/margin_mean": 720.6719970703125,
"margin_dpo/margin_std": 759.9878540039062,
"step": 656
},
{
"KL/chosen_KL_mean": -651.868896484375,
"KL/mean": -958.4510498046875,
"KL/rejected_KL_mean": -1265.033203125,
"KL/std": 730.9705810546875,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.0006971274269744754,
"fcm_dpo/delta": -0.03567207232117653,
"fcm_dpo/margin": 613.1644287109375,
"fcm_dpo/q_t": 0.4013122618198395,
"grad_norm": 41.946922302246094,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -1.2377593517303467,
"logits/rejected": -1.309966802597046,
"logps/chosen": -698.5694580078125,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -1362.947998046875,
"loss": 1.0675,
"margin_dpo/margin_mean": 613.1643676757812,
"margin_dpo/margin_std": 760.7774658203125,
"step": 657
},
{
"KL/chosen_KL_mean": -706.0740966796875,
"KL/mean": -946.1854248046875,
"KL/rejected_KL_mean": -1186.2967529296875,
"KL/std": 566.0189208984375,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.0007183150737546384,
"fcm_dpo/delta": 0.06623679399490356,
"fcm_dpo/margin": 480.22265625,
"fcm_dpo/q_t": 0.42133376002311707,
"grad_norm": 38.043453216552734,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -1.1505177021026611,
"logits/rejected": -1.187692642211914,
"logps/chosen": -767.0323486328125,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -1282.2362060546875,
"loss": 1.1244,
"margin_dpo/margin_mean": 480.2226257324219,
"margin_dpo/margin_std": 638.424072265625,
"step": 658
},
{
"KL/chosen_KL_mean": -624.73291015625,
"KL/mean": -855.0606689453125,
"KL/rejected_KL_mean": -1085.388427734375,
"KL/std": 586.4114990234375,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.0007668976904824376,
"fcm_dpo/delta": 0.0567876435816288,
"fcm_dpo/margin": 460.6553649902344,
"fcm_dpo/q_t": 0.4216195344924927,
"grad_norm": 27.48058319091797,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -1.0954115390777588,
"logits/rejected": -1.0741057395935059,
"logps/chosen": -701.4759521484375,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -1172.859375,
"loss": 1.1233,
"margin_dpo/margin_mean": 460.6553955078125,
"margin_dpo/margin_std": 662.3754272460938,
"step": 659
},
{
"KL/chosen_KL_mean": -714.209228515625,
"KL/mean": -1003.320068359375,
"KL/rejected_KL_mean": -1292.430908203125,
"KL/std": 690.353515625,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.0007632029009982944,
"fcm_dpo/delta": -0.052839018404483795,
"fcm_dpo/margin": 578.2216796875,
"fcm_dpo/q_t": 0.3979361057281494,
"grad_norm": 33.15445327758789,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -1.1898115873336792,
"logits/rejected": -1.206876277923584,
"logps/chosen": -773.257080078125,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -1368.3909912109375,
"loss": 1.0682,
"margin_dpo/margin_mean": 578.2216796875,
"margin_dpo/margin_std": 753.8338623046875,
"step": 660
},
{
"KL/chosen_KL_mean": -610.59130859375,
"KL/mean": -878.7642211914062,
"KL/rejected_KL_mean": -1146.9371337890625,
"KL/std": 757.8616943359375,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.0007565034320577979,
"fcm_dpo/delta": -0.0085027776658535,
"fcm_dpo/margin": 536.3458251953125,
"fcm_dpo/q_t": 0.40905699133872986,
"grad_norm": 47.93645477294922,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -1.2252672910690308,
"logits/rejected": -1.252467393875122,
"logps/chosen": -661.2652587890625,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -1232.94287109375,
"loss": 1.0877,
"margin_dpo/margin_mean": 536.3458251953125,
"margin_dpo/margin_std": 738.3463134765625,
"step": 661
},
{
"KL/chosen_KL_mean": -732.677490234375,
"KL/mean": -931.5089721679688,
"KL/rejected_KL_mean": -1130.3404541015625,
"KL/std": 597.563232421875,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.0007978413486853242,
"fcm_dpo/delta": 0.09812071919441223,
"fcm_dpo/margin": 397.6629333496094,
"fcm_dpo/q_t": 0.4306020140647888,
"grad_norm": 29.834253311157227,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -1.1183321475982666,
"logits/rejected": -1.1220099925994873,
"logps/chosen": -801.9385986328125,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -1219.3963623046875,
"loss": 1.178,
"margin_dpo/margin_mean": 397.66290283203125,
"margin_dpo/margin_std": 750.21630859375,
"step": 662
},
{
"KL/chosen_KL_mean": -667.935546875,
"KL/mean": -934.6746826171875,
"KL/rejected_KL_mean": -1201.4139404296875,
"KL/std": 711.580810546875,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.0007925699464976788,
"fcm_dpo/delta": -0.02965104579925537,
"fcm_dpo/margin": 533.4783935546875,
"fcm_dpo/q_t": 0.40630367398262024,
"grad_norm": 32.280399322509766,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -1.1441211700439453,
"logits/rejected": -1.1800665855407715,
"logps/chosen": -732.814453125,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -1315.33935546875,
"loss": 1.1028,
"margin_dpo/margin_mean": 533.4784545898438,
"margin_dpo/margin_std": 843.329345703125,
"step": 663
},
{
"KL/chosen_KL_mean": -713.7249145507812,
"KL/mean": -985.4025268554688,
"KL/rejected_KL_mean": -1257.0802001953125,
"KL/std": 668.9918823242188,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.0007669959450140595,
"fcm_dpo/delta": -0.022985609248280525,
"fcm_dpo/margin": 543.3553466796875,
"fcm_dpo/q_t": 0.40422388911247253,
"grad_norm": 36.076908111572266,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -1.1553318500518799,
"logits/rejected": -1.198824167251587,
"logps/chosen": -774.6134033203125,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -1362.6019287109375,
"loss": 1.0837,
"margin_dpo/margin_mean": 543.3553466796875,
"margin_dpo/margin_std": 723.924072265625,
"step": 664
},
{
"KL/chosen_KL_mean": -651.1171875,
"KL/mean": -885.782470703125,
"KL/rejected_KL_mean": -1120.4478759765625,
"KL/std": 580.0274658203125,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.0007947899866849184,
"fcm_dpo/delta": 0.033155716955661774,
"fcm_dpo/margin": 469.3306579589844,
"fcm_dpo/q_t": 0.4146922826766968,
"grad_norm": 52.002296447753906,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -1.2848122119903564,
"logits/rejected": -1.3209171295166016,
"logps/chosen": -711.6812744140625,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -1205.256591796875,
"loss": 1.1028,
"margin_dpo/margin_mean": 469.33062744140625,
"margin_dpo/margin_std": 604.48974609375,
"step": 665
},
{
"KL/chosen_KL_mean": -704.703125,
"KL/mean": -937.7362060546875,
"KL/rejected_KL_mean": -1170.769287109375,
"KL/std": 603.8134765625,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.0008114855154417455,
"fcm_dpo/delta": 0.026824505999684334,
"fcm_dpo/margin": 466.06622314453125,
"fcm_dpo/q_t": 0.4145530164241791,
"grad_norm": 36.94413375854492,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -1.0528377294540405,
"logits/rejected": -1.0693092346191406,
"logps/chosen": -769.123046875,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -1266.660888671875,
"loss": 1.1138,
"margin_dpo/margin_mean": 466.06622314453125,
"margin_dpo/margin_std": 662.6096801757812,
"step": 666
},
{
"KL/chosen_KL_mean": -802.0358276367188,
"KL/mean": -1084.480712890625,
"KL/rejected_KL_mean": -1366.925537109375,
"KL/std": 679.0506591796875,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.0007717009866610169,
"fcm_dpo/delta": -0.05106900632381439,
"fcm_dpo/margin": 564.8895874023438,
"fcm_dpo/q_t": 0.39843225479125977,
"grad_norm": 29.158369064331055,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -1.1201040744781494,
"logits/rejected": -1.1252251863479614,
"logps/chosen": -871.3128662109375,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -1454.760986328125,
"loss": 1.0729,
"margin_dpo/margin_mean": 564.8895874023438,
"margin_dpo/margin_std": 735.8818359375,
"step": 667
},
{
"KL/chosen_KL_mean": -843.2930297851562,
"KL/mean": -1036.6741943359375,
"KL/rejected_KL_mean": -1230.05517578125,
"KL/std": 693.7163696289062,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.0008507584570907056,
"fcm_dpo/delta": 0.08011265844106674,
"fcm_dpo/margin": 386.7622375488281,
"fcm_dpo/q_t": 0.43072831630706787,
"grad_norm": 44.3238410949707,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -1.2108023166656494,
"logits/rejected": -1.2172951698303223,
"logps/chosen": -915.8970336914062,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -1333.7943115234375,
"loss": 1.1988,
"margin_dpo/margin_mean": 386.7622375488281,
"margin_dpo/margin_std": 833.273681640625,
"step": 668
},
{
"KL/chosen_KL_mean": -618.487060546875,
"KL/mean": -896.7724609375,
"KL/rejected_KL_mean": -1175.057861328125,
"KL/std": 641.3170166015625,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.0008110044291242957,
"fcm_dpo/delta": -0.06620398163795471,
"fcm_dpo/margin": 556.57080078125,
"fcm_dpo/q_t": 0.39469441771507263,
"grad_norm": 24.252483367919922,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -1.144885778427124,
"logits/rejected": -1.1587302684783936,
"logps/chosen": -664.603515625,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -1252.9822998046875,
"loss": 1.0677,
"margin_dpo/margin_mean": 556.57080078125,
"margin_dpo/margin_std": 739.805419921875,
"step": 669
},
{
"KL/chosen_KL_mean": -602.510009765625,
"KL/mean": -883.6866455078125,
"KL/rejected_KL_mean": -1164.86328125,
"KL/std": 612.798583984375,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.0007813957636244595,
"fcm_dpo/delta": -0.051431987434625626,
"fcm_dpo/margin": 562.3533935546875,
"fcm_dpo/q_t": 0.3977850079536438,
"grad_norm": 31.681093215942383,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -1.0778193473815918,
"logits/rejected": -1.1026027202606201,
"logps/chosen": -664.8557739257812,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -1261.803955078125,
"loss": 1.0741,
"margin_dpo/margin_mean": 562.3533935546875,
"margin_dpo/margin_std": 736.040771484375,
"step": 670
},
{
"KL/chosen_KL_mean": -771.7396240234375,
"KL/mean": -993.390380859375,
"KL/rejected_KL_mean": -1215.041015625,
"KL/std": 630.7474365234375,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.0008008808945305645,
"fcm_dpo/delta": 0.051010433584451675,
"fcm_dpo/margin": 443.301513671875,
"fcm_dpo/q_t": 0.4195772409439087,
"grad_norm": 30.4383602142334,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -1.2151209115982056,
"logits/rejected": -1.2516989707946777,
"logps/chosen": -819.73974609375,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -1298.8603515625,
"loss": 1.1545,
"margin_dpo/margin_mean": 443.301513671875,
"margin_dpo/margin_std": 778.079833984375,
"step": 671
},
{
"KL/chosen_KL_mean": -849.791015625,
"KL/mean": -1072.2750244140625,
"KL/rejected_KL_mean": -1294.759033203125,
"KL/std": 755.86083984375,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.0008099371916614473,
"fcm_dpo/delta": 0.04807715862989426,
"fcm_dpo/margin": 444.9681396484375,
"fcm_dpo/q_t": 0.42253392934799194,
"grad_norm": 44.19379425048828,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -1.274925947189331,
"logits/rejected": -1.2912979125976562,
"logps/chosen": -908.374267578125,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -1387.899169921875,
"loss": 1.1696,
"margin_dpo/margin_mean": 444.9681091308594,
"margin_dpo/margin_std": 849.6900634765625,
"step": 672
},
{
"KL/chosen_KL_mean": -751.1322631835938,
"KL/mean": -956.3372802734375,
"KL/rejected_KL_mean": -1161.542236328125,
"KL/std": 581.4315795898438,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.0008629739750176668,
"fcm_dpo/delta": 0.05584951490163803,
"fcm_dpo/margin": 410.40997314453125,
"fcm_dpo/q_t": 0.4219851493835449,
"grad_norm": 32.456729888916016,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -1.208233118057251,
"logits/rejected": -1.214905023574829,
"logps/chosen": -797.85546875,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -1246.83837890625,
"loss": 1.1363,
"margin_dpo/margin_mean": 410.40997314453125,
"margin_dpo/margin_std": 664.936279296875,
"step": 673
},
{
"KL/chosen_KL_mean": -606.1369018554688,
"KL/mean": -891.3228759765625,
"KL/rejected_KL_mean": -1176.5089111328125,
"KL/std": 615.0496826171875,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.0008298626053147018,
"fcm_dpo/delta": -0.09556920826435089,
"fcm_dpo/margin": 570.3719482421875,
"fcm_dpo/q_t": 0.39160943031311035,
"grad_norm": 39.19919204711914,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -1.1228229999542236,
"logits/rejected": -1.1515860557556152,
"logps/chosen": -651.5823974609375,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -1246.5548095703125,
"loss": 1.0523,
"margin_dpo/margin_mean": 570.3719482421875,
"margin_dpo/margin_std": 732.308349609375,
"step": 674
},
{
"KL/chosen_KL_mean": -702.86669921875,
"KL/mean": -980.7730712890625,
"KL/rejected_KL_mean": -1258.679443359375,
"KL/std": 685.6355590820312,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.0007509513525292277,
"fcm_dpo/delta": -0.030970744788646698,
"fcm_dpo/margin": 555.8126220703125,
"fcm_dpo/q_t": 0.404343843460083,
"grad_norm": 24.052825927734375,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -1.1530423164367676,
"logits/rejected": -1.1733019351959229,
"logps/chosen": -747.04296875,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -1332.7713623046875,
"loss": 1.0862,
"margin_dpo/margin_mean": 555.8126831054688,
"margin_dpo/margin_std": 725.0802001953125,
"step": 675
},
{
"KL/chosen_KL_mean": -762.6793823242188,
"KL/mean": -1022.1181030273438,
"KL/rejected_KL_mean": -1281.556884765625,
"KL/std": 637.0001220703125,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.0007771970704197884,
"fcm_dpo/delta": -0.0041960906237363815,
"fcm_dpo/margin": 518.8775024414062,
"fcm_dpo/q_t": 0.40673425793647766,
"grad_norm": 22.74205780029297,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -1.0892168283462524,
"logits/rejected": -1.10811448097229,
"logps/chosen": -834.077880859375,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -1369.9156494140625,
"loss": 1.0849,
"margin_dpo/margin_mean": 518.8775024414062,
"margin_dpo/margin_std": 657.755859375,
"step": 676
},
{
"KL/chosen_KL_mean": -713.2413940429688,
"KL/mean": -988.1437377929688,
"KL/rejected_KL_mean": -1263.046142578125,
"KL/std": 681.3233032226562,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.0007561305537819862,
"fcm_dpo/delta": -0.020175673067569733,
"fcm_dpo/margin": 549.8046875,
"fcm_dpo/q_t": 0.40893790125846863,
"grad_norm": 24.729129791259766,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -1.1915206909179688,
"logits/rejected": -1.1915239095687866,
"logps/chosen": -769.768798828125,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -1341.2725830078125,
"loss": 1.0947,
"margin_dpo/margin_mean": 549.8046875,
"margin_dpo/margin_std": 822.973876953125,
"step": 677
},
{
"KL/chosen_KL_mean": -606.6762084960938,
"KL/mean": -909.5905151367188,
"KL/rejected_KL_mean": -1212.5048828125,
"KL/std": 725.3612060546875,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.000741387193556875,
"fcm_dpo/delta": -0.06460094451904297,
"fcm_dpo/margin": 605.8285522460938,
"fcm_dpo/q_t": 0.396203875541687,
"grad_norm": 33.879825592041016,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -1.1325280666351318,
"logits/rejected": -1.1821608543395996,
"logps/chosen": -652.8106689453125,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -1293.109375,
"loss": 1.0706,
"margin_dpo/margin_mean": 605.82861328125,
"margin_dpo/margin_std": 814.54296875,
"step": 678
},
{
"KL/chosen_KL_mean": -715.3004150390625,
"KL/mean": -929.665771484375,
"KL/rejected_KL_mean": -1144.03125,
"KL/std": 562.1583251953125,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.0007575766649097204,
"fcm_dpo/delta": 0.0897865891456604,
"fcm_dpo/margin": 428.7308349609375,
"fcm_dpo/q_t": 0.4284992218017578,
"grad_norm": 30.023422241210938,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -1.1566033363342285,
"logits/rejected": -1.156817078590393,
"logps/chosen": -765.5953369140625,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -1220.62939453125,
"loss": 1.1495,
"margin_dpo/margin_mean": 428.7308349609375,
"margin_dpo/margin_std": 679.2525634765625,
"step": 679
},
{
"KL/chosen_KL_mean": -714.9935302734375,
"KL/mean": -986.1622314453125,
"KL/rejected_KL_mean": -1257.3310546875,
"KL/std": 710.556884765625,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.0007529498543590307,
"fcm_dpo/delta": -0.012800686061382294,
"fcm_dpo/margin": 542.3373413085938,
"fcm_dpo/q_t": 0.4060339331626892,
"grad_norm": 35.72113800048828,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -1.134705901145935,
"logits/rejected": -1.1759812831878662,
"logps/chosen": -791.9093017578125,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -1369.7158203125,
"loss": 1.1036,
"margin_dpo/margin_mean": 542.3373413085938,
"margin_dpo/margin_std": 799.07666015625,
"step": 680
},
{
"KL/chosen_KL_mean": -731.4549560546875,
"KL/mean": -950.778076171875,
"KL/rejected_KL_mean": -1170.10107421875,
"KL/std": 619.8388671875,
"epoch": 1.0,
"fcm_dpo/beta": 0.000749021302908659,
"fcm_dpo/delta": -0.04601984843611717,
"fcm_dpo/margin": 438.6461181640625,
"fcm_dpo/q_t": 0.4235824942588806,
"grad_norm": 26.493305206298828,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -1.2273634672164917,
"logits/rejected": -1.2576552629470825,
"logps/chosen": -792.4122314453125,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -1258.6590576171875,
"loss": 1.1554,
"margin_dpo/margin_mean": 438.6461486816406,
"margin_dpo/margin_std": 686.915771484375,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 1.1259927408285604,
"train_runtime": 1736.5167,
"train_samples_per_second": 25.107,
"train_steps_per_second": 0.392
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink