W-61/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.45-s_star-0.35-20260428-045924
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
ddb093e65006fccb41f4b1e4a7a5585727dda8d5
llama-3-8b-base-new-dpo-ult…/trainer_state.json
ModelHub XC ddb093e650 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.45-s_star-0.35-20260428-045924
Source: Original Platform
2026-05-26 11:58:21 +08:00

11057 lines
414 KiB
JSON
Raw Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9989528795811519,
"eval_steps": 200,
"global_step": 477,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": -0.05443763732910156,
"KL/mean": -0.04046645760536194,
"KL/rejected_KL_mean": -0.026498794555664062,
"KL/std": 0.47091516852378845,
"epoch": 0.0020942408376963353,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02793481945991516,
"fcm_dpo/q_t": 0.500069797039032,
"grad_norm": 28.59322738647461,
"learning_rate": 0.0,
"logits/chosen": -0.5898098945617676,
"logits/rejected": -0.604260265827179,
"logps/chosen": -275.28570556640625,
"logps/ref_chosen": -275.2312927246094,
"logps/ref_rejected": -222.9380340576172,
"logps/rejected": -222.96453857421875,
"loss": 5.5463,
"margin_dpo/margin_mean": -0.02793477475643158,
"margin_dpo/margin_std": 0.5724214911460876,
"step": 1
},
{
"KL/chosen_KL_mean": 0.04458808898925781,
"KL/mean": 0.0374295711517334,
"KL/rejected_KL_mean": 0.030269622802734375,
"KL/std": 0.49036121368408203,
"epoch": 0.004188481675392671,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014312177896499634,
"fcm_dpo/q_t": 0.4999642074108124,
"grad_norm": 27.875282287597656,
"learning_rate": 1.0416666666666666e-08,
"logits/chosen": -0.6574729681015015,
"logits/rejected": -0.6464410424232483,
"logps/chosen": -264.7165222167969,
"logps/ref_chosen": -264.7611083984375,
"logps/ref_rejected": -242.5597686767578,
"logps/rejected": -242.52951049804688,
"loss": 5.5446,
"margin_dpo/margin_mean": 0.014312252402305603,
"margin_dpo/margin_std": 0.6423971652984619,
"step": 2
},
{
"KL/chosen_KL_mean": -0.00445556640625,
"KL/mean": -0.034483589231967926,
"KL/rejected_KL_mean": -0.06451225280761719,
"KL/std": 0.4977545440196991,
"epoch": 0.0062827225130890054,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0600539892911911,
"fcm_dpo/q_t": 0.4998498558998108,
"grad_norm": 25.79319953918457,
"learning_rate": 2.083333333333333e-08,
"logits/chosen": -0.6840305328369141,
"logits/rejected": -0.735154390335083,
"logps/chosen": -274.1063537597656,
"logps/ref_chosen": -274.1018981933594,
"logps/ref_rejected": -286.5882568359375,
"logps/rejected": -286.65277099609375,
"loss": 5.5428,
"margin_dpo/margin_mean": 0.060050785541534424,
"margin_dpo/margin_std": 0.7622858881950378,
"step": 3
},
{
"KL/chosen_KL_mean": 0.017669677734375,
"KL/mean": -0.01155301183462143,
"KL/rejected_KL_mean": -0.04076957702636719,
"KL/std": 0.5529603958129883,
"epoch": 0.008376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05844058096408844,
"fcm_dpo/q_t": 0.4998539090156555,
"grad_norm": 31.700010299682617,
"learning_rate": 3.125e-08,
"logits/chosen": -0.6172097325325012,
"logits/rejected": -0.611587405204773,
"logps/chosen": -329.8205871582031,
"logps/ref_chosen": -329.8382568359375,
"logps/ref_rejected": -303.2850646972656,
"logps/rejected": -303.3258361816406,
"loss": 5.5429,
"margin_dpo/margin_mean": 0.05844086408615112,
"margin_dpo/margin_std": 0.7090508937835693,
"step": 4
},
{
"KL/chosen_KL_mean": 0.07551956176757812,
"KL/mean": 0.07383932173252106,
"KL/rejected_KL_mean": 0.07216453552246094,
"KL/std": 0.533249020576477,
"epoch": 0.010471204188481676,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0033641159534454346,
"fcm_dpo/q_t": 0.49999159574508667,
"grad_norm": 29.5593318939209,
"learning_rate": 4.166666666666666e-08,
"logits/chosen": -0.5748654007911682,
"logits/rejected": -0.591567873954773,
"logps/chosen": -301.6634216308594,
"logps/ref_chosen": -301.7389221191406,
"logps/ref_rejected": -274.7654724121094,
"logps/rejected": -274.69329833984375,
"loss": 5.5451,
"margin_dpo/margin_mean": 0.003364160656929016,
"margin_dpo/margin_std": 0.7680634260177612,
"step": 5
},
{
"KL/chosen_KL_mean": 0.05218315124511719,
"KL/mean": 0.025937549769878387,
"KL/rejected_KL_mean": -0.00030517578125,
"KL/std": 0.5092122554779053,
"epoch": 0.012565445026178011,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05248446762561798,
"fcm_dpo/q_t": 0.49986881017684937,
"grad_norm": 28.216808319091797,
"learning_rate": 5.208333333333333e-08,
"logits/chosen": -0.6817427277565002,
"logits/rejected": -0.644816517829895,
"logps/chosen": -285.6424865722656,
"logps/ref_chosen": -285.6946716308594,
"logps/ref_rejected": -245.8200225830078,
"logps/rejected": -245.82032775878906,
"loss": 5.5431,
"margin_dpo/margin_mean": 0.05248543620109558,
"margin_dpo/margin_std": 0.6747213006019592,
"step": 6
},
{
"KL/chosen_KL_mean": -0.06369400024414062,
"KL/mean": -0.026505768299102783,
"KL/rejected_KL_mean": 0.010677337646484375,
"KL/std": 0.5207120776176453,
"epoch": 0.014659685863874346,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.07438454031944275,
"fcm_dpo/q_t": 0.5001859664916992,
"grad_norm": 28.569318771362305,
"learning_rate": 6.25e-08,
"logits/chosen": -0.5781703591346741,
"logits/rejected": -0.6110429167747498,
"logps/chosen": -264.7191467285156,
"logps/ref_chosen": -264.65545654296875,
"logps/ref_rejected": -253.10305786132812,
"logps/rejected": -253.09237670898438,
"loss": 5.5482,
"margin_dpo/margin_mean": -0.0743849128484726,
"margin_dpo/margin_std": 0.7227006554603577,
"step": 7
},
{
"KL/chosen_KL_mean": -0.0766448974609375,
"KL/mean": -0.04857754707336426,
"KL/rejected_KL_mean": -0.020505905151367188,
"KL/std": 0.5684447288513184,
"epoch": 0.016753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.05614340305328369,
"fcm_dpo/q_t": 0.5001404285430908,
"grad_norm": 30.8109073638916,
"learning_rate": 7.291666666666667e-08,
"logits/chosen": -0.683033287525177,
"logits/rejected": -0.6907529234886169,
"logps/chosen": -354.2654113769531,
"logps/ref_chosen": -354.1887512207031,
"logps/ref_rejected": -282.9112243652344,
"logps/rejected": -282.93170166015625,
"loss": 5.5475,
"margin_dpo/margin_mean": -0.05614197254180908,
"margin_dpo/margin_std": 0.7655779123306274,
"step": 8
},
{
"KL/chosen_KL_mean": -0.01226043701171875,
"KL/mean": -0.01405593752861023,
"KL/rejected_KL_mean": -0.015859603881835938,
"KL/std": 0.49738261103630066,
"epoch": 0.018848167539267015,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.00360184907913208,
"fcm_dpo/q_t": 0.4999909996986389,
"grad_norm": 27.886255264282227,
"learning_rate": 8.333333333333333e-08,
"logits/chosen": -0.6334417462348938,
"logits/rejected": -0.6513369083404541,
"logps/chosen": -285.5624694824219,
"logps/ref_chosen": -285.5502014160156,
"logps/ref_rejected": -267.99664306640625,
"logps/rejected": -268.01251220703125,
"loss": 5.5451,
"margin_dpo/margin_mean": 0.0036015063524246216,
"margin_dpo/margin_std": 0.6722431778907776,
"step": 9
},
{
"KL/chosen_KL_mean": -0.012775421142578125,
"KL/mean": -0.01617148518562317,
"KL/rejected_KL_mean": -0.019565582275390625,
"KL/std": 0.4946087896823883,
"epoch": 0.020942408376963352,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.00680088996887207,
"fcm_dpo/q_t": 0.4999829828739166,
"grad_norm": 26.488203048706055,
"learning_rate": 9.375e-08,
"logits/chosen": -0.68070387840271,
"logits/rejected": -0.6747847199440002,
"logps/chosen": -251.9251708984375,
"logps/ref_chosen": -251.91238403320312,
"logps/ref_rejected": -226.45260620117188,
"logps/rejected": -226.47216796875,
"loss": 5.5449,
"margin_dpo/margin_mean": 0.006801143288612366,
"margin_dpo/margin_std": 0.651378870010376,
"step": 10
},
{
"KL/chosen_KL_mean": 0.04965972900390625,
"KL/mean": 0.05433201044797897,
"KL/rejected_KL_mean": 0.05899620056152344,
"KL/std": 0.48608970642089844,
"epoch": 0.023036649214659685,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.009331181645393372,
"fcm_dpo/q_t": 0.5000233054161072,
"grad_norm": 29.016658782958984,
"learning_rate": 1.0416666666666667e-07,
"logits/chosen": -0.5983105897903442,
"logits/rejected": -0.6540327072143555,
"logps/chosen": -301.0338134765625,
"logps/ref_chosen": -301.08343505859375,
"logps/ref_rejected": -259.546630859375,
"logps/rejected": -259.4876403808594,
"loss": 5.5456,
"margin_dpo/margin_mean": -0.009330227971076965,
"margin_dpo/margin_std": 0.6587917804718018,
"step": 11
},
{
"KL/chosen_KL_mean": -0.02674102783203125,
"KL/mean": -0.08727707713842392,
"KL/rejected_KL_mean": -0.14780616760253906,
"KL/std": 0.5344446897506714,
"epoch": 0.025130890052356022,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12106941640377045,
"fcm_dpo/q_t": 0.49969732761383057,
"grad_norm": 30.082069396972656,
"learning_rate": 1.1458333333333332e-07,
"logits/chosen": -0.5738207101821899,
"logits/rejected": -0.5335919857025146,
"logps/chosen": -287.5748596191406,
"logps/ref_chosen": -287.548095703125,
"logps/ref_rejected": -277.37945556640625,
"logps/rejected": -277.5272521972656,
"loss": 5.5404,
"margin_dpo/margin_mean": 0.12106986343860626,
"margin_dpo/margin_std": 0.7516077756881714,
"step": 12
},
{
"KL/chosen_KL_mean": -0.03533744812011719,
"KL/mean": -0.07945731282234192,
"KL/rejected_KL_mean": -0.12357711791992188,
"KL/std": 0.48769837617874146,
"epoch": 0.027225130890052355,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08824285864830017,
"fcm_dpo/q_t": 0.4997794032096863,
"grad_norm": 27.216075897216797,
"learning_rate": 1.25e-07,
"logits/chosen": -0.653461217880249,
"logits/rejected": -0.6608254313468933,
"logps/chosen": -270.7017517089844,
"logps/ref_chosen": -270.6664123535156,
"logps/ref_rejected": -274.6546936035156,
"logps/rejected": -274.77825927734375,
"loss": 5.5417,
"margin_dpo/margin_mean": 0.08824336528778076,
"margin_dpo/margin_std": 0.6230098009109497,
"step": 13
},
{
"KL/chosen_KL_mean": -0.006622314453125,
"KL/mean": -0.017597533762454987,
"KL/rejected_KL_mean": -0.028568267822265625,
"KL/std": 0.5604071617126465,
"epoch": 0.02931937172774869,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02194879949092865,
"fcm_dpo/q_t": 0.49994513392448425,
"grad_norm": 28.335783004760742,
"learning_rate": 1.3541666666666666e-07,
"logits/chosen": -0.6175643801689148,
"logits/rejected": -0.6454805731773376,
"logps/chosen": -281.5998229980469,
"logps/ref_chosen": -281.59320068359375,
"logps/ref_rejected": -263.52215576171875,
"logps/rejected": -263.55072021484375,
"loss": 5.5444,
"margin_dpo/margin_mean": 0.02194814383983612,
"margin_dpo/margin_std": 0.8026820421218872,
"step": 14
},
{
"KL/chosen_KL_mean": 0.022657394409179688,
"KL/mean": -0.011080130934715271,
"KL/rejected_KL_mean": -0.04481697082519531,
"KL/std": 0.47758567333221436,
"epoch": 0.031413612565445025,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06748214364051819,
"fcm_dpo/q_t": 0.49983131885528564,
"grad_norm": 30.399776458740234,
"learning_rate": 1.4583333333333335e-07,
"logits/chosen": -0.6331017017364502,
"logits/rejected": -0.644238293170929,
"logps/chosen": -298.4307861328125,
"logps/ref_chosen": -298.45343017578125,
"logps/ref_rejected": -227.17832946777344,
"logps/rejected": -227.22314453125,
"loss": 5.5425,
"margin_dpo/margin_mean": 0.06748141348361969,
"margin_dpo/margin_std": 0.6817619800567627,
"step": 15
},
{
"KL/chosen_KL_mean": 0.066680908203125,
"KL/mean": 0.02534003183245659,
"KL/rejected_KL_mean": -0.016004562377929688,
"KL/std": 0.5315845012664795,
"epoch": 0.033507853403141365,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08268719166517258,
"fcm_dpo/q_t": 0.49979326128959656,
"grad_norm": 30.135879516601562,
"learning_rate": 1.5624999999999999e-07,
"logits/chosen": -0.5993279218673706,
"logits/rejected": -0.5962104797363281,
"logps/chosen": -293.8999328613281,
"logps/ref_chosen": -293.96661376953125,
"logps/ref_rejected": -250.78443908691406,
"logps/rejected": -250.80044555664062,
"loss": 5.5419,
"margin_dpo/margin_mean": 0.08268788456916809,
"margin_dpo/margin_std": 0.7702502012252808,
"step": 16
},
{
"KL/chosen_KL_mean": 0.043094635009765625,
"KL/mean": 0.030858159065246582,
"KL/rejected_KL_mean": 0.018617630004882812,
"KL/std": 0.4950779974460602,
"epoch": 0.0356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02447649836540222,
"fcm_dpo/q_t": 0.4999387860298157,
"grad_norm": 27.67669105529785,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5722163319587708,
"logits/rejected": -0.5985718965530396,
"logps/chosen": -262.35089111328125,
"logps/ref_chosen": -262.39398193359375,
"logps/ref_rejected": -248.500244140625,
"logps/rejected": -248.48162841796875,
"loss": 5.5443,
"margin_dpo/margin_mean": 0.024476364254951477,
"margin_dpo/margin_std": 0.6870174407958984,
"step": 17
},
{
"KL/chosen_KL_mean": 0.02175140380859375,
"KL/mean": -0.027004405856132507,
"KL/rejected_KL_mean": -0.07576942443847656,
"KL/std": 0.5336968898773193,
"epoch": 0.03769633507853403,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.09750780463218689,
"fcm_dpo/q_t": 0.49975621700286865,
"grad_norm": 29.709609985351562,
"learning_rate": 1.7708333333333334e-07,
"logits/chosen": -0.6183552742004395,
"logits/rejected": -0.6245881915092468,
"logps/chosen": -293.6875,
"logps/ref_chosen": -293.709228515625,
"logps/ref_rejected": -274.5875244140625,
"logps/rejected": -274.663330078125,
"loss": 5.5413,
"margin_dpo/margin_mean": 0.09750711917877197,
"margin_dpo/margin_std": 0.7472649216651917,
"step": 18
},
{
"KL/chosen_KL_mean": 0.034343719482421875,
"KL/mean": 0.04221701622009277,
"KL/rejected_KL_mean": 0.05010414123535156,
"KL/std": 0.4715411067008972,
"epoch": 0.039790575916230364,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.015760570764541626,
"fcm_dpo/q_t": 0.5000393986701965,
"grad_norm": 28.117584228515625,
"learning_rate": 1.875e-07,
"logits/chosen": -0.6209872961044312,
"logits/rejected": -0.6168931722640991,
"logps/chosen": -280.2313232421875,
"logps/ref_chosen": -280.26568603515625,
"logps/ref_rejected": -259.9742736816406,
"logps/rejected": -259.9241638183594,
"loss": 5.5459,
"margin_dpo/margin_mean": -0.01576043665409088,
"margin_dpo/margin_std": 0.6759567260742188,
"step": 19
},
{
"KL/chosen_KL_mean": 0.1446208953857422,
"KL/mean": 0.09324289113283157,
"KL/rejected_KL_mean": 0.04186439514160156,
"KL/std": 0.5209404826164246,
"epoch": 0.041884816753926704,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10276265442371368,
"fcm_dpo/q_t": 0.49974310398101807,
"grad_norm": 29.73322868347168,
"learning_rate": 1.9791666666666664e-07,
"logits/chosen": -0.6250026822090149,
"logits/rejected": -0.6576300263404846,
"logps/chosen": -303.7508239746094,
"logps/ref_chosen": -303.8954162597656,
"logps/ref_rejected": -260.214599609375,
"logps/rejected": -260.1727600097656,
"loss": 5.5411,
"margin_dpo/margin_mean": 0.10276344418525696,
"margin_dpo/margin_std": 0.7132169008255005,
"step": 20
},
{
"KL/chosen_KL_mean": 0.1494312286376953,
"KL/mean": 0.05350485444068909,
"KL/rejected_KL_mean": -0.04242897033691406,
"KL/std": 0.5489564538002014,
"epoch": 0.04397905759162304,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19186845421791077,
"fcm_dpo/q_t": 0.49952030181884766,
"grad_norm": 34.93364715576172,
"learning_rate": 2.0833333333333333e-07,
"logits/chosen": -0.643299400806427,
"logits/rejected": -0.670960545539856,
"logps/chosen": -301.3840637207031,
"logps/ref_chosen": -301.5334777832031,
"logps/ref_rejected": -280.28900146484375,
"logps/rejected": -280.3314208984375,
"loss": 5.5376,
"margin_dpo/margin_mean": 0.1918683797121048,
"margin_dpo/margin_std": 0.7002838850021362,
"step": 21
},
{
"KL/chosen_KL_mean": 0.08012008666992188,
"KL/mean": 0.055122010409832,
"KL/rejected_KL_mean": 0.030115127563476562,
"KL/std": 0.5340926051139832,
"epoch": 0.04607329842931937,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.050002023577690125,
"fcm_dpo/q_t": 0.4998749792575836,
"grad_norm": 25.30601692199707,
"learning_rate": 2.1875e-07,
"logits/chosen": -0.6519259810447693,
"logits/rejected": -0.6534304022789001,
"logps/chosen": -259.9150390625,
"logps/ref_chosen": -259.9951477050781,
"logps/ref_rejected": -243.0721435546875,
"logps/rejected": -243.04205322265625,
"loss": 5.5432,
"margin_dpo/margin_mean": 0.05000239610671997,
"margin_dpo/margin_std": 0.7330818176269531,
"step": 22
},
{
"KL/chosen_KL_mean": 0.08829116821289062,
"KL/mean": 0.03367228806018829,
"KL/rejected_KL_mean": -0.0209503173828125,
"KL/std": 0.5275300145149231,
"epoch": 0.048167539267015703,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.10923662781715393,
"fcm_dpo/q_t": 0.49972692131996155,
"grad_norm": 27.844369888305664,
"learning_rate": 2.2916666666666663e-07,
"logits/chosen": -0.6188793778419495,
"logits/rejected": -0.6528036594390869,
"logps/chosen": -282.09246826171875,
"logps/ref_chosen": -282.1807556152344,
"logps/ref_rejected": -265.0758056640625,
"logps/rejected": -265.09674072265625,
"loss": 5.5409,
"margin_dpo/margin_mean": 0.10923708975315094,
"margin_dpo/margin_std": 0.6296786665916443,
"step": 23
},
{
"KL/chosen_KL_mean": 0.15508651733398438,
"KL/mean": 0.03027293086051941,
"KL/rejected_KL_mean": -0.09454917907714844,
"KL/std": 0.5798429846763611,
"epoch": 0.050261780104712044,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24962344765663147,
"fcm_dpo/q_t": 0.49937596917152405,
"grad_norm": 30.161964416503906,
"learning_rate": 2.3958333333333335e-07,
"logits/chosen": -0.6673365831375122,
"logits/rejected": -0.579649031162262,
"logps/chosen": -301.0245361328125,
"logps/ref_chosen": -301.17962646484375,
"logps/ref_rejected": -302.12786865234375,
"logps/rejected": -302.222412109375,
"loss": 5.5353,
"margin_dpo/margin_mean": 0.24962244927883148,
"margin_dpo/margin_std": 0.759114682674408,
"step": 24
},
{
"KL/chosen_KL_mean": 0.19303131103515625,
"KL/mean": 0.11812683194875717,
"KL/rejected_KL_mean": 0.04321861267089844,
"KL/std": 0.5453072190284729,
"epoch": 0.05235602094240838,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.14981667697429657,
"fcm_dpo/q_t": 0.49962544441223145,
"grad_norm": 27.210227966308594,
"learning_rate": 2.5e-07,
"logits/chosen": -0.602566123008728,
"logits/rejected": -0.6109638214111328,
"logps/chosen": -246.55343627929688,
"logps/ref_chosen": -246.74649047851562,
"logps/ref_rejected": -235.55638122558594,
"logps/rejected": -235.5131378173828,
"loss": 5.5392,
"margin_dpo/margin_mean": 0.14981591701507568,
"margin_dpo/margin_std": 0.7190626859664917,
"step": 25
},
{
"KL/chosen_KL_mean": 0.21875381469726562,
"KL/mean": 0.13972681760787964,
"KL/rejected_KL_mean": 0.06070518493652344,
"KL/std": 0.5684024691581726,
"epoch": 0.05445026178010471,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.15804807841777802,
"fcm_dpo/q_t": 0.4996049404144287,
"grad_norm": 28.61292266845703,
"learning_rate": 2.604166666666667e-07,
"logits/chosen": -0.6518624424934387,
"logits/rejected": -0.6661399602890015,
"logps/chosen": -281.97686767578125,
"logps/ref_chosen": -282.1955871582031,
"logps/ref_rejected": -235.3135528564453,
"logps/rejected": -235.25283813476562,
"loss": 5.5389,
"margin_dpo/margin_mean": 0.15804804861545563,
"margin_dpo/margin_std": 0.7699212431907654,
"step": 26
},
{
"KL/chosen_KL_mean": 0.2905387878417969,
"KL/mean": 0.16653002798557281,
"KL/rejected_KL_mean": 0.042522430419921875,
"KL/std": 0.6718394160270691,
"epoch": 0.05654450261780105,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24801474809646606,
"fcm_dpo/q_t": 0.4993799924850464,
"grad_norm": 27.75027084350586,
"learning_rate": 2.708333333333333e-07,
"logits/chosen": -0.6572801470756531,
"logits/rejected": -0.6775311827659607,
"logps/chosen": -323.5657958984375,
"logps/ref_chosen": -323.8563537597656,
"logps/ref_rejected": -245.968017578125,
"logps/rejected": -245.9254913330078,
"loss": 5.5354,
"margin_dpo/margin_mean": 0.24801427125930786,
"margin_dpo/margin_std": 0.9611263275146484,
"step": 27
},
{
"KL/chosen_KL_mean": 0.23924636840820312,
"KL/mean": 0.16367939114570618,
"KL/rejected_KL_mean": 0.088104248046875,
"KL/std": 0.585762083530426,
"epoch": 0.05863874345549738,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1511506289243698,
"fcm_dpo/q_t": 0.4996221363544464,
"grad_norm": 26.398744583129883,
"learning_rate": 2.8125e-07,
"logits/chosen": -0.6180057525634766,
"logits/rejected": -0.6270272731781006,
"logps/chosen": -248.00747680664062,
"logps/ref_chosen": -248.24673461914062,
"logps/ref_rejected": -240.0382080078125,
"logps/rejected": -239.95013427734375,
"loss": 5.5392,
"margin_dpo/margin_mean": 0.15115085244178772,
"margin_dpo/margin_std": 0.8178046941757202,
"step": 28
},
{
"KL/chosen_KL_mean": 0.3459911346435547,
"KL/mean": 0.24326342344284058,
"KL/rejected_KL_mean": 0.140533447265625,
"KL/std": 0.6447778940200806,
"epoch": 0.060732984293193716,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.20546211302280426,
"fcm_dpo/q_t": 0.49948635697364807,
"grad_norm": 30.924745559692383,
"learning_rate": 2.916666666666667e-07,
"logits/chosen": -0.5906850695610046,
"logits/rejected": -0.6126277446746826,
"logps/chosen": -317.910400390625,
"logps/ref_chosen": -318.2564392089844,
"logps/ref_rejected": -286.75848388671875,
"logps/rejected": -286.6180114746094,
"loss": 5.537,
"margin_dpo/margin_mean": 0.2054625153541565,
"margin_dpo/margin_std": 0.8641461133956909,
"step": 29
},
{
"KL/chosen_KL_mean": 0.3351631164550781,
"KL/mean": 0.2142288237810135,
"KL/rejected_KL_mean": 0.09329414367675781,
"KL/std": 0.6597960591316223,
"epoch": 0.06282722513089005,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24187538027763367,
"fcm_dpo/q_t": 0.49939531087875366,
"grad_norm": 28.964019775390625,
"learning_rate": 3.020833333333333e-07,
"logits/chosen": -0.6030691266059875,
"logits/rejected": -0.6204848885536194,
"logps/chosen": -252.71395874023438,
"logps/ref_chosen": -253.0491485595703,
"logps/ref_rejected": -261.30029296875,
"logps/rejected": -261.20697021484375,
"loss": 5.5356,
"margin_dpo/margin_mean": 0.24187535047531128,
"margin_dpo/margin_std": 0.8954479694366455,
"step": 30
},
{
"KL/chosen_KL_mean": 0.3509845733642578,
"KL/mean": 0.256618857383728,
"KL/rejected_KL_mean": 0.16225242614746094,
"KL/std": 0.726455569267273,
"epoch": 0.06492146596858639,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.18873509764671326,
"fcm_dpo/q_t": 0.499528169631958,
"grad_norm": 25.20211410522461,
"learning_rate": 3.1249999999999997e-07,
"logits/chosen": -0.6544696688652039,
"logits/rejected": -0.6891241073608398,
"logps/chosen": -247.80206298828125,
"logps/ref_chosen": -248.15301513671875,
"logps/ref_rejected": -203.17703247070312,
"logps/rejected": -203.01480102539062,
"loss": 5.5377,
"margin_dpo/margin_mean": 0.1887347549200058,
"margin_dpo/margin_std": 0.9368091225624084,
"step": 31
},
{
"KL/chosen_KL_mean": 0.5199089050292969,
"KL/mean": 0.3558507561683655,
"KL/rejected_KL_mean": 0.1917896270751953,
"KL/std": 0.686607837677002,
"epoch": 0.06701570680628273,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.32811465859413147,
"fcm_dpo/q_t": 0.4991797208786011,
"grad_norm": 29.66213607788086,
"learning_rate": 3.2291666666666666e-07,
"logits/chosen": -0.6054428219795227,
"logits/rejected": -0.6095200777053833,
"logps/chosen": -305.0200500488281,
"logps/ref_chosen": -305.5399475097656,
"logps/ref_rejected": -267.6527099609375,
"logps/rejected": -267.4609069824219,
"loss": 5.5322,
"margin_dpo/margin_mean": 0.3281139135360718,
"margin_dpo/margin_std": 0.8977186679840088,
"step": 32
},
{
"KL/chosen_KL_mean": 0.6279258728027344,
"KL/mean": 0.4263458847999573,
"KL/rejected_KL_mean": 0.2247791290283203,
"KL/std": 0.8318162560462952,
"epoch": 0.06910994764397906,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4031338691711426,
"fcm_dpo/q_t": 0.4989922046661377,
"grad_norm": 28.33131217956543,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.6404253840446472,
"logits/rejected": -0.6528037190437317,
"logps/chosen": -285.6055908203125,
"logps/ref_chosen": -286.2335205078125,
"logps/ref_rejected": -255.38748168945312,
"logps/rejected": -255.16270446777344,
"loss": 5.5292,
"margin_dpo/margin_mean": 0.40313342213630676,
"margin_dpo/margin_std": 1.0211021900177002,
"step": 33
},
{
"KL/chosen_KL_mean": 0.7115211486816406,
"KL/mean": 0.37938636541366577,
"KL/rejected_KL_mean": 0.04724884033203125,
"KL/std": 1.0121040344238281,
"epoch": 0.0712041884816754,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6642796993255615,
"fcm_dpo/q_t": 0.49833938479423523,
"grad_norm": 31.110395431518555,
"learning_rate": 3.4375e-07,
"logits/chosen": -0.6285020112991333,
"logits/rejected": -0.6383810043334961,
"logps/chosen": -340.8804931640625,
"logps/ref_chosen": -341.5920104980469,
"logps/ref_rejected": -278.8866882324219,
"logps/rejected": -278.8394470214844,
"loss": 5.5188,
"margin_dpo/margin_mean": 0.6642793416976929,
"margin_dpo/margin_std": 1.1777315139770508,
"step": 34
},
{
"KL/chosen_KL_mean": 0.6163215637207031,
"KL/mean": 0.4386853873729706,
"KL/rejected_KL_mean": 0.26105690002441406,
"KL/std": 1.0218110084533691,
"epoch": 0.07329842931937172,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.35526472330093384,
"fcm_dpo/q_t": 0.4991118907928467,
"grad_norm": 26.492263793945312,
"learning_rate": 3.541666666666667e-07,
"logits/chosen": -0.6304330229759216,
"logits/rejected": -0.6505473256111145,
"logps/chosen": -264.4632263183594,
"logps/ref_chosen": -265.0795593261719,
"logps/ref_rejected": -264.4876708984375,
"logps/rejected": -264.22662353515625,
"loss": 5.5312,
"margin_dpo/margin_mean": 0.3552646338939667,
"margin_dpo/margin_std": 1.282766580581665,
"step": 35
},
{
"KL/chosen_KL_mean": 0.8185024261474609,
"KL/mean": 0.5113904476165771,
"KL/rejected_KL_mean": 0.2042827606201172,
"KL/std": 1.0968939065933228,
"epoch": 0.07539267015706806,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6142216920852661,
"fcm_dpo/q_t": 0.49846455454826355,
"grad_norm": 31.88554573059082,
"learning_rate": 3.645833333333333e-07,
"logits/chosen": -0.5922667980194092,
"logits/rejected": -0.6093138456344604,
"logps/chosen": -296.50762939453125,
"logps/ref_chosen": -297.3261413574219,
"logps/ref_rejected": -282.09515380859375,
"logps/rejected": -281.890869140625,
"loss": 5.5209,
"margin_dpo/margin_mean": 0.6142206788063049,
"margin_dpo/margin_std": 1.4092711210250854,
"step": 36
},
{
"KL/chosen_KL_mean": 0.7976875305175781,
"KL/mean": 0.5208411812782288,
"KL/rejected_KL_mean": 0.24400711059570312,
"KL/std": 1.1177294254302979,
"epoch": 0.0774869109947644,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5536773204803467,
"fcm_dpo/q_t": 0.49861592054367065,
"grad_norm": 30.80563735961914,
"learning_rate": 3.75e-07,
"logits/chosen": -0.6051210761070251,
"logits/rejected": -0.6199328899383545,
"logps/chosen": -313.236328125,
"logps/ref_chosen": -314.0340270996094,
"logps/ref_rejected": -299.3437805175781,
"logps/rejected": -299.09979248046875,
"loss": 5.5233,
"margin_dpo/margin_mean": 0.5536766052246094,
"margin_dpo/margin_std": 1.531832218170166,
"step": 37
},
{
"KL/chosen_KL_mean": 0.9667644500732422,
"KL/mean": 0.6877572536468506,
"KL/rejected_KL_mean": 0.4087562561035156,
"KL/std": 1.1976516246795654,
"epoch": 0.07958115183246073,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5580183267593384,
"fcm_dpo/q_t": 0.49860504269599915,
"grad_norm": 28.460735321044922,
"learning_rate": 3.8541666666666665e-07,
"logits/chosen": -0.6329147219657898,
"logits/rejected": -0.6447269916534424,
"logps/chosen": -281.5744323730469,
"logps/ref_chosen": -282.54119873046875,
"logps/ref_rejected": -269.7773132324219,
"logps/rejected": -269.3685302734375,
"loss": 5.5231,
"margin_dpo/margin_mean": 0.5580188632011414,
"margin_dpo/margin_std": 1.5111653804779053,
"step": 38
},
{
"KL/chosen_KL_mean": 1.3426704406738281,
"KL/mean": 0.6943768262863159,
"KL/rejected_KL_mean": 0.04608726501464844,
"KL/std": 1.4975932836532593,
"epoch": 0.08167539267015707,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2965832948684692,
"fcm_dpo/q_t": 0.49675899744033813,
"grad_norm": 29.592939376831055,
"learning_rate": 3.958333333333333e-07,
"logits/chosen": -0.6133351922035217,
"logits/rejected": -0.6270293593406677,
"logps/chosen": -275.4302673339844,
"logps/ref_chosen": -276.7729187011719,
"logps/ref_rejected": -249.95889282226562,
"logps/rejected": -249.91278076171875,
"loss": 5.4938,
"margin_dpo/margin_mean": 1.2965826988220215,
"margin_dpo/margin_std": 1.7473759651184082,
"step": 39
},
{
"KL/chosen_KL_mean": 1.0723190307617188,
"KL/mean": 0.676328182220459,
"KL/rejected_KL_mean": 0.2803363800048828,
"KL/std": 1.3713598251342773,
"epoch": 0.08376963350785341,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7919808030128479,
"fcm_dpo/q_t": 0.49802035093307495,
"grad_norm": 27.339101791381836,
"learning_rate": 4.0625e-07,
"logits/chosen": -0.6305021047592163,
"logits/rejected": -0.6675591468811035,
"logps/chosen": -283.2347717285156,
"logps/ref_chosen": -284.30706787109375,
"logps/ref_rejected": -244.4459991455078,
"logps/rejected": -244.16563415527344,
"loss": 5.5139,
"margin_dpo/margin_mean": 0.7919806241989136,
"margin_dpo/margin_std": 1.7995089292526245,
"step": 40
},
{
"KL/chosen_KL_mean": 1.1165599822998047,
"KL/mean": 0.7140932679176331,
"KL/rejected_KL_mean": 0.3116283416748047,
"KL/std": 1.5022192001342773,
"epoch": 0.08586387434554973,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8049377202987671,
"fcm_dpo/q_t": 0.49798786640167236,
"grad_norm": 30.32787322998047,
"learning_rate": 4.1666666666666667e-07,
"logits/chosen": -0.6256983876228333,
"logits/rejected": -0.6507747769355774,
"logps/chosen": -292.6986389160156,
"logps/ref_chosen": -293.8151550292969,
"logps/ref_rejected": -252.16815185546875,
"logps/rejected": -251.85653686523438,
"loss": 5.5134,
"margin_dpo/margin_mean": 0.8049373626708984,
"margin_dpo/margin_std": 1.754093050956726,
"step": 41
},
{
"KL/chosen_KL_mean": 1.208505630493164,
"KL/mean": 0.7730397582054138,
"KL/rejected_KL_mean": 0.33758544921875,
"KL/std": 1.6781818866729736,
"epoch": 0.08795811518324607,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8709163665771484,
"fcm_dpo/q_t": 0.4978230595588684,
"grad_norm": 27.442352294921875,
"learning_rate": 4.270833333333333e-07,
"logits/chosen": -0.632148265838623,
"logits/rejected": -0.6499121189117432,
"logps/chosen": -251.55172729492188,
"logps/ref_chosen": -252.76023864746094,
"logps/ref_rejected": -261.0414733886719,
"logps/rejected": -260.7038879394531,
"loss": 5.511,
"margin_dpo/margin_mean": 0.8709160089492798,
"margin_dpo/margin_std": 2.178144693374634,
"step": 42
},
{
"KL/chosen_KL_mean": 1.2755184173583984,
"KL/mean": 0.6735259890556335,
"KL/rejected_KL_mean": 0.07152366638183594,
"KL/std": 1.878867268562317,
"epoch": 0.09005235602094241,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2040010690689087,
"fcm_dpo/q_t": 0.4969903826713562,
"grad_norm": 30.027923583984375,
"learning_rate": 4.375e-07,
"logits/chosen": -0.5912747383117676,
"logits/rejected": -0.60638427734375,
"logps/chosen": -315.5592041015625,
"logps/ref_chosen": -316.8347473144531,
"logps/ref_rejected": -273.7649230957031,
"logps/rejected": -273.693359375,
"loss": 5.4977,
"margin_dpo/margin_mean": 1.2040014266967773,
"margin_dpo/margin_std": 2.2428250312805176,
"step": 43
},
{
"KL/chosen_KL_mean": 1.544687271118164,
"KL/mean": 0.6966075897216797,
"KL/rejected_KL_mean": -0.15147972106933594,
"KL/std": 2.2357659339904785,
"epoch": 0.09214659685863874,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.696164846420288,
"fcm_dpo/q_t": 0.49576056003570557,
"grad_norm": 30.86996078491211,
"learning_rate": 4.479166666666667e-07,
"logits/chosen": -0.5922445058822632,
"logits/rejected": -0.5905203223228455,
"logps/chosen": -285.33099365234375,
"logps/ref_chosen": -286.8757019042969,
"logps/ref_rejected": -282.4681396484375,
"logps/rejected": -282.6196594238281,
"loss": 5.4786,
"margin_dpo/margin_mean": 1.696164846420288,
"margin_dpo/margin_std": 3.0431594848632812,
"step": 44
},
{
"KL/chosen_KL_mean": 1.7514400482177734,
"KL/mean": 1.0452030897140503,
"KL/rejected_KL_mean": 0.33896636962890625,
"KL/std": 2.3070363998413086,
"epoch": 0.09424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4124752283096313,
"fcm_dpo/q_t": 0.49646997451782227,
"grad_norm": 28.983610153198242,
"learning_rate": 4.5833333333333327e-07,
"logits/chosen": -0.6918033957481384,
"logits/rejected": -0.7159820199012756,
"logps/chosen": -322.511962890625,
"logps/ref_chosen": -324.2633972167969,
"logps/ref_rejected": -293.09466552734375,
"logps/rejected": -292.7557067871094,
"loss": 5.4897,
"margin_dpo/margin_mean": 1.412475347518921,
"margin_dpo/margin_std": 2.8487558364868164,
"step": 45
},
{
"KL/chosen_KL_mean": 1.676168441772461,
"KL/mean": 0.9087100028991699,
"KL/rejected_KL_mean": 0.14125442504882812,
"KL/std": 2.392753839492798,
"epoch": 0.09633507853403141,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5349185466766357,
"fcm_dpo/q_t": 0.4961636960506439,
"grad_norm": 30.31817626953125,
"learning_rate": 4.6874999999999996e-07,
"logits/chosen": -0.6293231248855591,
"logits/rejected": -0.643032968044281,
"logps/chosen": -296.6595764160156,
"logps/ref_chosen": -298.3357238769531,
"logps/ref_rejected": -267.66204833984375,
"logps/rejected": -267.520751953125,
"loss": 5.4847,
"margin_dpo/margin_mean": 1.5349185466766357,
"margin_dpo/margin_std": 2.6055755615234375,
"step": 46
},
{
"KL/chosen_KL_mean": 1.4842796325683594,
"KL/mean": 0.9708749055862427,
"KL/rejected_KL_mean": 0.4574623107910156,
"KL/std": 2.3160178661346436,
"epoch": 0.09842931937172775,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0268166065216064,
"fcm_dpo/q_t": 0.49743345379829407,
"grad_norm": 26.303861618041992,
"learning_rate": 4.791666666666667e-07,
"logits/chosen": -0.6067807078361511,
"logits/rejected": -0.627469539642334,
"logps/chosen": -261.0826416015625,
"logps/ref_chosen": -262.5669250488281,
"logps/ref_rejected": -258.70989990234375,
"logps/rejected": -258.2524108886719,
"loss": 5.5054,
"margin_dpo/margin_mean": 1.0268152952194214,
"margin_dpo/margin_std": 3.289706230163574,
"step": 47
},
{
"KL/chosen_KL_mean": 1.8775215148925781,
"KL/mean": 1.1411019563674927,
"KL/rejected_KL_mean": 0.4046955108642578,
"KL/std": 2.5993833541870117,
"epoch": 0.10052356020942409,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4728214740753174,
"fcm_dpo/q_t": 0.49632006883621216,
"grad_norm": 27.548425674438477,
"learning_rate": 4.895833333333333e-07,
"logits/chosen": -0.6069186925888062,
"logits/rejected": -0.631554126739502,
"logps/chosen": -267.61572265625,
"logps/ref_chosen": -269.4932556152344,
"logps/ref_rejected": -241.888916015625,
"logps/rejected": -241.48423767089844,
"loss": 5.4876,
"margin_dpo/margin_mean": 1.4728212356567383,
"margin_dpo/margin_std": 3.0481390953063965,
"step": 48
},
{
"KL/chosen_KL_mean": 2.2043685913085938,
"KL/mean": 1.3517543077468872,
"KL/rejected_KL_mean": 0.4991436004638672,
"KL/std": 2.6367743015289307,
"epoch": 0.10261780104712041,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.7052276134490967,
"fcm_dpo/q_t": 0.4957382082939148,
"grad_norm": 27.595603942871094,
"learning_rate": 5e-07,
"logits/chosen": -0.6787562966346741,
"logits/rejected": -0.6629523038864136,
"logps/chosen": -255.68011474609375,
"logps/ref_chosen": -257.8844909667969,
"logps/ref_rejected": -256.8912048339844,
"logps/rejected": -256.3920593261719,
"loss": 5.4785,
"margin_dpo/margin_mean": 1.7052276134490967,
"margin_dpo/margin_std": 3.387364625930786,
"step": 49
},
{
"KL/chosen_KL_mean": 1.974029541015625,
"KL/mean": 1.4002656936645508,
"KL/rejected_KL_mean": 0.826507568359375,
"KL/std": 3.0437886714935303,
"epoch": 0.10471204188481675,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.147522211074829,
"fcm_dpo/q_t": 0.49713265895843506,
"grad_norm": 27.980398178100586,
"learning_rate": 4.999932966293553e-07,
"logits/chosen": -0.633937418460846,
"logits/rejected": -0.6559914350509644,
"logps/chosen": -299.6548156738281,
"logps/ref_chosen": -301.62884521484375,
"logps/ref_rejected": -298.2716064453125,
"logps/rejected": -297.4450988769531,
"loss": 5.5011,
"margin_dpo/margin_mean": 1.1475228071212769,
"margin_dpo/margin_std": 3.857665777206421,
"step": 50
},
{
"KL/chosen_KL_mean": 2.1326828002929688,
"KL/mean": 1.1401723623275757,
"KL/rejected_KL_mean": 0.1476593017578125,
"KL/std": 3.5523159503936768,
"epoch": 0.1068062827225131,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9850341081619263,
"fcm_dpo/q_t": 0.4950404763221741,
"grad_norm": 29.13739585876465,
"learning_rate": 4.999731868769026e-07,
"logits/chosen": -0.6125076413154602,
"logits/rejected": -0.6065410375595093,
"logps/chosen": -267.2397155761719,
"logps/ref_chosen": -269.37237548828125,
"logps/ref_rejected": -297.0167541503906,
"logps/rejected": -296.8690490722656,
"loss": 5.4685,
"margin_dpo/margin_mean": 1.9850351810455322,
"margin_dpo/margin_std": 4.482949256896973,
"step": 51
},
{
"KL/chosen_KL_mean": 2.7480640411376953,
"KL/mean": 1.210340976715088,
"KL/rejected_KL_mean": -0.3273906707763672,
"KL/std": 3.7830421924591064,
"epoch": 0.10890052356020942,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.0754599571228027,
"fcm_dpo/q_t": 0.49231770634651184,
"grad_norm": 30.282175064086914,
"learning_rate": 4.99939671821067e-07,
"logits/chosen": -0.6546225547790527,
"logits/rejected": -0.661052942276001,
"logps/chosen": -304.15472412109375,
"logps/ref_chosen": -306.9028015136719,
"logps/ref_rejected": -281.24737548828125,
"logps/rejected": -281.5747375488281,
"loss": 5.4256,
"margin_dpo/margin_mean": 3.075460195541382,
"margin_dpo/margin_std": 4.735509395599365,
"step": 52
},
{
"KL/chosen_KL_mean": 2.7503910064697266,
"KL/mean": 1.6823850870132446,
"KL/rejected_KL_mean": 0.6143932342529297,
"KL/std": 4.089887619018555,
"epoch": 0.11099476439790576,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.136000633239746,
"fcm_dpo/q_t": 0.49466413259506226,
"grad_norm": 29.992427825927734,
"learning_rate": 4.998927532591591e-07,
"logits/chosen": -0.6430725455284119,
"logits/rejected": -0.6831983327865601,
"logps/chosen": -283.2255554199219,
"logps/ref_chosen": -285.9759521484375,
"logps/ref_rejected": -273.9073486328125,
"logps/rejected": -273.29296875,
"loss": 5.4633,
"margin_dpo/margin_mean": 2.136000156402588,
"margin_dpo/margin_std": 5.067354679107666,
"step": 53
},
{
"KL/chosen_KL_mean": 2.6715049743652344,
"KL/mean": 1.728779911994934,
"KL/rejected_KL_mean": 0.7860507965087891,
"KL/std": 4.300192356109619,
"epoch": 0.1130890052356021,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8854495286941528,
"fcm_dpo/q_t": 0.4952917695045471,
"grad_norm": 26.448970794677734,
"learning_rate": 4.998324337072792e-07,
"logits/chosen": -0.6894734501838684,
"logits/rejected": -0.6976596713066101,
"logps/chosen": -303.8331298828125,
"logps/ref_chosen": -306.504638671875,
"logps/ref_rejected": -272.67431640625,
"logps/rejected": -271.8882751464844,
"loss": 5.4733,
"margin_dpo/margin_mean": 1.8854488134384155,
"margin_dpo/margin_std": 5.352308750152588,
"step": 54
},
{
"KL/chosen_KL_mean": 1.8048782348632812,
"KL/mean": 0.6566358804702759,
"KL/rejected_KL_mean": -0.49161338806152344,
"KL/std": 3.7019548416137695,
"epoch": 0.11518324607329843,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.296487808227539,
"fcm_dpo/q_t": 0.494264155626297,
"grad_norm": 24.89438819885254,
"learning_rate": 4.997587164001815e-07,
"logits/chosen": -0.6316308975219727,
"logits/rejected": -0.6350709199905396,
"logps/chosen": -220.52525329589844,
"logps/ref_chosen": -222.33013916015625,
"logps/ref_rejected": -206.59571838378906,
"logps/rejected": -207.0873260498047,
"loss": 5.4564,
"margin_dpo/margin_mean": 2.2964882850646973,
"margin_dpo/margin_std": 4.970940113067627,
"step": 55
},
{
"KL/chosen_KL_mean": 2.7975711822509766,
"KL/mean": 1.2214081287384033,
"KL/rejected_KL_mean": -0.35475921630859375,
"KL/std": 4.772672653198242,
"epoch": 0.11727748691099477,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.1523289680480957,
"fcm_dpo/q_t": 0.4921315908432007,
"grad_norm": 27.497465133666992,
"learning_rate": 4.996716052911017e-07,
"logits/chosen": -0.6138368844985962,
"logits/rejected": -0.6288657188415527,
"logps/chosen": -247.6805877685547,
"logps/ref_chosen": -250.47816467285156,
"logps/ref_rejected": -228.25848388671875,
"logps/rejected": -228.61325073242188,
"loss": 5.4244,
"margin_dpo/margin_mean": 3.1523287296295166,
"margin_dpo/margin_std": 5.94834041595459,
"step": 56
},
{
"KL/chosen_KL_mean": 3.8411941528320312,
"KL/mean": 1.778662919998169,
"KL/rejected_KL_mean": -0.2838726043701172,
"KL/std": 4.771164417266846,
"epoch": 0.1193717277486911,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.125072479248047,
"fcm_dpo/q_t": 0.48969873785972595,
"grad_norm": 30.785301208496094,
"learning_rate": 4.99571105051544e-07,
"logits/chosen": -0.6994465589523315,
"logits/rejected": -0.6702845096588135,
"logps/chosen": -311.2783508300781,
"logps/ref_chosen": -315.1195373535156,
"logps/ref_rejected": -272.755615234375,
"logps/rejected": -273.03948974609375,
"loss": 5.3852,
"margin_dpo/margin_mean": 4.125072002410889,
"margin_dpo/margin_std": 5.533132553100586,
"step": 57
},
{
"KL/chosen_KL_mean": 2.443532943725586,
"KL/mean": 1.277455449104309,
"KL/rejected_KL_mean": 0.11137962341308594,
"KL/std": 4.433131217956543,
"epoch": 0.12146596858638743,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 2.3321542739868164,
"fcm_dpo/q_t": 0.49417322874069214,
"grad_norm": 27.45433235168457,
"learning_rate": 4.994572210710314e-07,
"logits/chosen": -0.6160473823547363,
"logits/rejected": -0.6402156352996826,
"logps/chosen": -262.7381591796875,
"logps/ref_chosen": -265.1816711425781,
"logps/ref_rejected": -268.2203369140625,
"logps/rejected": -268.10894775390625,
"loss": 5.456,
"margin_dpo/margin_mean": 2.3321540355682373,
"margin_dpo/margin_std": 5.656576156616211,
"step": 58
},
{
"KL/chosen_KL_mean": 2.1098995208740234,
"KL/mean": 1.2807146310806274,
"KL/rejected_KL_mean": 0.4515361785888672,
"KL/std": 5.291559219360352,
"epoch": 0.12356020942408377,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6583688259124756,
"fcm_dpo/q_t": 0.49586057662963867,
"grad_norm": 29.933673858642578,
"learning_rate": 4.993299594568162e-07,
"logits/chosen": -0.6065900921821594,
"logits/rejected": -0.5998431444168091,
"logps/chosen": -284.2440490722656,
"logps/ref_chosen": -286.35394287109375,
"logps/ref_rejected": -260.6757507324219,
"logps/rejected": -260.2242126464844,
"loss": 5.485,
"margin_dpo/margin_mean": 1.6583688259124756,
"margin_dpo/margin_std": 7.080748558044434,
"step": 59
},
{
"KL/chosen_KL_mean": 2.876659393310547,
"KL/mean": 1.1580764055252075,
"KL/rejected_KL_mean": -0.5605106353759766,
"KL/std": 5.181823253631592,
"epoch": 0.1256544502617801,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.4371724128723145,
"fcm_dpo/q_t": 0.49141788482666016,
"grad_norm": 28.205102920532227,
"learning_rate": 4.991893270335525e-07,
"logits/chosen": -0.687022864818573,
"logits/rejected": -0.7121449708938599,
"logps/chosen": -255.8719482421875,
"logps/ref_chosen": -258.74859619140625,
"logps/ref_rejected": -255.04893493652344,
"logps/rejected": -255.60943603515625,
"loss": 5.4144,
"margin_dpo/margin_mean": 3.4371743202209473,
"margin_dpo/margin_std": 7.137161731719971,
"step": 60
},
{
"KL/chosen_KL_mean": 3.009805679321289,
"KL/mean": 1.4096331596374512,
"KL/rejected_KL_mean": -0.19052886962890625,
"KL/std": 5.557041645050049,
"epoch": 0.12774869109947645,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.200338363647461,
"fcm_dpo/q_t": 0.4920124113559723,
"grad_norm": 29.904375076293945,
"learning_rate": 4.990353313429303e-07,
"logits/chosen": -0.6517542004585266,
"logits/rejected": -0.668937087059021,
"logps/chosen": -275.4580993652344,
"logps/ref_chosen": -278.4678955078125,
"logps/ref_rejected": -252.02720642089844,
"logps/rejected": -252.21771240234375,
"loss": 5.4249,
"margin_dpo/margin_mean": 3.2003390789031982,
"margin_dpo/margin_std": 7.634793281555176,
"step": 61
},
{
"KL/chosen_KL_mean": 3.993856430053711,
"KL/mean": 2.264561414718628,
"KL/rejected_KL_mean": 0.5352687835693359,
"KL/std": 5.766189098358154,
"epoch": 0.12984293193717278,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.458585739135742,
"fcm_dpo/q_t": 0.49137410521507263,
"grad_norm": 26.703571319580078,
"learning_rate": 4.988679806432711e-07,
"logits/chosen": -0.6075385808944702,
"logits/rejected": -0.6490654945373535,
"logps/chosen": -268.9304504394531,
"logps/ref_chosen": -272.92431640625,
"logps/ref_rejected": -260.7935485839844,
"logps/rejected": -260.2582702636719,
"loss": 5.4139,
"margin_dpo/margin_mean": 3.458585739135742,
"margin_dpo/margin_std": 7.508395195007324,
"step": 62
},
{
"KL/chosen_KL_mean": 3.311128616333008,
"KL/mean": 1.574736475944519,
"KL/rejected_KL_mean": -0.1616668701171875,
"KL/std": 5.540885925292969,
"epoch": 0.1319371727748691,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.472792863845825,
"fcm_dpo/q_t": 0.4913320541381836,
"grad_norm": 28.29745864868164,
"learning_rate": 4.986872839090852e-07,
"logits/chosen": -0.6676257848739624,
"logits/rejected": -0.6745007038116455,
"logps/chosen": -273.7778625488281,
"logps/ref_chosen": -277.0889892578125,
"logps/ref_rejected": -273.3413391113281,
"logps/rejected": -273.50299072265625,
"loss": 5.4129,
"margin_dpo/margin_mean": 3.4727916717529297,
"margin_dpo/margin_std": 7.09786319732666,
"step": 63
},
{
"KL/chosen_KL_mean": 4.158119201660156,
"KL/mean": 1.989778757095337,
"KL/rejected_KL_mean": -0.1785602569580078,
"KL/std": 6.2935590744018555,
"epoch": 0.13403141361256546,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.336686611175537,
"fcm_dpo/q_t": 0.4891809821128845,
"grad_norm": 28.39674186706543,
"learning_rate": 4.9849325083059e-07,
"logits/chosen": -0.6363846063613892,
"logits/rejected": -0.6319074630737305,
"logps/chosen": -279.6663818359375,
"logps/ref_chosen": -283.8244934082031,
"logps/ref_rejected": -263.29351806640625,
"logps/rejected": -263.4720764160156,
"loss": 5.381,
"margin_dpo/margin_mean": 4.336686611175537,
"margin_dpo/margin_std": 8.268450736999512,
"step": 64
},
{
"KL/chosen_KL_mean": 3.4316043853759766,
"KL/mean": 1.7727466821670532,
"KL/rejected_KL_mean": 0.11389350891113281,
"KL/std": 5.717898368835449,
"epoch": 0.13612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.3177053928375244,
"fcm_dpo/q_t": 0.49171876907348633,
"grad_norm": 27.772890090942383,
"learning_rate": 4.982858918131906e-07,
"logits/chosen": -0.696242094039917,
"logits/rejected": -0.6643742918968201,
"logps/chosen": -261.4383239746094,
"logps/ref_chosen": -264.8699645996094,
"logps/ref_rejected": -268.5076904296875,
"logps/rejected": -268.393798828125,
"loss": 5.4193,
"margin_dpo/margin_mean": 3.317704439163208,
"margin_dpo/margin_std": 7.324665069580078,
"step": 65
},
{
"KL/chosen_KL_mean": 3.131479263305664,
"KL/mean": 1.0381340980529785,
"KL/rejected_KL_mean": -1.0552043914794922,
"KL/std": 7.445024490356445,
"epoch": 0.1382198952879581,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.186667442321777,
"fcm_dpo/q_t": 0.4895693361759186,
"grad_norm": 27.830961227416992,
"learning_rate": 4.980652179769217e-07,
"logits/chosen": -0.6735014319419861,
"logits/rejected": -0.6931784749031067,
"logps/chosen": -269.7968444824219,
"logps/ref_chosen": -272.9283142089844,
"logps/ref_rejected": -280.94696044921875,
"logps/rejected": -282.0021667480469,
"loss": 5.3901,
"margin_dpo/margin_mean": 4.186666965484619,
"margin_dpo/margin_std": 10.143901824951172,
"step": 66
},
{
"KL/chosen_KL_mean": 4.024763107299805,
"KL/mean": 2.15836501121521,
"KL/rejected_KL_mean": 0.29196739196777344,
"KL/std": 7.101201057434082,
"epoch": 0.14031413612565444,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 3.7328035831451416,
"fcm_dpo/q_t": 0.49068784713745117,
"grad_norm": 25.62310791015625,
"learning_rate": 4.978312411558517e-07,
"logits/chosen": -0.6985008716583252,
"logits/rejected": -0.7321080565452576,
"logps/chosen": -262.16217041015625,
"logps/ref_chosen": -266.18695068359375,
"logps/ref_rejected": -250.17405700683594,
"logps/rejected": -249.88209533691406,
"loss": 5.4053,
"margin_dpo/margin_mean": 3.7328031063079834,
"margin_dpo/margin_std": 8.70307445526123,
"step": 67
},
{
"KL/chosen_KL_mean": 2.9730300903320312,
"KL/mean": 0.6432544589042664,
"KL/rejected_KL_mean": -1.6865234375,
"KL/std": 8.092517852783203,
"epoch": 0.1424083769633508,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.65955924987793,
"fcm_dpo/q_t": 0.4884108304977417,
"grad_norm": 28.221860885620117,
"learning_rate": 4.975839738974473e-07,
"logits/chosen": -0.68064284324646,
"logits/rejected": -0.6935175657272339,
"logps/chosen": -294.9655456542969,
"logps/ref_chosen": -297.9385986328125,
"logps/ref_rejected": -261.5141296386719,
"logps/rejected": -263.20062255859375,
"loss": 5.373,
"margin_dpo/margin_mean": 4.659559726715088,
"margin_dpo/margin_std": 10.586312294006348,
"step": 68
},
{
"KL/chosen_KL_mean": 3.898843765258789,
"KL/mean": 0.8900386095046997,
"KL/rejected_KL_mean": -2.1187610626220703,
"KL/std": 7.852825164794922,
"epoch": 0.14450261780104712,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.017608642578125,
"fcm_dpo/q_t": 0.48500216007232666,
"grad_norm": 28.46092414855957,
"learning_rate": 4.97323429461901e-07,
"logits/chosen": -0.7054140567779541,
"logits/rejected": -0.7388426065444946,
"logps/chosen": -261.71868896484375,
"logps/ref_chosen": -265.6175231933594,
"logps/ref_rejected": -236.8287353515625,
"logps/rejected": -238.947509765625,
"loss": 5.3179,
"margin_dpo/margin_mean": 6.017608642578125,
"margin_dpo/margin_std": 9.605355262756348,
"step": 69
},
{
"KL/chosen_KL_mean": 4.312845230102539,
"KL/mean": 1.372788429260254,
"KL/rejected_KL_mean": -1.5672664642333984,
"KL/std": 8.651166915893555,
"epoch": 0.14659685863874344,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.880106449127197,
"fcm_dpo/q_t": 0.4853578507900238,
"grad_norm": 28.572994232177734,
"learning_rate": 4.970496218214204e-07,
"logits/chosen": -0.6805024147033691,
"logits/rejected": -0.7144273519515991,
"logps/chosen": -291.9130859375,
"logps/ref_chosen": -296.2259216308594,
"logps/ref_rejected": -254.68496704101562,
"logps/rejected": -256.252197265625,
"loss": 5.3263,
"margin_dpo/margin_mean": 5.8801045417785645,
"margin_dpo/margin_std": 10.940442085266113,
"step": 70
},
{
"KL/chosen_KL_mean": 5.105655670166016,
"KL/mean": 2.5064656734466553,
"KL/rejected_KL_mean": -0.09272193908691406,
"KL/std": 9.356485366821289,
"epoch": 0.1486910994764398,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.19838285446167,
"fcm_dpo/q_t": 0.48704421520233154,
"grad_norm": 28.81658172607422,
"learning_rate": 4.967625656594781e-07,
"logits/chosen": -0.6425278782844543,
"logits/rejected": -0.630567729473114,
"logps/chosen": -283.82159423828125,
"logps/ref_chosen": -288.92724609375,
"logps/ref_rejected": -278.6405334472656,
"logps/rejected": -278.7332458496094,
"loss": 5.3565,
"margin_dpo/margin_mean": 5.198383331298828,
"margin_dpo/margin_std": 12.317588806152344,
"step": 71
},
{
"KL/chosen_KL_mean": 3.2886581420898438,
"KL/mean": 0.7096061706542969,
"KL/rejected_KL_mean": -1.8694496154785156,
"KL/std": 8.412724494934082,
"epoch": 0.15078534031413612,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.158100128173828,
"fcm_dpo/q_t": 0.48713764548301697,
"grad_norm": 28.224729537963867,
"learning_rate": 4.964622763700252e-07,
"logits/chosen": -0.6948481202125549,
"logits/rejected": -0.7061551809310913,
"logps/chosen": -233.7566375732422,
"logps/ref_chosen": -237.0452880859375,
"logps/ref_rejected": -252.7946319580078,
"logps/rejected": -254.66407775878906,
"loss": 5.354,
"margin_dpo/margin_mean": 5.158100128173828,
"margin_dpo/margin_std": 10.428442001342773,
"step": 72
},
{
"KL/chosen_KL_mean": 4.298160552978516,
"KL/mean": 1.9073495864868164,
"KL/rejected_KL_mean": -0.48345375061035156,
"KL/std": 8.987354278564453,
"epoch": 0.15287958115183245,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.781611442565918,
"fcm_dpo/q_t": 0.48807430267333984,
"grad_norm": 27.856719970703125,
"learning_rate": 4.961487700566646e-07,
"logits/chosen": -0.6573762893676758,
"logits/rejected": -0.6748519539833069,
"logps/chosen": -268.75494384765625,
"logps/ref_chosen": -273.0531005859375,
"logps/ref_rejected": -246.8330841064453,
"logps/rejected": -247.3165283203125,
"loss": 5.3727,
"margin_dpo/margin_mean": 4.781611442565918,
"margin_dpo/margin_std": 12.327835083007812,
"step": 73
},
{
"KL/chosen_KL_mean": 3.3608856201171875,
"KL/mean": 1.0955123901367188,
"KL/rejected_KL_mean": -1.1698570251464844,
"KL/std": 9.356354713439941,
"epoch": 0.1549738219895288,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 4.530752182006836,
"fcm_dpo/q_t": 0.48871979117393494,
"grad_norm": 30.288177490234375,
"learning_rate": 4.958220635317885e-07,
"logits/chosen": -0.7295544147491455,
"logits/rejected": -0.7080293893814087,
"logps/chosen": -338.9210205078125,
"logps/ref_chosen": -342.2818908691406,
"logps/ref_rejected": -330.0293884277344,
"logps/rejected": -331.1991882324219,
"loss": 5.3808,
"margin_dpo/margin_mean": 4.530751705169678,
"margin_dpo/margin_std": 11.701099395751953,
"step": 74
},
{
"KL/chosen_KL_mean": 4.55201530456543,
"KL/mean": 1.2454302310943604,
"KL/rejected_KL_mean": -2.061159133911133,
"KL/std": 9.440528869628906,
"epoch": 0.15706806282722513,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.613184928894043,
"fcm_dpo/q_t": 0.4835236966609955,
"grad_norm": 29.505849838256836,
"learning_rate": 4.954821743156767e-07,
"logits/chosen": -0.6339945197105408,
"logits/rejected": -0.6352519989013672,
"logps/chosen": -262.3121337890625,
"logps/ref_chosen": -266.8641662597656,
"logps/ref_rejected": -276.8699951171875,
"logps/rejected": -278.9311828613281,
"loss": 5.2986,
"margin_dpo/margin_mean": 6.613184452056885,
"margin_dpo/margin_std": 11.05253791809082,
"step": 75
},
{
"KL/chosen_KL_mean": 4.158893585205078,
"KL/mean": 0.5089197158813477,
"KL/rejected_KL_mean": -3.1410446166992188,
"KL/std": 9.98751163482666,
"epoch": 0.15916230366492146,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.2999348640441895,
"fcm_dpo/q_t": 0.4818638563156128,
"grad_norm": 29.17070198059082,
"learning_rate": 4.951291206355559e-07,
"logits/chosen": -0.7195841670036316,
"logits/rejected": -0.7272598743438721,
"logps/chosen": -277.01568603515625,
"logps/ref_chosen": -281.174560546875,
"logps/ref_rejected": -263.6067199707031,
"logps/rejected": -266.7477722167969,
"loss": 5.2763,
"margin_dpo/margin_mean": 7.299935340881348,
"margin_dpo/margin_std": 12.812186241149902,
"step": 76
},
{
"KL/chosen_KL_mean": 3.6959686279296875,
"KL/mean": 0.8079696893692017,
"KL/rejected_KL_mean": -2.0800418853759766,
"KL/std": 11.053658485412598,
"epoch": 0.1612565445026178,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.776014804840088,
"fcm_dpo/q_t": 0.48565778136253357,
"grad_norm": 32.9556884765625,
"learning_rate": 4.947629214246236e-07,
"logits/chosen": -0.564210057258606,
"logits/rejected": -0.5733780860900879,
"logps/chosen": -302.39935302734375,
"logps/ref_chosen": -306.09527587890625,
"logps/ref_rejected": -253.49569702148438,
"logps/rejected": -255.57574462890625,
"loss": 5.3386,
"margin_dpo/margin_mean": 5.77601432800293,
"margin_dpo/margin_std": 13.992176055908203,
"step": 77
},
{
"KL/chosen_KL_mean": 3.927644729614258,
"KL/mean": -0.44709402322769165,
"KL/rejected_KL_mean": -4.821832656860352,
"KL/std": 11.035544395446777,
"epoch": 0.16335078534031414,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.749481201171875,
"fcm_dpo/q_t": 0.4782907962799072,
"grad_norm": 29.68499755859375,
"learning_rate": 4.943835963210323e-07,
"logits/chosen": -0.6741328239440918,
"logits/rejected": -0.6679074764251709,
"logps/chosen": -252.97471618652344,
"logps/ref_chosen": -256.90234375,
"logps/ref_rejected": -211.57154846191406,
"logps/rejected": -216.39340209960938,
"loss": 5.2247,
"margin_dpo/margin_mean": 8.749481201171875,
"margin_dpo/margin_std": 14.491506576538086,
"step": 78
},
{
"KL/chosen_KL_mean": 3.0539989471435547,
"KL/mean": -1.0648549795150757,
"KL/rejected_KL_mean": -5.183706283569336,
"KL/std": 11.793110847473145,
"epoch": 0.16544502617801046,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.237702369689941,
"fcm_dpo/q_t": 0.47956132888793945,
"grad_norm": 29.673128128051758,
"learning_rate": 4.939911656668361e-07,
"logits/chosen": -0.6683970093727112,
"logits/rejected": -0.6898422241210938,
"logps/chosen": -263.2195739746094,
"logps/ref_chosen": -266.2735595703125,
"logps/ref_rejected": -251.57257080078125,
"logps/rejected": -256.75628662109375,
"loss": 5.2466,
"margin_dpo/margin_mean": 8.237702369689941,
"margin_dpo/margin_std": 15.015762329101562,
"step": 79
},
{
"KL/chosen_KL_mean": 1.7665481567382812,
"KL/mean": -1.5028220415115356,
"KL/rejected_KL_mean": -4.772197723388672,
"KL/std": 11.338288307189941,
"epoch": 0.16753926701570682,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 6.538745880126953,
"fcm_dpo/q_t": 0.4837731420993805,
"grad_norm": 28.959936141967773,
"learning_rate": 4.935856505068998e-07,
"logits/chosen": -0.6686798334121704,
"logits/rejected": -0.6984753608703613,
"logps/chosen": -286.0844421386719,
"logps/ref_chosen": -287.8509826660156,
"logps/ref_rejected": -256.0766296386719,
"logps/rejected": -260.84881591796875,
"loss": 5.3077,
"margin_dpo/margin_mean": 6.538746356964111,
"margin_dpo/margin_std": 13.19536304473877,
"step": 80
},
{
"KL/chosen_KL_mean": 2.343219757080078,
"KL/mean": -1.2019355297088623,
"KL/rejected_KL_mean": -4.747091293334961,
"KL/std": 12.21540355682373,
"epoch": 0.16963350785340314,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.090311050415039,
"fcm_dpo/q_t": 0.4824158251285553,
"grad_norm": 28.09380340576172,
"learning_rate": 4.93167072587771e-07,
"logits/chosen": -0.639075756072998,
"logits/rejected": -0.6379343271255493,
"logps/chosen": -266.17999267578125,
"logps/ref_chosen": -268.5232238769531,
"logps/ref_rejected": -237.81137084960938,
"logps/rejected": -242.5584716796875,
"loss": 5.2961,
"margin_dpo/margin_mean": 7.090311527252197,
"margin_dpo/margin_std": 16.69711685180664,
"step": 81
},
{
"KL/chosen_KL_mean": 2.3644046783447266,
"KL/mean": -1.5669686794281006,
"KL/rejected_KL_mean": -5.49833869934082,
"KL/std": 12.20776653289795,
"epoch": 0.17172774869109947,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.862754821777344,
"fcm_dpo/q_t": 0.4805517792701721,
"grad_norm": 27.840482711791992,
"learning_rate": 4.92735454356513e-07,
"logits/chosen": -0.7332218885421753,
"logits/rejected": -0.7412782907485962,
"logps/chosen": -276.9995422363281,
"logps/ref_chosen": -279.36395263671875,
"logps/ref_rejected": -236.51365661621094,
"logps/rejected": -242.01199340820312,
"loss": 5.2652,
"margin_dpo/margin_mean": 7.862753391265869,
"margin_dpo/margin_std": 15.764188766479492,
"step": 82
},
{
"KL/chosen_KL_mean": 0.2912254333496094,
"KL/mean": -4.046411514282227,
"KL/rejected_KL_mean": -8.384056091308594,
"KL/std": 13.611123085021973,
"epoch": 0.17382198952879582,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.675275802612305,
"fcm_dpo/q_t": 0.4784584641456604,
"grad_norm": 30.742509841918945,
"learning_rate": 4.922908189595017e-07,
"logits/chosen": -0.6875962615013123,
"logits/rejected": -0.670936107635498,
"logps/chosen": -273.92803955078125,
"logps/ref_chosen": -274.21923828125,
"logps/ref_rejected": -276.2212219238281,
"logps/rejected": -284.6052551269531,
"loss": 5.2421,
"margin_dpo/margin_mean": 8.675275802612305,
"margin_dpo/margin_std": 18.091768264770508,
"step": 83
},
{
"KL/chosen_KL_mean": 0.62109375,
"KL/mean": -3.0680785179138184,
"KL/rejected_KL_mean": -6.757253646850586,
"KL/std": 13.313782691955566,
"epoch": 0.17591623036649215,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 7.378340721130371,
"fcm_dpo/q_t": 0.4817003905773163,
"grad_norm": 29.882280349731445,
"learning_rate": 4.918331902411841e-07,
"logits/chosen": -0.7385432124137878,
"logits/rejected": -0.7533390522003174,
"logps/chosen": -293.7764587402344,
"logps/ref_chosen": -294.3975524902344,
"logps/ref_rejected": -279.81884765625,
"logps/rejected": -286.5760803222656,
"loss": 5.2865,
"margin_dpo/margin_mean": 7.378340721130371,
"margin_dpo/margin_std": 16.830930709838867,
"step": 84
},
{
"KL/chosen_KL_mean": -1.5447559356689453,
"KL/mean": -4.431204795837402,
"KL/rejected_KL_mean": -7.317655563354492,
"KL/std": 12.28141975402832,
"epoch": 0.17801047120418848,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 5.7729082107543945,
"fcm_dpo/q_t": 0.48567745089530945,
"grad_norm": 29.436279296875,
"learning_rate": 4.913625927427995e-07,
"logits/chosen": -0.6729092597961426,
"logits/rejected": -0.6811429262161255,
"logps/chosen": -245.2069549560547,
"logps/ref_chosen": -243.66220092773438,
"logps/ref_rejected": -263.9421691894531,
"logps/rejected": -271.25982666015625,
"loss": 5.3443,
"margin_dpo/margin_mean": 5.772907257080078,
"margin_dpo/margin_std": 15.37360954284668,
"step": 85
},
{
"KL/chosen_KL_mean": 1.2405624389648438,
"KL/mean": -3.23270845413208,
"KL/rejected_KL_mean": -7.705974578857422,
"KL/std": 13.27261734008789,
"epoch": 0.18010471204188483,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.946547508239746,
"fcm_dpo/q_t": 0.47779226303100586,
"grad_norm": 35.33269500732422,
"learning_rate": 4.908790517010636e-07,
"logits/chosen": -0.6816504001617432,
"logits/rejected": -0.6742711067199707,
"logps/chosen": -308.19012451171875,
"logps/ref_chosen": -309.4306945800781,
"logps/ref_rejected": -290.91278076171875,
"logps/rejected": -298.6187744140625,
"loss": 5.227,
"margin_dpo/margin_mean": 8.94654655456543,
"margin_dpo/margin_std": 17.33509063720703,
"step": 86
},
{
"KL/chosen_KL_mean": -0.1603870391845703,
"KL/mean": -5.2997941970825195,
"KL/rejected_KL_mean": -10.439201354980469,
"KL/std": 15.276622772216797,
"epoch": 0.18219895287958116,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.278806686401367,
"fcm_dpo/q_t": 0.4746187925338745,
"grad_norm": 29.66010856628418,
"learning_rate": 4.903825930468148e-07,
"logits/chosen": -0.7536052465438843,
"logits/rejected": -0.747842013835907,
"logps/chosen": -278.1881103515625,
"logps/ref_chosen": -278.0277099609375,
"logps/ref_rejected": -245.70123291015625,
"logps/rejected": -256.1404113769531,
"loss": 5.1829,
"margin_dpo/margin_mean": 10.27880573272705,
"margin_dpo/margin_std": 18.86574363708496,
"step": 87
},
{
"KL/chosen_KL_mean": -2.069059371948242,
"KL/mean": -6.533992290496826,
"KL/rejected_KL_mean": -10.998920440673828,
"KL/std": 15.059403419494629,
"epoch": 0.18429319371727748,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.929862976074219,
"fcm_dpo/q_t": 0.4779248833656311,
"grad_norm": 28.824947357177734,
"learning_rate": 4.898732434036243e-07,
"logits/chosen": -0.7699937224388123,
"logits/rejected": -0.786287248134613,
"logps/chosen": -268.5838623046875,
"logps/ref_chosen": -266.5148010253906,
"logps/ref_rejected": -265.90081787109375,
"logps/rejected": -276.89971923828125,
"loss": 5.2342,
"margin_dpo/margin_mean": 8.929862976074219,
"margin_dpo/margin_std": 19.146970748901367,
"step": 88
},
{
"KL/chosen_KL_mean": 0.019695281982421875,
"KL/mean": -4.6023640632629395,
"KL/rejected_KL_mean": -9.224431991577148,
"KL/std": 14.222090721130371,
"epoch": 0.18638743455497384,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.244122505187988,
"fcm_dpo/q_t": 0.4771014451980591,
"grad_norm": 30.576059341430664,
"learning_rate": 4.893510300863676e-07,
"logits/chosen": -0.7487041354179382,
"logits/rejected": -0.7396792769432068,
"logps/chosen": -265.6695861816406,
"logps/ref_chosen": -265.6893005371094,
"logps/ref_rejected": -251.49314880371094,
"logps/rejected": -260.7175598144531,
"loss": 5.2212,
"margin_dpo/margin_mean": 9.244121551513672,
"margin_dpo/margin_std": 18.273433685302734,
"step": 89
},
{
"KL/chosen_KL_mean": -1.189279556274414,
"KL/mean": -5.295719146728516,
"KL/rejected_KL_mean": -9.402151107788086,
"KL/std": 14.302242279052734,
"epoch": 0.18848167539267016,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 8.212868690490723,
"fcm_dpo/q_t": 0.47964057326316833,
"grad_norm": 29.81761360168457,
"learning_rate": 4.8881598109976e-07,
"logits/chosen": -0.7451858520507812,
"logits/rejected": -0.7541492581367493,
"logps/chosen": -308.6142883300781,
"logps/ref_chosen": -307.4250183105469,
"logps/ref_rejected": -265.7172546386719,
"logps/rejected": -275.119384765625,
"loss": 5.2572,
"margin_dpo/margin_mean": 8.212869644165039,
"margin_dpo/margin_std": 17.621723175048828,
"step": 90
},
{
"KL/chosen_KL_mean": -2.184307098388672,
"KL/mean": -6.940348148345947,
"KL/rejected_KL_mean": -11.696388244628906,
"KL/std": 14.84321403503418,
"epoch": 0.1905759162303665,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.512090682983398,
"fcm_dpo/q_t": 0.4765446186065674,
"grad_norm": 32.97541427612305,
"learning_rate": 4.882681251368548e-07,
"logits/chosen": -0.6837319135665894,
"logits/rejected": -0.7010940909385681,
"logps/chosen": -237.92529296875,
"logps/ref_chosen": -235.74098205566406,
"logps/ref_rejected": -226.6428985595703,
"logps/rejected": -238.33929443359375,
"loss": 5.2161,
"margin_dpo/margin_mean": 9.512090682983398,
"margin_dpo/margin_std": 19.28335189819336,
"step": 91
},
{
"KL/chosen_KL_mean": -3.1826534271240234,
"KL/mean": -8.19556713104248,
"KL/rejected_KL_mean": -13.208467483520508,
"KL/std": 17.41613006591797,
"epoch": 0.19267015706806281,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.025823593139648,
"fcm_dpo/q_t": 0.47521984577178955,
"grad_norm": 33.7353401184082,
"learning_rate": 4.877074915775048e-07,
"logits/chosen": -0.7360749840736389,
"logits/rejected": -0.7194338440895081,
"logps/chosen": -286.63018798828125,
"logps/ref_chosen": -283.4475402832031,
"logps/ref_rejected": -273.134033203125,
"logps/rejected": -286.3424987792969,
"loss": 5.2016,
"margin_dpo/margin_mean": 10.025823593139648,
"margin_dpo/margin_std": 21.176162719726562,
"step": 92
},
{
"KL/chosen_KL_mean": -2.424844741821289,
"KL/mean": -7.178375720977783,
"KL/rejected_KL_mean": -11.93189811706543,
"KL/std": 15.435590744018555,
"epoch": 0.19476439790575917,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.507054328918457,
"fcm_dpo/q_t": 0.4765450060367584,
"grad_norm": 28.56822395324707,
"learning_rate": 4.871341104867864e-07,
"logits/chosen": -0.739149272441864,
"logits/rejected": -0.7635706067085266,
"logps/chosen": -235.76199340820312,
"logps/ref_chosen": -233.33714294433594,
"logps/ref_rejected": -230.54273986816406,
"logps/rejected": -242.47462463378906,
"loss": 5.215,
"margin_dpo/margin_mean": 9.507055282592773,
"margin_dpo/margin_std": 19.51372528076172,
"step": 93
},
{
"KL/chosen_KL_mean": -2.452543258666992,
"KL/mean": -7.112139701843262,
"KL/rejected_KL_mean": -11.771745681762695,
"KL/std": 16.46709442138672,
"epoch": 0.1968586387434555,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.31920051574707,
"fcm_dpo/q_t": 0.4769359230995178,
"grad_norm": 31.440210342407227,
"learning_rate": 4.865480126133871e-07,
"logits/chosen": -0.6917638182640076,
"logits/rejected": -0.7137020230293274,
"logps/chosen": -297.1053466796875,
"logps/ref_chosen": -294.6528015136719,
"logps/ref_rejected": -283.657958984375,
"logps/rejected": -295.42974853515625,
"loss": 5.2323,
"margin_dpo/margin_mean": 9.319201469421387,
"margin_dpo/margin_std": 21.84139633178711,
"step": 94
},
{
"KL/chosen_KL_mean": -3.3814926147460938,
"KL/mean": -8.43643569946289,
"KL/rejected_KL_mean": -13.491373062133789,
"KL/std": 18.077224731445312,
"epoch": 0.19895287958115182,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.109877586364746,
"fcm_dpo/q_t": 0.47511792182922363,
"grad_norm": 33.59014892578125,
"learning_rate": 4.859492293879573e-07,
"logits/chosen": -0.7207019925117493,
"logits/rejected": -0.741344153881073,
"logps/chosen": -315.05120849609375,
"logps/ref_chosen": -311.6697082519531,
"logps/ref_rejected": -262.7471923828125,
"logps/rejected": -276.23858642578125,
"loss": 5.2086,
"margin_dpo/margin_mean": 10.109875679016113,
"margin_dpo/margin_std": 22.540496826171875,
"step": 95
},
{
"KL/chosen_KL_mean": -4.737796783447266,
"KL/mean": -9.849888801574707,
"KL/rejected_KL_mean": -14.961980819702148,
"KL/std": 19.32669448852539,
"epoch": 0.20104712041884817,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 10.224173545837402,
"fcm_dpo/q_t": 0.4748472571372986,
"grad_norm": 37.64308547973633,
"learning_rate": 4.853377929214243e-07,
"logits/chosen": -0.7003790736198425,
"logits/rejected": -0.712866485118866,
"logps/chosen": -287.29376220703125,
"logps/ref_chosen": -282.55596923828125,
"logps/ref_rejected": -242.71588134765625,
"logps/rejected": -257.6778564453125,
"loss": 5.205,
"margin_dpo/margin_mean": 10.224174499511719,
"margin_dpo/margin_std": 23.328632354736328,
"step": 96
},
{
"KL/chosen_KL_mean": -4.413461685180664,
"KL/mean": -10.396341323852539,
"KL/rejected_KL_mean": -16.37922477722168,
"KL/std": 17.304428100585938,
"epoch": 0.2031413612565445,
"fcm_dpo/beta": 0.009999998845160007,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 11.965749740600586,
"fcm_dpo/q_t": 0.47052669525146484,
"grad_norm": 32.066131591796875,
"learning_rate": 4.847137360032699e-07,
"logits/chosen": -0.7437675595283508,
"logits/rejected": -0.7308604717254639,
"logps/chosen": -307.99127197265625,
"logps/ref_chosen": -303.57781982421875,
"logps/ref_rejected": -264.22491455078125,
"logps/rejected": -280.6041564941406,
"loss": 5.1338,
"margin_dpo/margin_mean": 11.965749740600586,
"margin_dpo/margin_std": 22.346803665161133,
"step": 97
},
{
"KL/chosen_KL_mean": -4.811254501342773,
"KL/mean": -11.00230884552002,
"KL/rejected_KL_mean": -17.19336700439453,
"KL/std": 18.988019943237305,
"epoch": 0.20523560209424083,
"fcm_dpo/beta": 0.01010863482952118,
"fcm_dpo/delta": 0.024685721844434738,
"fcm_dpo/margin": 12.38210678100586,
"fcm_dpo/q_t": 0.4695638120174408,
"grad_norm": 34.899879455566406,
"learning_rate": 4.84077092099773e-07,
"logits/chosen": -0.771165132522583,
"logits/rejected": -0.7832750082015991,
"logps/chosen": -291.6416015625,
"logps/ref_chosen": -286.8303527832031,
"logps/ref_rejected": -278.08331298828125,
"logps/rejected": -295.27667236328125,
"loss": 5.1274,
"margin_dpo/margin_mean": 12.382107734680176,
"margin_dpo/margin_std": 22.810455322265625,
"step": 98
},
{
"KL/chosen_KL_mean": -5.324771881103516,
"KL/mean": -11.61580753326416,
"KL/rejected_KL_mean": -17.906835556030273,
"KL/std": 19.748458862304688,
"epoch": 0.20732984293193718,
"fcm_dpo/beta": 0.010278570465743542,
"fcm_dpo/delta": 0.01541962381452322,
"fcm_dpo/margin": 12.582066535949707,
"fcm_dpo/q_t": 0.4683588445186615,
"grad_norm": 33.6032600402832,
"learning_rate": 4.834278953522137e-07,
"logits/chosen": -0.7313593626022339,
"logits/rejected": -0.7438006401062012,
"logps/chosen": -285.2459716796875,
"logps/ref_chosen": -279.92120361328125,
"logps/ref_rejected": -250.3365478515625,
"logps/rejected": -268.2433776855469,
"loss": 5.127,
"margin_dpo/margin_mean": 12.582066535949707,
"margin_dpo/margin_std": 27.021068572998047,
"step": 99
},
{
"KL/chosen_KL_mean": -7.78460693359375,
"KL/mean": -14.085283279418945,
"KL/rejected_KL_mean": -20.385950088500977,
"KL/std": 21.98175811767578,
"epoch": 0.2094240837696335,
"fcm_dpo/beta": 0.010492552071809769,
"fcm_dpo/delta": 0.02715744450688362,
"fcm_dpo/margin": 12.601343154907227,
"fcm_dpo/q_t": 0.4678565263748169,
"grad_norm": 37.04026794433594,
"learning_rate": 4.827661805750437e-07,
"logits/chosen": -0.7743514776229858,
"logits/rejected": -0.7868786454200745,
"logps/chosen": -304.6123046875,
"logps/ref_chosen": -296.8276672363281,
"logps/ref_rejected": -275.56146240234375,
"logps/rejected": -295.9474182128906,
"loss": 5.1047,
"margin_dpo/margin_mean": 12.601341247558594,
"margin_dpo/margin_std": 24.13147735595703,
"step": 100
},
{
"KL/chosen_KL_mean": -5.093202590942383,
"KL/mean": -12.606334686279297,
"KL/rejected_KL_mean": -20.11946678161621,
"KL/std": 22.76483154296875,
"epoch": 0.21151832460732983,
"fcm_dpo/beta": 0.010632730089128017,
"fcm_dpo/delta": 0.02228887937963009,
"fcm_dpo/margin": 15.02625846862793,
"fcm_dpo/q_t": 0.46118059754371643,
"grad_norm": 34.257389068603516,
"learning_rate": 4.820919832540181e-07,
"logits/chosen": -0.7760720252990723,
"logits/rejected": -0.7899767160415649,
"logps/chosen": -257.8352355957031,
"logps/ref_chosen": -252.74203491210938,
"logps/ref_rejected": -276.4185485839844,
"logps/rejected": -296.53802490234375,
"loss": 5.0165,
"margin_dpo/margin_mean": 15.026259422302246,
"margin_dpo/margin_std": 26.347774505615234,
"step": 101
},
{
"KL/chosen_KL_mean": -5.112276077270508,
"KL/mean": -12.508711814880371,
"KL/rejected_KL_mean": -19.905141830444336,
"KL/std": 19.70211410522461,
"epoch": 0.2136125654450262,
"fcm_dpo/beta": 0.010945435613393784,
"fcm_dpo/delta": 0.04265664890408516,
"fcm_dpo/margin": 14.792869567871094,
"fcm_dpo/q_t": 0.4603489637374878,
"grad_norm": 34.85258102416992,
"learning_rate": 4.814053395442932e-07,
"logits/chosen": -0.7522796988487244,
"logits/rejected": -0.7484122514724731,
"logps/chosen": -224.666015625,
"logps/ref_chosen": -219.5537109375,
"logps/ref_rejected": -231.90853881835938,
"logps/rejected": -251.81369018554688,
"loss": 5.0038,
"margin_dpo/margin_mean": 14.792869567871094,
"margin_dpo/margin_std": 24.94087791442871,
"step": 102
},
{
"KL/chosen_KL_mean": -4.756736755371094,
"KL/mean": -11.560209274291992,
"KL/rejected_KL_mean": -18.363676071166992,
"KL/std": 21.59006690979004,
"epoch": 0.2157068062827225,
"fcm_dpo/beta": 0.011328417807817459,
"fcm_dpo/delta": 0.033762745559215546,
"fcm_dpo/margin": 13.606931686401367,
"fcm_dpo/q_t": 0.46292123198509216,
"grad_norm": 36.85382843017578,
"learning_rate": 4.807062862684873e-07,
"logits/chosen": -0.7776579856872559,
"logits/rejected": -0.7746756672859192,
"logps/chosen": -264.4317932128906,
"logps/ref_chosen": -259.6750793457031,
"logps/ref_rejected": -278.7400817871094,
"logps/rejected": -297.103759765625,
"loss": 5.0428,
"margin_dpo/margin_mean": 13.606931686401367,
"margin_dpo/margin_std": 25.625690460205078,
"step": 103
},
{
"KL/chosen_KL_mean": -9.137903213500977,
"KL/mean": -14.100648880004883,
"KL/rejected_KL_mean": -19.06338119506836,
"KL/std": 22.391727447509766,
"epoch": 0.21780104712041884,
"fcm_dpo/beta": 0.011419950984418392,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 9.92547607421875,
"fcm_dpo/q_t": 0.47238224744796753,
"grad_norm": 37.83346176147461,
"learning_rate": 4.799948609147061e-07,
"logits/chosen": -0.7624853253364563,
"logits/rejected": -0.7678467631340027,
"logps/chosen": -277.11212158203125,
"logps/ref_chosen": -267.9741516113281,
"logps/ref_rejected": -230.5306396484375,
"logps/rejected": -249.59400939941406,
"loss": 5.1971,
"margin_dpo/margin_mean": 9.92547607421875,
"margin_dpo/margin_std": 26.146018981933594,
"step": 104
},
{
"KL/chosen_KL_mean": -5.228021621704102,
"KL/mean": -15.545706748962402,
"KL/rejected_KL_mean": -25.8634033203125,
"KL/std": 25.212154388427734,
"epoch": 0.2198952879581152,
"fcm_dpo/beta": 0.011541016399860382,
"fcm_dpo/delta": 0.012324552051723003,
"fcm_dpo/margin": 20.63536834716797,
"fcm_dpo/q_t": 0.4421055316925049,
"grad_norm": 39.61387252807617,
"learning_rate": 4.792711016345321e-07,
"logits/chosen": -0.764461874961853,
"logits/rejected": -0.7763689756393433,
"logps/chosen": -327.48284912109375,
"logps/ref_chosen": -322.25482177734375,
"logps/ref_rejected": -279.02978515625,
"logps/rejected": -304.8931579589844,
"loss": 4.7572,
"margin_dpo/margin_mean": 20.63536834716797,
"margin_dpo/margin_std": 27.043453216552734,
"step": 105
},
{
"KL/chosen_KL_mean": -12.485261917114258,
"KL/mean": -18.77321434020996,
"KL/rejected_KL_mean": -25.061166763305664,
"KL/std": 23.474252700805664,
"epoch": 0.22198952879581152,
"fcm_dpo/beta": 0.011548945680260658,
"fcm_dpo/delta": 0.006849272176623344,
"fcm_dpo/margin": 12.575901985168457,
"fcm_dpo/q_t": 0.46476516127586365,
"grad_norm": 44.48746871948242,
"learning_rate": 4.785350472409791e-07,
"logits/chosen": -0.7404403686523438,
"logits/rejected": -0.7760816812515259,
"logps/chosen": -308.64306640625,
"logps/ref_chosen": -296.15777587890625,
"logps/ref_rejected": -266.2691650390625,
"logps/rejected": -291.3303527832031,
"loss": 5.1058,
"margin_dpo/margin_mean": 12.575901985168457,
"margin_dpo/margin_std": 29.33730697631836,
"step": 106
},
{
"KL/chosen_KL_mean": -4.258321762084961,
"KL/mean": -14.100729942321777,
"KL/rejected_KL_mean": -23.943143844604492,
"KL/std": 22.039447784423828,
"epoch": 0.22408376963350785,
"fcm_dpo/beta": 0.011992116458714008,
"fcm_dpo/delta": 0.07569944858551025,
"fcm_dpo/margin": 19.6848087310791,
"fcm_dpo/q_t": 0.44321852922439575,
"grad_norm": 44.562828063964844,
"learning_rate": 4.777867372064105e-07,
"logits/chosen": -0.7750982642173767,
"logits/rejected": -0.7679177522659302,
"logps/chosen": -311.254638671875,
"logps/ref_chosen": -306.996337890625,
"logps/ref_rejected": -296.79412841796875,
"logps/rejected": -320.73724365234375,
"loss": 4.7705,
"margin_dpo/margin_mean": 19.68480682373047,
"margin_dpo/margin_std": 27.532594680786133,
"step": 107
},
{
"KL/chosen_KL_mean": -5.4611663818359375,
"KL/mean": -14.556838989257812,
"KL/rejected_KL_mean": -23.65252113342285,
"KL/std": 25.107391357421875,
"epoch": 0.2261780104712042,
"fcm_dpo/beta": 0.012616816908121109,
"fcm_dpo/delta": 0.041766583919525146,
"fcm_dpo/margin": 18.191373825073242,
"fcm_dpo/q_t": 0.44564181566238403,
"grad_norm": 127.99562072753906,
"learning_rate": 4.770262116604223e-07,
"logits/chosen": -0.7681426405906677,
"logits/rejected": -0.7800797820091248,
"logps/chosen": -300.61383056640625,
"logps/ref_chosen": -295.1526794433594,
"logps/ref_rejected": -235.974853515625,
"logps/rejected": -259.62738037109375,
"loss": 4.8423,
"margin_dpo/margin_mean": 18.19137191772461,
"margin_dpo/margin_std": 30.33808708190918,
"step": 108
},
{
"KL/chosen_KL_mean": -8.192440032958984,
"KL/mean": -18.310985565185547,
"KL/rejected_KL_mean": -28.429523468017578,
"KL/std": 25.256664276123047,
"epoch": 0.22827225130890053,
"fcm_dpo/beta": 0.013039087876677513,
"fcm_dpo/delta": 0.0516541451215744,
"fcm_dpo/margin": 20.237075805664062,
"fcm_dpo/q_t": 0.4380655884742737,
"grad_norm": 48.19257736206055,
"learning_rate": 4.7625351138769166e-07,
"logits/chosen": -0.7955527901649475,
"logits/rejected": -0.7929811477661133,
"logps/chosen": -334.11724853515625,
"logps/ref_chosen": -325.9248046875,
"logps/ref_rejected": -279.15423583984375,
"logps/rejected": -307.5837707519531,
"loss": 4.7317,
"margin_dpo/margin_mean": 20.237077713012695,
"margin_dpo/margin_std": 31.672195434570312,
"step": 109
},
{
"KL/chosen_KL_mean": -8.757614135742188,
"KL/mean": -18.272449493408203,
"KL/rejected_KL_mean": -27.78728675842285,
"KL/std": 27.215492248535156,
"epoch": 0.23036649214659685,
"fcm_dpo/beta": 0.013565942645072937,
"fcm_dpo/delta": 0.046711843460798264,
"fcm_dpo/margin": 19.029666900634766,
"fcm_dpo/q_t": 0.43923109769821167,
"grad_norm": 44.99413299560547,
"learning_rate": 4.75468677825789e-07,
"logits/chosen": -0.8082597851753235,
"logits/rejected": -0.7972973585128784,
"logps/chosen": -283.1968078613281,
"logps/ref_chosen": -274.439208984375,
"logps/ref_rejected": -260.0552062988281,
"logps/rejected": -287.8424987792969,
"loss": 4.7868,
"margin_dpo/margin_mean": 19.029666900634766,
"margin_dpo/margin_std": 33.67991256713867,
"step": 110
},
{
"KL/chosen_KL_mean": -9.568780899047852,
"KL/mean": -19.672412872314453,
"KL/rejected_KL_mean": -29.776050567626953,
"KL/std": 27.85650062561035,
"epoch": 0.2324607329842932,
"fcm_dpo/beta": 0.013938742689788342,
"fcm_dpo/delta": 0.017976418137550354,
"fcm_dpo/margin": 20.20726776123047,
"fcm_dpo/q_t": 0.4342862367630005,
"grad_norm": 50.70798873901367,
"learning_rate": 4.7467175306295647e-07,
"logits/chosen": -0.8316294550895691,
"logits/rejected": -0.8110882639884949,
"logps/chosen": -338.804931640625,
"logps/ref_chosen": -329.2361755371094,
"logps/ref_rejected": -287.82830810546875,
"logps/rejected": -317.6043395996094,
"loss": 4.7335,
"margin_dpo/margin_mean": 20.20726776123047,
"margin_dpo/margin_std": 34.256099700927734,
"step": 111
},
{
"KL/chosen_KL_mean": -15.03680419921875,
"KL/mean": -21.586559295654297,
"KL/rejected_KL_mean": -28.13631820678711,
"KL/std": 26.71904182434082,
"epoch": 0.23455497382198953,
"fcm_dpo/beta": 0.013969598338007927,
"fcm_dpo/delta": 0.00925416685640812,
"fcm_dpo/margin": 13.099512100219727,
"fcm_dpo/q_t": 0.4575166404247284,
"grad_norm": 55.44192123413086,
"learning_rate": 4.7386277983585053e-07,
"logits/chosen": -0.7383916974067688,
"logits/rejected": -0.7683345675468445,
"logps/chosen": -272.09619140625,
"logps/ref_chosen": -257.0593566894531,
"logps/ref_rejected": -272.9595031738281,
"logps/rejected": -301.0958557128906,
"loss": 5.1261,
"margin_dpo/margin_mean": 13.099512100219727,
"margin_dpo/margin_std": 35.875091552734375,
"step": 112
},
{
"KL/chosen_KL_mean": -12.496185302734375,
"KL/mean": -24.628555297851562,
"KL/rejected_KL_mean": -36.76092529296875,
"KL/std": 30.702579498291016,
"epoch": 0.23664921465968586,
"fcm_dpo/beta": 0.013736647553741932,
"fcm_dpo/delta": -0.03855419531464577,
"fcm_dpo/margin": 24.26474380493164,
"fcm_dpo/q_t": 0.42343804240226746,
"grad_norm": 53.458412170410156,
"learning_rate": 4.7304180152725024e-07,
"logits/chosen": -0.8085811138153076,
"logits/rejected": -0.8125609159469604,
"logps/chosen": -298.537841796875,
"logps/ref_chosen": -286.0416564941406,
"logps/ref_rejected": -270.374267578125,
"logps/rejected": -307.13519287109375,
"loss": 4.6065,
"margin_dpo/margin_mean": 24.264739990234375,
"margin_dpo/margin_std": 38.84653854370117,
"step": 113
},
{
"KL/chosen_KL_mean": -16.22774887084961,
"KL/mean": -22.99600601196289,
"KL/rejected_KL_mean": -29.764259338378906,
"KL/std": 27.5384521484375,
"epoch": 0.2387434554973822,
"fcm_dpo/beta": 0.013897491618990898,
"fcm_dpo/delta": 0.028407400473952293,
"fcm_dpo/margin": 13.536506652832031,
"fcm_dpo/q_t": 0.4553212821483612,
"grad_norm": 53.780792236328125,
"learning_rate": 4.7220886216373085e-07,
"logits/chosen": -0.8557870388031006,
"logits/rejected": -0.8575012683868408,
"logps/chosen": -276.2362060546875,
"logps/ref_chosen": -260.0084533691406,
"logps/ref_rejected": -246.67190551757812,
"logps/rejected": -276.4361572265625,
"loss": 5.0764,
"margin_dpo/margin_mean": 13.536508560180664,
"margin_dpo/margin_std": 34.91930389404297,
"step": 114
},
{
"KL/chosen_KL_mean": -17.749019622802734,
"KL/mean": -24.61688232421875,
"KL/rejected_KL_mean": -31.484745025634766,
"KL/std": 30.253429412841797,
"epoch": 0.24083769633507854,
"fcm_dpo/beta": 0.013822369277477264,
"fcm_dpo/delta": -0.01768554374575615,
"fcm_dpo/margin": 13.73572826385498,
"fcm_dpo/q_t": 0.4572725296020508,
"grad_norm": 59.642250061035156,
"learning_rate": 4.7136400641330245e-07,
"logits/chosen": -0.848541796207428,
"logits/rejected": -0.8091092705726624,
"logps/chosen": -317.1719970703125,
"logps/ref_chosen": -299.4229736328125,
"logps/ref_rejected": -272.1186828613281,
"logps/rejected": -303.60345458984375,
"loss": 5.1056,
"margin_dpo/margin_mean": 13.735727310180664,
"margin_dpo/margin_std": 38.05146408081055,
"step": 115
},
{
"KL/chosen_KL_mean": -13.734819412231445,
"KL/mean": -22.632715225219727,
"KL/rejected_KL_mean": -31.530614852905273,
"KL/std": 28.271486282348633,
"epoch": 0.24293193717277486,
"fcm_dpo/beta": 0.014031772501766682,
"fcm_dpo/delta": 0.0374310202896595,
"fcm_dpo/margin": 17.79579734802246,
"fcm_dpo/q_t": 0.4412615895271301,
"grad_norm": 51.95737075805664,
"learning_rate": 4.70507279583015e-07,
"logits/chosen": -0.8612798452377319,
"logits/rejected": -0.8237401843070984,
"logps/chosen": -292.9987487792969,
"logps/ref_chosen": -279.263916015625,
"logps/ref_rejected": -253.6192169189453,
"logps/rejected": -285.14984130859375,
"loss": 4.845,
"margin_dpo/margin_mean": 17.795799255371094,
"margin_dpo/margin_std": 34.34822463989258,
"step": 116
},
{
"KL/chosen_KL_mean": -16.676069259643555,
"KL/mean": -26.847412109375,
"KL/rejected_KL_mean": -37.01875305175781,
"KL/std": 26.84279441833496,
"epoch": 0.2450261780104712,
"fcm_dpo/beta": 0.014229537919163704,
"fcm_dpo/delta": -0.02480526641011238,
"fcm_dpo/margin": 20.34267807006836,
"fcm_dpo/q_t": 0.43198737502098083,
"grad_norm": 57.02861785888672,
"learning_rate": 4.6963872761652834e-07,
"logits/chosen": -0.8200348615646362,
"logits/rejected": -0.8224045038223267,
"logps/chosen": -275.90093994140625,
"logps/ref_chosen": -259.2248840332031,
"logps/ref_rejected": -229.3042755126953,
"logps/rejected": -266.32305908203125,
"loss": 4.7122,
"margin_dpo/margin_mean": 20.34267807006836,
"margin_dpo/margin_std": 31.05514907836914,
"step": 117
},
{
"KL/chosen_KL_mean": -22.062801361083984,
"KL/mean": -33.46045684814453,
"KL/rejected_KL_mean": -44.85810852050781,
"KL/std": 33.74718475341797,
"epoch": 0.24712041884816754,
"fcm_dpo/beta": 0.01404194999486208,
"fcm_dpo/delta": -0.0025224993005394936,
"fcm_dpo/margin": 22.795305252075195,
"fcm_dpo/q_t": 0.42747047543525696,
"grad_norm": 59.17610168457031,
"learning_rate": 4.687583970916486e-07,
"logits/chosen": -0.8000219464302063,
"logits/rejected": -0.78817218542099,
"logps/chosen": -289.1335144042969,
"logps/ref_chosen": -267.0707092285156,
"logps/ref_rejected": -272.7322082519531,
"logps/rejected": -317.59027099609375,
"loss": 4.7095,
"margin_dpo/margin_mean": 22.795305252075195,
"margin_dpo/margin_std": 41.762020111083984,
"step": 118
},
{
"KL/chosen_KL_mean": -26.935039520263672,
"KL/mean": -35.30348587036133,
"KL/rejected_KL_mean": -43.671932220458984,
"KL/std": 33.14229965209961,
"epoch": 0.24921465968586387,
"fcm_dpo/beta": 0.014066150411963463,
"fcm_dpo/delta": 0.007536234334111214,
"fcm_dpo/margin": 16.736879348754883,
"fcm_dpo/q_t": 0.44677790999412537,
"grad_norm": 61.08521270751953,
"learning_rate": 4.6786633521783005e-07,
"logits/chosen": -0.8812325596809387,
"logits/rejected": -0.8813977241516113,
"logps/chosen": -351.6116943359375,
"logps/ref_chosen": -324.6766357421875,
"logps/ref_rejected": -306.0322265625,
"logps/rejected": -349.70416259765625,
"loss": 5.0037,
"margin_dpo/margin_mean": 16.73687744140625,
"margin_dpo/margin_std": 40.269229888916016,
"step": 119
},
{
"KL/chosen_KL_mean": -23.385488510131836,
"KL/mean": -32.74627685546875,
"KL/rejected_KL_mean": -42.10707092285156,
"KL/std": 31.856447219848633,
"epoch": 0.2513089005235602,
"fcm_dpo/beta": 0.013914698734879494,
"fcm_dpo/delta": -0.018124153837561607,
"fcm_dpo/margin": 18.72157859802246,
"fcm_dpo/q_t": 0.44076991081237793,
"grad_norm": 54.6278076171875,
"learning_rate": 4.669625898336438e-07,
"logits/chosen": -0.8311907052993774,
"logits/rejected": -0.8439901471138,
"logps/chosen": -338.6472473144531,
"logps/ref_chosen": -315.2617492675781,
"logps/ref_rejected": -265.32501220703125,
"logps/rejected": -307.4320983886719,
"loss": 4.9043,
"margin_dpo/margin_mean": 18.72157859802246,
"margin_dpo/margin_std": 39.45494842529297,
"step": 120
},
{
"KL/chosen_KL_mean": -27.836078643798828,
"KL/mean": -34.881961822509766,
"KL/rejected_KL_mean": -41.9278564453125,
"KL/std": 30.415508270263672,
"epoch": 0.2534031413612565,
"fcm_dpo/beta": 0.014034891501069069,
"fcm_dpo/delta": 0.04258127138018608,
"fcm_dpo/margin": 14.091768264770508,
"fcm_dpo/q_t": 0.4556926190853119,
"grad_norm": 70.71554565429688,
"learning_rate": 4.6604720940421207e-07,
"logits/chosen": -0.8554950952529907,
"logits/rejected": -0.8707772493362427,
"logps/chosen": -250.83218383789062,
"logps/ref_chosen": -222.99609375,
"logps/ref_rejected": -226.92860412597656,
"logps/rejected": -268.8564453125,
"loss": 5.0876,
"margin_dpo/margin_mean": 14.091768264770508,
"margin_dpo/margin_std": 38.64990234375,
"step": 121
},
{
"KL/chosen_KL_mean": -28.256216049194336,
"KL/mean": -37.67997741699219,
"KL/rejected_KL_mean": -47.10374069213867,
"KL/std": 35.624656677246094,
"epoch": 0.2554973821989529,
"fcm_dpo/beta": 0.014204684644937515,
"fcm_dpo/delta": -0.0006600477499887347,
"fcm_dpo/margin": 18.847537994384766,
"fcm_dpo/q_t": 0.4392344355583191,
"grad_norm": 65.4048080444336,
"learning_rate": 4.651202430186092e-07,
"logits/chosen": -0.9069045186042786,
"logits/rejected": -0.8689112663269043,
"logps/chosen": -304.28253173828125,
"logps/ref_chosen": -276.02630615234375,
"logps/ref_rejected": -277.97418212890625,
"logps/rejected": -325.0779113769531,
"loss": 4.9306,
"margin_dpo/margin_mean": 18.847537994384766,
"margin_dpo/margin_std": 43.85592269897461,
"step": 122
},
{
"KL/chosen_KL_mean": -21.264556884765625,
"KL/mean": -33.50984191894531,
"KL/rejected_KL_mean": -45.75513458251953,
"KL/std": 34.91896438598633,
"epoch": 0.25759162303664923,
"fcm_dpo/beta": 0.014176392927765846,
"fcm_dpo/delta": -0.01918771117925644,
"fcm_dpo/margin": 24.49056625366211,
"fcm_dpo/q_t": 0.41995978355407715,
"grad_norm": 63.334815979003906,
"learning_rate": 4.6418174038722924e-07,
"logits/chosen": -0.8300783634185791,
"logits/rejected": -0.8240232467651367,
"logps/chosen": -349.419189453125,
"logps/ref_chosen": -328.1546325683594,
"logps/ref_rejected": -280.6911315917969,
"logps/rejected": -326.4462890625,
"loss": 4.6042,
"margin_dpo/margin_mean": 24.490564346313477,
"margin_dpo/margin_std": 40.045387268066406,
"step": 123
},
{
"KL/chosen_KL_mean": -22.935548782348633,
"KL/mean": -33.28853988647461,
"KL/rejected_KL_mean": -43.64153289794922,
"KL/std": 33.49969482421875,
"epoch": 0.25968586387434556,
"fcm_dpo/beta": 0.013861719518899918,
"fcm_dpo/delta": -0.010028579272329807,
"fcm_dpo/margin": 20.705991744995117,
"fcm_dpo/q_t": 0.4327910542488098,
"grad_norm": 66.91712951660156,
"learning_rate": 4.6323175183912023e-07,
"logits/chosen": -0.8569639325141907,
"logits/rejected": -0.82343590259552,
"logps/chosen": -298.6317443847656,
"logps/ref_chosen": -275.6961975097656,
"logps/ref_rejected": -225.361572265625,
"logps/rejected": -269.0030822753906,
"loss": 4.7679,
"margin_dpo/margin_mean": 20.705991744995117,
"margin_dpo/margin_std": 37.01435089111328,
"step": 124
},
{
"KL/chosen_KL_mean": -24.424352645874023,
"KL/mean": -34.008094787597656,
"KL/rejected_KL_mean": -43.59183883666992,
"KL/std": 34.53946304321289,
"epoch": 0.2617801047120419,
"fcm_dpo/beta": 0.013881498947739601,
"fcm_dpo/delta": -0.015007440000772476,
"fcm_dpo/margin": 19.167491912841797,
"fcm_dpo/q_t": 0.4410821795463562,
"grad_norm": 67.82720184326172,
"learning_rate": 4.6227032831928483e-07,
"logits/chosen": -0.8077659606933594,
"logits/rejected": -0.7630800008773804,
"logps/chosen": -302.4941101074219,
"logps/ref_chosen": -278.06976318359375,
"logps/ref_rejected": -265.63873291015625,
"logps/rejected": -309.2305908203125,
"loss": 4.9802,
"margin_dpo/margin_mean": 19.16749382019043,
"margin_dpo/margin_std": 47.179725646972656,
"step": 125
},
{
"KL/chosen_KL_mean": -20.58001136779785,
"KL/mean": -31.229753494262695,
"KL/rejected_KL_mean": -41.87950134277344,
"KL/std": 35.75967788696289,
"epoch": 0.2638743455497382,
"fcm_dpo/beta": 0.013768978416919708,
"fcm_dpo/delta": 0.018436769023537636,
"fcm_dpo/margin": 21.299489974975586,
"fcm_dpo/q_t": 0.4325592815876007,
"grad_norm": 67.8341064453125,
"learning_rate": 4.612975213859487e-07,
"logits/chosen": -0.8235605359077454,
"logits/rejected": -0.8356186151504517,
"logps/chosen": -341.9761047363281,
"logps/ref_chosen": -321.3960876464844,
"logps/ref_rejected": -285.37664794921875,
"logps/rejected": -327.256103515625,
"loss": 4.7876,
"margin_dpo/margin_mean": 21.299489974975586,
"margin_dpo/margin_std": 41.721378326416016,
"step": 126
},
{
"KL/chosen_KL_mean": -18.004310607910156,
"KL/mean": -30.651229858398438,
"KL/rejected_KL_mean": -43.29817199707031,
"KL/std": 36.886497497558594,
"epoch": 0.26596858638743454,
"fcm_dpo/beta": 0.01383438054472208,
"fcm_dpo/delta": -0.06396237015724182,
"fcm_dpo/margin": 25.293867111206055,
"fcm_dpo/q_t": 0.42087969183921814,
"grad_norm": 68.39196014404297,
"learning_rate": 4.603133832077953e-07,
"logits/chosen": -0.8953632116317749,
"logits/rejected": -0.845830500125885,
"logps/chosen": -324.5630798339844,
"logps/ref_chosen": -306.55877685546875,
"logps/ref_rejected": -274.8651428222656,
"logps/rejected": -318.163330078125,
"loss": 4.6431,
"margin_dpo/margin_mean": 25.293869018554688,
"margin_dpo/margin_std": 41.969058990478516,
"step": 127
},
{
"KL/chosen_KL_mean": -12.162698745727539,
"KL/mean": -26.459373474121094,
"KL/rejected_KL_mean": -40.75605010986328,
"KL/std": 38.046356201171875,
"epoch": 0.2680628272251309,
"fcm_dpo/beta": 0.01290260162204504,
"fcm_dpo/delta": -0.04873104393482208,
"fcm_dpo/margin": 28.59334945678711,
"fcm_dpo/q_t": 0.4156576991081238,
"grad_norm": 50.67338180541992,
"learning_rate": 4.5931796656116837e-07,
"logits/chosen": -0.7757830023765564,
"logits/rejected": -0.7743943929672241,
"logps/chosen": -277.56005859375,
"logps/ref_chosen": -265.3973693847656,
"logps/ref_rejected": -250.9737548828125,
"logps/rejected": -291.72979736328125,
"loss": 4.5472,
"margin_dpo/margin_mean": 28.59334945678711,
"margin_dpo/margin_std": 45.63541030883789,
"step": 128
},
{
"KL/chosen_KL_mean": -13.00929069519043,
"KL/mean": -25.526004791259766,
"KL/rejected_KL_mean": -38.042720794677734,
"KL/std": 35.623844146728516,
"epoch": 0.27015706806282724,
"fcm_dpo/beta": 0.012600157409906387,
"fcm_dpo/delta": -0.035920120775699615,
"fcm_dpo/margin": 25.033435821533203,
"fcm_dpo/q_t": 0.42803478240966797,
"grad_norm": 64.80278778076172,
"learning_rate": 4.5831132482724193e-07,
"logits/chosen": -0.8124359846115112,
"logits/rejected": -0.8133723139762878,
"logps/chosen": -316.167724609375,
"logps/ref_chosen": -303.158447265625,
"logps/ref_rejected": -275.9891052246094,
"logps/rejected": -314.0318298339844,
"loss": 4.7028,
"margin_dpo/margin_mean": 25.03343391418457,
"margin_dpo/margin_std": 44.670989990234375,
"step": 129
},
{
"KL/chosen_KL_mean": -14.777093887329102,
"KL/mean": -26.76191520690918,
"KL/rejected_KL_mean": -38.746734619140625,
"KL/std": 34.12180709838867,
"epoch": 0.27225130890052357,
"fcm_dpo/beta": 0.01246493961662054,
"fcm_dpo/delta": 0.026410242542624474,
"fcm_dpo/margin": 23.96963119506836,
"fcm_dpo/q_t": 0.4313068389892578,
"grad_norm": 67.26573181152344,
"learning_rate": 4.5729351198915705e-07,
"logits/chosen": -0.7755675911903381,
"logits/rejected": -0.8148837685585022,
"logps/chosen": -301.1844177246094,
"logps/ref_chosen": -286.4073486328125,
"logps/ref_rejected": -294.38665771484375,
"logps/rejected": -333.1334228515625,
"loss": 4.7318,
"margin_dpo/margin_mean": 23.96963119506836,
"margin_dpo/margin_std": 43.78425216674805,
"step": 130
},
{
"KL/chosen_KL_mean": -17.329355239868164,
"KL/mean": -27.278770446777344,
"KL/rejected_KL_mean": -37.22817611694336,
"KL/std": 37.274391174316406,
"epoch": 0.2743455497382199,
"fcm_dpo/beta": 0.01295158639550209,
"fcm_dpo/delta": 0.03556347265839577,
"fcm_dpo/margin": 19.898832321166992,
"fcm_dpo/q_t": 0.44100892543792725,
"grad_norm": 71.38201904296875,
"learning_rate": 4.5626458262912735e-07,
"logits/chosen": -0.8666002750396729,
"logits/rejected": -0.8158248662948608,
"logps/chosen": -328.8944091796875,
"logps/ref_chosen": -311.5650634765625,
"logps/ref_rejected": -291.62432861328125,
"logps/rejected": -328.8525390625,
"loss": 4.9313,
"margin_dpo/margin_mean": 19.898832321166992,
"margin_dpo/margin_std": 44.99689483642578,
"step": 131
},
{
"KL/chosen_KL_mean": -13.132219314575195,
"KL/mean": -27.211585998535156,
"KL/rejected_KL_mean": -41.290950775146484,
"KL/std": 37.97029113769531,
"epoch": 0.2764397905759162,
"fcm_dpo/beta": 0.013177872635424137,
"fcm_dpo/delta": -0.022556345909833908,
"fcm_dpo/margin": 28.158733367919922,
"fcm_dpo/q_t": 0.4168829023838043,
"grad_norm": 86.56053924560547,
"learning_rate": 4.5522459192551166e-07,
"logits/chosen": -0.8170298933982849,
"logits/rejected": -0.8009600639343262,
"logps/chosen": -283.2140197753906,
"logps/ref_chosen": -270.0818176269531,
"logps/ref_rejected": -284.3084411621094,
"logps/rejected": -325.5993957519531,
"loss": 4.6122,
"margin_dpo/margin_mean": 28.158733367919922,
"margin_dpo/margin_std": 48.239803314208984,
"step": 132
},
{
"KL/chosen_KL_mean": -14.192306518554688,
"KL/mean": -27.241323471069336,
"KL/rejected_KL_mean": -40.290340423583984,
"KL/std": 36.0876350402832,
"epoch": 0.27853403141361255,
"fcm_dpo/beta": 0.012713887728750706,
"fcm_dpo/delta": -0.01462561171501875,
"fcm_dpo/margin": 26.098039627075195,
"fcm_dpo/q_t": 0.4238502085208893,
"grad_norm": 55.55501174926758,
"learning_rate": 4.541735956498554e-07,
"logits/chosen": -0.8563445806503296,
"logits/rejected": -0.8587122559547424,
"logps/chosen": -299.81365966796875,
"logps/ref_chosen": -285.6213684082031,
"logps/ref_rejected": -251.19386291503906,
"logps/rejected": -291.48419189453125,
"loss": 4.6574,
"margin_dpo/margin_mean": 26.098039627075195,
"margin_dpo/margin_std": 44.765777587890625,
"step": 133
},
{
"KL/chosen_KL_mean": -25.40647315979004,
"KL/mean": -35.70955276489258,
"KL/rejected_KL_mean": -46.01262664794922,
"KL/std": 37.336177825927734,
"epoch": 0.2806282722513089,
"fcm_dpo/beta": 0.01256817951798439,
"fcm_dpo/delta": -0.014759279787540436,
"fcm_dpo/margin": 20.606157302856445,
"fcm_dpo/q_t": 0.43928804993629456,
"grad_norm": 53.031646728515625,
"learning_rate": 4.5311165016389914e-07,
"logits/chosen": -0.8518386483192444,
"logits/rejected": -0.8516722917556763,
"logps/chosen": -344.3273010253906,
"logps/ref_chosen": -318.92083740234375,
"logps/ref_rejected": -293.1894836425781,
"logps/rejected": -339.20208740234375,
"loss": 4.8628,
"margin_dpo/margin_mean": 20.606155395507812,
"margin_dpo/margin_std": 41.08130645751953,
"step": 134
},
{
"KL/chosen_KL_mean": -21.542409896850586,
"KL/mean": -33.6240234375,
"KL/rejected_KL_mean": -45.70563888549805,
"KL/std": 38.229713439941406,
"epoch": 0.28272251308900526,
"fcm_dpo/beta": 0.012417946010828018,
"fcm_dpo/delta": -0.01869252882897854,
"fcm_dpo/margin": 24.16322898864746,
"fcm_dpo/q_t": 0.4295629858970642,
"grad_norm": 71.39109802246094,
"learning_rate": 4.520388124165564e-07,
"logits/chosen": -0.7575780153274536,
"logits/rejected": -0.7987253665924072,
"logps/chosen": -314.3641662597656,
"logps/ref_chosen": -292.8217468261719,
"logps/ref_rejected": -269.2896728515625,
"logps/rejected": -314.995361328125,
"loss": 4.7495,
"margin_dpo/margin_mean": 24.16322898864746,
"margin_dpo/margin_std": 42.811241149902344,
"step": 135
},
{
"KL/chosen_KL_mean": -23.46628189086914,
"KL/mean": -37.211727142333984,
"KL/rejected_KL_mean": -50.957183837890625,
"KL/std": 39.38726043701172,
"epoch": 0.2848167539267016,
"fcm_dpo/beta": 0.012112447991967201,
"fcm_dpo/delta": -0.032658714801073074,
"fcm_dpo/margin": 27.49091339111328,
"fcm_dpo/q_t": 0.4244113862514496,
"grad_norm": 68.2113265991211,
"learning_rate": 4.5095513994085974e-07,
"logits/chosen": -0.8045265078544617,
"logits/rejected": -0.7990159392356873,
"logps/chosen": -296.3188171386719,
"logps/ref_chosen": -272.8525390625,
"logps/ref_rejected": -252.68202209472656,
"logps/rejected": -303.63922119140625,
"loss": 4.6882,
"margin_dpo/margin_mean": 27.49091339111328,
"margin_dpo/margin_std": 49.20502853393555,
"step": 136
},
{
"KL/chosen_KL_mean": -27.184465408325195,
"KL/mean": -38.95940017700195,
"KL/rejected_KL_mean": -50.734336853027344,
"KL/std": 41.108158111572266,
"epoch": 0.2869109947643979,
"fcm_dpo/beta": 0.011971182189881802,
"fcm_dpo/delta": -0.009735875762999058,
"fcm_dpo/margin": 23.549869537353516,
"fcm_dpo/q_t": 0.43623051047325134,
"grad_norm": 58.9945068359375,
"learning_rate": 4.498606908508753e-07,
"logits/chosen": -0.865675687789917,
"logits/rejected": -0.8515555262565613,
"logps/chosen": -327.9366760253906,
"logps/ref_chosen": -300.7522277832031,
"logps/ref_rejected": -286.1935119628906,
"logps/rejected": -336.9278869628906,
"loss": 4.816,
"margin_dpo/margin_mean": 23.54987144470215,
"margin_dpo/margin_std": 47.051055908203125,
"step": 137
},
{
"KL/chosen_KL_mean": -24.572195053100586,
"KL/mean": -37.86249923706055,
"KL/rejected_KL_mean": -51.15279006958008,
"KL/std": 41.30282211303711,
"epoch": 0.28900523560209423,
"fcm_dpo/beta": 0.011851192452013493,
"fcm_dpo/delta": 0.0014142822474241257,
"fcm_dpo/margin": 26.580608367919922,
"fcm_dpo/q_t": 0.43020325899124146,
"grad_norm": 57.485626220703125,
"learning_rate": 4.487555238385862e-07,
"logits/chosen": -0.770022988319397,
"logits/rejected": -0.7489472031593323,
"logps/chosen": -313.5091552734375,
"logps/ref_chosen": -288.9369812011719,
"logps/ref_rejected": -263.7076416015625,
"logps/rejected": -314.8604431152344,
"loss": 4.7718,
"margin_dpo/margin_mean": 26.580608367919922,
"margin_dpo/margin_std": 53.411861419677734,
"step": 138
},
{
"KL/chosen_KL_mean": -23.968965530395508,
"KL/mean": -33.10724639892578,
"KL/rejected_KL_mean": -42.24552917480469,
"KL/std": 37.81224060058594,
"epoch": 0.29109947643979056,
"fcm_dpo/beta": 0.011845817789435387,
"fcm_dpo/delta": 0.011396631598472595,
"fcm_dpo/margin": 18.276565551757812,
"fcm_dpo/q_t": 0.45067816972732544,
"grad_norm": 62.92908477783203,
"learning_rate": 4.476396981707453e-07,
"logits/chosen": -0.8066970109939575,
"logits/rejected": -0.8366684913635254,
"logps/chosen": -294.0132751464844,
"logps/ref_chosen": -270.0443115234375,
"logps/ref_rejected": -267.3226013183594,
"logps/rejected": -309.568115234375,
"loss": 5.0408,
"margin_dpo/margin_mean": 18.276565551757812,
"margin_dpo/margin_std": 45.98987579345703,
"step": 139
},
{
"KL/chosen_KL_mean": -23.92278289794922,
"KL/mean": -38.3636360168457,
"KL/rejected_KL_mean": -52.80449676513672,
"KL/std": 41.12156295776367,
"epoch": 0.2931937172774869,
"fcm_dpo/beta": 0.011858167126774788,
"fcm_dpo/delta": -0.036420539021492004,
"fcm_dpo/margin": 28.881723403930664,
"fcm_dpo/q_t": 0.4207428991794586,
"grad_norm": 62.07208251953125,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": -0.892544150352478,
"logits/rejected": -0.8596428632736206,
"logps/chosen": -306.87835693359375,
"logps/ref_chosen": -282.9555969238281,
"logps/ref_rejected": -251.17181396484375,
"logps/rejected": -303.976318359375,
"loss": 4.5731,
"margin_dpo/margin_mean": 28.881723403930664,
"margin_dpo/margin_std": 43.74916458129883,
"step": 140
},
{
"KL/chosen_KL_mean": -23.3916072845459,
"KL/mean": -38.7611198425293,
"KL/rejected_KL_mean": -54.1306266784668,
"KL/std": 40.98480224609375,
"epoch": 0.29528795811518327,
"fcm_dpo/beta": 0.01144687831401825,
"fcm_dpo/delta": -0.031413860619068146,
"fcm_dpo/margin": 30.739025115966797,
"fcm_dpo/q_t": 0.4180885851383209,
"grad_norm": 58.84656524658203,
"learning_rate": 4.453763107901675e-07,
"logits/chosen": -0.8045722842216492,
"logits/rejected": -0.7984085083007812,
"logps/chosen": -319.6917724609375,
"logps/ref_chosen": -296.3001708984375,
"logps/ref_rejected": -279.8486633300781,
"logps/rejected": -333.979248046875,
"loss": 4.6001,
"margin_dpo/margin_mean": 30.739025115966797,
"margin_dpo/margin_std": 50.728179931640625,
"step": 141
},
{
"KL/chosen_KL_mean": -25.941381454467773,
"KL/mean": -38.507537841796875,
"KL/rejected_KL_mean": -51.07369613647461,
"KL/std": 42.384246826171875,
"epoch": 0.2973821989528796,
"fcm_dpo/beta": 0.011349150910973549,
"fcm_dpo/delta": -0.005858212243765593,
"fcm_dpo/margin": 25.132293701171875,
"fcm_dpo/q_t": 0.4345892667770386,
"grad_norm": 58.35767364501953,
"learning_rate": 4.4422887045602674e-07,
"logits/chosen": -0.8434002995491028,
"logits/rejected": -0.8453127145767212,
"logps/chosen": -326.50726318359375,
"logps/ref_chosen": -300.56585693359375,
"logps/ref_rejected": -231.43316650390625,
"logps/rejected": -282.5068359375,
"loss": 4.8534,
"margin_dpo/margin_mean": 25.132295608520508,
"margin_dpo/margin_std": 53.17611312866211,
"step": 142
},
{
"KL/chosen_KL_mean": -27.10928726196289,
"KL/mean": -42.68886947631836,
"KL/rejected_KL_mean": -58.268463134765625,
"KL/std": 41.513275146484375,
"epoch": 0.2994764397905759,
"fcm_dpo/beta": 0.010968414135277271,
"fcm_dpo/delta": -0.057835567742586136,
"fcm_dpo/margin": 31.15917205810547,
"fcm_dpo/q_t": 0.42122355103492737,
"grad_norm": 57.15718078613281,
"learning_rate": 4.4307101421701755e-07,
"logits/chosen": -0.8285028338432312,
"logits/rejected": -0.8069489002227783,
"logps/chosen": -323.84161376953125,
"logps/ref_chosen": -296.73236083984375,
"logps/ref_rejected": -266.45257568359375,
"logps/rejected": -324.7210693359375,
"loss": 4.5791,
"margin_dpo/margin_mean": 31.15917205810547,
"margin_dpo/margin_std": 47.604496002197266,
"step": 143
},
{
"KL/chosen_KL_mean": -29.586715698242188,
"KL/mean": -42.25658416748047,
"KL/rejected_KL_mean": -54.92646026611328,
"KL/std": 44.707191467285156,
"epoch": 0.30157068062827225,
"fcm_dpo/beta": 0.010684658773243427,
"fcm_dpo/delta": -0.006853851955384016,
"fcm_dpo/margin": 25.339746475219727,
"fcm_dpo/q_t": 0.4385736882686615,
"grad_norm": 52.378929138183594,
"learning_rate": 4.419028041654559e-07,
"logits/chosen": -0.8826330304145813,
"logits/rejected": -0.8678580522537231,
"logps/chosen": -328.43072509765625,
"logps/ref_chosen": -298.843994140625,
"logps/ref_rejected": -266.120849609375,
"logps/rejected": -321.04730224609375,
"loss": 4.8628,
"margin_dpo/margin_mean": 25.33974838256836,
"margin_dpo/margin_std": 52.99594497680664,
"step": 144
},
{
"KL/chosen_KL_mean": -25.344947814941406,
"KL/mean": -42.59326934814453,
"KL/rejected_KL_mean": -59.84159469604492,
"KL/std": 41.871673583984375,
"epoch": 0.3036649214659686,
"fcm_dpo/beta": 0.01033711526542902,
"fcm_dpo/delta": -0.07404731959104538,
"fcm_dpo/margin": 34.49665069580078,
"fcm_dpo/q_t": 0.41699421405792236,
"grad_norm": 55.13352584838867,
"learning_rate": 4.4072430294890166e-07,
"logits/chosen": -0.8775890469551086,
"logits/rejected": -0.8793016076087952,
"logps/chosen": -301.0978088378906,
"logps/ref_chosen": -275.7528381347656,
"logps/ref_rejected": -214.74807739257812,
"logps/rejected": -274.58966064453125,
"loss": 4.5283,
"margin_dpo/margin_mean": 34.49665069580078,
"margin_dpo/margin_std": 49.740142822265625,
"step": 145
},
{
"KL/chosen_KL_mean": -35.13841247558594,
"KL/mean": -51.969417572021484,
"KL/rejected_KL_mean": -68.8004150390625,
"KL/std": 45.30426788330078,
"epoch": 0.3057591623036649,
"fcm_dpo/beta": 0.010088625364005566,
"fcm_dpo/delta": -0.013300293125212193,
"fcm_dpo/margin": 33.662017822265625,
"fcm_dpo/q_t": 0.4215752184391022,
"grad_norm": 58.701690673828125,
"learning_rate": 4.395355737667985e-07,
"logits/chosen": -0.8522456288337708,
"logits/rejected": -0.8450497984886169,
"logps/chosen": -312.23663330078125,
"logps/ref_chosen": -277.09820556640625,
"logps/ref_rejected": -265.41046142578125,
"logps/rejected": -334.2108459472656,
"loss": 4.5732,
"margin_dpo/margin_mean": 33.66202163696289,
"margin_dpo/margin_std": 50.35722351074219,
"step": 146
},
{
"KL/chosen_KL_mean": -37.00726318359375,
"KL/mean": -49.70895767211914,
"KL/rejected_KL_mean": -62.41065216064453,
"KL/std": 45.74989318847656,
"epoch": 0.3078534031413613,
"fcm_dpo/beta": 0.010042143985629082,
"fcm_dpo/delta": -0.03261498734354973,
"fcm_dpo/margin": 25.403392791748047,
"fcm_dpo/q_t": 0.44056451320648193,
"grad_norm": 73.16529846191406,
"learning_rate": 4.3833668036708483e-07,
"logits/chosen": -0.8551448583602905,
"logits/rejected": -0.8509636521339417,
"logps/chosen": -328.42584228515625,
"logps/ref_chosen": -291.4185791015625,
"logps/ref_rejected": -253.43051147460938,
"logps/rejected": -315.84112548828125,
"loss": 4.9002,
"margin_dpo/margin_mean": 25.403392791748047,
"margin_dpo/margin_std": 51.63163375854492,
"step": 147
},
{
"KL/chosen_KL_mean": -39.83953857421875,
"KL/mean": -53.04331588745117,
"KL/rejected_KL_mean": -66.24710083007812,
"KL/std": 47.25356674194336,
"epoch": 0.3099476439790576,
"fcm_dpo/beta": 0.009962659329175949,
"fcm_dpo/delta": 0.04581095650792122,
"fcm_dpo/margin": 26.407564163208008,
"fcm_dpo/q_t": 0.4414011240005493,
"grad_norm": 59.49394226074219,
"learning_rate": 4.3712768704277524e-07,
"logits/chosen": -0.9048643112182617,
"logits/rejected": -0.9068287014961243,
"logps/chosen": -276.5880432128906,
"logps/ref_chosen": -236.74850463867188,
"logps/ref_rejected": -231.4674072265625,
"logps/rejected": -297.7144775390625,
"loss": 4.9041,
"margin_dpo/margin_mean": 26.407562255859375,
"margin_dpo/margin_std": 59.63642120361328,
"step": 148
},
{
"KL/chosen_KL_mean": -41.80915069580078,
"KL/mean": -59.93357849121094,
"KL/rejected_KL_mean": -78.05801391601562,
"KL/std": 44.09037399291992,
"epoch": 0.31204188481675393,
"fcm_dpo/beta": 0.010042773559689522,
"fcm_dpo/delta": -0.01487391721457243,
"fcm_dpo/margin": 36.248878479003906,
"fcm_dpo/q_t": 0.4150589406490326,
"grad_norm": 70.69794464111328,
"learning_rate": 4.3590865862851263e-07,
"logits/chosen": -0.8720038533210754,
"logits/rejected": -0.8556748628616333,
"logps/chosen": -361.7376403808594,
"logps/ref_chosen": -319.9284973144531,
"logps/ref_rejected": -308.20233154296875,
"logps/rejected": -386.2603454589844,
"loss": 4.4518,
"margin_dpo/margin_mean": 36.248878479003906,
"margin_dpo/margin_std": 48.31082534790039,
"step": 149
},
{
"KL/chosen_KL_mean": -50.01300048828125,
"KL/mean": -63.49955749511719,
"KL/rejected_KL_mean": -76.98611450195312,
"KL/std": 46.77826690673828,
"epoch": 0.31413612565445026,
"fcm_dpo/beta": 0.010052911005914211,
"fcm_dpo/delta": -0.0005218717269599438,
"fcm_dpo/margin": 26.973114013671875,
"fcm_dpo/q_t": 0.4368109703063965,
"grad_norm": 59.045284271240234,
"learning_rate": 4.346796604970912e-07,
"logits/chosen": -0.8413239121437073,
"logits/rejected": -0.8248977661132812,
"logps/chosen": -326.33123779296875,
"logps/ref_chosen": -276.3182373046875,
"logps/ref_rejected": -273.02215576171875,
"logps/rejected": -350.00823974609375,
"loss": 4.8136,
"margin_dpo/margin_mean": 26.97311019897461,
"margin_dpo/margin_std": 52.10138702392578,
"step": 150
},
{
"KL/chosen_KL_mean": -46.77452087402344,
"KL/mean": -66.79248809814453,
"KL/rejected_KL_mean": -86.81045532226562,
"KL/std": 46.96579360961914,
"epoch": 0.3162303664921466,
"fcm_dpo/beta": 0.009873086586594582,
"fcm_dpo/delta": -0.04695701599121094,
"fcm_dpo/margin": 40.035926818847656,
"fcm_dpo/q_t": 0.40897297859191895,
"grad_norm": 81.46723175048828,
"learning_rate": 4.3344075855595097e-07,
"logits/chosen": -0.8678506016731262,
"logits/rejected": -0.8601349592208862,
"logps/chosen": -344.0873718261719,
"logps/ref_chosen": -297.31280517578125,
"logps/ref_rejected": -266.1003723144531,
"logps/rejected": -352.91082763671875,
"loss": 4.4272,
"margin_dpo/margin_mean": 40.03593063354492,
"margin_dpo/margin_std": 55.97598648071289,
"step": 151
},
{
"KL/chosen_KL_mean": -37.11475372314453,
"KL/mean": -55.36869812011719,
"KL/rejected_KL_mean": -73.62263488769531,
"KL/std": 49.92714309692383,
"epoch": 0.3183246073298429,
"fcm_dpo/beta": 0.009343523532152176,
"fcm_dpo/delta": -0.04294053837656975,
"fcm_dpo/margin": 36.50787353515625,
"fcm_dpo/q_t": 0.4220908284187317,
"grad_norm": 58.823143005371094,
"learning_rate": 4.3219201924364323e-07,
"logits/chosen": -0.880072832107544,
"logits/rejected": -0.8768435716629028,
"logps/chosen": -307.36175537109375,
"logps/ref_chosen": -270.2470397949219,
"logps/ref_rejected": -269.7749328613281,
"logps/rejected": -343.3975830078125,
"loss": 4.6093,
"margin_dpo/margin_mean": 36.50787353515625,
"margin_dpo/margin_std": 58.238258361816406,
"step": 152
},
{
"KL/chosen_KL_mean": -42.54793167114258,
"KL/mean": -67.99817657470703,
"KL/rejected_KL_mean": -93.44841003417969,
"KL/std": 53.13435363769531,
"epoch": 0.3204188481675393,
"fcm_dpo/beta": 0.008867593482136726,
"fcm_dpo/delta": -0.1064264178276062,
"fcm_dpo/margin": 50.900489807128906,
"fcm_dpo/q_t": 0.3951815664768219,
"grad_norm": 61.95179748535156,
"learning_rate": 4.309335095262675e-07,
"logits/chosen": -0.8620288372039795,
"logits/rejected": -0.8489632606506348,
"logps/chosen": -316.3269958496094,
"logps/ref_chosen": -273.779052734375,
"logps/ref_rejected": -280.9530944824219,
"logps/rejected": -374.4014892578125,
"loss": 4.1803,
"margin_dpo/margin_mean": 50.90049362182617,
"margin_dpo/margin_std": 55.47189712524414,
"step": 153
},
{
"KL/chosen_KL_mean": -41.75373458862305,
"KL/mean": -61.51596450805664,
"KL/rejected_KL_mean": -81.27819061279297,
"KL/std": 55.93893051147461,
"epoch": 0.3225130890052356,
"fcm_dpo/beta": 0.008387413807213306,
"fcm_dpo/delta": -0.043754659593105316,
"fcm_dpo/margin": 39.52445983886719,
"fcm_dpo/q_t": 0.4235866665840149,
"grad_norm": 64.17202758789062,
"learning_rate": 4.2966529689388064e-07,
"logits/chosen": -0.9106939435005188,
"logits/rejected": -0.8913959860801697,
"logps/chosen": -331.6569519042969,
"logps/ref_chosen": -289.9031982421875,
"logps/ref_rejected": -261.5166320800781,
"logps/rejected": -342.7948303222656,
"loss": 4.6569,
"margin_dpo/margin_mean": 39.52445983886719,
"margin_dpo/margin_std": 66.74655151367188,
"step": 154
},
{
"KL/chosen_KL_mean": -45.44877624511719,
"KL/mean": -62.710758209228516,
"KL/rejected_KL_mean": -79.9727554321289,
"KL/std": 53.82670974731445,
"epoch": 0.32460732984293195,
"fcm_dpo/beta": 0.008284756913781166,
"fcm_dpo/delta": 0.005428697448223829,
"fcm_dpo/margin": 34.52397918701172,
"fcm_dpo/q_t": 0.43525511026382446,
"grad_norm": 67.82777404785156,
"learning_rate": 4.2838744935687716e-07,
"logits/chosen": -0.8425617814064026,
"logits/rejected": -0.8398025035858154,
"logps/chosen": -331.3099670410156,
"logps/ref_chosen": -285.8612060546875,
"logps/ref_rejected": -300.1272888183594,
"logps/rejected": -380.10003662109375,
"loss": 4.7548,
"margin_dpo/margin_mean": 34.52397918701172,
"margin_dpo/margin_std": 62.337467193603516,
"step": 155
},
{
"KL/chosen_KL_mean": -40.039371490478516,
"KL/mean": -63.92396926879883,
"KL/rejected_KL_mean": -87.80857849121094,
"KL/std": 56.946449279785156,
"epoch": 0.3267015706806283,
"fcm_dpo/beta": 0.008112832903862,
"fcm_dpo/delta": -0.08456975221633911,
"fcm_dpo/margin": 47.769203186035156,
"fcm_dpo/q_t": 0.41121193766593933,
"grad_norm": 50.074623107910156,
"learning_rate": 4.271000354423425e-07,
"logits/chosen": -0.8586286902427673,
"logits/rejected": -0.8539766073226929,
"logps/chosen": -319.0748596191406,
"logps/ref_chosen": -279.0354919433594,
"logps/ref_rejected": -244.2198486328125,
"logps/rejected": -332.0284423828125,
"loss": 4.4928,
"margin_dpo/margin_mean": 47.769203186035156,
"margin_dpo/margin_std": 71.89561462402344,
"step": 156
},
{
"KL/chosen_KL_mean": -43.30219268798828,
"KL/mean": -62.22343826293945,
"KL/rejected_KL_mean": -81.14466857910156,
"KL/std": 56.71215057373047,
"epoch": 0.3287958115183246,
"fcm_dpo/beta": 0.007705520838499069,
"fcm_dpo/delta": -0.007637109607458115,
"fcm_dpo/margin": 37.84247589111328,
"fcm_dpo/q_t": 0.4317454695701599,
"grad_norm": 59.33927917480469,
"learning_rate": 4.258031241903777e-07,
"logits/chosen": -0.9295624494552612,
"logits/rejected": -0.9276785254478455,
"logps/chosen": -314.13250732421875,
"logps/ref_chosen": -270.830322265625,
"logps/ref_rejected": -259.08319091796875,
"logps/rejected": -340.2278747558594,
"loss": 4.7054,
"margin_dpo/margin_mean": 37.84247589111328,
"margin_dpo/margin_std": 61.984832763671875,
"step": 157
},
{
"KL/chosen_KL_mean": -42.21181869506836,
"KL/mean": -62.8458137512207,
"KL/rejected_KL_mean": -83.47979736328125,
"KL/std": 57.90792465209961,
"epoch": 0.3308900523560209,
"fcm_dpo/beta": 0.007645631209015846,
"fcm_dpo/delta": -0.030163168907165527,
"fcm_dpo/margin": 41.267974853515625,
"fcm_dpo/q_t": 0.4256693720817566,
"grad_norm": 70.06487274169922,
"learning_rate": 4.2449678515039743e-07,
"logits/chosen": -0.8575935959815979,
"logits/rejected": -0.8421310186386108,
"logps/chosen": -332.1781311035156,
"logps/ref_chosen": -289.9663391113281,
"logps/ref_rejected": -271.335693359375,
"logps/rejected": -354.8154602050781,
"loss": 4.6531,
"margin_dpo/margin_mean": 41.267974853515625,
"margin_dpo/margin_std": 65.3279037475586,
"step": 158
},
{
"KL/chosen_KL_mean": -51.21124267578125,
"KL/mean": -63.622154235839844,
"KL/rejected_KL_mean": -76.03305053710938,
"KL/std": 54.81487274169922,
"epoch": 0.33298429319371725,
"fcm_dpo/beta": 0.007572945207357407,
"fcm_dpo/delta": 0.028481464833021164,
"fcm_dpo/margin": 24.821823120117188,
"fcm_dpo/q_t": 0.45764467120170593,
"grad_norm": 76.31413269042969,
"learning_rate": 4.2318108837739986e-07,
"logits/chosen": -0.9422204494476318,
"logits/rejected": -0.8938316702842712,
"logps/chosen": -372.589599609375,
"logps/ref_chosen": -321.37835693359375,
"logps/ref_rejected": -250.45652770996094,
"logps/rejected": -326.4895935058594,
"loss": 5.1345,
"margin_dpo/margin_mean": 24.821823120117188,
"margin_dpo/margin_std": 70.63139343261719,
"step": 159
},
{
"KL/chosen_KL_mean": -41.84352111816406,
"KL/mean": -66.21701049804688,
"KL/rejected_KL_mean": -90.59048461914062,
"KL/std": 57.292686462402344,
"epoch": 0.33507853403141363,
"fcm_dpo/beta": 0.007637364789843559,
"fcm_dpo/delta": -0.02358035184442997,
"fcm_dpo/margin": 48.74696350097656,
"fcm_dpo/q_t": 0.4126546382904053,
"grad_norm": 54.5738525390625,
"learning_rate": 4.218561044282098e-07,
"logits/chosen": -0.8767799139022827,
"logits/rejected": -0.8782601952552795,
"logps/chosen": -318.12701416015625,
"logps/ref_chosen": -276.28350830078125,
"logps/ref_rejected": -262.7477722167969,
"logps/rejected": -353.3382568359375,
"loss": 4.3985,
"margin_dpo/margin_mean": 48.74696350097656,
"margin_dpo/margin_std": 58.39624786376953,
"step": 160
},
{
"KL/chosen_KL_mean": -50.77786636352539,
"KL/mean": -72.51949310302734,
"KL/rejected_KL_mean": -94.26113891601562,
"KL/std": 60.983299255371094,
"epoch": 0.33717277486910996,
"fcm_dpo/beta": 0.007549113593995571,
"fcm_dpo/delta": -0.019491776823997498,
"fcm_dpo/margin": 43.4832649230957,
"fcm_dpo/q_t": 0.4252697825431824,
"grad_norm": 69.1987075805664,
"learning_rate": 4.2052190435769554e-07,
"logits/chosen": -0.8953877687454224,
"logits/rejected": -0.8785583972930908,
"logps/chosen": -361.2706604003906,
"logps/ref_chosen": -310.4927978515625,
"logps/ref_rejected": -250.25347900390625,
"logps/rejected": -344.5146179199219,
"loss": 4.6394,
"margin_dpo/margin_mean": 43.48326873779297,
"margin_dpo/margin_std": 71.65884399414062,
"step": 161
},
{
"KL/chosen_KL_mean": -47.96656036376953,
"KL/mean": -67.32000732421875,
"KL/rejected_KL_mean": -86.6734619140625,
"KL/std": 56.53418731689453,
"epoch": 0.3392670157068063,
"fcm_dpo/beta": 0.007303684484213591,
"fcm_dpo/delta": -0.009029125794768333,
"fcm_dpo/margin": 38.7069091796875,
"fcm_dpo/q_t": 0.43392330408096313,
"grad_norm": 63.3481559753418,
"learning_rate": 4.1917855971495763e-07,
"logits/chosen": -0.8692272901535034,
"logits/rejected": -0.8613343238830566,
"logps/chosen": -344.0770263671875,
"logps/ref_chosen": -296.1105041503906,
"logps/ref_rejected": -253.4247589111328,
"logps/rejected": -340.0982360839844,
"loss": 4.7335,
"margin_dpo/margin_mean": 38.7069091796875,
"margin_dpo/margin_std": 64.84658813476562,
"step": 162
},
{
"KL/chosen_KL_mean": -51.703914642333984,
"KL/mean": -73.81686401367188,
"KL/rejected_KL_mean": -95.9298095703125,
"KL/std": 55.789180755615234,
"epoch": 0.3413612565445026,
"fcm_dpo/beta": 0.007537513505667448,
"fcm_dpo/delta": 0.01612706668674946,
"fcm_dpo/margin": 44.22589111328125,
"fcm_dpo/q_t": 0.42153996229171753,
"grad_norm": 68.57307434082031,
"learning_rate": 4.1782614253949255e-07,
"logits/chosen": -0.8940120339393616,
"logits/rejected": -0.8899393081665039,
"logps/chosen": -345.2038269042969,
"logps/ref_chosen": -293.4999084472656,
"logps/ref_rejected": -266.7116394042969,
"logps/rejected": -362.6415100097656,
"loss": 4.5561,
"margin_dpo/margin_mean": 44.22589111328125,
"margin_dpo/margin_std": 61.894046783447266,
"step": 163
},
{
"KL/chosen_KL_mean": -60.08200454711914,
"KL/mean": -80.8095703125,
"KL/rejected_KL_mean": -101.5371322631836,
"KL/std": 61.46062469482422,
"epoch": 0.34345549738219894,
"fcm_dpo/beta": 0.0075241196900606155,
"fcm_dpo/delta": -0.012014145031571388,
"fcm_dpo/margin": 41.455135345458984,
"fcm_dpo/q_t": 0.42829614877700806,
"grad_norm": 60.644813537597656,
"learning_rate": 4.164647253573289e-07,
"logits/chosen": -0.8615102171897888,
"logits/rejected": -0.8748703598976135,
"logps/chosen": -327.13153076171875,
"logps/ref_chosen": -267.04949951171875,
"logps/ref_rejected": -215.9768829345703,
"logps/rejected": -317.5140075683594,
"loss": 4.6851,
"margin_dpo/margin_mean": 41.45513153076172,
"margin_dpo/margin_std": 70.61214447021484,
"step": 164
},
{
"KL/chosen_KL_mean": -49.37308120727539,
"KL/mean": -63.531410217285156,
"KL/rejected_KL_mean": -77.68974304199219,
"KL/std": 52.045166015625,
"epoch": 0.34554973821989526,
"fcm_dpo/beta": 0.0073788780719041824,
"fcm_dpo/delta": -0.004405612125992775,
"fcm_dpo/margin": 28.3166561126709,
"fcm_dpo/q_t": 0.4508272409439087,
"grad_norm": 80.6587142944336,
"learning_rate": 4.1509438117713863e-07,
"logits/chosen": -0.8801563382148743,
"logits/rejected": -0.8534449934959412,
"logps/chosen": -327.4346008300781,
"logps/ref_chosen": -278.06146240234375,
"logps/ref_rejected": -260.4288635253906,
"logps/rejected": -338.11865234375,
"loss": 4.9606,
"margin_dpo/margin_mean": 28.31665802001953,
"margin_dpo/margin_std": 59.50096893310547,
"step": 165
},
{
"KL/chosen_KL_mean": -45.839622497558594,
"KL/mean": -64.79864501953125,
"KL/rejected_KL_mean": -83.75764465332031,
"KL/std": 55.16622543334961,
"epoch": 0.34764397905759165,
"fcm_dpo/beta": 0.007531080860644579,
"fcm_dpo/delta": 0.06673787534236908,
"fcm_dpo/margin": 37.91801071166992,
"fcm_dpo/q_t": 0.43460068106651306,
"grad_norm": 61.27188491821289,
"learning_rate": 4.137151834863213e-07,
"logits/chosen": -0.8442600965499878,
"logits/rejected": -0.809601366519928,
"logps/chosen": -321.7886962890625,
"logps/ref_chosen": -275.9490661621094,
"logps/ref_rejected": -232.13473510742188,
"logps/rejected": -315.8923645019531,
"loss": 4.7461,
"margin_dpo/margin_mean": 37.91801452636719,
"margin_dpo/margin_std": 67.47180938720703,
"step": 166
},
{
"KL/chosen_KL_mean": -56.37845993041992,
"KL/mean": -77.35397338867188,
"KL/rejected_KL_mean": -98.32949829101562,
"KL/std": 60.27275848388672,
"epoch": 0.34973821989528797,
"fcm_dpo/beta": 0.007845787331461906,
"fcm_dpo/delta": 0.00037165917456150055,
"fcm_dpo/margin": 41.95103073120117,
"fcm_dpo/q_t": 0.4235421419143677,
"grad_norm": 73.83167266845703,
"learning_rate": 4.123272062470633e-07,
"logits/chosen": -0.8352253437042236,
"logits/rejected": -0.8208370208740234,
"logps/chosen": -336.92987060546875,
"logps/ref_chosen": -280.5514221191406,
"logps/ref_rejected": -255.2896728515625,
"logps/rejected": -353.6191711425781,
"loss": 4.6424,
"margin_dpo/margin_mean": 41.95103073120117,
"margin_dpo/margin_std": 69.98577117919922,
"step": 167
},
{
"KL/chosen_KL_mean": -55.98714828491211,
"KL/mean": -79.81024169921875,
"KL/rejected_KL_mean": -103.6333236694336,
"KL/std": 63.85597610473633,
"epoch": 0.3518324607329843,
"fcm_dpo/beta": 0.0075763678178191185,
"fcm_dpo/delta": -0.04661968722939491,
"fcm_dpo/margin": 47.646175384521484,
"fcm_dpo/q_t": 0.4152388274669647,
"grad_norm": 115.36837005615234,
"learning_rate": 4.1093052389237174e-07,
"logits/chosen": -0.8275383710861206,
"logits/rejected": -0.7986543774604797,
"logps/chosen": -371.7853698730469,
"logps/ref_chosen": -315.7982177734375,
"logps/ref_rejected": -291.48406982421875,
"logps/rejected": -395.11737060546875,
"loss": 4.5223,
"margin_dpo/margin_mean": 47.64617156982422,
"margin_dpo/margin_std": 70.27813720703125,
"step": 168
},
{
"KL/chosen_KL_mean": -55.89307403564453,
"KL/mean": -84.45292663574219,
"KL/rejected_KL_mean": -113.01277160644531,
"KL/std": 64.26335144042969,
"epoch": 0.3539267015706806,
"fcm_dpo/beta": 0.0072573344223201275,
"fcm_dpo/delta": -0.09110445529222488,
"fcm_dpo/margin": 57.11970520019531,
"fcm_dpo/q_t": 0.4041180908679962,
"grad_norm": 88.24478912353516,
"learning_rate": 4.0952521132208267e-07,
"logits/chosen": -0.8238295912742615,
"logits/rejected": -0.8293823599815369,
"logps/chosen": -316.95733642578125,
"logps/ref_chosen": -261.06427001953125,
"logps/ref_rejected": -235.40663146972656,
"logps/rejected": -348.41943359375,
"loss": 4.3353,
"margin_dpo/margin_mean": 57.11970520019531,
"margin_dpo/margin_std": 70.19021606445312,
"step": 169
},
{
"KL/chosen_KL_mean": -56.9837760925293,
"KL/mean": -80.9883804321289,
"KL/rejected_KL_mean": -104.99298095703125,
"KL/std": 67.8764877319336,
"epoch": 0.35602094240837695,
"fcm_dpo/beta": 0.006932388059794903,
"fcm_dpo/delta": -0.0026629664935171604,
"fcm_dpo/margin": 48.00920104980469,
"fcm_dpo/q_t": 0.4246680736541748,
"grad_norm": 63.6226806640625,
"learning_rate": 4.081113438988443e-07,
"logits/chosen": -0.8085747957229614,
"logits/rejected": -0.8082205653190613,
"logps/chosen": -365.95098876953125,
"logps/ref_chosen": -308.96722412109375,
"logps/ref_rejected": -263.8466796875,
"logps/rejected": -368.83966064453125,
"loss": 4.6691,
"margin_dpo/margin_mean": 48.00919723510742,
"margin_dpo/margin_std": 86.9151840209961,
"step": 170
},
{
"KL/chosen_KL_mean": -52.376243591308594,
"KL/mean": -80.6495132446289,
"KL/rejected_KL_mean": -108.92279052734375,
"KL/std": 65.0744857788086,
"epoch": 0.3581151832460733,
"fcm_dpo/beta": 0.006713580340147018,
"fcm_dpo/delta": -0.06937012076377869,
"fcm_dpo/margin": 56.54655456542969,
"fcm_dpo/q_t": 0.41104498505592346,
"grad_norm": 81.89509582519531,
"learning_rate": 4.0668899744407567e-07,
"logits/chosen": -0.8374527096748352,
"logits/rejected": -0.8505634069442749,
"logps/chosen": -311.2652587890625,
"logps/ref_chosen": -258.8890380859375,
"logps/ref_rejected": -262.19140625,
"logps/rejected": -371.11419677734375,
"loss": 4.3971,
"margin_dpo/margin_mean": 56.54655456542969,
"margin_dpo/margin_std": 69.68020629882812,
"step": 171
},
{
"KL/chosen_KL_mean": -56.283409118652344,
"KL/mean": -72.89511108398438,
"KL/rejected_KL_mean": -89.50682067871094,
"KL/std": 59.91847610473633,
"epoch": 0.36020942408376966,
"fcm_dpo/beta": 0.006686648353934288,
"fcm_dpo/delta": 0.03564419597387314,
"fcm_dpo/margin": 33.223411560058594,
"fcm_dpo/q_t": 0.447513610124588,
"grad_norm": 90.56462097167969,
"learning_rate": 4.0525824823390043e-07,
"logits/chosen": -0.8602209687232971,
"logits/rejected": -0.8738598227500916,
"logps/chosen": -395.3057556152344,
"logps/ref_chosen": -339.0223388671875,
"logps/ref_rejected": -295.78759765625,
"logps/rejected": -385.29443359375,
"loss": 4.955,
"margin_dpo/margin_mean": 33.22341537475586,
"margin_dpo/margin_std": 74.95690155029297,
"step": 172
},
{
"KL/chosen_KL_mean": -58.3000602722168,
"KL/mean": -80.27344512939453,
"KL/rejected_KL_mean": -102.24684143066406,
"KL/std": 63.42533874511719,
"epoch": 0.362303664921466,
"fcm_dpo/beta": 0.0067140692844986916,
"fcm_dpo/delta": -0.014882495626807213,
"fcm_dpo/margin": 43.94677734375,
"fcm_dpo/q_t": 0.43222668766975403,
"grad_norm": 49.26839828491211,
"learning_rate": 4.0381917299505686e-07,
"logits/chosen": -0.88132643699646,
"logits/rejected": -0.8753910660743713,
"logps/chosen": -358.4115295410156,
"logps/ref_chosen": -300.1114501953125,
"logps/ref_rejected": -273.78460693359375,
"logps/rejected": -376.03143310546875,
"loss": 4.7171,
"margin_dpo/margin_mean": 43.946781158447266,
"margin_dpo/margin_std": 74.02472686767578,
"step": 173
},
{
"KL/chosen_KL_mean": -59.025245666503906,
"KL/mean": -84.17713165283203,
"KL/rejected_KL_mean": -109.32901000976562,
"KL/std": 63.81253433227539,
"epoch": 0.3643979057591623,
"fcm_dpo/beta": 0.006741848308593035,
"fcm_dpo/delta": 0.010899264365434647,
"fcm_dpo/margin": 50.303775787353516,
"fcm_dpo/q_t": 0.419879287481308,
"grad_norm": 70.78109741210938,
"learning_rate": 4.0237184890078243e-07,
"logits/chosen": -0.8532913327217102,
"logits/rejected": -0.8315014839172363,
"logps/chosen": -394.0791015625,
"logps/ref_chosen": -335.0538635253906,
"logps/ref_rejected": -257.4646911621094,
"logps/rejected": -366.7936706542969,
"loss": 4.5267,
"margin_dpo/margin_mean": 50.303775787353516,
"margin_dpo/margin_std": 70.04698944091797,
"step": 174
},
{
"KL/chosen_KL_mean": -71.4371109008789,
"KL/mean": -94.12638854980469,
"KL/rejected_KL_mean": -116.815673828125,
"KL/std": 69.9000473022461,
"epoch": 0.36649214659685864,
"fcm_dpo/beta": 0.006678528618067503,
"fcm_dpo/delta": -0.048241063952445984,
"fcm_dpo/margin": 45.37855529785156,
"fcm_dpo/q_t": 0.432115375995636,
"grad_norm": 63.40766143798828,
"learning_rate": 4.00916353566676e-07,
"logits/chosen": -0.8552145957946777,
"logits/rejected": -0.8552535772323608,
"logps/chosen": -355.8326721191406,
"logps/ref_chosen": -284.39556884765625,
"logps/ref_rejected": -283.3876647949219,
"logps/rejected": -400.20330810546875,
"loss": 4.7497,
"margin_dpo/margin_mean": 45.37855529785156,
"margin_dpo/margin_std": 81.3629379272461,
"step": 175
},
{
"KL/chosen_KL_mean": -65.49942016601562,
"KL/mean": -83.41252136230469,
"KL/rejected_KL_mean": -101.32563781738281,
"KL/std": 64.79817199707031,
"epoch": 0.36858638743455496,
"fcm_dpo/beta": 0.006564862094819546,
"fcm_dpo/delta": 0.004861448425799608,
"fcm_dpo/margin": 35.82622528076172,
"fcm_dpo/q_t": 0.4460398554801941,
"grad_norm": 69.52070617675781,
"learning_rate": 3.994527650465352e-07,
"logits/chosen": -0.8273112773895264,
"logits/rejected": -0.832463264465332,
"logps/chosen": -317.31219482421875,
"logps/ref_chosen": -251.81280517578125,
"logps/ref_rejected": -242.05328369140625,
"logps/rejected": -343.3789367675781,
"loss": 4.9677,
"margin_dpo/margin_mean": 35.82622528076172,
"margin_dpo/margin_std": 83.5301513671875,
"step": 176
},
{
"KL/chosen_KL_mean": -72.5023193359375,
"KL/mean": -90.53446960449219,
"KL/rejected_KL_mean": -108.56661224365234,
"KL/std": 65.00486755371094,
"epoch": 0.3706806282722513,
"fcm_dpo/beta": 0.006526026874780655,
"fcm_dpo/delta": 0.008697102777659893,
"fcm_dpo/margin": 36.06428909301758,
"fcm_dpo/q_t": 0.44549474120140076,
"grad_norm": 63.3103141784668,
"learning_rate": 3.979811618281705e-07,
"logits/chosen": -0.9081556797027588,
"logits/rejected": -0.8767774105072021,
"logps/chosen": -371.14874267578125,
"logps/ref_chosen": -298.6463928222656,
"logps/ref_rejected": -295.66534423828125,
"logps/rejected": -404.2319641113281,
"loss": 4.9694,
"margin_dpo/margin_mean": 36.06428909301758,
"margin_dpo/margin_std": 79.99176025390625,
"step": 177
},
{
"KL/chosen_KL_mean": -60.73016357421875,
"KL/mean": -88.53191375732422,
"KL/rejected_KL_mean": -116.33367156982422,
"KL/std": 69.4561538696289,
"epoch": 0.37277486910994767,
"fcm_dpo/beta": 0.006494695320725441,
"fcm_dpo/delta": -0.011969889514148235,
"fcm_dpo/margin": 55.60350799560547,
"fcm_dpo/q_t": 0.41702789068222046,
"grad_norm": 54.077415466308594,
"learning_rate": 3.9650162282919654e-07,
"logits/chosen": -0.8326258659362793,
"logits/rejected": -0.8247045874595642,
"logps/chosen": -346.98779296875,
"logps/ref_chosen": -286.2576599121094,
"logps/ref_rejected": -243.97491455078125,
"logps/rejected": -360.3085632324219,
"loss": 4.4967,
"margin_dpo/margin_mean": 55.60350799560547,
"margin_dpo/margin_std": 79.79682922363281,
"step": 178
},
{
"KL/chosen_KL_mean": -70.25981903076172,
"KL/mean": -92.2850341796875,
"KL/rejected_KL_mean": -114.31026458740234,
"KL/std": 69.08260345458984,
"epoch": 0.374869109947644,
"fcm_dpo/beta": 0.006461943034082651,
"fcm_dpo/delta": -0.009359869174659252,
"fcm_dpo/margin": 44.050445556640625,
"fcm_dpo/q_t": 0.4336419403553009,
"grad_norm": 72.00945281982422,
"learning_rate": 3.9501422739279953e-07,
"logits/chosen": -0.8239375352859497,
"logits/rejected": -0.8243504166603088,
"logps/chosen": -329.99688720703125,
"logps/ref_chosen": -259.737060546875,
"logps/ref_rejected": -277.8813171386719,
"logps/rejected": -392.19158935546875,
"loss": 4.7656,
"margin_dpo/margin_mean": 44.05044174194336,
"margin_dpo/margin_std": 78.93538665771484,
"step": 179
},
{
"KL/chosen_KL_mean": -71.84423828125,
"KL/mean": -98.30107879638672,
"KL/rejected_KL_mean": -124.75789642333984,
"KL/std": 66.42559814453125,
"epoch": 0.3769633507853403,
"fcm_dpo/beta": 0.006444687955081463,
"fcm_dpo/delta": -0.013784291222691536,
"fcm_dpo/margin": 52.91366958618164,
"fcm_dpo/q_t": 0.42224639654159546,
"grad_norm": 78.1937026977539,
"learning_rate": 3.935190552834828e-07,
"logits/chosen": -0.8528253436088562,
"logits/rejected": -0.8767141699790955,
"logps/chosen": -339.15313720703125,
"logps/ref_chosen": -267.30889892578125,
"logps/ref_rejected": -230.4376983642578,
"logps/rejected": -355.19561767578125,
"loss": 4.5563,
"margin_dpo/margin_mean": 52.91366958618164,
"margin_dpo/margin_std": 78.59088134765625,
"step": 180
},
{
"KL/chosen_KL_mean": -83.44524383544922,
"KL/mean": -106.03001403808594,
"KL/rejected_KL_mean": -128.61477661132812,
"KL/std": 69.1161880493164,
"epoch": 0.37905759162303665,
"fcm_dpo/beta": 0.006374704651534557,
"fcm_dpo/delta": 0.011338986456394196,
"fcm_dpo/margin": 45.169532775878906,
"fcm_dpo/q_t": 0.4334385395050049,
"grad_norm": 76.543212890625,
"learning_rate": 3.920161866827889e-07,
"logits/chosen": -0.8567452430725098,
"logits/rejected": -0.8574219346046448,
"logps/chosen": -383.9366149902344,
"logps/ref_chosen": -300.49139404296875,
"logps/ref_rejected": -278.98284912109375,
"logps/rejected": -407.5976257324219,
"loss": 4.7631,
"margin_dpo/margin_mean": 45.169532775878906,
"margin_dpo/margin_std": 83.6402587890625,
"step": 181
},
{
"KL/chosen_KL_mean": -84.8715591430664,
"KL/mean": -116.56951904296875,
"KL/rejected_KL_mean": -148.26747131347656,
"KL/std": 69.70707702636719,
"epoch": 0.381151832460733,
"fcm_dpo/beta": 0.006300204433500767,
"fcm_dpo/delta": -0.08057114481925964,
"fcm_dpo/margin": 63.39593505859375,
"fcm_dpo/q_t": 0.40720659494400024,
"grad_norm": 86.77213287353516,
"learning_rate": 3.90505702185e-07,
"logits/chosen": -0.8535815477371216,
"logits/rejected": -0.877009391784668,
"logps/chosen": -364.3696594238281,
"logps/ref_chosen": -279.4981689453125,
"logps/ref_rejected": -263.6926574707031,
"logps/rejected": -411.9601135253906,
"loss": 4.3766,
"margin_dpo/margin_mean": 63.39593505859375,
"margin_dpo/margin_std": 80.48890686035156,
"step": 182
},
{
"KL/chosen_KL_mean": -83.44680786132812,
"KL/mean": -110.89114379882812,
"KL/rejected_KL_mean": -138.33546447753906,
"KL/std": 72.1974105834961,
"epoch": 0.3832460732984293,
"fcm_dpo/beta": 0.0060380203649401665,
"fcm_dpo/delta": 0.019243724644184113,
"fcm_dpo/margin": 54.8886604309082,
"fcm_dpo/q_t": 0.4239034652709961,
"grad_norm": 63.45649719238281,
"learning_rate": 3.889876827928156e-07,
"logits/chosen": -0.8888998031616211,
"logits/rejected": -0.8892589807510376,
"logps/chosen": -354.6525573730469,
"logps/ref_chosen": -271.2057189941406,
"logps/ref_rejected": -243.91549682617188,
"logps/rejected": -382.2509460449219,
"loss": 4.6048,
"margin_dpo/margin_mean": 54.8886604309082,
"margin_dpo/margin_std": 87.88034057617188,
"step": 183
},
{
"KL/chosen_KL_mean": -82.51263427734375,
"KL/mean": -119.50811767578125,
"KL/rejected_KL_mean": -156.5036163330078,
"KL/std": 79.93283081054688,
"epoch": 0.38534031413612563,
"fcm_dpo/beta": 0.005751221906393766,
"fcm_dpo/delta": -0.10398561507463455,
"fcm_dpo/margin": 73.99097442626953,
"fcm_dpo/q_t": 0.400656133890152,
"grad_norm": 59.62614822387695,
"learning_rate": 3.874622099130087e-07,
"logits/chosen": -0.9280270934104919,
"logits/rejected": -0.9025266766548157,
"logps/chosen": -400.95843505859375,
"logps/ref_chosen": -318.4457702636719,
"logps/ref_rejected": -266.640869140625,
"logps/rejected": -423.14447021484375,
"loss": 4.3096,
"margin_dpo/margin_mean": 73.990966796875,
"margin_dpo/margin_std": 93.23393249511719,
"step": 184
},
{
"KL/chosen_KL_mean": -96.60318756103516,
"KL/mean": -123.63733673095703,
"KL/rejected_KL_mean": -150.67147827148438,
"KL/std": 84.32405090332031,
"epoch": 0.387434554973822,
"fcm_dpo/beta": 0.005654371343553066,
"fcm_dpo/delta": 0.013005169108510017,
"fcm_dpo/margin": 54.06830978393555,
"fcm_dpo/q_t": 0.42959728837013245,
"grad_norm": 81.64273834228516,
"learning_rate": 3.859293653520604e-07,
"logits/chosen": -0.8964764475822449,
"logits/rejected": -0.8889636397361755,
"logps/chosen": -370.9120178222656,
"logps/ref_chosen": -274.308837890625,
"logps/ref_rejected": -260.7274169921875,
"logps/rejected": -411.3988952636719,
"loss": 4.6766,
"margin_dpo/margin_mean": 54.06830978393555,
"margin_dpo/margin_std": 89.609130859375,
"step": 185
},
{
"KL/chosen_KL_mean": -102.949951171875,
"KL/mean": -129.26104736328125,
"KL/rejected_KL_mean": -155.57212829589844,
"KL/std": 75.34392547607422,
"epoch": 0.38952879581151834,
"fcm_dpo/beta": 0.0056923748925328255,
"fcm_dpo/delta": -0.009608768858015537,
"fcm_dpo/margin": 52.622169494628906,
"fcm_dpo/q_t": 0.42984485626220703,
"grad_norm": 63.738101959228516,
"learning_rate": 3.8438923131177237e-07,
"logits/chosen": -0.9191085696220398,
"logits/rejected": -0.9174607396125793,
"logps/chosen": -401.955322265625,
"logps/ref_chosen": -299.00537109375,
"logps/ref_rejected": -274.4014587402344,
"logps/rejected": -429.97357177734375,
"loss": 4.6882,
"margin_dpo/margin_mean": 52.622169494628906,
"margin_dpo/margin_std": 85.38917541503906,
"step": 186
},
{
"KL/chosen_KL_mean": -112.53751373291016,
"KL/mean": -135.30740356445312,
"KL/rejected_KL_mean": -158.07728576660156,
"KL/std": 80.25025939941406,
"epoch": 0.39162303664921466,
"fcm_dpo/beta": 0.005760915577411652,
"fcm_dpo/delta": 0.046012088656425476,
"fcm_dpo/margin": 45.53976821899414,
"fcm_dpo/q_t": 0.4372592568397522,
"grad_norm": 98.26136016845703,
"learning_rate": 3.828418903848593e-07,
"logits/chosen": -0.8597753047943115,
"logits/rejected": -0.8412461876869202,
"logps/chosen": -442.3628234863281,
"logps/ref_chosen": -329.8253173828125,
"logps/ref_rejected": -263.73175048828125,
"logps/rejected": -421.8090515136719,
"loss": 4.8902,
"margin_dpo/margin_mean": 45.53976821899414,
"margin_dpo/margin_std": 97.40721893310547,
"step": 187
},
{
"KL/chosen_KL_mean": -101.80508422851562,
"KL/mean": -128.22146606445312,
"KL/rejected_KL_mean": -154.6378173828125,
"KL/std": 82.52928924560547,
"epoch": 0.393717277486911,
"fcm_dpo/beta": 0.005808525253087282,
"fcm_dpo/delta": -0.002299257554113865,
"fcm_dpo/margin": 52.83273696899414,
"fcm_dpo/q_t": 0.43033504486083984,
"grad_norm": 82.98176574707031,
"learning_rate": 3.812874255505191e-07,
"logits/chosen": -0.8709114789962769,
"logits/rejected": -0.8592625260353088,
"logps/chosen": -364.8106689453125,
"logps/ref_chosen": -263.005615234375,
"logps/ref_rejected": -247.08668518066406,
"logps/rejected": -401.7244873046875,
"loss": 4.7836,
"margin_dpo/margin_mean": 52.83273696899414,
"margin_dpo/margin_std": 103.43292236328125,
"step": 188
},
{
"KL/chosen_KL_mean": -88.54216003417969,
"KL/mean": -119.99163818359375,
"KL/rejected_KL_mean": -151.44114685058594,
"KL/std": 81.04593658447266,
"epoch": 0.3958115183246073,
"fcm_dpo/beta": 0.005683474242687225,
"fcm_dpo/delta": -0.05930778384208679,
"fcm_dpo/margin": 62.898963928222656,
"fcm_dpo/q_t": 0.41748058795928955,
"grad_norm": 80.47416687011719,
"learning_rate": 3.797259201699833e-07,
"logits/chosen": -0.8913860321044922,
"logits/rejected": -0.8976064920425415,
"logps/chosen": -361.5025634765625,
"logps/ref_chosen": -272.96038818359375,
"logps/ref_rejected": -275.13238525390625,
"logps/rejected": -426.5735168457031,
"loss": 4.503,
"margin_dpo/margin_mean": 62.898963928222656,
"margin_dpo/margin_std": 89.14469909667969,
"step": 189
},
{
"KL/chosen_KL_mean": -90.09049987792969,
"KL/mean": -119.01117706298828,
"KL/rejected_KL_mean": -147.93182373046875,
"KL/std": 78.23188781738281,
"epoch": 0.39790575916230364,
"fcm_dpo/beta": 0.005622372031211853,
"fcm_dpo/delta": 0.02536490187048912,
"fcm_dpo/margin": 57.84133529663086,
"fcm_dpo/q_t": 0.4240185022354126,
"grad_norm": 57.77339172363281,
"learning_rate": 3.781574579820464e-07,
"logits/chosen": -0.884880542755127,
"logits/rejected": -0.8418174386024475,
"logps/chosen": -347.8880615234375,
"logps/ref_chosen": -257.79754638671875,
"logps/ref_rejected": -225.2164306640625,
"logps/rejected": -373.14825439453125,
"loss": 4.5946,
"margin_dpo/margin_mean": 57.84133529663086,
"margin_dpo/margin_std": 86.56686401367188,
"step": 190
},
{
"KL/chosen_KL_mean": -82.45503234863281,
"KL/mean": -112.23388671875,
"KL/rejected_KL_mean": -142.0127410888672,
"KL/std": 79.87920379638672,
"epoch": 0.4,
"fcm_dpo/beta": 0.005705860909074545,
"fcm_dpo/delta": -0.009763660840690136,
"fcm_dpo/margin": 59.557716369628906,
"fcm_dpo/q_t": 0.4211188554763794,
"grad_norm": 77.27084350585938,
"learning_rate": 3.765821230985757e-07,
"logits/chosen": -0.9118991494178772,
"logits/rejected": -0.9118152856826782,
"logps/chosen": -326.3135681152344,
"logps/ref_chosen": -243.8585205078125,
"logps/ref_rejected": -245.12136840820312,
"logps/rejected": -387.1341247558594,
"loss": 4.5937,
"margin_dpo/margin_mean": 59.55772018432617,
"margin_dpo/margin_std": 94.2480239868164,
"step": 191
},
{
"KL/chosen_KL_mean": -96.14141845703125,
"KL/mean": -120.22209167480469,
"KL/rejected_KL_mean": -144.30276489257812,
"KL/std": 74.34886932373047,
"epoch": 0.40209424083769635,
"fcm_dpo/beta": 0.005650951061397791,
"fcm_dpo/delta": -0.001796043012291193,
"fcm_dpo/margin": 48.16132354736328,
"fcm_dpo/q_t": 0.43616360425949097,
"grad_norm": 85.8841781616211,
"learning_rate": 3.75e-07,
"logits/chosen": -0.8480498194694519,
"logits/rejected": -0.8291639685630798,
"logps/chosen": -363.121337890625,
"logps/ref_chosen": -266.9799499511719,
"logps/ref_rejected": -260.1697082519531,
"logps/rejected": -404.472412109375,
"loss": 4.7714,
"margin_dpo/margin_mean": 48.16132354736328,
"margin_dpo/margin_std": 84.6034927368164,
"step": 192
},
{
"KL/chosen_KL_mean": -110.70806884765625,
"KL/mean": -140.7467498779297,
"KL/rejected_KL_mean": -170.78546142578125,
"KL/std": 84.00237274169922,
"epoch": 0.4041884816753927,
"fcm_dpo/beta": 0.00559256924316287,
"fcm_dpo/delta": -0.024292614310979843,
"fcm_dpo/margin": 60.07738494873047,
"fcm_dpo/q_t": 0.4226481318473816,
"grad_norm": 79.06672668457031,
"learning_rate": 3.734111735307796e-07,
"logits/chosen": -0.9012806415557861,
"logits/rejected": -0.8710051774978638,
"logps/chosen": -390.9613037109375,
"logps/ref_chosen": -280.25323486328125,
"logps/ref_rejected": -291.0348815917969,
"logps/rejected": -461.8203430175781,
"loss": 4.5952,
"margin_dpo/margin_mean": 60.07738494873047,
"margin_dpo/margin_std": 93.01579284667969,
"step": 193
},
{
"KL/chosen_KL_mean": -119.90608215332031,
"KL/mean": -141.48089599609375,
"KL/rejected_KL_mean": -163.0557098388672,
"KL/std": 80.69340515136719,
"epoch": 0.406282722513089,
"fcm_dpo/beta": 0.0055352929048240185,
"fcm_dpo/delta": 0.009452687576413155,
"fcm_dpo/margin": 43.14963150024414,
"fcm_dpo/q_t": 0.44507163763046265,
"grad_norm": 83.3067398071289,
"learning_rate": 3.7181572889485623e-07,
"logits/chosen": -0.8725247383117676,
"logits/rejected": -0.8608178496360779,
"logps/chosen": -408.0455322265625,
"logps/ref_chosen": -288.13946533203125,
"logps/ref_rejected": -251.31529235839844,
"logps/rejected": -414.3709716796875,
"loss": 4.8879,
"margin_dpo/margin_mean": 43.14963150024414,
"margin_dpo/margin_std": 87.68283081054688,
"step": 194
},
{
"KL/chosen_KL_mean": -126.77201843261719,
"KL/mean": -146.34735107421875,
"KL/rejected_KL_mean": -165.9226837158203,
"KL/std": 80.25843811035156,
"epoch": 0.4083769633507853,
"fcm_dpo/beta": 0.005730912555009127,
"fcm_dpo/delta": 0.06066245585680008,
"fcm_dpo/margin": 39.1506462097168,
"fcm_dpo/q_t": 0.44943714141845703,
"grad_norm": 81.02775573730469,
"learning_rate": 3.7021375165108377e-07,
"logits/chosen": -0.8785435557365417,
"logits/rejected": -0.8827537894248962,
"logps/chosen": -400.77264404296875,
"logps/ref_chosen": -274.0006408691406,
"logps/ref_rejected": -280.22723388671875,
"logps/rejected": -446.14996337890625,
"loss": 4.9734,
"margin_dpo/margin_mean": 39.1506462097168,
"margin_dpo/margin_std": 91.88790893554688,
"step": 195
},
{
"KL/chosen_KL_mean": -105.7139892578125,
"KL/mean": -136.76834106445312,
"KL/rejected_KL_mean": -167.8226776123047,
"KL/std": 83.81902313232422,
"epoch": 0.41047120418848165,
"fcm_dpo/beta": 0.005761865060776472,
"fcm_dpo/delta": -0.02817281149327755,
"fcm_dpo/margin": 62.10868453979492,
"fcm_dpo/q_t": 0.4175757169723511,
"grad_norm": 77.755859375,
"learning_rate": 3.6860532770864005e-07,
"logits/chosen": -0.8623827695846558,
"logits/rejected": -0.8653976321220398,
"logps/chosen": -380.61468505859375,
"logps/ref_chosen": -274.90069580078125,
"logps/ref_rejected": -248.7281951904297,
"logps/rejected": -416.5509033203125,
"loss": 4.5488,
"margin_dpo/margin_mean": 62.10868453979492,
"margin_dpo/margin_std": 96.91848754882812,
"step": 196
},
{
"KL/chosen_KL_mean": -113.64167785644531,
"KL/mean": -146.5870361328125,
"KL/rejected_KL_mean": -179.53237915039062,
"KL/std": 85.77356719970703,
"epoch": 0.41256544502617803,
"fcm_dpo/beta": 0.005556808784604073,
"fcm_dpo/delta": -0.06945441663265228,
"fcm_dpo/margin": 65.89070129394531,
"fcm_dpo/q_t": 0.4156161844730377,
"grad_norm": 101.23210144042969,
"learning_rate": 3.6699054332241985e-07,
"logits/chosen": -0.8872265219688416,
"logits/rejected": -0.8704795837402344,
"logps/chosen": -423.176513671875,
"logps/ref_chosen": -309.5348205566406,
"logps/ref_rejected": -264.3179931640625,
"logps/rejected": -443.85040283203125,
"loss": 4.5075,
"margin_dpo/margin_mean": 65.89070129394531,
"margin_dpo/margin_std": 94.22657012939453,
"step": 197
},
{
"KL/chosen_KL_mean": -101.82292175292969,
"KL/mean": -137.69024658203125,
"KL/rejected_KL_mean": -173.55758666992188,
"KL/std": 86.97761535644531,
"epoch": 0.41465968586387436,
"fcm_dpo/beta": 0.005258793011307716,
"fcm_dpo/delta": -0.04758100211620331,
"fcm_dpo/margin": 71.73465728759766,
"fcm_dpo/q_t": 0.41255441308021545,
"grad_norm": 57.259925842285156,
"learning_rate": 3.653694850884091e-07,
"logits/chosen": -0.8913165330886841,
"logits/rejected": -0.8648770451545715,
"logps/chosen": -402.8363952636719,
"logps/ref_chosen": -301.0134582519531,
"logps/ref_rejected": -292.84185791015625,
"logps/rejected": -466.3994140625,
"loss": 4.4359,
"margin_dpo/margin_mean": 71.73465728759766,
"margin_dpo/margin_std": 96.48065948486328,
"step": 198
},
{
"KL/chosen_KL_mean": -107.54205322265625,
"KL/mean": -136.6196746826172,
"KL/rejected_KL_mean": -165.6973114013672,
"KL/std": 78.0857925415039,
"epoch": 0.4167539267015707,
"fcm_dpo/beta": 0.005284877959638834,
"fcm_dpo/delta": 0.017607517540454865,
"fcm_dpo/margin": 58.15523910522461,
"fcm_dpo/q_t": 0.42847687005996704,
"grad_norm": 71.95082092285156,
"learning_rate": 3.6374223993904124e-07,
"logits/chosen": -0.8611741065979004,
"logits/rejected": -0.817335307598114,
"logps/chosen": -372.1479187011719,
"logps/ref_chosen": -264.6058654785156,
"logps/ref_rejected": -214.9014892578125,
"logps/rejected": -380.5987854003906,
"loss": 4.6595,
"margin_dpo/margin_mean": 58.155242919921875,
"margin_dpo/margin_std": 94.97657775878906,
"step": 199
},
{
"KL/chosen_KL_mean": -113.04508972167969,
"KL/mean": -136.30540466308594,
"KL/rejected_KL_mean": -159.56573486328125,
"KL/std": 83.99603271484375,
"epoch": 0.418848167539267,
"fcm_dpo/beta": 0.00527481734752655,
"fcm_dpo/delta": -0.007252190262079239,
"fcm_dpo/margin": 46.5206184387207,
"fcm_dpo/q_t": 0.4431796073913574,
"grad_norm": 118.91153717041016,
"learning_rate": 3.621088951385353e-07,
"logits/chosen": -0.9174572229385376,
"logits/rejected": -0.8933016061782837,
"logps/chosen": -437.20391845703125,
"logps/ref_chosen": -324.1588134765625,
"logps/ref_rejected": -277.80218505859375,
"logps/rejected": -437.367919921875,
"loss": 4.959,
"margin_dpo/margin_mean": 46.52062225341797,
"margin_dpo/margin_std": 107.42146301269531,
"step": 200
},
{
"epoch": 0.418848167539267,
"eval_KL/chosen_KL_mean": -112.52688598632812,
"eval_KL/mean": -140.7783966064453,
"eval_KL/rejected_KL_mean": -169.0298614501953,
"eval_KL/std": 83.77688598632812,
"eval_fcm_dpo/beta": 0.005265017040073872,
"eval_logits/chosen": -0.8929662704467773,
"eval_logits/rejected": -0.8754425644874573,
"eval_logps/chosen": -400.35369873046875,
"eval_logps/ref_chosen": -287.8268127441406,
"eval_logps/ref_rejected": -266.9300231933594,
"eval_logps/rejected": -435.9598693847656,
"eval_loss": 0.5908177495002747,
"eval_margin_dpo/margin_mean": 56.50297546386719,
"eval_margin_dpo/margin_std": 93.3614501953125,
"eval_runtime": 78.0509,
"eval_samples_per_second": 25.624,
"eval_steps_per_second": 3.203,
"step": 200
},
{
"KL/chosen_KL_mean": -113.03704071044922,
"KL/mean": -142.47430419921875,
"KL/rejected_KL_mean": -171.91156005859375,
"KL/std": 85.64922332763672,
"epoch": 0.42094240837696334,
"fcm_dpo/beta": 0.005312280263751745,
"fcm_dpo/delta": -0.0002721776254475117,
"fcm_dpo/margin": 58.8745231628418,
"fcm_dpo/q_t": 0.4275677502155304,
"grad_norm": 103.92790985107422,
"learning_rate": 3.604695382782159e-07,
"logits/chosen": -0.8811385631561279,
"logits/rejected": -0.8665943145751953,
"logps/chosen": -384.5327453613281,
"logps/ref_chosen": -271.49566650390625,
"logps/ref_rejected": -245.71414184570312,
"logps/rejected": -417.62567138671875,
"loss": 4.6766,
"margin_dpo/margin_mean": 58.8745231628418,
"margin_dpo/margin_std": 98.85687255859375,
"step": 201
},
{
"KL/chosen_KL_mean": -106.796630859375,
"KL/mean": -135.2665557861328,
"KL/rejected_KL_mean": -163.73648071289062,
"KL/std": 82.64311218261719,
"epoch": 0.42303664921465967,
"fcm_dpo/beta": 0.005215301178395748,
"fcm_dpo/delta": -0.007278631906956434,
"fcm_dpo/margin": 56.93985366821289,
"fcm_dpo/q_t": 0.4311002492904663,
"grad_norm": 85.00394439697266,
"learning_rate": 3.588242572718162e-07,
"logits/chosen": -0.8881719708442688,
"logits/rejected": -0.8747313022613525,
"logps/chosen": -378.8945617675781,
"logps/ref_chosen": -272.0979309082031,
"logps/ref_rejected": -235.94805908203125,
"logps/rejected": -399.6845703125,
"loss": 4.7285,
"margin_dpo/margin_mean": 56.93985366821289,
"margin_dpo/margin_std": 100.85719299316406,
"step": 202
},
{
"KL/chosen_KL_mean": -109.50445556640625,
"KL/mean": -132.5556182861328,
"KL/rejected_KL_mean": -155.60678100585938,
"KL/std": 81.00904846191406,
"epoch": 0.42513089005235605,
"fcm_dpo/beta": 0.0052531929686665535,
"fcm_dpo/delta": 0.02188851684331894,
"fcm_dpo/margin": 46.10232925415039,
"fcm_dpo/q_t": 0.4425586462020874,
"grad_norm": 88.16043853759766,
"learning_rate": 3.571731403507635e-07,
"logits/chosen": -0.8671609163284302,
"logits/rejected": -0.8737747669219971,
"logps/chosen": -389.72662353515625,
"logps/ref_chosen": -280.2221374511719,
"logps/ref_rejected": -251.79798889160156,
"logps/rejected": -407.40478515625,
"loss": 4.8388,
"margin_dpo/margin_mean": 46.10232925415039,
"margin_dpo/margin_std": 84.27774047851562,
"step": 203
},
{
"KL/chosen_KL_mean": -97.49296569824219,
"KL/mean": -128.00454711914062,
"KL/rejected_KL_mean": -158.51614379882812,
"KL/std": 79.80975341796875,
"epoch": 0.4272251308900524,
"fcm_dpo/beta": 0.005305150058120489,
"fcm_dpo/delta": 0.006939949467778206,
"fcm_dpo/margin": 61.023170471191406,
"fcm_dpo/q_t": 0.42425647377967834,
"grad_norm": 79.25830841064453,
"learning_rate": 3.5551627605944746e-07,
"logits/chosen": -0.9088394045829773,
"logits/rejected": -0.8813303709030151,
"logps/chosen": -416.28900146484375,
"logps/ref_chosen": -318.7960510253906,
"logps/ref_rejected": -269.69921875,
"logps/rejected": -428.21533203125,
"loss": 4.5763,
"margin_dpo/margin_mean": 61.023170471191406,
"margin_dpo/margin_std": 89.2691879272461,
"step": 204
},
{
"KL/chosen_KL_mean": -97.44485473632812,
"KL/mean": -133.10438537597656,
"KL/rejected_KL_mean": -168.76394653320312,
"KL/std": 87.8086166381836,
"epoch": 0.4293193717277487,
"fcm_dpo/beta": 0.005317580420523882,
"fcm_dpo/delta": -0.030647534877061844,
"fcm_dpo/margin": 71.319091796875,
"fcm_dpo/q_t": 0.4137461483478546,
"grad_norm": 81.98143005371094,
"learning_rate": 3.5385375325047163e-07,
"logits/chosen": -0.8436496257781982,
"logits/rejected": -0.8121804594993591,
"logps/chosen": -381.2069091796875,
"logps/ref_chosen": -283.7620544433594,
"logps/ref_rejected": -297.69439697265625,
"logps/rejected": -466.4584045410156,
"loss": 4.4529,
"margin_dpo/margin_mean": 71.31909942626953,
"margin_dpo/margin_std": 99.79702758789062,
"step": 205
},
{
"KL/chosen_KL_mean": -106.53814697265625,
"KL/mean": -134.1820831298828,
"KL/rejected_KL_mean": -161.8260040283203,
"KL/std": 88.01177978515625,
"epoch": 0.431413612565445,
"fcm_dpo/beta": 0.005157336127012968,
"fcm_dpo/delta": -0.01993716135621071,
"fcm_dpo/margin": 55.28785705566406,
"fcm_dpo/q_t": 0.4342867136001587,
"grad_norm": 79.73020935058594,
"learning_rate": 3.5218566107988867e-07,
"logits/chosen": -0.8805474042892456,
"logits/rejected": -0.9017478823661804,
"logps/chosen": -400.20196533203125,
"logps/ref_chosen": -293.66387939453125,
"logps/ref_rejected": -291.3056640625,
"logps/rejected": -453.13165283203125,
"loss": 4.7925,
"margin_dpo/margin_mean": 55.28785705566406,
"margin_dpo/margin_std": 103.69197082519531,
"step": 206
},
{
"KL/chosen_KL_mean": -99.68655395507812,
"KL/mean": -127.54710388183594,
"KL/rejected_KL_mean": -155.40768432617188,
"KL/std": 94.75768280029297,
"epoch": 0.43350785340314135,
"fcm_dpo/beta": 0.005091050639748573,
"fcm_dpo/delta": -0.01264068391174078,
"fcm_dpo/margin": 55.72111511230469,
"fcm_dpo/q_t": 0.4338788688182831,
"grad_norm": 86.8250961303711,
"learning_rate": 3.505120890024195e-07,
"logits/chosen": -0.8275532722473145,
"logits/rejected": -0.8351794481277466,
"logps/chosen": -370.22161865234375,
"logps/ref_chosen": -270.5350646972656,
"logps/ref_rejected": -278.7747497558594,
"logps/rejected": -434.18243408203125,
"loss": 4.8577,
"margin_dpo/margin_mean": 55.72111511230469,
"margin_dpo/margin_std": 117.74676513671875,
"step": 207
},
{
"KL/chosen_KL_mean": -105.37781524658203,
"KL/mean": -140.57730102539062,
"KL/rejected_KL_mean": -175.77679443359375,
"KL/std": 94.64569854736328,
"epoch": 0.4356020942408377,
"fcm_dpo/beta": 0.005056449212133884,
"fcm_dpo/delta": -0.029296772554516792,
"fcm_dpo/margin": 70.39897155761719,
"fcm_dpo/q_t": 0.4191908836364746,
"grad_norm": 78.56848907470703,
"learning_rate": 3.4883312676665534e-07,
"logits/chosen": -0.8986639380455017,
"logits/rejected": -0.8506540060043335,
"logps/chosen": -384.9605712890625,
"logps/ref_chosen": -279.582763671875,
"logps/ref_rejected": -290.041015625,
"logps/rejected": -465.8177795410156,
"loss": 4.5134,
"margin_dpo/margin_mean": 70.39897155761719,
"margin_dpo/margin_std": 102.34636688232422,
"step": 208
},
{
"KL/chosen_KL_mean": -119.72714233398438,
"KL/mean": -140.69674682617188,
"KL/rejected_KL_mean": -161.66632080078125,
"KL/std": 89.64631652832031,
"epoch": 0.437696335078534,
"fcm_dpo/beta": 0.005014811176806688,
"fcm_dpo/delta": 0.030134355649352074,
"fcm_dpo/margin": 41.939170837402344,
"fcm_dpo/q_t": 0.4505680203437805,
"grad_norm": 74.9662094116211,
"learning_rate": 3.4714886441024573e-07,
"logits/chosen": -0.816275954246521,
"logits/rejected": -0.8116741180419922,
"logps/chosen": -438.59967041015625,
"logps/ref_chosen": -318.8725280761719,
"logps/ref_rejected": -270.64324951171875,
"logps/rejected": -432.3095703125,
"loss": 4.9977,
"margin_dpo/margin_mean": 41.939170837402344,
"margin_dpo/margin_std": 98.76667785644531,
"step": 209
},
{
"KL/chosen_KL_mean": -100.22869873046875,
"KL/mean": -132.45101928710938,
"KL/rejected_KL_mean": -164.67333984375,
"KL/std": 86.8050537109375,
"epoch": 0.4397905759162304,
"fcm_dpo/beta": 0.005045687779784203,
"fcm_dpo/delta": -0.00024997442960739136,
"fcm_dpo/margin": 64.44464111328125,
"fcm_dpo/q_t": 0.4242168068885803,
"grad_norm": 76.91278839111328,
"learning_rate": 3.454593922550693e-07,
"logits/chosen": -0.8455177545547485,
"logits/rejected": -0.8349478244781494,
"logps/chosen": -383.3689880371094,
"logps/ref_chosen": -283.14031982421875,
"logps/ref_rejected": -287.2986755371094,
"logps/rejected": -451.97198486328125,
"loss": 4.605,
"margin_dpo/margin_mean": 64.44464111328125,
"margin_dpo/margin_std": 99.05096435546875,
"step": 210
},
{
"KL/chosen_KL_mean": -90.99891662597656,
"KL/mean": -128.32627868652344,
"KL/rejected_KL_mean": -165.65362548828125,
"KL/std": 85.75665283203125,
"epoch": 0.4418848167539267,
"fcm_dpo/beta": 0.004931907169520855,
"fcm_dpo/delta": -0.0706772655248642,
"fcm_dpo/margin": 74.65471649169922,
"fcm_dpo/q_t": 0.41302767395973206,
"grad_norm": 84.34500885009766,
"learning_rate": 3.4376480090239047e-07,
"logits/chosen": -0.8675280213356018,
"logits/rejected": -0.8453372716903687,
"logps/chosen": -367.42181396484375,
"logps/ref_chosen": -276.4228515625,
"logps/ref_rejected": -252.40603637695312,
"logps/rejected": -418.0596923828125,
"loss": 4.4164,
"margin_dpo/margin_mean": 74.65471649169922,
"margin_dpo/margin_std": 88.93736267089844,
"step": 211
},
{
"KL/chosen_KL_mean": -103.40162658691406,
"KL/mean": -126.14122009277344,
"KL/rejected_KL_mean": -148.88082885742188,
"KL/std": 79.97456359863281,
"epoch": 0.44397905759162304,
"fcm_dpo/beta": 0.0047816019505262375,
"fcm_dpo/delta": 0.0027326405979692936,
"fcm_dpo/margin": 45.479190826416016,
"fcm_dpo/q_t": 0.44804397225379944,
"grad_norm": 68.33330535888672,
"learning_rate": 3.4206518122800055e-07,
"logits/chosen": -0.8594481945037842,
"logits/rejected": -0.8569087982177734,
"logps/chosen": -375.107177734375,
"logps/ref_chosen": -271.7055358886719,
"logps/ref_rejected": -241.18511962890625,
"logps/rejected": -390.06597900390625,
"loss": 4.9313,
"margin_dpo/margin_mean": 45.479190826416016,
"margin_dpo/margin_std": 90.3721694946289,
"step": 212
},
{
"KL/chosen_KL_mean": -110.37431335449219,
"KL/mean": -135.97512817382812,
"KL/rejected_KL_mean": -161.57594299316406,
"KL/std": 97.51154327392578,
"epoch": 0.44607329842931936,
"fcm_dpo/beta": 0.004937951453030109,
"fcm_dpo/delta": 0.034974243491888046,
"fcm_dpo/margin": 51.201629638671875,
"fcm_dpo/q_t": 0.44280922412872314,
"grad_norm": 66.99137115478516,
"learning_rate": 3.403606243773448e-07,
"logits/chosen": -0.8500252962112427,
"logits/rejected": -0.8643661141395569,
"logps/chosen": -412.6719665527344,
"logps/ref_chosen": -302.2976379394531,
"logps/ref_rejected": -303.6202087402344,
"logps/rejected": -465.1961364746094,
"loss": 4.874,
"margin_dpo/margin_mean": 51.201629638671875,
"margin_dpo/margin_std": 106.69091796875,
"step": 213
},
{
"KL/chosen_KL_mean": -113.40092468261719,
"KL/mean": -147.15130615234375,
"KL/rejected_KL_mean": -180.90167236328125,
"KL/std": 88.04362487792969,
"epoch": 0.4481675392670157,
"fcm_dpo/beta": 0.004911818541586399,
"fcm_dpo/delta": -0.007429293356835842,
"fcm_dpo/margin": 67.50076293945312,
"fcm_dpo/q_t": 0.4230693280696869,
"grad_norm": 136.9283905029297,
"learning_rate": 3.3865122176063385e-07,
"logits/chosen": -0.8651580810546875,
"logits/rejected": -0.8647922873497009,
"logps/chosen": -385.5335388183594,
"logps/ref_chosen": -272.13262939453125,
"logps/ref_rejected": -294.82354736328125,
"logps/rejected": -475.7251892089844,
"loss": 4.5422,
"margin_dpo/margin_mean": 67.50076293945312,
"margin_dpo/margin_std": 91.87295532226562,
"step": 214
},
{
"KL/chosen_KL_mean": -122.17478942871094,
"KL/mean": -142.13975524902344,
"KL/rejected_KL_mean": -162.104736328125,
"KL/std": 98.62191009521484,
"epoch": 0.450261780104712,
"fcm_dpo/beta": 0.004899848252534866,
"fcm_dpo/delta": -0.021466845646500587,
"fcm_dpo/margin": 39.92993927001953,
"fcm_dpo/q_t": 0.45550331473350525,
"grad_norm": 86.05668640136719,
"learning_rate": 3.3693706504794243e-07,
"logits/chosen": -0.9035891890525818,
"logits/rejected": -0.8835508823394775,
"logps/chosen": -413.5531005859375,
"logps/ref_chosen": -291.3782958984375,
"logps/ref_rejected": -261.05792236328125,
"logps/rejected": -423.1626281738281,
"loss": 5.0789,
"margin_dpo/margin_mean": 39.92993927001953,
"margin_dpo/margin_std": 105.15361785888672,
"step": 215
},
{
"KL/chosen_KL_mean": -113.71073913574219,
"KL/mean": -144.38978576660156,
"KL/rejected_KL_mean": -175.06881713867188,
"KL/std": 93.9161376953125,
"epoch": 0.4523560209424084,
"fcm_dpo/beta": 0.004838687367737293,
"fcm_dpo/delta": -0.0036501260474324226,
"fcm_dpo/margin": 61.358089447021484,
"fcm_dpo/q_t": 0.4316501319408417,
"grad_norm": 91.55261993408203,
"learning_rate": 3.3521824616429284e-07,
"logits/chosen": -0.9281271696090698,
"logits/rejected": -0.920557975769043,
"logps/chosen": -452.2161560058594,
"logps/ref_chosen": -338.50543212890625,
"logps/ref_rejected": -305.76104736328125,
"logps/rejected": -480.829833984375,
"loss": 4.721,
"margin_dpo/margin_mean": 61.358089447021484,
"margin_dpo/margin_std": 107.93409729003906,
"step": 216
},
{
"KL/chosen_KL_mean": -120.99028015136719,
"KL/mean": -158.77626037597656,
"KL/rejected_KL_mean": -196.56224060058594,
"KL/std": 99.00094604492188,
"epoch": 0.4544502617801047,
"fcm_dpo/beta": 0.004801592789590359,
"fcm_dpo/delta": -0.013928312808275223,
"fcm_dpo/margin": 75.57192993164062,
"fcm_dpo/q_t": 0.4170202612876892,
"grad_norm": 60.47035217285156,
"learning_rate": 3.334948572847253e-07,
"logits/chosen": -0.8282374739646912,
"logits/rejected": -0.7875553369522095,
"logps/chosen": -414.5400695800781,
"logps/ref_chosen": -293.5498046875,
"logps/ref_rejected": -256.7830810546875,
"logps/rejected": -453.34527587890625,
"loss": 4.5201,
"margin_dpo/margin_mean": 75.57192993164062,
"margin_dpo/margin_std": 112.79179382324219,
"step": 217
},
{
"KL/chosen_KL_mean": -120.90603637695312,
"KL/mean": -155.9094696044922,
"KL/rejected_KL_mean": -190.91290283203125,
"KL/std": 98.04553985595703,
"epoch": 0.45654450261780105,
"fcm_dpo/beta": 0.00477956049144268,
"fcm_dpo/delta": 0.015686698257923126,
"fcm_dpo/margin": 70.00685119628906,
"fcm_dpo/q_t": 0.4229409098625183,
"grad_norm": 81.7936019897461,
"learning_rate": 3.317669908293554e-07,
"logits/chosen": -0.8649943470954895,
"logits/rejected": -0.8808245658874512,
"logps/chosen": -441.4853820800781,
"logps/ref_chosen": -320.579345703125,
"logps/ref_rejected": -294.0381164550781,
"logps/rejected": -484.95098876953125,
"loss": 4.576,
"margin_dpo/margin_mean": 70.00685119628906,
"margin_dpo/margin_std": 107.72746276855469,
"step": 218
},
{
"KL/chosen_KL_mean": -120.40277099609375,
"KL/mean": -156.69488525390625,
"KL/rejected_KL_mean": -192.9869842529297,
"KL/std": 93.7581558227539,
"epoch": 0.4586387434554974,
"fcm_dpo/beta": 0.0046853115782141685,
"fcm_dpo/delta": -0.0629052147269249,
"fcm_dpo/margin": 72.58422088623047,
"fcm_dpo/q_t": 0.42137354612350464,
"grad_norm": 91.5604019165039,
"learning_rate": 3.300347394584172e-07,
"logits/chosen": -0.8439445495605469,
"logits/rejected": -0.8702387809753418,
"logps/chosen": -388.8214111328125,
"logps/ref_chosen": -268.4186096191406,
"logps/ref_rejected": -265.7808837890625,
"logps/rejected": -458.7678527832031,
"loss": 4.6062,
"margin_dpo/margin_mean": 72.58422088623047,
"margin_dpo/margin_std": 113.50650024414062,
"step": 219
},
{
"KL/chosen_KL_mean": -122.6368408203125,
"KL/mean": -160.08717346191406,
"KL/rejected_KL_mean": -197.53753662109375,
"KL/std": 94.09228515625,
"epoch": 0.4607329842931937,
"fcm_dpo/beta": 0.00458397064357996,
"fcm_dpo/delta": 0.006707936059683561,
"fcm_dpo/margin": 74.90068817138672,
"fcm_dpo/q_t": 0.4198194146156311,
"grad_norm": 66.64740753173828,
"learning_rate": 3.2829819606729477e-07,
"logits/chosen": -0.9155178070068359,
"logits/rejected": -0.8841468095779419,
"logps/chosen": -435.5233154296875,
"logps/ref_chosen": -312.8864440917969,
"logps/ref_rejected": -259.5191955566406,
"logps/rejected": -457.05670166015625,
"loss": 4.5305,
"margin_dpo/margin_mean": 74.90068817138672,
"margin_dpo/margin_std": 107.72557830810547,
"step": 220
},
{
"KL/chosen_KL_mean": -135.22842407226562,
"KL/mean": -166.27349853515625,
"KL/rejected_KL_mean": -197.3185272216797,
"KL/std": 99.48567962646484,
"epoch": 0.46282722513089003,
"fcm_dpo/beta": 0.004662664607167244,
"fcm_dpo/delta": -0.014790516346693039,
"fcm_dpo/margin": 62.09008026123047,
"fcm_dpo/q_t": 0.43384507298469543,
"grad_norm": 71.38502502441406,
"learning_rate": 3.265574537815398e-07,
"logits/chosen": -0.8199173212051392,
"logits/rejected": -0.8298671245574951,
"logps/chosen": -435.5542907714844,
"logps/ref_chosen": -300.32586669921875,
"logps/ref_rejected": -286.312255859375,
"logps/rejected": -483.6307678222656,
"loss": 4.7667,
"margin_dpo/margin_mean": 62.09008026123047,
"margin_dpo/margin_std": 114.19816589355469,
"step": 221
},
{
"KL/chosen_KL_mean": -132.97164916992188,
"KL/mean": -162.91835021972656,
"KL/rejected_KL_mean": -192.86505126953125,
"KL/std": 89.18901062011719,
"epoch": 0.4649214659685864,
"fcm_dpo/beta": 0.004599052481353283,
"fcm_dpo/delta": 0.0036037182435393333,
"fcm_dpo/margin": 59.89339065551758,
"fcm_dpo/q_t": 0.4353114664554596,
"grad_norm": 79.5625991821289,
"learning_rate": 3.248126059518784e-07,
"logits/chosen": -0.9127927422523499,
"logits/rejected": -0.8806558847427368,
"logps/chosen": -430.0830383300781,
"logps/ref_chosen": -297.1113586425781,
"logps/ref_rejected": -235.53146362304688,
"logps/rejected": -428.396484375,
"loss": 4.7627,
"margin_dpo/margin_mean": 59.89339065551758,
"margin_dpo/margin_std": 105.86309814453125,
"step": 222
},
{
"KL/chosen_KL_mean": -142.52178955078125,
"KL/mean": -175.47720336914062,
"KL/rejected_KL_mean": -208.4326171875,
"KL/std": 96.71148681640625,
"epoch": 0.46701570680628274,
"fcm_dpo/beta": 0.004580706357955933,
"fcm_dpo/delta": -0.0016177373472601175,
"fcm_dpo/margin": 65.91083526611328,
"fcm_dpo/q_t": 0.42940470576286316,
"grad_norm": 90.15703582763672,
"learning_rate": 3.230637461492043e-07,
"logits/chosen": -0.8521276712417603,
"logits/rejected": -0.8184829950332642,
"logps/chosen": -428.9368896484375,
"logps/ref_chosen": -286.41510009765625,
"logps/ref_rejected": -241.1181640625,
"logps/rejected": -449.5508117675781,
"loss": 4.6955,
"margin_dpo/margin_mean": 65.91083526611328,
"margin_dpo/margin_std": 111.98548126220703,
"step": 223
},
{
"KL/chosen_KL_mean": -137.31179809570312,
"KL/mean": -171.8541717529297,
"KL/rejected_KL_mean": -206.39657592773438,
"KL/std": 92.82299041748047,
"epoch": 0.46910994764397906,
"fcm_dpo/beta": 0.0045858751982450485,
"fcm_dpo/delta": -0.019513994455337524,
"fcm_dpo/margin": 69.08476257324219,
"fcm_dpo/q_t": 0.4261767268180847,
"grad_norm": 83.94529724121094,
"learning_rate": 3.213109681595612e-07,
"logits/chosen": -0.8192363977432251,
"logits/rejected": -0.836337149143219,
"logps/chosen": -386.80413818359375,
"logps/ref_chosen": -249.49234008789062,
"logps/ref_rejected": -233.10752868652344,
"logps/rejected": -439.50408935546875,
"loss": 4.6084,
"margin_dpo/margin_mean": 69.08476257324219,
"margin_dpo/margin_std": 102.27616119384766,
"step": 224
},
{
"KL/chosen_KL_mean": -160.03781127929688,
"KL/mean": -191.45758056640625,
"KL/rejected_KL_mean": -222.87734985351562,
"KL/std": 104.55242156982422,
"epoch": 0.4712041884816754,
"fcm_dpo/beta": 0.004542301408946514,
"fcm_dpo/delta": 0.026309635490179062,
"fcm_dpo/margin": 62.83953857421875,
"fcm_dpo/q_t": 0.4349502921104431,
"grad_norm": 82.01798248291016,
"learning_rate": 3.1955436597911315e-07,
"logits/chosen": -0.8548203706741333,
"logits/rejected": -0.8286932706832886,
"logps/chosen": -471.8962097167969,
"logps/ref_chosen": -311.8583679199219,
"logps/ref_rejected": -336.8523864746094,
"logps/rejected": -559.729736328125,
"loss": 4.8119,
"margin_dpo/margin_mean": 62.83953857421875,
"margin_dpo/margin_std": 125.79779052734375,
"step": 225
},
{
"KL/chosen_KL_mean": -149.28944396972656,
"KL/mean": -175.85955810546875,
"KL/rejected_KL_mean": -202.42967224121094,
"KL/std": 98.32502746582031,
"epoch": 0.4732984293193717,
"fcm_dpo/beta": 0.0046951621770858765,
"fcm_dpo/delta": 0.028832029551267624,
"fcm_dpo/margin": 53.14021301269531,
"fcm_dpo/q_t": 0.44293373823165894,
"grad_norm": 128.95950317382812,
"learning_rate": 3.1779403380910425e-07,
"logits/chosen": -0.8814980387687683,
"logits/rejected": -0.8697863221168518,
"logps/chosen": -401.49072265625,
"logps/ref_chosen": -252.20123291015625,
"logps/ref_rejected": -254.41162109375,
"logps/rejected": -456.8412780761719,
"loss": 4.9116,
"margin_dpo/margin_mean": 53.14021301269531,
"margin_dpo/margin_std": 117.36741638183594,
"step": 226
},
{
"KL/chosen_KL_mean": -137.6566162109375,
"KL/mean": -174.5099334716797,
"KL/rejected_KL_mean": -211.36326599121094,
"KL/std": 95.45277404785156,
"epoch": 0.47539267015706804,
"fcm_dpo/beta": 0.004618903622031212,
"fcm_dpo/delta": -0.009899847209453583,
"fcm_dpo/margin": 73.70665740966797,
"fcm_dpo/q_t": 0.42046865820884705,
"grad_norm": 75.73706817626953,
"learning_rate": 3.160300660508064e-07,
"logits/chosen": -0.8298121690750122,
"logits/rejected": -0.8260351419448853,
"logps/chosen": -422.916015625,
"logps/ref_chosen": -285.25946044921875,
"logps/ref_rejected": -261.3220520019531,
"logps/rejected": -472.685302734375,
"loss": 4.5876,
"margin_dpo/margin_mean": 73.70665740966797,
"margin_dpo/margin_std": 115.51934814453125,
"step": 227
},
{
"KL/chosen_KL_mean": -124.57246398925781,
"KL/mean": -160.07022094726562,
"KL/rejected_KL_mean": -195.56796264648438,
"KL/std": 91.25104522705078,
"epoch": 0.4774869109947644,
"fcm_dpo/beta": 0.004695215728133917,
"fcm_dpo/delta": -0.007118703797459602,
"fcm_dpo/margin": 70.99549102783203,
"fcm_dpo/q_t": 0.422940731048584,
"grad_norm": 94.28306579589844,
"learning_rate": 3.1426255730045695e-07,
"logits/chosen": -0.8550602197647095,
"logits/rejected": -0.8216891288757324,
"logps/chosen": -438.3912658691406,
"logps/ref_chosen": -313.81878662109375,
"logps/ref_rejected": -258.07061767578125,
"logps/rejected": -453.6386413574219,
"loss": 4.5961,
"margin_dpo/margin_mean": 70.99549865722656,
"margin_dpo/margin_std": 109.40447235107422,
"step": 228
},
{
"KL/chosen_KL_mean": -136.54019165039062,
"KL/mean": -179.30322265625,
"KL/rejected_KL_mean": -222.06625366210938,
"KL/std": 98.02401733398438,
"epoch": 0.47958115183246075,
"fcm_dpo/beta": 0.0044661094434559345,
"fcm_dpo/delta": -0.0841682031750679,
"fcm_dpo/margin": 85.52606964111328,
"fcm_dpo/q_t": 0.409898966550827,
"grad_norm": 84.5251235961914,
"learning_rate": 3.1249160234418644e-07,
"logits/chosen": -0.8241239190101624,
"logits/rejected": -0.8290253281593323,
"logps/chosen": -428.5109558105469,
"logps/ref_chosen": -291.9707946777344,
"logps/ref_rejected": -263.42059326171875,
"logps/rejected": -485.4868469238281,
"loss": 4.4071,
"margin_dpo/margin_mean": 85.52606964111328,
"margin_dpo/margin_std": 106.6671142578125,
"step": 229
},
{
"KL/chosen_KL_mean": -126.73824310302734,
"KL/mean": -161.21286010742188,
"KL/rejected_KL_mean": -195.6875,
"KL/std": 102.5704574584961,
"epoch": 0.4816753926701571,
"fcm_dpo/beta": 0.00430025951936841,
"fcm_dpo/delta": -0.009406418539583683,
"fcm_dpo/margin": 68.94926452636719,
"fcm_dpo/q_t": 0.43157389760017395,
"grad_norm": 98.18880462646484,
"learning_rate": 3.1071729615293424e-07,
"logits/chosen": -0.8713237643241882,
"logits/rejected": -0.8723396062850952,
"logps/chosen": -359.99835205078125,
"logps/ref_chosen": -233.2601318359375,
"logps/ref_rejected": -238.922119140625,
"logps/rejected": -434.6096496582031,
"loss": 4.7,
"margin_dpo/margin_mean": 68.94926452636719,
"margin_dpo/margin_std": 116.23746490478516,
"step": 230
},
{
"KL/chosen_KL_mean": -126.22959899902344,
"KL/mean": -155.4517822265625,
"KL/rejected_KL_mean": -184.67396545410156,
"KL/std": 88.7551040649414,
"epoch": 0.4837696335078534,
"fcm_dpo/beta": 0.004372628405690193,
"fcm_dpo/delta": 0.038051147013902664,
"fcm_dpo/margin": 58.44437026977539,
"fcm_dpo/q_t": 0.43943536281585693,
"grad_norm": 81.86189270019531,
"learning_rate": 3.0893973387735683e-07,
"logits/chosen": -0.8475738167762756,
"logits/rejected": -0.8374499082565308,
"logps/chosen": -448.384765625,
"logps/ref_chosen": -322.1551818847656,
"logps/ref_rejected": -280.97613525390625,
"logps/rejected": -465.650146484375,
"loss": 4.7883,
"margin_dpo/margin_mean": 58.44437026977539,
"margin_dpo/margin_std": 99.115478515625,
"step": 231
},
{
"KL/chosen_KL_mean": -125.79296875,
"KL/mean": -156.00680541992188,
"KL/rejected_KL_mean": -186.22064208984375,
"KL/std": 96.2881088256836,
"epoch": 0.48586387434554973,
"fcm_dpo/beta": 0.004428422078490257,
"fcm_dpo/delta": 0.006661738269031048,
"fcm_dpo/margin": 60.42768096923828,
"fcm_dpo/q_t": 0.43733125925064087,
"grad_norm": 86.83077239990234,
"learning_rate": 3.071590108427243e-07,
"logits/chosen": -0.8142211437225342,
"logits/rejected": -0.7967170476913452,
"logps/chosen": -397.5367736816406,
"logps/ref_chosen": -271.7437744140625,
"logps/ref_rejected": -249.94981384277344,
"logps/rejected": -436.1704406738281,
"loss": 4.8043,
"margin_dpo/margin_mean": 60.427677154541016,
"margin_dpo/margin_std": 109.37926483154297,
"step": 232
},
{
"KL/chosen_KL_mean": -114.01582336425781,
"KL/mean": -150.8794403076172,
"KL/rejected_KL_mean": -187.74307250976562,
"KL/std": 90.44303131103516,
"epoch": 0.48795811518324606,
"fcm_dpo/beta": 0.004417214542627335,
"fcm_dpo/delta": -0.0340757891535759,
"fcm_dpo/margin": 73.72722625732422,
"fcm_dpo/q_t": 0.4235742688179016,
"grad_norm": 75.02037811279297,
"learning_rate": 3.05375222543809e-07,
"logits/chosen": -0.8514370322227478,
"logits/rejected": -0.8423394560813904,
"logps/chosen": -399.3581848144531,
"logps/ref_chosen": -285.3423156738281,
"logps/ref_rejected": -266.34320068359375,
"logps/rejected": -454.08624267578125,
"loss": 4.5624,
"margin_dpo/margin_mean": 73.72722625732422,
"margin_dpo/margin_std": 98.64462280273438,
"step": 233
},
{
"KL/chosen_KL_mean": -114.3414077758789,
"KL/mean": -146.4625244140625,
"KL/rejected_KL_mean": -178.58364868164062,
"KL/std": 90.29096984863281,
"epoch": 0.4900523560209424,
"fcm_dpo/beta": 0.004416363779455423,
"fcm_dpo/delta": 0.037968117743730545,
"fcm_dpo/margin": 64.24222564697266,
"fcm_dpo/q_t": 0.4340072572231293,
"grad_norm": 104.63549041748047,
"learning_rate": 3.035884646397637e-07,
"logits/chosen": -0.8319493532180786,
"logits/rejected": -0.8197311758995056,
"logps/chosen": -409.2471618652344,
"logps/ref_chosen": -294.9057312011719,
"logps/ref_rejected": -299.37054443359375,
"logps/rejected": -477.9541931152344,
"loss": 4.7523,
"margin_dpo/margin_mean": 64.24222564697266,
"margin_dpo/margin_std": 115.23040008544922,
"step": 234
},
{
"KL/chosen_KL_mean": -115.91151428222656,
"KL/mean": -149.10227966308594,
"KL/rejected_KL_mean": -182.29306030273438,
"KL/std": 96.69937133789062,
"epoch": 0.49214659685863876,
"fcm_dpo/beta": 0.004568690899759531,
"fcm_dpo/delta": 0.02875841036438942,
"fcm_dpo/margin": 66.38154602050781,
"fcm_dpo/q_t": 0.4287912845611572,
"grad_norm": 80.04872131347656,
"learning_rate": 3.017988329489923e-07,
"logits/chosen": -0.8510345816612244,
"logits/rejected": -0.8468698263168335,
"logps/chosen": -405.40911865234375,
"logps/ref_chosen": -289.49755859375,
"logps/ref_rejected": -247.55076599121094,
"logps/rejected": -429.8438415527344,
"loss": 4.6593,
"margin_dpo/margin_mean": 66.38154602050781,
"margin_dpo/margin_std": 106.72305297851562,
"step": 235
},
{
"KL/chosen_KL_mean": -108.14529418945312,
"KL/mean": -142.6337432861328,
"KL/rejected_KL_mean": -177.12220764160156,
"KL/std": 94.26268005371094,
"epoch": 0.4942408376963351,
"fcm_dpo/beta": 0.004645336419343948,
"fcm_dpo/delta": 0.005949061363935471,
"fcm_dpo/margin": 68.97691345214844,
"fcm_dpo/q_t": 0.4257998764514923,
"grad_norm": 63.3080940246582,
"learning_rate": 3.000064234440111e-07,
"logits/chosen": -0.8691312670707703,
"logits/rejected": -0.8709132671356201,
"logps/chosen": -397.0299377441406,
"logps/ref_chosen": -288.8846435546875,
"logps/ref_rejected": -242.0452880859375,
"logps/rejected": -419.16748046875,
"loss": 4.6332,
"margin_dpo/margin_mean": 68.97691345214844,
"margin_dpo/margin_std": 111.18565368652344,
"step": 236
},
{
"KL/chosen_KL_mean": -112.85758972167969,
"KL/mean": -146.0996856689453,
"KL/rejected_KL_mean": -179.341796875,
"KL/std": 80.62757110595703,
"epoch": 0.4963350785340314,
"fcm_dpo/beta": 0.004666031338274479,
"fcm_dpo/delta": 0.014529314823448658,
"fcm_dpo/margin": 66.48421478271484,
"fcm_dpo/q_t": 0.42739030718803406,
"grad_norm": 97.25940704345703,
"learning_rate": 2.9821133224630223e-07,
"logits/chosen": -0.8377714157104492,
"logits/rejected": -0.8173346519470215,
"logps/chosen": -378.3363037109375,
"logps/ref_chosen": -265.47869873046875,
"logps/ref_rejected": -267.9891357421875,
"logps/rejected": -447.3309326171875,
"loss": 4.6172,
"margin_dpo/margin_mean": 66.48422241210938,
"margin_dpo/margin_std": 99.97410583496094,
"step": 237
},
{
"KL/chosen_KL_mean": -119.30398559570312,
"KL/mean": -151.64341735839844,
"KL/rejected_KL_mean": -183.98287963867188,
"KL/std": 90.0185546875,
"epoch": 0.49842931937172774,
"fcm_dpo/beta": 0.004634943325072527,
"fcm_dpo/delta": -0.022365430369973183,
"fcm_dpo/margin": 64.67887878417969,
"fcm_dpo/q_t": 0.43083974719047546,
"grad_norm": 97.27252197265625,
"learning_rate": 2.964136556211588e-07,
"logits/chosen": -0.8281529545783997,
"logits/rejected": -0.7977491617202759,
"logps/chosen": -431.306640625,
"logps/ref_chosen": -312.0026550292969,
"logps/ref_rejected": -270.0257263183594,
"logps/rejected": -454.0085754394531,
"loss": 4.6711,
"margin_dpo/margin_mean": 64.67887878417969,
"margin_dpo/margin_std": 103.0881576538086,
"step": 238
},
{
"KL/chosen_KL_mean": -106.01922607421875,
"KL/mean": -135.8646240234375,
"KL/rejected_KL_mean": -165.71002197265625,
"KL/std": 91.91876220703125,
"epoch": 0.5005235602094241,
"fcm_dpo/beta": 0.004592178855091333,
"fcm_dpo/delta": 0.017465800046920776,
"fcm_dpo/margin": 59.690773010253906,
"fcm_dpo/q_t": 0.43653061985969543,
"grad_norm": 95.7003402709961,
"learning_rate": 2.946134899725226e-07,
"logits/chosen": -0.837583065032959,
"logits/rejected": -0.8786738514900208,
"logps/chosen": -373.18646240234375,
"logps/ref_chosen": -267.167236328125,
"logps/ref_rejected": -275.99468994140625,
"logps/rejected": -441.7046813964844,
"loss": 4.8234,
"margin_dpo/margin_mean": 59.690773010253906,
"margin_dpo/margin_std": 118.9461441040039,
"step": 239
},
{
"KL/chosen_KL_mean": -104.64605712890625,
"KL/mean": -141.4502410888672,
"KL/rejected_KL_mean": -178.25445556640625,
"KL/std": 90.54521179199219,
"epoch": 0.5026178010471204,
"fcm_dpo/beta": 0.004639900289475918,
"fcm_dpo/delta": 0.008587292395532131,
"fcm_dpo/margin": 73.60839080810547,
"fcm_dpo/q_t": 0.42163485288619995,
"grad_norm": 61.846168518066406,
"learning_rate": 2.9281093183781403e-07,
"logits/chosen": -0.8871752023696899,
"logits/rejected": -0.8817972540855408,
"logps/chosen": -390.62567138671875,
"logps/ref_chosen": -285.9796142578125,
"logps/ref_rejected": -256.8258056640625,
"logps/rejected": -435.0802307128906,
"loss": 4.5299,
"margin_dpo/margin_mean": 73.60839080810547,
"margin_dpo/margin_std": 106.27149963378906,
"step": 240
},
{
"KL/chosen_KL_mean": -121.47247314453125,
"KL/mean": -149.4210968017578,
"KL/rejected_KL_mean": -177.36972045898438,
"KL/std": 95.52164459228516,
"epoch": 0.5047120418848168,
"fcm_dpo/beta": 0.004665836691856384,
"fcm_dpo/delta": -0.0019118525087833405,
"fcm_dpo/margin": 55.897247314453125,
"fcm_dpo/q_t": 0.44022229313850403,
"grad_norm": 55.96895217895508,
"learning_rate": 2.910060778827554e-07,
"logits/chosen": -0.7976040840148926,
"logits/rejected": -0.773979127407074,
"logps/chosen": -382.9893493652344,
"logps/ref_chosen": -261.516845703125,
"logps/ref_rejected": -250.2250518798828,
"logps/rejected": -427.59478759765625,
"loss": 4.8268,
"margin_dpo/margin_mean": 55.897247314453125,
"margin_dpo/margin_std": 105.24579620361328,
"step": 241
},
{
"KL/chosen_KL_mean": -109.387451171875,
"KL/mean": -138.6495361328125,
"KL/rejected_KL_mean": -167.91165161132812,
"KL/std": 94.68675994873047,
"epoch": 0.506806282722513,
"fcm_dpo/beta": 0.004686980042606592,
"fcm_dpo/delta": 0.028351204469799995,
"fcm_dpo/margin": 58.5241813659668,
"fcm_dpo/q_t": 0.4375962018966675,
"grad_norm": 69.91267395019531,
"learning_rate": 2.891990248961871e-07,
"logits/chosen": -0.8706403374671936,
"logits/rejected": -0.8489304184913635,
"logps/chosen": -379.90142822265625,
"logps/ref_chosen": -270.51397705078125,
"logps/ref_rejected": -244.8560791015625,
"logps/rejected": -412.7677001953125,
"loss": 4.7544,
"margin_dpo/margin_mean": 58.52418518066406,
"margin_dpo/margin_std": 103.48672485351562,
"step": 242
},
{
"KL/chosen_KL_mean": -106.11418914794922,
"KL/mean": -139.8125762939453,
"KL/rejected_KL_mean": -173.510986328125,
"KL/std": 88.66896057128906,
"epoch": 0.5089005235602094,
"fcm_dpo/beta": 0.004790696781128645,
"fcm_dpo/delta": -0.01932116597890854,
"fcm_dpo/margin": 67.39679718017578,
"fcm_dpo/q_t": 0.4278520345687866,
"grad_norm": 73.60661315917969,
"learning_rate": 2.873898697848762e-07,
"logits/chosen": -0.8736940622329712,
"logits/rejected": -0.8598724603652954,
"logps/chosen": -430.7962646484375,
"logps/ref_chosen": -324.68206787109375,
"logps/ref_rejected": -307.1111755371094,
"logps/rejected": -480.62213134765625,
"loss": 4.6179,
"margin_dpo/margin_mean": 67.39679718017578,
"margin_dpo/margin_std": 106.52936553955078,
"step": 243
},
{
"KL/chosen_KL_mean": -112.37324523925781,
"KL/mean": -146.05435180664062,
"KL/rejected_KL_mean": -179.73544311523438,
"KL/std": 87.45367431640625,
"epoch": 0.5109947643979058,
"fcm_dpo/beta": 0.0046486621722579,
"fcm_dpo/delta": -0.014583555981516838,
"fcm_dpo/margin": 67.36219024658203,
"fcm_dpo/q_t": 0.42639923095703125,
"grad_norm": 69.93437957763672,
"learning_rate": 2.8557870956832133e-07,
"logits/chosen": -0.861052930355072,
"logits/rejected": -0.8165071606636047,
"logps/chosen": -431.35247802734375,
"logps/ref_chosen": -318.979248046875,
"logps/ref_rejected": -269.67572021484375,
"logps/rejected": -449.4111633300781,
"loss": 4.6283,
"margin_dpo/margin_mean": 67.36219024658203,
"margin_dpo/margin_std": 100.07360076904297,
"step": 244
},
{
"KL/chosen_KL_mean": -108.40132141113281,
"KL/mean": -140.7490997314453,
"KL/rejected_KL_mean": -173.09686279296875,
"KL/std": 80.6366958618164,
"epoch": 0.5130890052356021,
"fcm_dpo/beta": 0.004631055053323507,
"fcm_dpo/delta": -0.014426006004214287,
"fcm_dpo/margin": 64.69554138183594,
"fcm_dpo/q_t": 0.42882615327835083,
"grad_norm": 97.62570190429688,
"learning_rate": 2.837656413735479e-07,
"logits/chosen": -0.880599856376648,
"logits/rejected": -0.8871563076972961,
"logps/chosen": -403.29937744140625,
"logps/ref_chosen": -294.8980712890625,
"logps/ref_rejected": -239.8111114501953,
"logps/rejected": -412.907958984375,
"loss": 4.6392,
"margin_dpo/margin_mean": 64.6955337524414,
"margin_dpo/margin_std": 92.92266845703125,
"step": 245
},
{
"KL/chosen_KL_mean": -128.1768341064453,
"KL/mean": -153.97454833984375,
"KL/rejected_KL_mean": -179.77224731445312,
"KL/std": 83.72673034667969,
"epoch": 0.5151832460732985,
"fcm_dpo/beta": 0.004627012647688389,
"fcm_dpo/delta": 0.025665562599897385,
"fcm_dpo/margin": 51.595394134521484,
"fcm_dpo/q_t": 0.44522804021835327,
"grad_norm": 74.21184539794922,
"learning_rate": 2.8195076242990116e-07,
"logits/chosen": -0.8659073114395142,
"logits/rejected": -0.8697488307952881,
"logps/chosen": -408.86224365234375,
"logps/ref_chosen": -280.6854248046875,
"logps/ref_rejected": -253.65382385253906,
"logps/rejected": -433.426025390625,
"loss": 4.8917,
"margin_dpo/margin_mean": 51.595394134521484,
"margin_dpo/margin_std": 106.63719177246094,
"step": 246
},
{
"KL/chosen_KL_mean": -124.93351745605469,
"KL/mean": -155.44081115722656,
"KL/rejected_KL_mean": -185.94808959960938,
"KL/std": 90.46797180175781,
"epoch": 0.5172774869109947,
"fcm_dpo/beta": 0.00462773721665144,
"fcm_dpo/delta": -0.03191925212740898,
"fcm_dpo/margin": 61.01456832885742,
"fcm_dpo/q_t": 0.4344664216041565,
"grad_norm": 82.88591766357422,
"learning_rate": 2.801341700638307e-07,
"logits/chosen": -0.8671166896820068,
"logits/rejected": -0.8641787171363831,
"logps/chosen": -406.0426025390625,
"logps/ref_chosen": -281.1091003417969,
"logps/ref_rejected": -260.3700866699219,
"logps/rejected": -446.318115234375,
"loss": 4.736,
"margin_dpo/margin_mean": 61.01456832885742,
"margin_dpo/margin_std": 101.25210571289062,
"step": 247
},
{
"KL/chosen_KL_mean": -126.19422149658203,
"KL/mean": -153.88031005859375,
"KL/rejected_KL_mean": -181.56640625,
"KL/std": 91.33638763427734,
"epoch": 0.5193717277486911,
"fcm_dpo/beta": 0.004508162848651409,
"fcm_dpo/delta": -0.018666669726371765,
"fcm_dpo/margin": 55.37216567993164,
"fcm_dpo/q_t": 0.441021203994751,
"grad_norm": 84.99152374267578,
"learning_rate": 2.7831596169367227e-07,
"logits/chosen": -0.8335204124450684,
"logits/rejected": -0.8391017913818359,
"logps/chosen": -396.5125732421875,
"logps/ref_chosen": -270.318359375,
"logps/ref_rejected": -233.46778869628906,
"logps/rejected": -415.0342102050781,
"loss": 4.8294,
"margin_dpo/margin_mean": 55.37216567993164,
"margin_dpo/margin_std": 100.47364807128906,
"step": 248
},
{
"KL/chosen_KL_mean": -144.4632110595703,
"KL/mean": -169.6751251220703,
"KL/rejected_KL_mean": -194.88705444335938,
"KL/std": 96.40766143798828,
"epoch": 0.5214659685863874,
"fcm_dpo/beta": 0.0045876894146203995,
"fcm_dpo/delta": 0.0523696206510067,
"fcm_dpo/margin": 50.42381286621094,
"fcm_dpo/q_t": 0.4456580579280853,
"grad_norm": 70.08956146240234,
"learning_rate": 2.7649623482442274e-07,
"logits/chosen": -0.8723926544189453,
"logits/rejected": -0.8454214930534363,
"logps/chosen": -420.2720642089844,
"logps/ref_chosen": -275.8088684082031,
"logps/ref_rejected": -243.45138549804688,
"logps/rejected": -438.33843994140625,
"loss": 4.9733,
"margin_dpo/margin_mean": 50.42381286621094,
"margin_dpo/margin_std": 120.47492980957031,
"step": 249
},
{
"KL/chosen_KL_mean": -147.3335418701172,
"KL/mean": -185.5907440185547,
"KL/rejected_KL_mean": -223.84793090820312,
"KL/std": 113.06914520263672,
"epoch": 0.5235602094240838,
"fcm_dpo/beta": 0.004560520872473717,
"fcm_dpo/delta": -0.05923834070563316,
"fcm_dpo/margin": 76.51438903808594,
"fcm_dpo/q_t": 0.4200321435928345,
"grad_norm": 77.23839569091797,
"learning_rate": 2.7467508704251135e-07,
"logits/chosen": -0.8727688193321228,
"logits/rejected": -0.8649075031280518,
"logps/chosen": -439.8280944824219,
"logps/ref_chosen": -292.4945373535156,
"logps/ref_rejected": -284.2869567871094,
"logps/rejected": -508.13482666015625,
"loss": 4.6123,
"margin_dpo/margin_mean": 76.51438903808594,
"margin_dpo/margin_std": 127.33292388916016,
"step": 250
},
{
"KL/chosen_KL_mean": -137.64854431152344,
"KL/mean": -172.20977783203125,
"KL/rejected_KL_mean": -206.77101135253906,
"KL/std": 109.15238189697266,
"epoch": 0.5256544502617801,
"fcm_dpo/beta": 0.004414653405547142,
"fcm_dpo/delta": -0.04583483934402466,
"fcm_dpo/margin": 69.12245178222656,
"fcm_dpo/q_t": 0.433123379945755,
"grad_norm": 99.51129913330078,
"learning_rate": 2.7285261601056697e-07,
"logits/chosen": -0.8708928823471069,
"logits/rejected": -0.8535991311073303,
"logps/chosen": -419.3851318359375,
"logps/ref_chosen": -281.736572265625,
"logps/ref_rejected": -255.9419708251953,
"logps/rejected": -462.7129821777344,
"loss": 4.7186,
"margin_dpo/margin_mean": 69.12245178222656,
"margin_dpo/margin_std": 121.37847900390625,
"step": 251
},
{
"KL/chosen_KL_mean": -150.81507873535156,
"KL/mean": -186.76541137695312,
"KL/rejected_KL_mean": -222.71575927734375,
"KL/std": 103.89976501464844,
"epoch": 0.5277486910994764,
"fcm_dpo/beta": 0.004231244325637817,
"fcm_dpo/delta": -0.02651745080947876,
"fcm_dpo/margin": 71.90065002441406,
"fcm_dpo/q_t": 0.4281459152698517,
"grad_norm": 55.069679260253906,
"learning_rate": 2.7102891946217994e-07,
"logits/chosen": -0.9138679504394531,
"logits/rejected": -0.8856754302978516,
"logps/chosen": -446.7825622558594,
"logps/ref_chosen": -295.9674072265625,
"logps/ref_rejected": -280.111572265625,
"logps/rejected": -502.82733154296875,
"loss": 4.6809,
"margin_dpo/margin_mean": 71.90065002441406,
"margin_dpo/margin_std": 118.01815032958984,
"step": 252
},
{
"KL/chosen_KL_mean": -162.62567138671875,
"KL/mean": -196.12326049804688,
"KL/rejected_KL_mean": -229.620849609375,
"KL/std": 113.87892150878906,
"epoch": 0.5298429319371728,
"fcm_dpo/beta": 0.004193556495010853,
"fcm_dpo/delta": -0.004063830710947514,
"fcm_dpo/margin": 66.99517822265625,
"fcm_dpo/q_t": 0.43573546409606934,
"grad_norm": 64.6878890991211,
"learning_rate": 2.692040951966617e-07,
"logits/chosen": -0.8893136978149414,
"logits/rejected": -0.8789063692092896,
"logps/chosen": -439.69793701171875,
"logps/ref_chosen": -277.072265625,
"logps/ref_rejected": -247.31643676757812,
"logps/rejected": -476.9372863769531,
"loss": 4.8051,
"margin_dpo/margin_mean": 66.99517822265625,
"margin_dpo/margin_std": 127.16492462158203,
"step": 253
},
{
"KL/chosen_KL_mean": -155.98829650878906,
"KL/mean": -191.45701599121094,
"KL/rejected_KL_mean": -226.92575073242188,
"KL/std": 111.76846313476562,
"epoch": 0.5319371727748691,
"fcm_dpo/beta": 0.00419703358784318,
"fcm_dpo/delta": -0.028339367359876633,
"fcm_dpo/margin": 70.93742370605469,
"fcm_dpo/q_t": 0.43127933144569397,
"grad_norm": 61.05302047729492,
"learning_rate": 2.6737824107379947e-07,
"logits/chosen": -0.8169664740562439,
"logits/rejected": -0.7953270673751831,
"logps/chosen": -425.9361572265625,
"logps/ref_chosen": -269.9478454589844,
"logps/ref_rejected": -249.45005798339844,
"logps/rejected": -476.37579345703125,
"loss": 4.701,
"margin_dpo/margin_mean": 70.93742370605469,
"margin_dpo/margin_std": 119.53722381591797,
"step": 254
},
{
"KL/chosen_KL_mean": -152.83241271972656,
"KL/mean": -194.11044311523438,
"KL/rejected_KL_mean": -235.38848876953125,
"KL/std": 110.81584930419922,
"epoch": 0.5340314136125655,
"fcm_dpo/beta": 0.004109766334295273,
"fcm_dpo/delta": -0.013929899781942368,
"fcm_dpo/margin": 82.55609130859375,
"fcm_dpo/q_t": 0.4215297996997833,
"grad_norm": 83.60579681396484,
"learning_rate": 2.655514550086086e-07,
"logits/chosen": -0.842009961605072,
"logits/rejected": -0.8048016428947449,
"logps/chosen": -459.4877014160156,
"logps/ref_chosen": -306.6552734375,
"logps/ref_rejected": -254.47528076171875,
"logps/rejected": -489.8637390136719,
"loss": 4.6142,
"margin_dpo/margin_mean": 82.55609893798828,
"margin_dpo/margin_std": 136.48744201660156,
"step": 255
},
{
"KL/chosen_KL_mean": -155.2074432373047,
"KL/mean": -192.99703979492188,
"KL/rejected_KL_mean": -230.78665161132812,
"KL/std": 112.66366577148438,
"epoch": 0.5361256544502618,
"fcm_dpo/beta": 0.003994643688201904,
"fcm_dpo/delta": -0.06211162358522415,
"fcm_dpo/margin": 75.57919311523438,
"fcm_dpo/q_t": 0.42736783623695374,
"grad_norm": 89.4056167602539,
"learning_rate": 2.6372383496608186e-07,
"logits/chosen": -0.8736791014671326,
"logits/rejected": -0.8580365180969238,
"logps/chosen": -478.9256591796875,
"logps/ref_chosen": -323.7181701660156,
"logps/ref_rejected": -254.1871337890625,
"logps/rejected": -484.9737854003906,
"loss": 4.7341,
"margin_dpo/margin_mean": 75.57919311523438,
"margin_dpo/margin_std": 127.94612884521484,
"step": 256
},
{
"KL/chosen_KL_mean": -156.5220184326172,
"KL/mean": -196.63253784179688,
"KL/rejected_KL_mean": -236.7430419921875,
"KL/std": 107.44888305664062,
"epoch": 0.5382198952879581,
"fcm_dpo/beta": 0.003933961037546396,
"fcm_dpo/delta": 0.035381704568862915,
"fcm_dpo/margin": 80.22103881835938,
"fcm_dpo/q_t": 0.4262939393520355,
"grad_norm": 69.32088470458984,
"learning_rate": 2.618954789559356e-07,
"logits/chosen": -0.857764720916748,
"logits/rejected": -0.8485167622566223,
"logps/chosen": -423.73406982421875,
"logps/ref_chosen": -267.21209716796875,
"logps/ref_rejected": -249.12579345703125,
"logps/rejected": -485.86883544921875,
"loss": 4.6149,
"margin_dpo/margin_mean": 80.22103881835938,
"margin_dpo/margin_std": 123.36433410644531,
"step": 257
},
{
"KL/chosen_KL_mean": -161.85557556152344,
"KL/mean": -204.7264404296875,
"KL/rejected_KL_mean": -247.5973358154297,
"KL/std": 105.87276458740234,
"epoch": 0.5403141361256545,
"fcm_dpo/beta": 0.0038711142260581255,
"fcm_dpo/delta": -0.05333390459418297,
"fcm_dpo/margin": 85.74173736572266,
"fcm_dpo/q_t": 0.4211893677711487,
"grad_norm": 59.2056770324707,
"learning_rate": 2.600664850273538e-07,
"logits/chosen": -0.8781223297119141,
"logits/rejected": -0.8470626473426819,
"logps/chosen": -439.53826904296875,
"logps/ref_chosen": -277.6827392578125,
"logps/ref_rejected": -250.73385620117188,
"logps/rejected": -498.3311767578125,
"loss": 4.5362,
"margin_dpo/margin_mean": 85.74173736572266,
"margin_dpo/margin_std": 114.76727294921875,
"step": 258
},
{
"KL/chosen_KL_mean": -154.62147521972656,
"KL/mean": -193.3902587890625,
"KL/rejected_KL_mean": -232.1590576171875,
"KL/std": 112.23604583740234,
"epoch": 0.5424083769633508,
"fcm_dpo/beta": 0.0038862007204443216,
"fcm_dpo/delta": 0.00880090519785881,
"fcm_dpo/margin": 77.53759765625,
"fcm_dpo/q_t": 0.4306592345237732,
"grad_norm": 74.10926055908203,
"learning_rate": 2.582369512637302e-07,
"logits/chosen": -0.8891596794128418,
"logits/rejected": -0.8823885321617126,
"logps/chosen": -449.2314453125,
"logps/ref_chosen": -294.6099853515625,
"logps/ref_rejected": -272.2725830078125,
"logps/rejected": -504.431640625,
"loss": 4.6745,
"margin_dpo/margin_mean": 77.53760528564453,
"margin_dpo/margin_std": 126.21345520019531,
"step": 259
},
{
"KL/chosen_KL_mean": -165.0343780517578,
"KL/mean": -180.45216369628906,
"KL/rejected_KL_mean": -195.8699951171875,
"KL/std": 95.3846206665039,
"epoch": 0.5445026178010471,
"fcm_dpo/beta": 0.003850242355838418,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 30.835628509521484,
"fcm_dpo/q_t": 0.47213658690452576,
"grad_norm": 68.87381744384766,
"learning_rate": 2.5640697577740815e-07,
"logits/chosen": -0.8860092163085938,
"logits/rejected": -0.8798715472221375,
"logps/chosen": -455.8914794921875,
"logps/ref_chosen": -290.85711669921875,
"logps/ref_rejected": -277.5970153808594,
"logps/rejected": -473.4670104980469,
"loss": 5.273,
"margin_dpo/margin_mean": 30.83563232421875,
"margin_dpo/margin_std": 111.492431640625,
"step": 260
},
{
"KL/chosen_KL_mean": -178.47940063476562,
"KL/mean": -209.2558135986328,
"KL/rejected_KL_mean": -240.03221130371094,
"KL/std": 104.58702087402344,
"epoch": 0.5465968586387434,
"fcm_dpo/beta": 0.0038688078057020903,
"fcm_dpo/delta": 0.007620506454259157,
"fcm_dpo/margin": 61.552818298339844,
"fcm_dpo/q_t": 0.44480204582214355,
"grad_norm": 58.4471549987793,
"learning_rate": 2.5457665670441937e-07,
"logits/chosen": -0.7781463861465454,
"logits/rejected": -0.7877897620201111,
"logps/chosen": -429.6116027832031,
"logps/ref_chosen": -251.13223266601562,
"logps/ref_rejected": -244.76016235351562,
"logps/rejected": -484.7923583984375,
"loss": 4.9127,
"margin_dpo/margin_mean": 61.552818298339844,
"margin_dpo/margin_std": 131.39944458007812,
"step": 261
},
{
"KL/chosen_KL_mean": -166.16873168945312,
"KL/mean": -208.1382598876953,
"KL/rejected_KL_mean": -250.1077880859375,
"KL/std": 105.5480728149414,
"epoch": 0.5486910994764398,
"fcm_dpo/beta": 0.003931888844817877,
"fcm_dpo/delta": -0.0042154379189014435,
"fcm_dpo/margin": 83.93904113769531,
"fcm_dpo/q_t": 0.42236289381980896,
"grad_norm": 64.91275024414062,
"learning_rate": 2.527460921992209e-07,
"logits/chosen": -0.8072105646133423,
"logits/rejected": -0.7947403788566589,
"logps/chosen": -465.8905029296875,
"logps/ref_chosen": -299.7217712402344,
"logps/ref_rejected": -277.0969543457031,
"logps/rejected": -527.2047119140625,
"loss": 4.55,
"margin_dpo/margin_mean": 83.93904113769531,
"margin_dpo/margin_std": 116.44705963134766,
"step": 262
},
{
"KL/chosen_KL_mean": -173.62734985351562,
"KL/mean": -207.27186584472656,
"KL/rejected_KL_mean": -240.91635131835938,
"KL/std": 108.08099365234375,
"epoch": 0.5507853403141362,
"fcm_dpo/beta": 0.0038349460810422897,
"fcm_dpo/delta": -0.013341857120394707,
"fcm_dpo/margin": 67.28901672363281,
"fcm_dpo/q_t": 0.43885383009910583,
"grad_norm": 56.40167999267578,
"learning_rate": 2.509153804294318e-07,
"logits/chosen": -0.813066840171814,
"logits/rejected": -0.7947369813919067,
"logps/chosen": -453.5799560546875,
"logps/ref_chosen": -279.95257568359375,
"logps/ref_rejected": -256.5327453613281,
"logps/rejected": -497.4490661621094,
"loss": 4.8042,
"margin_dpo/margin_mean": 67.28901672363281,
"margin_dpo/margin_std": 117.43939208984375,
"step": 263
},
{
"KL/chosen_KL_mean": -166.5303955078125,
"KL/mean": -209.10252380371094,
"KL/rejected_KL_mean": -251.6746826171875,
"KL/std": 108.8144302368164,
"epoch": 0.5528795811518324,
"fcm_dpo/beta": 0.0037783209700137377,
"fcm_dpo/delta": -0.005399790126830339,
"fcm_dpo/margin": 85.1442642211914,
"fcm_dpo/q_t": 0.42472028732299805,
"grad_norm": 69.8880386352539,
"learning_rate": 2.4908461957056825e-07,
"logits/chosen": -0.8240674734115601,
"logits/rejected": -0.8171445727348328,
"logps/chosen": -427.06549072265625,
"logps/ref_chosen": -260.53509521484375,
"logps/ref_rejected": -255.53799438476562,
"logps/rejected": -507.212646484375,
"loss": 4.5679,
"margin_dpo/margin_mean": 85.1442642211914,
"margin_dpo/margin_std": 118.58462524414062,
"step": 264
},
{
"KL/chosen_KL_mean": -170.61724853515625,
"KL/mean": -213.73841857910156,
"KL/rejected_KL_mean": -256.859619140625,
"KL/std": 115.99640655517578,
"epoch": 0.5549738219895288,
"fcm_dpo/beta": 0.003773211035877466,
"fcm_dpo/delta": -0.012947505339980125,
"fcm_dpo/margin": 86.2424087524414,
"fcm_dpo/q_t": 0.424190491437912,
"grad_norm": 53.74254608154297,
"learning_rate": 2.4725390780077905e-07,
"logits/chosen": -0.8812572956085205,
"logits/rejected": -0.8822892308235168,
"logps/chosen": -454.33026123046875,
"logps/ref_chosen": -283.7130432128906,
"logps/ref_rejected": -270.3209533691406,
"logps/rejected": -527.1806030273438,
"loss": 4.5936,
"margin_dpo/margin_mean": 86.2424087524414,
"margin_dpo/margin_std": 130.76058959960938,
"step": 265
},
{
"KL/chosen_KL_mean": -154.7050323486328,
"KL/mean": -199.395751953125,
"KL/rejected_KL_mean": -244.08648681640625,
"KL/std": 108.90046691894531,
"epoch": 0.5570680628272251,
"fcm_dpo/beta": 0.0037543033249676228,
"fcm_dpo/delta": -0.023054908961057663,
"fcm_dpo/margin": 89.3814468383789,
"fcm_dpo/q_t": 0.4214840829372406,
"grad_norm": 75.81797790527344,
"learning_rate": 2.454233432955807e-07,
"logits/chosen": -0.9104347229003906,
"logits/rejected": -0.8814761638641357,
"logps/chosen": -432.8043212890625,
"logps/ref_chosen": -278.09930419921875,
"logps/ref_rejected": -260.6734619140625,
"logps/rejected": -504.75994873046875,
"loss": 4.5175,
"margin_dpo/margin_mean": 89.3814468383789,
"margin_dpo/margin_std": 117.37722778320312,
"step": 266
},
{
"KL/chosen_KL_mean": -182.2346649169922,
"KL/mean": -214.27894592285156,
"KL/rejected_KL_mean": -246.32321166992188,
"KL/std": 112.4916763305664,
"epoch": 0.5591623036649215,
"fcm_dpo/beta": 0.0037524704821407795,
"fcm_dpo/delta": 0.04089874029159546,
"fcm_dpo/margin": 64.08850860595703,
"fcm_dpo/q_t": 0.44417688250541687,
"grad_norm": 61.133914947509766,
"learning_rate": 2.435930242225919e-07,
"logits/chosen": -0.8439475893974304,
"logits/rejected": -0.8501834869384766,
"logps/chosen": -462.5679016113281,
"logps/ref_chosen": -280.33319091796875,
"logps/ref_rejected": -247.78099060058594,
"logps/rejected": -494.1041564941406,
"loss": 4.8717,
"margin_dpo/margin_mean": 64.08850860595703,
"margin_dpo/margin_std": 123.6415786743164,
"step": 267
},
{
"KL/chosen_KL_mean": -163.2926025390625,
"KL/mean": -205.69686889648438,
"KL/rejected_KL_mean": -248.1011199951172,
"KL/std": 111.8238296508789,
"epoch": 0.5612565445026177,
"fcm_dpo/beta": 0.003843301208689809,
"fcm_dpo/delta": 0.0008840234950184822,
"fcm_dpo/margin": 84.80854034423828,
"fcm_dpo/q_t": 0.42401742935180664,
"grad_norm": 68.5013656616211,
"learning_rate": 2.4176304873626984e-07,
"logits/chosen": -0.8200739026069641,
"logits/rejected": -0.7995492815971375,
"logps/chosen": -467.4713134765625,
"logps/ref_chosen": -304.1787109375,
"logps/ref_rejected": -272.80316162109375,
"logps/rejected": -520.904296875,
"loss": 4.5785,
"margin_dpo/margin_mean": 84.80854034423828,
"margin_dpo/margin_std": 123.63158416748047,
"step": 268
},
{
"KL/chosen_KL_mean": -163.21693420410156,
"KL/mean": -194.07969665527344,
"KL/rejected_KL_mean": -224.94247436523438,
"KL/std": 99.83782196044922,
"epoch": 0.5633507853403141,
"fcm_dpo/beta": 0.003911444451659918,
"fcm_dpo/delta": 0.05910227820277214,
"fcm_dpo/margin": 61.725563049316406,
"fcm_dpo/q_t": 0.4430586099624634,
"grad_norm": 87.58391571044922,
"learning_rate": 2.399335149726463e-07,
"logits/chosen": -0.8339531421661377,
"logits/rejected": -0.8274367451667786,
"logps/chosen": -413.0619812011719,
"logps/ref_chosen": -249.84512329101562,
"logps/ref_rejected": -223.37356567382812,
"logps/rejected": -448.3160400390625,
"loss": 4.867,
"margin_dpo/margin_mean": 61.725563049316406,
"margin_dpo/margin_std": 122.53068542480469,
"step": 269
},
{
"KL/chosen_KL_mean": -168.89588928222656,
"KL/mean": -206.75592041015625,
"KL/rejected_KL_mean": -244.61593627929688,
"KL/std": 114.02747344970703,
"epoch": 0.5654450261780105,
"fcm_dpo/beta": 0.004022765904664993,
"fcm_dpo/delta": 0.02421419322490692,
"fcm_dpo/margin": 75.72004699707031,
"fcm_dpo/q_t": 0.43018603324890137,
"grad_norm": 64.52881622314453,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": -0.9270073175430298,
"logits/rejected": -0.9333524703979492,
"logps/chosen": -487.458251953125,
"logps/ref_chosen": -318.5623779296875,
"logps/ref_rejected": -281.1880798339844,
"logps/rejected": -525.8040161132812,
"loss": 4.7208,
"margin_dpo/margin_mean": 75.72004699707031,
"margin_dpo/margin_std": 138.9295654296875,
"step": 270
},
{
"KL/chosen_KL_mean": -167.02001953125,
"KL/mean": -201.0181884765625,
"KL/rejected_KL_mean": -235.01632690429688,
"KL/std": 112.24214935302734,
"epoch": 0.5675392670157068,
"fcm_dpo/beta": 0.00413860660046339,
"fcm_dpo/delta": 0.023125268518924713,
"fcm_dpo/margin": 67.99632263183594,
"fcm_dpo/q_t": 0.4350787103176117,
"grad_norm": 74.27021789550781,
"learning_rate": 2.3627616503391812e-07,
"logits/chosen": -0.7775533199310303,
"logits/rejected": -0.7718729972839355,
"logps/chosen": -451.1247253417969,
"logps/ref_chosen": -284.104736328125,
"logps/ref_rejected": -253.9580535888672,
"logps/rejected": -488.97442626953125,
"loss": 4.7303,
"margin_dpo/margin_mean": 67.99632263183594,
"margin_dpo/margin_std": 115.17955017089844,
"step": 271
},
{
"KL/chosen_KL_mean": -157.8939666748047,
"KL/mean": -197.00889587402344,
"KL/rejected_KL_mean": -236.12380981445312,
"KL/std": 115.54269409179688,
"epoch": 0.5696335078534032,
"fcm_dpo/beta": 0.004133047070354223,
"fcm_dpo/delta": -0.008325421251356602,
"fcm_dpo/margin": 78.22982788085938,
"fcm_dpo/q_t": 0.42606019973754883,
"grad_norm": 58.807376861572266,
"learning_rate": 2.344485449913914e-07,
"logits/chosen": -0.8697534203529358,
"logits/rejected": -0.8527165651321411,
"logps/chosen": -455.25299072265625,
"logps/ref_chosen": -297.3590087890625,
"logps/ref_rejected": -279.20196533203125,
"logps/rejected": -515.3258056640625,
"loss": 4.6817,
"margin_dpo/margin_mean": 78.22982788085938,
"margin_dpo/margin_std": 139.56898498535156,
"step": 272
},
{
"KL/chosen_KL_mean": -158.95762634277344,
"KL/mean": -199.33731079101562,
"KL/rejected_KL_mean": -239.71701049804688,
"KL/std": 105.8140640258789,
"epoch": 0.5717277486910994,
"fcm_dpo/beta": 0.004161749500781298,
"fcm_dpo/delta": 0.013979857787489891,
"fcm_dpo/margin": 80.75938415527344,
"fcm_dpo/q_t": 0.42355871200561523,
"grad_norm": 85.51101684570312,
"learning_rate": 2.3262175892620062e-07,
"logits/chosen": -0.8654077053070068,
"logits/rejected": -0.8781574964523315,
"logps/chosen": -452.16339111328125,
"logps/ref_chosen": -293.20574951171875,
"logps/ref_rejected": -274.7646789550781,
"logps/rejected": -514.481689453125,
"loss": 4.6187,
"margin_dpo/margin_mean": 80.75936889648438,
"margin_dpo/margin_std": 134.3832244873047,
"step": 273
},
{
"KL/chosen_KL_mean": -156.28903198242188,
"KL/mean": -206.505859375,
"KL/rejected_KL_mean": -256.7226867675781,
"KL/std": 118.53926086425781,
"epoch": 0.5738219895287958,
"fcm_dpo/beta": 0.004055425524711609,
"fcm_dpo/delta": -0.060026153922080994,
"fcm_dpo/margin": 100.43367767333984,
"fcm_dpo/q_t": 0.40615060925483704,
"grad_norm": 54.804561614990234,
"learning_rate": 2.3079590480333827e-07,
"logits/chosen": -0.8232536315917969,
"logits/rejected": -0.7915986776351929,
"logps/chosen": -426.84771728515625,
"logps/ref_chosen": -270.55865478515625,
"logps/ref_rejected": -239.47048950195312,
"logps/rejected": -496.19317626953125,
"loss": 4.3666,
"margin_dpo/margin_mean": 100.43367767333984,
"margin_dpo/margin_std": 134.73028564453125,
"step": 274
},
{
"KL/chosen_KL_mean": -144.99569702148438,
"KL/mean": -192.33421325683594,
"KL/rejected_KL_mean": -239.67274475097656,
"KL/std": 111.46051025390625,
"epoch": 0.5759162303664922,
"fcm_dpo/beta": 0.003937349654734135,
"fcm_dpo/delta": -0.02433641441166401,
"fcm_dpo/margin": 94.67705535888672,
"fcm_dpo/q_t": 0.4138934016227722,
"grad_norm": 48.31379699707031,
"learning_rate": 2.2897108053782e-07,
"logits/chosen": -0.8615505695343018,
"logits/rejected": -0.8430612683296204,
"logps/chosen": -395.31494140625,
"logps/ref_chosen": -250.31922912597656,
"logps/ref_rejected": -249.3187255859375,
"logps/rejected": -488.991455078125,
"loss": 4.4327,
"margin_dpo/margin_mean": 94.67705535888672,
"margin_dpo/margin_std": 124.92469787597656,
"step": 275
},
{
"KL/chosen_KL_mean": -152.50244140625,
"KL/mean": -198.66302490234375,
"KL/rejected_KL_mean": -244.82359313964844,
"KL/std": 118.38050842285156,
"epoch": 0.5780104712041885,
"fcm_dpo/beta": 0.003826065920293331,
"fcm_dpo/delta": -0.004235291387885809,
"fcm_dpo/margin": 92.32111358642578,
"fcm_dpo/q_t": 0.4181724786758423,
"grad_norm": 62.1019287109375,
"learning_rate": 2.2714738398943308e-07,
"logits/chosen": -0.926426887512207,
"logits/rejected": -0.9037998914718628,
"logps/chosen": -450.1335144042969,
"logps/ref_chosen": -297.6310729980469,
"logps/ref_rejected": -295.225830078125,
"logps/rejected": -540.0493774414062,
"loss": 4.5166,
"margin_dpo/margin_mean": 92.32112121582031,
"margin_dpo/margin_std": 130.4283905029297,
"step": 276
},
{
"KL/chosen_KL_mean": -176.41529846191406,
"KL/mean": -207.66116333007812,
"KL/rejected_KL_mean": -238.90696716308594,
"KL/std": 117.95841979980469,
"epoch": 0.5801047120418849,
"fcm_dpo/beta": 0.0038804924115538597,
"fcm_dpo/delta": 0.004320847801864147,
"fcm_dpo/margin": 62.49165725708008,
"fcm_dpo/q_t": 0.44412410259246826,
"grad_norm": 92.11759948730469,
"learning_rate": 2.2532491295748865e-07,
"logits/chosen": -0.8612473011016846,
"logits/rejected": -0.8609473705291748,
"logps/chosen": -442.7757568359375,
"logps/ref_chosen": -266.3604736328125,
"logps/ref_rejected": -253.36767578125,
"logps/rejected": -492.2746276855469,
"loss": 4.9139,
"margin_dpo/margin_mean": 62.49165344238281,
"margin_dpo/margin_std": 134.01393127441406,
"step": 277
},
{
"KL/chosen_KL_mean": -189.94204711914062,
"KL/mean": -214.97512817382812,
"KL/rejected_KL_mean": -240.0082550048828,
"KL/std": 124.60601806640625,
"epoch": 0.5821989528795811,
"fcm_dpo/beta": 0.003933742176741362,
"fcm_dpo/delta": 0.009704717434942722,
"fcm_dpo/margin": 50.06622314453125,
"fcm_dpo/q_t": 0.45811718702316284,
"grad_norm": 66.18705749511719,
"learning_rate": 2.2350376517557726e-07,
"logits/chosen": -0.8861530423164368,
"logits/rejected": -0.8472384810447693,
"logps/chosen": -457.34930419921875,
"logps/ref_chosen": -267.40728759765625,
"logps/ref_rejected": -229.5758514404297,
"logps/rejected": -469.5841369628906,
"loss": 5.1834,
"margin_dpo/margin_mean": 50.06621551513672,
"margin_dpo/margin_std": 154.64942932128906,
"step": 278
},
{
"KL/chosen_KL_mean": -173.27903747558594,
"KL/mean": -222.08213806152344,
"KL/rejected_KL_mean": -270.8852233886719,
"KL/std": 131.5963897705078,
"epoch": 0.5842931937172775,
"fcm_dpo/beta": 0.0038174251094460487,
"fcm_dpo/delta": -0.06448332965373993,
"fcm_dpo/margin": 97.60619354248047,
"fcm_dpo/q_t": 0.4153634309768677,
"grad_norm": 117.3631591796875,
"learning_rate": 2.2168403830632769e-07,
"logits/chosen": -0.8119494915008545,
"logits/rejected": -0.7960024476051331,
"logps/chosen": -486.6468200683594,
"logps/ref_chosen": -313.3677978515625,
"logps/ref_rejected": -299.1744384765625,
"logps/rejected": -570.0596923828125,
"loss": 4.5018,
"margin_dpo/margin_mean": 97.60618591308594,
"margin_dpo/margin_std": 146.39854431152344,
"step": 279
},
{
"KL/chosen_KL_mean": -160.8683319091797,
"KL/mean": -198.24644470214844,
"KL/rejected_KL_mean": -235.62454223632812,
"KL/std": 106.50706481933594,
"epoch": 0.5863874345549738,
"fcm_dpo/beta": 0.003840838558971882,
"fcm_dpo/delta": 0.046352073550224304,
"fcm_dpo/margin": 74.75621032714844,
"fcm_dpo/q_t": 0.43327394127845764,
"grad_norm": 51.44595718383789,
"learning_rate": 2.1986582993616925e-07,
"logits/chosen": -0.8889455199241638,
"logits/rejected": -0.9012906551361084,
"logps/chosen": -426.4241638183594,
"logps/ref_chosen": -265.5558166503906,
"logps/ref_rejected": -247.1573944091797,
"logps/rejected": -482.781982421875,
"loss": 4.73,
"margin_dpo/margin_mean": 74.75621032714844,
"margin_dpo/margin_std": 131.78170776367188,
"step": 280
},
{
"KL/chosen_KL_mean": -190.2058868408203,
"KL/mean": -233.55860900878906,
"KL/rejected_KL_mean": -276.91131591796875,
"KL/std": 123.68882751464844,
"epoch": 0.5884816753926702,
"fcm_dpo/beta": 0.0038038038183003664,
"fcm_dpo/delta": -0.03660136088728905,
"fcm_dpo/margin": 86.70545959472656,
"fcm_dpo/q_t": 0.42573872208595276,
"grad_norm": 55.746944427490234,
"learning_rate": 2.1804923757009882e-07,
"logits/chosen": -0.8607216477394104,
"logits/rejected": -0.8701354265213013,
"logps/chosen": -485.5054931640625,
"logps/ref_chosen": -295.2995910644531,
"logps/ref_rejected": -293.80877685546875,
"logps/rejected": -570.7200927734375,
"loss": 4.6412,
"margin_dpo/margin_mean": 86.70547485351562,
"margin_dpo/margin_std": 145.6509552001953,
"step": 281
},
{
"KL/chosen_KL_mean": -186.29031372070312,
"KL/mean": -226.07289123535156,
"KL/rejected_KL_mean": -265.8554992675781,
"KL/std": 120.34805297851562,
"epoch": 0.5905759162303665,
"fcm_dpo/beta": 0.003840283490717411,
"fcm_dpo/delta": 0.045822735875844955,
"fcm_dpo/margin": 79.565185546875,
"fcm_dpo/q_t": 0.4292564392089844,
"grad_norm": 84.25727081298828,
"learning_rate": 2.1623435862645205e-07,
"logits/chosen": -0.8561784625053406,
"logits/rejected": -0.8544781804084778,
"logps/chosen": -504.927490234375,
"logps/ref_chosen": -318.63714599609375,
"logps/ref_rejected": -273.5943603515625,
"logps/rejected": -539.4498291015625,
"loss": 4.6817,
"margin_dpo/margin_mean": 79.565185546875,
"margin_dpo/margin_std": 134.56092834472656,
"step": 282
},
{
"KL/chosen_KL_mean": -190.91969299316406,
"KL/mean": -228.52896118164062,
"KL/rejected_KL_mean": -266.13824462890625,
"KL/std": 117.83441925048828,
"epoch": 0.5926701570680628,
"fcm_dpo/beta": 0.0039502596482634544,
"fcm_dpo/delta": -0.001253846101462841,
"fcm_dpo/margin": 75.21856689453125,
"fcm_dpo/q_t": 0.43303558230400085,
"grad_norm": 61.768646240234375,
"learning_rate": 2.1442129043167873e-07,
"logits/chosen": -0.8777590990066528,
"logits/rejected": -0.8666056394577026,
"logps/chosen": -445.58026123046875,
"logps/ref_chosen": -254.66053771972656,
"logps/ref_rejected": -236.8627166748047,
"logps/rejected": -503.0010070800781,
"loss": 4.7327,
"margin_dpo/margin_mean": 75.21856689453125,
"margin_dpo/margin_std": 134.68295288085938,
"step": 283
},
{
"KL/chosen_KL_mean": -199.0657501220703,
"KL/mean": -239.81517028808594,
"KL/rejected_KL_mean": -280.5645751953125,
"KL/std": 125.28971099853516,
"epoch": 0.5947643979057592,
"fcm_dpo/beta": 0.0038177259266376495,
"fcm_dpo/delta": -0.03368262201547623,
"fcm_dpo/margin": 81.49883270263672,
"fcm_dpo/q_t": 0.4287629723548889,
"grad_norm": 65.97899627685547,
"learning_rate": 2.1261013021512378e-07,
"logits/chosen": -0.850072979927063,
"logits/rejected": -0.8169124126434326,
"logps/chosen": -472.4210205078125,
"logps/ref_chosen": -273.355224609375,
"logps/ref_rejected": -259.84759521484375,
"logps/rejected": -540.4122314453125,
"loss": 4.732,
"margin_dpo/margin_mean": 81.49883270263672,
"margin_dpo/margin_std": 149.6010284423828,
"step": 284
},
{
"KL/chosen_KL_mean": -206.79808044433594,
"KL/mean": -235.41844177246094,
"KL/rejected_KL_mean": -264.038818359375,
"KL/std": 118.39861297607422,
"epoch": 0.5968586387434555,
"fcm_dpo/beta": 0.00384325347840786,
"fcm_dpo/delta": 0.00668741762638092,
"fcm_dpo/margin": 57.24073028564453,
"fcm_dpo/q_t": 0.4500173330307007,
"grad_norm": 73.33934783935547,
"learning_rate": 2.1080097510381294e-07,
"logits/chosen": -0.8570336103439331,
"logits/rejected": -0.85038161277771,
"logps/chosen": -516.6002807617188,
"logps/ref_chosen": -309.8022155761719,
"logps/ref_rejected": -279.11846923828125,
"logps/rejected": -543.1572875976562,
"loss": 5.0613,
"margin_dpo/margin_mean": 57.24073028564453,
"margin_dpo/margin_std": 148.24229431152344,
"step": 285
},
{
"KL/chosen_KL_mean": -200.6256866455078,
"KL/mean": -238.26754760742188,
"KL/rejected_KL_mean": -275.909423828125,
"KL/std": 126.28824615478516,
"epoch": 0.5989528795811518,
"fcm_dpo/beta": 0.0037976307794451714,
"fcm_dpo/delta": 0.0033064894378185272,
"fcm_dpo/margin": 75.28372192382812,
"fcm_dpo/q_t": 0.43621984124183655,
"grad_norm": 100.08869934082031,
"learning_rate": 2.089939221172446e-07,
"logits/chosen": -0.8294232487678528,
"logits/rejected": -0.8224241733551025,
"logps/chosen": -472.09124755859375,
"logps/ref_chosen": -271.4655456542969,
"logps/ref_rejected": -279.531494140625,
"logps/rejected": -555.44091796875,
"loss": 4.7871,
"margin_dpo/margin_mean": 75.28372192382812,
"margin_dpo/margin_std": 145.39089965820312,
"step": 286
},
{
"KL/chosen_KL_mean": -205.491943359375,
"KL/mean": -242.21157836914062,
"KL/rejected_KL_mean": -278.93121337890625,
"KL/std": 122.3520736694336,
"epoch": 0.6010471204188481,
"fcm_dpo/beta": 0.00381746469065547,
"fcm_dpo/delta": -0.0022038668394088745,
"fcm_dpo/margin": 73.43925476074219,
"fcm_dpo/q_t": 0.4349508583545685,
"grad_norm": 90.85375213623047,
"learning_rate": 2.0718906816218595e-07,
"logits/chosen": -0.853791356086731,
"logits/rejected": -0.8370929956436157,
"logps/chosen": -482.5851745605469,
"logps/ref_chosen": -277.0932312011719,
"logps/ref_rejected": -233.55599975585938,
"logps/rejected": -512.4871826171875,
"loss": 4.8313,
"margin_dpo/margin_mean": 73.43925476074219,
"margin_dpo/margin_std": 150.7272186279297,
"step": 287
},
{
"KL/chosen_KL_mean": -201.13818359375,
"KL/mean": -238.72735595703125,
"KL/rejected_KL_mean": -276.3164978027344,
"KL/std": 125.49322509765625,
"epoch": 0.6031413612565445,
"fcm_dpo/beta": 0.00389170553535223,
"fcm_dpo/delta": 0.011016139760613441,
"fcm_dpo/margin": 75.17829132080078,
"fcm_dpo/q_t": 0.43270570039749146,
"grad_norm": 89.63787841796875,
"learning_rate": 2.053865100274774e-07,
"logits/chosen": -0.8803871273994446,
"logits/rejected": -0.8856675028800964,
"logps/chosen": -494.306396484375,
"logps/ref_chosen": -293.1681823730469,
"logps/ref_rejected": -263.4059143066406,
"logps/rejected": -539.722412109375,
"loss": 4.7564,
"margin_dpo/margin_mean": 75.17829132080078,
"margin_dpo/margin_std": 140.85711669921875,
"step": 288
},
{
"KL/chosen_KL_mean": -218.83831787109375,
"KL/mean": -242.6974639892578,
"KL/rejected_KL_mean": -266.5566101074219,
"KL/std": 120.14765930175781,
"epoch": 0.6052356020942409,
"fcm_dpo/beta": 0.0038702820893377066,
"fcm_dpo/delta": 0.004241670481860638,
"fcm_dpo/margin": 47.71830368041992,
"fcm_dpo/q_t": 0.45726847648620605,
"grad_norm": 69.4831771850586,
"learning_rate": 2.035863443788411e-07,
"logits/chosen": -0.8640040159225464,
"logits/rejected": -0.8451948165893555,
"logps/chosen": -548.7957153320312,
"logps/ref_chosen": -329.9574279785156,
"logps/ref_rejected": -276.7565002441406,
"logps/rejected": -543.3131103515625,
"loss": 5.1264,
"margin_dpo/margin_mean": 47.71830368041992,
"margin_dpo/margin_std": 134.6345977783203,
"step": 289
},
{
"KL/chosen_KL_mean": -200.6356658935547,
"KL/mean": -238.46449279785156,
"KL/rejected_KL_mean": -276.29327392578125,
"KL/std": 125.73330688476562,
"epoch": 0.6073298429319371,
"fcm_dpo/beta": 0.0037858528085052967,
"fcm_dpo/delta": -0.024414364248514175,
"fcm_dpo/margin": 75.65762329101562,
"fcm_dpo/q_t": 0.4347863495349884,
"grad_norm": 63.50288391113281,
"learning_rate": 2.0178866775369774e-07,
"logits/chosen": -0.8688110113143921,
"logits/rejected": -0.8021400570869446,
"logps/chosen": -525.3047485351562,
"logps/ref_chosen": -324.6690673828125,
"logps/ref_rejected": -311.8439636230469,
"logps/rejected": -588.1372680664062,
"loss": 4.8057,
"margin_dpo/margin_mean": 75.65762329101562,
"margin_dpo/margin_std": 148.36460876464844,
"step": 290
},
{
"KL/chosen_KL_mean": -192.0414276123047,
"KL/mean": -240.45831298828125,
"KL/rejected_KL_mean": -288.8752136230469,
"KL/std": 125.70685577392578,
"epoch": 0.6094240837696335,
"fcm_dpo/beta": 0.0037065076176077127,
"fcm_dpo/delta": -0.04703481122851372,
"fcm_dpo/margin": 96.83379364013672,
"fcm_dpo/q_t": 0.41666173934936523,
"grad_norm": 62.59638214111328,
"learning_rate": 1.9999357655598891e-07,
"logits/chosen": -0.8442043662071228,
"logits/rejected": -0.8339248299598694,
"logps/chosen": -466.18548583984375,
"logps/ref_chosen": -274.1440734863281,
"logps/ref_rejected": -278.07208251953125,
"logps/rejected": -566.9473266601562,
"loss": 4.5137,
"margin_dpo/margin_mean": 96.83379364013672,
"margin_dpo/margin_std": 137.9904327392578,
"step": 291
},
{
"KL/chosen_KL_mean": -184.49508666992188,
"KL/mean": -225.76486206054688,
"KL/rejected_KL_mean": -267.03466796875,
"KL/std": 128.42660522460938,
"epoch": 0.6115183246073298,
"fcm_dpo/beta": 0.003625905141234398,
"fcm_dpo/delta": 0.005497709847986698,
"fcm_dpo/margin": 82.53958129882812,
"fcm_dpo/q_t": 0.4316753149032593,
"grad_norm": 77.57347869873047,
"learning_rate": 1.9820116705100775e-07,
"logits/chosen": -0.8392362594604492,
"logits/rejected": -0.8352776169776917,
"logps/chosen": -443.8587951660156,
"logps/ref_chosen": -259.3636779785156,
"logps/ref_rejected": -279.30218505859375,
"logps/rejected": -546.3368530273438,
"loss": 4.734,
"margin_dpo/margin_mean": 82.53958129882812,
"margin_dpo/margin_std": 148.260986328125,
"step": 292
},
{
"KL/chosen_KL_mean": -198.5830078125,
"KL/mean": -244.17982482910156,
"KL/rejected_KL_mean": -289.776611328125,
"KL/std": 132.66905212402344,
"epoch": 0.6136125654450262,
"fcm_dpo/beta": 0.003596771974116564,
"fcm_dpo/delta": -0.03300413116812706,
"fcm_dpo/margin": 91.19361877441406,
"fcm_dpo/q_t": 0.4237741231918335,
"grad_norm": 80.38446044921875,
"learning_rate": 1.9641153536023642e-07,
"logits/chosen": -0.9296824932098389,
"logits/rejected": -0.8887497186660767,
"logps/chosen": -502.35382080078125,
"logps/ref_chosen": -303.77081298828125,
"logps/ref_rejected": -270.07513427734375,
"logps/rejected": -559.8517456054688,
"loss": 4.5962,
"margin_dpo/margin_mean": 91.19361877441406,
"margin_dpo/margin_std": 136.49072265625,
"step": 293
},
{
"KL/chosen_KL_mean": -188.66331481933594,
"KL/mean": -226.62930297851562,
"KL/rejected_KL_mean": -264.59527587890625,
"KL/std": 125.87104034423828,
"epoch": 0.6157068062827226,
"fcm_dpo/beta": 0.0035779972095042467,
"fcm_dpo/delta": 0.004179379902780056,
"fcm_dpo/margin": 75.93196105957031,
"fcm_dpo/q_t": 0.4386540353298187,
"grad_norm": 63.021671295166016,
"learning_rate": 1.9462477745619106e-07,
"logits/chosen": -0.837997317314148,
"logits/rejected": -0.8435596823692322,
"logps/chosen": -428.90167236328125,
"logps/ref_chosen": -240.23831176757812,
"logps/ref_rejected": -229.187744140625,
"logps/rejected": -493.78302001953125,
"loss": 4.8281,
"margin_dpo/margin_mean": 75.93196105957031,
"margin_dpo/margin_std": 151.36424255371094,
"step": 294
},
{
"KL/chosen_KL_mean": -184.72071838378906,
"KL/mean": -222.63592529296875,
"KL/rejected_KL_mean": -260.5511779785156,
"KL/std": 110.61774444580078,
"epoch": 0.6178010471204188,
"fcm_dpo/beta": 0.0036432219203561544,
"fcm_dpo/delta": 0.03179817646741867,
"fcm_dpo/margin": 75.83045196533203,
"fcm_dpo/q_t": 0.4368009567260742,
"grad_norm": 66.03601837158203,
"learning_rate": 1.928409891572757e-07,
"logits/chosen": -0.8257983922958374,
"logits/rejected": -0.8278652429580688,
"logps/chosen": -435.73040771484375,
"logps/ref_chosen": -251.00970458984375,
"logps/ref_rejected": -244.15142822265625,
"logps/rejected": -504.7026062011719,
"loss": 4.7656,
"margin_dpo/margin_mean": 75.83045196533203,
"margin_dpo/margin_std": 136.33737182617188,
"step": 295
},
{
"KL/chosen_KL_mean": -192.8872528076172,
"KL/mean": -244.12698364257812,
"KL/rejected_KL_mean": -295.36669921875,
"KL/std": 126.94731140136719,
"epoch": 0.6198952879581152,
"fcm_dpo/beta": 0.0035800463519990444,
"fcm_dpo/delta": -0.06783003360033035,
"fcm_dpo/margin": 102.47940826416016,
"fcm_dpo/q_t": 0.41508862376213074,
"grad_norm": 56.43666076660156,
"learning_rate": 1.9106026612264315e-07,
"logits/chosen": -0.8277036547660828,
"logits/rejected": -0.7932828664779663,
"logps/chosen": -486.76788330078125,
"logps/ref_chosen": -293.880615234375,
"logps/ref_rejected": -283.4175720214844,
"logps/rejected": -578.7842407226562,
"loss": 4.5237,
"margin_dpo/margin_mean": 102.47940826416016,
"margin_dpo/margin_std": 155.32875061035156,
"step": 296
},
{
"KL/chosen_KL_mean": -191.3452911376953,
"KL/mean": -227.1857147216797,
"KL/rejected_KL_mean": -263.0261535644531,
"KL/std": 114.53092956542969,
"epoch": 0.6219895287958115,
"fcm_dpo/beta": 0.0034403197932988405,
"fcm_dpo/delta": -0.010270388796925545,
"fcm_dpo/margin": 71.68084716796875,
"fcm_dpo/q_t": 0.4418776333332062,
"grad_norm": 72.24175262451172,
"learning_rate": 1.8928270384706582e-07,
"logits/chosen": -0.9125591516494751,
"logits/rejected": -0.9058645963668823,
"logps/chosen": -480.8053894042969,
"logps/ref_chosen": -289.4600830078125,
"logps/ref_rejected": -283.69110107421875,
"logps/rejected": -546.7172241210938,
"loss": 4.8233,
"margin_dpo/margin_mean": 71.68084716796875,
"margin_dpo/margin_std": 126.96052551269531,
"step": 297
},
{
"KL/chosen_KL_mean": -180.29208374023438,
"KL/mean": -228.24099731445312,
"KL/rejected_KL_mean": -276.1899108886719,
"KL/std": 122.46294403076172,
"epoch": 0.6240837696335079,
"fcm_dpo/beta": 0.0034188858699053526,
"fcm_dpo/delta": -0.038795486092567444,
"fcm_dpo/margin": 95.89783477783203,
"fcm_dpo/q_t": 0.4239664673805237,
"grad_norm": 59.304649353027344,
"learning_rate": 1.875083976558136e-07,
"logits/chosen": -0.8275625109672546,
"logits/rejected": -0.8134176135063171,
"logps/chosen": -486.80706787109375,
"logps/ref_chosen": -306.5150146484375,
"logps/ref_rejected": -280.6969909667969,
"logps/rejected": -556.8869018554688,
"loss": 4.6075,
"margin_dpo/margin_mean": 95.89783477783203,
"margin_dpo/margin_std": 146.3650360107422,
"step": 298
},
{
"KL/chosen_KL_mean": -206.77996826171875,
"KL/mean": -242.58572387695312,
"KL/rejected_KL_mean": -278.3914489746094,
"KL/std": 122.42256164550781,
"epoch": 0.6261780104712041,
"fcm_dpo/beta": 0.003332537831738591,
"fcm_dpo/delta": 0.0029420889914035797,
"fcm_dpo/margin": 71.61149597167969,
"fcm_dpo/q_t": 0.44390517473220825,
"grad_norm": 48.41752624511719,
"learning_rate": 1.8573744269954297e-07,
"logits/chosen": -0.8100135326385498,
"logits/rejected": -0.7927089333534241,
"logps/chosen": -488.1437683105469,
"logps/ref_chosen": -281.36376953125,
"logps/ref_rejected": -270.39508056640625,
"logps/rejected": -548.7865600585938,
"loss": 4.8385,
"margin_dpo/margin_mean": 71.61149597167969,
"margin_dpo/margin_std": 119.70535278320312,
"step": 299
},
{
"KL/chosen_KL_mean": -212.83030700683594,
"KL/mean": -249.0923614501953,
"KL/rejected_KL_mean": -285.35443115234375,
"KL/std": 132.61465454101562,
"epoch": 0.6282722513089005,
"fcm_dpo/beta": 0.00347234308719635,
"fcm_dpo/delta": 0.05787147581577301,
"fcm_dpo/margin": 72.52412414550781,
"fcm_dpo/q_t": 0.4418582320213318,
"grad_norm": 63.956878662109375,
"learning_rate": 1.839699339491937e-07,
"logits/chosen": -0.8397761583328247,
"logits/rejected": -0.8176465034484863,
"logps/chosen": -527.666015625,
"logps/ref_chosen": -314.83575439453125,
"logps/ref_rejected": -269.1154479980469,
"logps/rejected": -554.4698486328125,
"loss": 4.8907,
"margin_dpo/margin_mean": 72.52412414550781,
"margin_dpo/margin_std": 154.9835968017578,
"step": 300
},
{
"KL/chosen_KL_mean": -199.2548828125,
"KL/mean": -235.5850372314453,
"KL/rejected_KL_mean": -271.9151916503906,
"KL/std": 123.72215270996094,
"epoch": 0.6303664921465969,
"fcm_dpo/beta": 0.0036008688621222973,
"fcm_dpo/delta": 0.018657753244042397,
"fcm_dpo/margin": 72.66029357910156,
"fcm_dpo/q_t": 0.4398142397403717,
"grad_norm": 48.57915115356445,
"learning_rate": 1.8220596619089573e-07,
"logits/chosen": -0.8675850629806519,
"logits/rejected": -0.8598907589912415,
"logps/chosen": -479.1494140625,
"logps/ref_chosen": -279.89453125,
"logps/ref_rejected": -271.6694641113281,
"logps/rejected": -543.5846557617188,
"loss": 4.8263,
"margin_dpo/margin_mean": 72.6603012084961,
"margin_dpo/margin_std": 138.31997680664062,
"step": 301
},
{
"KL/chosen_KL_mean": -193.967529296875,
"KL/mean": -242.45010375976562,
"KL/rejected_KL_mean": -290.93267822265625,
"KL/std": 130.0727081298828,
"epoch": 0.6324607329842932,
"fcm_dpo/beta": 0.0035522691905498505,
"fcm_dpo/delta": -0.026153553277254105,
"fcm_dpo/margin": 96.96512603759766,
"fcm_dpo/q_t": 0.41999223828315735,
"grad_norm": 60.82284927368164,
"learning_rate": 1.8044563402088682e-07,
"logits/chosen": -0.8301497101783752,
"logits/rejected": -0.8174096345901489,
"logps/chosen": -465.2993469238281,
"logps/ref_chosen": -271.3318176269531,
"logps/ref_rejected": -256.5587158203125,
"logps/rejected": -547.4913940429688,
"loss": 4.5419,
"margin_dpo/margin_mean": 96.96513366699219,
"margin_dpo/margin_std": 141.36141967773438,
"step": 302
},
{
"KL/chosen_KL_mean": -196.64830017089844,
"KL/mean": -238.0702362060547,
"KL/rejected_KL_mean": -279.4921569824219,
"KL/std": 119.1928939819336,
"epoch": 0.6345549738219896,
"fcm_dpo/beta": 0.003514134092256427,
"fcm_dpo/delta": -0.009628637693822384,
"fcm_dpo/margin": 82.84388732910156,
"fcm_dpo/q_t": 0.4314088523387909,
"grad_norm": 71.00035095214844,
"learning_rate": 1.7868903184043885e-07,
"logits/chosen": -0.813162624835968,
"logits/rejected": -0.789709210395813,
"logps/chosen": -501.5293273925781,
"logps/ref_chosen": -304.88104248046875,
"logps/ref_rejected": -269.063720703125,
"logps/rejected": -548.555908203125,
"loss": 4.6955,
"margin_dpo/margin_mean": 82.84388732910156,
"margin_dpo/margin_std": 131.7555389404297,
"step": 303
},
{
"KL/chosen_KL_mean": -209.07229614257812,
"KL/mean": -253.82827758789062,
"KL/rejected_KL_mean": -298.584228515625,
"KL/std": 142.47499084472656,
"epoch": 0.6366492146596858,
"fcm_dpo/beta": 0.003528868081048131,
"fcm_dpo/delta": 0.03502277284860611,
"fcm_dpo/margin": 89.51190185546875,
"fcm_dpo/q_t": 0.42928797006607056,
"grad_norm": 62.49873733520508,
"learning_rate": 1.7693625385079574e-07,
"logits/chosen": -0.8146846294403076,
"logits/rejected": -0.8206278681755066,
"logps/chosen": -499.7832946777344,
"logps/ref_chosen": -290.7109680175781,
"logps/ref_rejected": -237.6885986328125,
"logps/rejected": -536.2728271484375,
"loss": 4.681,
"margin_dpo/margin_mean": 89.51190185546875,
"margin_dpo/margin_std": 158.9447784423828,
"step": 304
},
{
"KL/chosen_KL_mean": -181.38778686523438,
"KL/mean": -248.25180053710938,
"KL/rejected_KL_mean": -315.1158142089844,
"KL/std": 137.24256896972656,
"epoch": 0.6387434554973822,
"fcm_dpo/beta": 0.0033798436634242535,
"fcm_dpo/delta": -0.12808626890182495,
"fcm_dpo/margin": 133.72805786132812,
"fcm_dpo/q_t": 0.396328866481781,
"grad_norm": 83.69937896728516,
"learning_rate": 1.7518739404812155e-07,
"logits/chosen": -0.864880383014679,
"logits/rejected": -0.8337042927742004,
"logps/chosen": -437.8717956542969,
"logps/ref_chosen": -256.4839782714844,
"logps/ref_rejected": -266.4063415527344,
"logps/rejected": -581.5220947265625,
"loss": 4.2357,
"margin_dpo/margin_mean": 133.72805786132812,
"margin_dpo/margin_std": 159.96530151367188,
"step": 305
},
{
"KL/chosen_KL_mean": -192.40550231933594,
"KL/mean": -232.20289611816406,
"KL/rejected_KL_mean": -272.00030517578125,
"KL/std": 119.71340942382812,
"epoch": 0.6408376963350786,
"fcm_dpo/beta": 0.0031991961877793074,
"fcm_dpo/delta": -0.0037949923425912857,
"fcm_dpo/margin": 79.59476470947266,
"fcm_dpo/q_t": 0.43988752365112305,
"grad_norm": 56.67892837524414,
"learning_rate": 1.7344254621846017e-07,
"logits/chosen": -0.8629582524299622,
"logits/rejected": -0.8421335220336914,
"logps/chosen": -513.0547485351562,
"logps/ref_chosen": -320.6492004394531,
"logps/ref_rejected": -273.36773681640625,
"logps/rejected": -545.3679809570312,
"loss": 4.7771,
"margin_dpo/margin_mean": 79.59476470947266,
"margin_dpo/margin_std": 127.43058013916016,
"step": 306
},
{
"KL/chosen_KL_mean": -220.43991088867188,
"KL/mean": -261.191650390625,
"KL/rejected_KL_mean": -301.943359375,
"KL/std": 134.89817810058594,
"epoch": 0.6429319371727749,
"fcm_dpo/beta": 0.0032339305616915226,
"fcm_dpo/delta": 0.020983368158340454,
"fcm_dpo/margin": 81.5034408569336,
"fcm_dpo/q_t": 0.4373091161251068,
"grad_norm": 75.93914031982422,
"learning_rate": 1.717018039327053e-07,
"logits/chosen": -0.7927024960517883,
"logits/rejected": -0.8132081031799316,
"logps/chosen": -499.89410400390625,
"logps/ref_chosen": -279.4541931152344,
"logps/ref_rejected": -240.3796844482422,
"logps/rejected": -542.3230590820312,
"loss": 4.7445,
"margin_dpo/margin_mean": 81.5034408569336,
"margin_dpo/margin_std": 128.4541015625,
"step": 307
},
{
"KL/chosen_KL_mean": -240.52467346191406,
"KL/mean": -269.99462890625,
"KL/rejected_KL_mean": -299.4645690917969,
"KL/std": 134.7300262451172,
"epoch": 0.6450261780104712,
"fcm_dpo/beta": 0.0033486539032310247,
"fcm_dpo/delta": 0.0349406898021698,
"fcm_dpo/margin": 58.93989944458008,
"fcm_dpo/q_t": 0.45389360189437866,
"grad_norm": 67.70240020751953,
"learning_rate": 1.699652605415828e-07,
"logits/chosen": -0.8504436612129211,
"logits/rejected": -0.8614956140518188,
"logps/chosen": -537.123046875,
"logps/ref_chosen": -296.598388671875,
"logps/ref_rejected": -258.6953430175781,
"logps/rejected": -558.1598510742188,
"loss": 5.03,
"margin_dpo/margin_mean": 58.93989944458008,
"margin_dpo/margin_std": 142.15562438964844,
"step": 308
},
{
"KL/chosen_KL_mean": -226.738525390625,
"KL/mean": -276.4004211425781,
"KL/rejected_KL_mean": -326.06231689453125,
"KL/std": 132.22732543945312,
"epoch": 0.6471204188481675,
"fcm_dpo/beta": 0.003401699708774686,
"fcm_dpo/delta": 0.012193283997476101,
"fcm_dpo/margin": 99.32380676269531,
"fcm_dpo/q_t": 0.4232228100299835,
"grad_norm": 67.57255554199219,
"learning_rate": 1.6823300917064458e-07,
"logits/chosen": -0.8523184061050415,
"logits/rejected": -0.8619851469993591,
"logps/chosen": -508.12664794921875,
"logps/ref_chosen": -281.3881530761719,
"logps/ref_rejected": -262.458740234375,
"logps/rejected": -588.5210571289062,
"loss": 4.5812,
"margin_dpo/margin_mean": 99.32379150390625,
"margin_dpo/margin_std": 157.58938598632812,
"step": 309
},
{
"KL/chosen_KL_mean": -215.33587646484375,
"KL/mean": -265.105712890625,
"KL/rejected_KL_mean": -314.87554931640625,
"KL/std": 138.22256469726562,
"epoch": 0.6492146596858639,
"fcm_dpo/beta": 0.003396518062800169,
"fcm_dpo/delta": -0.010896272957324982,
"fcm_dpo/margin": 99.53965759277344,
"fcm_dpo/q_t": 0.42174240946769714,
"grad_norm": 65.10143280029297,
"learning_rate": 1.6650514271527465e-07,
"logits/chosen": -0.8462041616439819,
"logits/rejected": -0.8201505541801453,
"logps/chosen": -494.52313232421875,
"logps/ref_chosen": -279.1872863769531,
"logps/ref_rejected": -261.8279724121094,
"logps/rejected": -576.7034912109375,
"loss": 4.5525,
"margin_dpo/margin_mean": 99.53965759277344,
"margin_dpo/margin_std": 145.5522918701172,
"step": 310
},
{
"KL/chosen_KL_mean": -224.9881134033203,
"KL/mean": -273.8262023925781,
"KL/rejected_KL_mean": -322.664306640625,
"KL/std": 144.56954956054688,
"epoch": 0.6513089005235602,
"fcm_dpo/beta": 0.0033532544039189816,
"fcm_dpo/delta": -0.006738318130373955,
"fcm_dpo/margin": 97.67620849609375,
"fcm_dpo/q_t": 0.4256284534931183,
"grad_norm": 72.03314208984375,
"learning_rate": 1.647817538357072e-07,
"logits/chosen": -0.8415148854255676,
"logits/rejected": -0.8207394480705261,
"logps/chosen": -496.3862609863281,
"logps/ref_chosen": -271.39813232421875,
"logps/ref_rejected": -266.12701416015625,
"logps/rejected": -588.7913208007812,
"loss": 4.6702,
"margin_dpo/margin_mean": 97.67620849609375,
"margin_dpo/margin_std": 168.5247039794922,
"step": 311
},
{
"KL/chosen_KL_mean": -223.4862823486328,
"KL/mean": -267.6231689453125,
"KL/rejected_KL_mean": -311.76007080078125,
"KL/std": 139.96295166015625,
"epoch": 0.6534031413612565,
"fcm_dpo/beta": 0.003314582398161292,
"fcm_dpo/delta": -0.019153205677866936,
"fcm_dpo/margin": 88.27379608154297,
"fcm_dpo/q_t": 0.43480026721954346,
"grad_norm": 102.6431655883789,
"learning_rate": 1.6306293495205755e-07,
"logits/chosen": -0.8480397462844849,
"logits/rejected": -0.8192158341407776,
"logps/chosen": -505.8712463378906,
"logps/ref_chosen": -282.3850402832031,
"logps/ref_rejected": -246.35389709472656,
"logps/rejected": -558.114013671875,
"loss": 4.8413,
"margin_dpo/margin_mean": 88.27378845214844,
"margin_dpo/margin_std": 183.87039184570312,
"step": 312
},
{
"KL/chosen_KL_mean": -232.0310516357422,
"KL/mean": -274.9604797363281,
"KL/rejected_KL_mean": -317.889892578125,
"KL/std": 148.04339599609375,
"epoch": 0.6554973821989529,
"fcm_dpo/beta": 0.0033472059294581413,
"fcm_dpo/delta": -6.6105276346206665e-06,
"fcm_dpo/margin": 85.85885620117188,
"fcm_dpo/q_t": 0.437633752822876,
"grad_norm": 60.15560531616211,
"learning_rate": 1.6134877823936607e-07,
"logits/chosen": -0.8933418393135071,
"logits/rejected": -0.8773050308227539,
"logps/chosen": -535.6619262695312,
"logps/ref_chosen": -303.630859375,
"logps/ref_rejected": -273.1156921386719,
"logps/rejected": -591.005615234375,
"loss": 4.8209,
"margin_dpo/margin_mean": 85.85885620117188,
"margin_dpo/margin_std": 170.81617736816406,
"step": 313
},
{
"KL/chosen_KL_mean": -220.53932189941406,
"KL/mean": -266.6279296875,
"KL/rejected_KL_mean": -312.716552734375,
"KL/std": 129.1936492919922,
"epoch": 0.6575916230366492,
"fcm_dpo/beta": 0.00330774555914104,
"fcm_dpo/delta": 0.004261258989572525,
"fcm_dpo/margin": 92.1772232055664,
"fcm_dpo/q_t": 0.4281480610370636,
"grad_norm": 62.16466522216797,
"learning_rate": 1.5963937562265522e-07,
"logits/chosen": -0.8956902623176575,
"logits/rejected": -0.8859307169914246,
"logps/chosen": -522.8436279296875,
"logps/ref_chosen": -302.3042907714844,
"logps/ref_rejected": -273.6416015625,
"logps/rejected": -586.358154296875,
"loss": 4.6366,
"margin_dpo/margin_mean": 92.1772232055664,
"margin_dpo/margin_std": 139.03298950195312,
"step": 314
},
{
"KL/chosen_KL_mean": -219.21998596191406,
"KL/mean": -273.90521240234375,
"KL/rejected_KL_mean": -328.5904541015625,
"KL/std": 133.44537353515625,
"epoch": 0.6596858638743456,
"fcm_dpo/beta": 0.003281098324805498,
"fcm_dpo/delta": -0.02703600376844406,
"fcm_dpo/margin": 109.3704833984375,
"fcm_dpo/q_t": 0.41646474599838257,
"grad_norm": 83.7893295288086,
"learning_rate": 1.5793481877199943e-07,
"logits/chosen": -0.8796918392181396,
"logits/rejected": -0.8612196445465088,
"logps/chosen": -521.9492797851562,
"logps/ref_chosen": -302.729248046875,
"logps/ref_rejected": -270.26910400390625,
"logps/rejected": -598.859619140625,
"loss": 4.4913,
"margin_dpo/margin_mean": 109.37049102783203,
"margin_dpo/margin_std": 153.03680419921875,
"step": 315
},
{
"KL/chosen_KL_mean": -230.92750549316406,
"KL/mean": -278.43603515625,
"KL/rejected_KL_mean": -325.944580078125,
"KL/std": 137.39004516601562,
"epoch": 0.6617801047120419,
"fcm_dpo/beta": 0.003247649408876896,
"fcm_dpo/delta": 0.0010586827993392944,
"fcm_dpo/margin": 95.01707458496094,
"fcm_dpo/q_t": 0.42847543954849243,
"grad_norm": 57.49567413330078,
"learning_rate": 1.562351990976095e-07,
"logits/chosen": -0.899001955986023,
"logits/rejected": -0.8805509209632874,
"logps/chosen": -541.4981689453125,
"logps/ref_chosen": -310.5706481933594,
"logps/ref_rejected": -272.9354553222656,
"logps/rejected": -598.8800659179688,
"loss": 4.6608,
"margin_dpo/margin_mean": 95.0170669555664,
"margin_dpo/margin_std": 155.4547882080078,
"step": 316
},
{
"KL/chosen_KL_mean": -204.7845001220703,
"KL/mean": -249.20443725585938,
"KL/rejected_KL_mean": -293.6243591308594,
"KL/std": 122.82341003417969,
"epoch": 0.6638743455497382,
"fcm_dpo/beta": 0.003311583073809743,
"fcm_dpo/delta": 0.05764345824718475,
"fcm_dpo/margin": 88.83987426757812,
"fcm_dpo/q_t": 0.4310056269168854,
"grad_norm": 67.7326431274414,
"learning_rate": 1.5454060774493065e-07,
"logits/chosen": -0.8872113823890686,
"logits/rejected": -0.8559964895248413,
"logps/chosen": -458.68487548828125,
"logps/ref_chosen": -253.90036010742188,
"logps/ref_rejected": -218.74078369140625,
"logps/rejected": -512.3651123046875,
"loss": 4.6495,
"margin_dpo/margin_mean": 88.83988189697266,
"margin_dpo/margin_std": 131.25308227539062,
"step": 317
},
{
"KL/chosen_KL_mean": -213.66708374023438,
"KL/mean": -270.26947021484375,
"KL/rejected_KL_mean": -326.87188720703125,
"KL/std": 135.44711303710938,
"epoch": 0.6659685863874345,
"fcm_dpo/beta": 0.003273027017712593,
"fcm_dpo/delta": -0.044196274131536484,
"fcm_dpo/margin": 113.20478820800781,
"fcm_dpo/q_t": 0.4135770797729492,
"grad_norm": 75.95118713378906,
"learning_rate": 1.5285113558975427e-07,
"logits/chosen": -0.9056518077850342,
"logits/rejected": -0.8769103288650513,
"logps/chosen": -484.4899597167969,
"logps/ref_chosen": -270.8228759765625,
"logps/ref_rejected": -255.30972290039062,
"logps/rejected": -582.1815795898438,
"loss": 4.4434,
"margin_dpo/margin_mean": 113.20478820800781,
"margin_dpo/margin_std": 149.13006591796875,
"step": 318
},
{
"KL/chosen_KL_mean": -231.12075805664062,
"KL/mean": -284.0456237792969,
"KL/rejected_KL_mean": -336.970458984375,
"KL/std": 135.72552490234375,
"epoch": 0.6680628272251309,
"fcm_dpo/beta": 0.0032491928432136774,
"fcm_dpo/delta": 0.006155331619083881,
"fcm_dpo/margin": 105.84971618652344,
"fcm_dpo/q_t": 0.4182014465332031,
"grad_norm": 47.199466705322266,
"learning_rate": 1.5116687323334464e-07,
"logits/chosen": -0.8861814141273499,
"logits/rejected": -0.8539734482765198,
"logps/chosen": -532.1235961914062,
"logps/ref_chosen": -301.0028076171875,
"logps/ref_rejected": -242.39002990722656,
"logps/rejected": -579.3605346679688,
"loss": 4.4814,
"margin_dpo/margin_mean": 105.8497085571289,
"margin_dpo/margin_std": 133.62188720703125,
"step": 319
},
{
"KL/chosen_KL_mean": -230.05516052246094,
"KL/mean": -276.97979736328125,
"KL/rejected_KL_mean": -323.9044189453125,
"KL/std": 141.29583740234375,
"epoch": 0.6701570680628273,
"fcm_dpo/beta": 0.003300814889371395,
"fcm_dpo/delta": 0.019661886617541313,
"fcm_dpo/margin": 93.84925842285156,
"fcm_dpo/q_t": 0.42925646901130676,
"grad_norm": 68.85742950439453,
"learning_rate": 1.4948791099758052e-07,
"logits/chosen": -0.855350911617279,
"logits/rejected": -0.8459681272506714,
"logps/chosen": -533.677734375,
"logps/ref_chosen": -303.6225891113281,
"logps/ref_rejected": -280.85174560546875,
"logps/rejected": -604.7561645507812,
"loss": 4.6843,
"margin_dpo/margin_mean": 93.84925842285156,
"margin_dpo/margin_std": 161.83663940429688,
"step": 320
},
{
"KL/chosen_KL_mean": -245.5141143798828,
"KL/mean": -282.2264099121094,
"KL/rejected_KL_mean": -318.938720703125,
"KL/std": 139.87318420410156,
"epoch": 0.6722513089005235,
"fcm_dpo/beta": 0.0033561254385858774,
"fcm_dpo/delta": 0.00966968759894371,
"fcm_dpo/margin": 73.42462158203125,
"fcm_dpo/q_t": 0.4437595009803772,
"grad_norm": 83.1728286743164,
"learning_rate": 1.478143389201113e-07,
"logits/chosen": -0.8710657954216003,
"logits/rejected": -0.8343586325645447,
"logps/chosen": -534.4999389648438,
"logps/ref_chosen": -288.98583984375,
"logps/ref_rejected": -241.1822052001953,
"logps/rejected": -560.1209106445312,
"loss": 4.9071,
"margin_dpo/margin_mean": 73.42462158203125,
"margin_dpo/margin_std": 156.1352996826172,
"step": 321
},
{
"KL/chosen_KL_mean": -258.0967712402344,
"KL/mean": -300.7222595214844,
"KL/rejected_KL_mean": -343.3477783203125,
"KL/std": 140.47586059570312,
"epoch": 0.6743455497382199,
"fcm_dpo/beta": 0.0033736673649400473,
"fcm_dpo/delta": -0.00021159928292036057,
"fcm_dpo/margin": 85.25101470947266,
"fcm_dpo/q_t": 0.4336232841014862,
"grad_norm": 78.91681671142578,
"learning_rate": 1.461462467495284e-07,
"logits/chosen": -0.9282336831092834,
"logits/rejected": -0.8913758397102356,
"logps/chosen": -566.6402587890625,
"logps/ref_chosen": -308.54345703125,
"logps/ref_rejected": -269.7995910644531,
"logps/rejected": -613.1473388671875,
"loss": 4.7848,
"margin_dpo/margin_mean": 85.25101470947266,
"margin_dpo/margin_std": 164.19068908691406,
"step": 322
},
{
"KL/chosen_KL_mean": -243.4940185546875,
"KL/mean": -277.5121765136719,
"KL/rejected_KL_mean": -311.53033447265625,
"KL/std": 131.02301025390625,
"epoch": 0.6764397905759162,
"fcm_dpo/beta": 0.0033622642513364553,
"fcm_dpo/delta": -0.0014576911926269531,
"fcm_dpo/margin": 68.03628540039062,
"fcm_dpo/q_t": 0.4480839669704437,
"grad_norm": 83.17462158203125,
"learning_rate": 1.4448372394055246e-07,
"logits/chosen": -0.8875012993812561,
"logits/rejected": -0.8782394528388977,
"logps/chosen": -525.9876708984375,
"logps/ref_chosen": -282.49365234375,
"logps/ref_rejected": -227.7105255126953,
"logps/rejected": -539.2408447265625,
"loss": 4.963,
"margin_dpo/margin_mean": 68.03628540039062,
"margin_dpo/margin_std": 155.5357208251953,
"step": 323
},
{
"KL/chosen_KL_mean": -221.84173583984375,
"KL/mean": -273.66180419921875,
"KL/rejected_KL_mean": -325.4819030761719,
"KL/std": 140.5488739013672,
"epoch": 0.6785340314136126,
"fcm_dpo/beta": 0.0032545297872275114,
"fcm_dpo/delta": -0.03785352408885956,
"fcm_dpo/margin": 103.64017486572266,
"fcm_dpo/q_t": 0.4208952486515045,
"grad_norm": 73.04549407958984,
"learning_rate": 1.428268596492364e-07,
"logits/chosen": -0.8201410174369812,
"logits/rejected": -0.8138597011566162,
"logps/chosen": -461.18011474609375,
"logps/ref_chosen": -239.33836364746094,
"logps/ref_rejected": -230.53775024414062,
"logps/rejected": -556.0196533203125,
"loss": 4.5327,
"margin_dpo/margin_mean": 103.64017486572266,
"margin_dpo/margin_std": 140.1855010986328,
"step": 324
},
{
"KL/chosen_KL_mean": -243.33395385742188,
"KL/mean": -289.827392578125,
"KL/rejected_KL_mean": -336.32080078125,
"KL/std": 146.8050537109375,
"epoch": 0.680628272251309,
"fcm_dpo/beta": 0.003209222573786974,
"fcm_dpo/delta": -0.013333482667803764,
"fcm_dpo/margin": 92.98680114746094,
"fcm_dpo/q_t": 0.4324532747268677,
"grad_norm": 74.4027328491211,
"learning_rate": 1.4117574272818386e-07,
"logits/chosen": -0.8348155617713928,
"logits/rejected": -0.8183823227882385,
"logps/chosen": -523.962890625,
"logps/ref_chosen": -280.62896728515625,
"logps/ref_rejected": -270.5085754394531,
"logps/rejected": -606.829345703125,
"loss": 4.7618,
"margin_dpo/margin_mean": 92.98680114746094,
"margin_dpo/margin_std": 173.72450256347656,
"step": 325
},
{
"KL/chosen_KL_mean": -229.41400146484375,
"KL/mean": -278.8251647949219,
"KL/rejected_KL_mean": -328.236328125,
"KL/std": 138.20025634765625,
"epoch": 0.6827225130890052,
"fcm_dpo/beta": 0.003266323357820511,
"fcm_dpo/delta": 0.0035640615969896317,
"fcm_dpo/margin": 98.82231903076172,
"fcm_dpo/q_t": 0.4255218505859375,
"grad_norm": 87.20018768310547,
"learning_rate": 1.3953046172178413e-07,
"logits/chosen": -0.9498153328895569,
"logits/rejected": -0.940109372138977,
"logps/chosen": -470.401123046875,
"logps/ref_chosen": -240.9871368408203,
"logps/ref_rejected": -261.0238342285156,
"logps/rejected": -589.2601318359375,
"loss": 4.6263,
"margin_dpo/margin_mean": 98.82231903076172,
"margin_dpo/margin_std": 158.7273406982422,
"step": 326
},
{
"KL/chosen_KL_mean": -219.74449157714844,
"KL/mean": -273.5055236816406,
"KL/rejected_KL_mean": -327.26654052734375,
"KL/std": 132.50523376464844,
"epoch": 0.6848167539267016,
"fcm_dpo/beta": 0.0032268206123262644,
"fcm_dpo/delta": -0.021940872073173523,
"fcm_dpo/margin": 107.5220718383789,
"fcm_dpo/q_t": 0.42012837529182434,
"grad_norm": 73.7422103881836,
"learning_rate": 1.3789110486146468e-07,
"logits/chosen": -0.8853604793548584,
"logits/rejected": -0.8608740568161011,
"logps/chosen": -499.2645263671875,
"logps/ref_chosen": -279.52001953125,
"logps/ref_rejected": -269.51824951171875,
"logps/rejected": -596.7847900390625,
"loss": 4.5388,
"margin_dpo/margin_mean": 107.5220718383789,
"margin_dpo/margin_std": 155.50799560546875,
"step": 327
},
{
"KL/chosen_KL_mean": -241.4763641357422,
"KL/mean": -290.4848327636719,
"KL/rejected_KL_mean": -339.4931945800781,
"KL/std": 136.66537475585938,
"epoch": 0.6869109947643979,
"fcm_dpo/beta": 0.0031155140604823828,
"fcm_dpo/delta": -0.02170320227742195,
"fcm_dpo/margin": 98.01687622070312,
"fcm_dpo/q_t": 0.42870888113975525,
"grad_norm": 52.95525360107422,
"learning_rate": 1.362577600609588e-07,
"logits/chosen": -0.8597935438156128,
"logits/rejected": -0.8578972220420837,
"logps/chosen": -542.5098266601562,
"logps/ref_chosen": -301.033447265625,
"logps/ref_rejected": -284.2101135253906,
"logps/rejected": -623.7033081054688,
"loss": 4.6399,
"margin_dpo/margin_mean": 98.01687622070312,
"margin_dpo/margin_std": 149.0442352294922,
"step": 328
},
{
"KL/chosen_KL_mean": -256.7224426269531,
"KL/mean": -309.82830810546875,
"KL/rejected_KL_mean": -362.9341125488281,
"KL/std": 172.28172302246094,
"epoch": 0.6890052356020943,
"fcm_dpo/beta": 0.0030725565738976,
"fcm_dpo/delta": -0.036677245050668716,
"fcm_dpo/margin": 106.211669921875,
"fcm_dpo/q_t": 0.42879265546798706,
"grad_norm": 60.5069694519043,
"learning_rate": 1.3463051491159093e-07,
"logits/chosen": -0.8657263517379761,
"logits/rejected": -0.8348018527030945,
"logps/chosen": -576.7113037109375,
"logps/ref_chosen": -319.9888610839844,
"logps/ref_rejected": -307.5588684082031,
"logps/rejected": -670.4929809570312,
"loss": 4.6954,
"margin_dpo/margin_mean": 106.21165466308594,
"margin_dpo/margin_std": 189.08535766601562,
"step": 329
},
{
"KL/chosen_KL_mean": -247.13705444335938,
"KL/mean": -297.3825378417969,
"KL/rejected_KL_mean": -347.6280517578125,
"KL/std": 152.5355224609375,
"epoch": 0.6910994764397905,
"fcm_dpo/beta": 0.00300287501886487,
"fcm_dpo/delta": -0.02195155620574951,
"fcm_dpo/margin": 100.49103546142578,
"fcm_dpo/q_t": 0.42996516823768616,
"grad_norm": 74.84465789794922,
"learning_rate": 1.3300945667758012e-07,
"logits/chosen": -0.8545703291893005,
"logits/rejected": -0.861172080039978,
"logps/chosen": -548.2518310546875,
"logps/ref_chosen": -301.11474609375,
"logps/ref_rejected": -299.673095703125,
"logps/rejected": -647.3011474609375,
"loss": 4.6614,
"margin_dpo/margin_mean": 100.49103546142578,
"margin_dpo/margin_std": 156.20176696777344,
"step": 330
},
{
"KL/chosen_KL_mean": -239.11048889160156,
"KL/mean": -285.6376647949219,
"KL/rejected_KL_mean": -332.16485595703125,
"KL/std": 151.9279022216797,
"epoch": 0.6931937172774869,
"fcm_dpo/beta": 0.003041815711185336,
"fcm_dpo/delta": 0.047408707439899445,
"fcm_dpo/margin": 93.05432891845703,
"fcm_dpo/q_t": 0.4363221824169159,
"grad_norm": 69.54380798339844,
"learning_rate": 1.3139467229135998e-07,
"logits/chosen": -0.8673312664031982,
"logits/rejected": -0.8451822996139526,
"logps/chosen": -516.7019653320312,
"logps/ref_chosen": -277.59149169921875,
"logps/ref_rejected": -256.025634765625,
"logps/rejected": -588.1904296875,
"loss": 4.7977,
"margin_dpo/margin_mean": 93.05433654785156,
"margin_dpo/margin_std": 185.36294555664062,
"step": 331
},
{
"KL/chosen_KL_mean": -246.1035919189453,
"KL/mean": -294.6551208496094,
"KL/rejected_KL_mean": -343.2066650390625,
"KL/std": 146.26475524902344,
"epoch": 0.6952879581151833,
"fcm_dpo/beta": 0.003083170158788562,
"fcm_dpo/delta": 0.0028741867281496525,
"fcm_dpo/margin": 97.10308837890625,
"fcm_dpo/q_t": 0.4316597878932953,
"grad_norm": 78.57029724121094,
"learning_rate": 1.2978624834891626e-07,
"logits/chosen": -0.8808923363685608,
"logits/rejected": -0.8539749383926392,
"logps/chosen": -516.0772705078125,
"logps/ref_chosen": -269.97369384765625,
"logps/ref_rejected": -235.03164672851562,
"logps/rejected": -578.2383422851562,
"loss": 4.7816,
"margin_dpo/margin_mean": 97.10308837890625,
"margin_dpo/margin_std": 189.04632568359375,
"step": 332
},
{
"KL/chosen_KL_mean": -242.9016571044922,
"KL/mean": -284.91748046875,
"KL/rejected_KL_mean": -326.9332580566406,
"KL/std": 140.8754119873047,
"epoch": 0.6973821989528796,
"fcm_dpo/beta": 0.0031835411209613085,
"fcm_dpo/delta": 0.03588256612420082,
"fcm_dpo/margin": 84.03166961669922,
"fcm_dpo/q_t": 0.4383671283721924,
"grad_norm": 63.44430160522461,
"learning_rate": 1.281842711051438e-07,
"logits/chosen": -0.9540477991104126,
"logits/rejected": -0.9159016609191895,
"logps/chosen": -539.6646728515625,
"logps/ref_chosen": -296.76300048828125,
"logps/ref_rejected": -265.97991943359375,
"logps/rejected": -592.9132080078125,
"loss": 4.808,
"margin_dpo/margin_mean": 84.03166198730469,
"margin_dpo/margin_std": 161.39566040039062,
"step": 333
},
{
"KL/chosen_KL_mean": -236.5258026123047,
"KL/mean": -283.31243896484375,
"KL/rejected_KL_mean": -330.0990295410156,
"KL/std": 143.63352966308594,
"epoch": 0.6994764397905759,
"fcm_dpo/beta": 0.0031920385081321,
"fcm_dpo/delta": -0.011491209268569946,
"fcm_dpo/margin": 93.57325744628906,
"fcm_dpo/q_t": 0.43058937788009644,
"grad_norm": 82.48439025878906,
"learning_rate": 1.2658882646922033e-07,
"logits/chosen": -0.8545200824737549,
"logits/rejected": -0.8265820145606995,
"logps/chosen": -537.5625610351562,
"logps/ref_chosen": -301.0367431640625,
"logps/ref_rejected": -268.87652587890625,
"logps/rejected": -598.9755859375,
"loss": 4.7247,
"margin_dpo/margin_mean": 93.57325744628906,
"margin_dpo/margin_std": 160.36260986328125,
"step": 334
},
{
"KL/chosen_KL_mean": -257.5293273925781,
"KL/mean": -303.568359375,
"KL/rejected_KL_mean": -349.6073913574219,
"KL/std": 131.11956787109375,
"epoch": 0.7015706806282722,
"fcm_dpo/beta": 0.003155500628054142,
"fcm_dpo/delta": 0.008357154205441475,
"fcm_dpo/margin": 92.07801818847656,
"fcm_dpo/q_t": 0.43331050872802734,
"grad_norm": 83.25331115722656,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.8444750308990479,
"logits/rejected": -0.8382730484008789,
"logps/chosen": -533.662109375,
"logps/ref_chosen": -276.13275146484375,
"logps/ref_rejected": -243.44203186035156,
"logps/rejected": -593.04931640625,
"loss": 4.7586,
"margin_dpo/margin_mean": 92.07801818847656,
"margin_dpo/margin_std": 167.62142944335938,
"step": 335
},
{
"KL/chosen_KL_mean": -253.9612579345703,
"KL/mean": -301.3621520996094,
"KL/rejected_KL_mean": -348.76300048828125,
"KL/std": 167.3890380859375,
"epoch": 0.7036649214659686,
"fcm_dpo/beta": 0.0031551639549434185,
"fcm_dpo/delta": -0.024496689438819885,
"fcm_dpo/margin": 94.80172729492188,
"fcm_dpo/q_t": 0.43416231870651245,
"grad_norm": 78.45794677734375,
"learning_rate": 1.2341787690142435e-07,
"logits/chosen": -0.8579354286193848,
"logits/rejected": -0.7981801629066467,
"logps/chosen": -500.22393798828125,
"logps/ref_chosen": -246.2626495361328,
"logps/ref_rejected": -261.0617980957031,
"logps/rejected": -609.8248901367188,
"loss": 4.7654,
"margin_dpo/margin_mean": 94.80172729492188,
"margin_dpo/margin_std": 181.49172973632812,
"step": 336
},
{
"KL/chosen_KL_mean": -232.01927185058594,
"KL/mean": -287.64947509765625,
"KL/rejected_KL_mean": -343.2796630859375,
"KL/std": 140.15916442871094,
"epoch": 0.7057591623036649,
"fcm_dpo/beta": 0.0031573735177516937,
"fcm_dpo/delta": -0.02179955132305622,
"fcm_dpo/margin": 111.26039123535156,
"fcm_dpo/q_t": 0.418662965297699,
"grad_norm": 85.16871643066406,
"learning_rate": 1.2184254201795363e-07,
"logits/chosen": -0.873991072177887,
"logits/rejected": -0.8448648452758789,
"logps/chosen": -499.0130615234375,
"logps/ref_chosen": -266.9937744140625,
"logps/ref_rejected": -253.015625,
"logps/rejected": -596.2952270507812,
"loss": 4.5231,
"margin_dpo/margin_mean": 111.26039123535156,
"margin_dpo/margin_std": 158.54148864746094,
"step": 337
},
{
"KL/chosen_KL_mean": -224.66920471191406,
"KL/mean": -278.88800048828125,
"KL/rejected_KL_mean": -333.1067810058594,
"KL/std": 147.4876708984375,
"epoch": 0.7078534031413612,
"fcm_dpo/beta": 0.003021388780325651,
"fcm_dpo/delta": -0.016518374904990196,
"fcm_dpo/margin": 108.43759155273438,
"fcm_dpo/q_t": 0.4243887960910797,
"grad_norm": 113.49219512939453,
"learning_rate": 1.202740798300168e-07,
"logits/chosen": -0.874689519405365,
"logits/rejected": -0.8554161190986633,
"logps/chosen": -501.26177978515625,
"logps/ref_chosen": -276.5925598144531,
"logps/ref_rejected": -233.979248046875,
"logps/rejected": -567.0860595703125,
"loss": 4.5851,
"margin_dpo/margin_mean": 108.4375991821289,
"margin_dpo/margin_std": 163.255859375,
"step": 338
},
{
"KL/chosen_KL_mean": -231.06105041503906,
"KL/mean": -289.5965576171875,
"KL/rejected_KL_mean": -348.13214111328125,
"KL/std": 148.72731018066406,
"epoch": 0.7099476439790576,
"fcm_dpo/beta": 0.0030209606047719717,
"fcm_dpo/delta": -0.028637029230594635,
"fcm_dpo/margin": 117.071044921875,
"fcm_dpo/q_t": 0.4185601770877838,
"grad_norm": 68.01354217529297,
"learning_rate": 1.1871257444948096e-07,
"logits/chosen": -0.8962268233299255,
"logits/rejected": -0.880439817905426,
"logps/chosen": -534.5888061523438,
"logps/ref_chosen": -303.5277404785156,
"logps/ref_rejected": -283.11676025390625,
"logps/rejected": -631.2489013671875,
"loss": 4.5252,
"margin_dpo/margin_mean": 117.071044921875,
"margin_dpo/margin_std": 171.6200408935547,
"step": 339
},
{
"KL/chosen_KL_mean": -246.8760986328125,
"KL/mean": -289.718017578125,
"KL/rejected_KL_mean": -332.5599670410156,
"KL/std": 158.76173400878906,
"epoch": 0.7120418848167539,
"fcm_dpo/beta": 0.0029858858324587345,
"fcm_dpo/delta": 0.03210965916514397,
"fcm_dpo/margin": 85.68387603759766,
"fcm_dpo/q_t": 0.44048047065734863,
"grad_norm": 123.83653259277344,
"learning_rate": 1.1715810961514072e-07,
"logits/chosen": -0.8333358764648438,
"logits/rejected": -0.8279672861099243,
"logps/chosen": -508.40185546875,
"logps/ref_chosen": -261.5257568359375,
"logps/ref_rejected": -259.39862060546875,
"logps/rejected": -591.9585571289062,
"loss": 4.9309,
"margin_dpo/margin_mean": 85.68387603759766,
"margin_dpo/margin_std": 192.60916137695312,
"step": 340
},
{
"KL/chosen_KL_mean": -277.3525390625,
"KL/mean": -318.425048828125,
"KL/rejected_KL_mean": -359.4975891113281,
"KL/std": 176.37754821777344,
"epoch": 0.7141361256544503,
"fcm_dpo/beta": 0.00305312592536211,
"fcm_dpo/delta": 0.0065925233066082,
"fcm_dpo/margin": 82.14506530761719,
"fcm_dpo/q_t": 0.44349971413612366,
"grad_norm": 63.160316467285156,
"learning_rate": 1.1561076868822755e-07,
"logits/chosen": -0.8536667227745056,
"logits/rejected": -0.8266552686691284,
"logps/chosen": -593.2560424804688,
"logps/ref_chosen": -315.903564453125,
"logps/ref_rejected": -308.02392578125,
"logps/rejected": -667.521484375,
"loss": 4.985,
"margin_dpo/margin_mean": 82.14506530761719,
"margin_dpo/margin_std": 199.24632263183594,
"step": 341
},
{
"KL/chosen_KL_mean": -232.24563598632812,
"KL/mean": -278.6511535644531,
"KL/rejected_KL_mean": -325.056640625,
"KL/std": 141.9452667236328,
"epoch": 0.7162303664921466,
"fcm_dpo/beta": 0.0031279418617486954,
"fcm_dpo/delta": 0.03749464452266693,
"fcm_dpo/margin": 92.81096649169922,
"fcm_dpo/q_t": 0.4330783486366272,
"grad_norm": 101.11497497558594,
"learning_rate": 1.1407063464793965e-07,
"logits/chosen": -0.8642405271530151,
"logits/rejected": -0.8597648739814758,
"logps/chosen": -501.42431640625,
"logps/ref_chosen": -269.17864990234375,
"logps/ref_rejected": -260.8977355957031,
"logps/rejected": -585.954345703125,
"loss": 4.7053,
"margin_dpo/margin_mean": 92.81096649169922,
"margin_dpo/margin_std": 154.14793395996094,
"step": 342
},
{
"KL/chosen_KL_mean": -244.72955322265625,
"KL/mean": -282.2915954589844,
"KL/rejected_KL_mean": -319.8536376953125,
"KL/std": 139.9641876220703,
"epoch": 0.7183246073298429,
"fcm_dpo/beta": 0.003217648947611451,
"fcm_dpo/delta": 0.0457039512693882,
"fcm_dpo/margin": 75.12411499023438,
"fcm_dpo/q_t": 0.4435161352157593,
"grad_norm": 102.95614624023438,
"learning_rate": 1.125377900869913e-07,
"logits/chosen": -0.859501302242279,
"logits/rejected": -0.8335797786712646,
"logps/chosen": -555.4495239257812,
"logps/ref_chosen": -310.719970703125,
"logps/ref_rejected": -263.5224914550781,
"logps/rejected": -583.3761596679688,
"loss": 4.8569,
"margin_dpo/margin_mean": 75.12410736083984,
"margin_dpo/margin_std": 147.29708862304688,
"step": 343
},
{
"KL/chosen_KL_mean": -240.10861206054688,
"KL/mean": -283.4324951171875,
"KL/rejected_KL_mean": -326.75640869140625,
"KL/std": 134.30642700195312,
"epoch": 0.7204188481675393,
"fcm_dpo/beta": 0.0033485074527561665,
"fcm_dpo/delta": 0.016562025994062424,
"fcm_dpo/margin": 86.64774322509766,
"fcm_dpo/q_t": 0.43293195962905884,
"grad_norm": 90.93418884277344,
"learning_rate": 1.110123172071844e-07,
"logits/chosen": -0.8508941531181335,
"logits/rejected": -0.8322650790214539,
"logps/chosen": -541.9085693359375,
"logps/ref_chosen": -301.7999267578125,
"logps/ref_rejected": -257.9061584472656,
"logps/rejected": -584.6625366210938,
"loss": 4.7599,
"margin_dpo/margin_mean": 86.64774322509766,
"margin_dpo/margin_std": 159.52081298828125,
"step": 344
},
{
"KL/chosen_KL_mean": -227.98085021972656,
"KL/mean": -269.60870361328125,
"KL/rejected_KL_mean": -311.236572265625,
"KL/std": 137.84278869628906,
"epoch": 0.7225130890052356,
"fcm_dpo/beta": 0.003270794404670596,
"fcm_dpo/delta": -0.04042273014783859,
"fcm_dpo/margin": 83.2557373046875,
"fcm_dpo/q_t": 0.4363884925842285,
"grad_norm": 61.989376068115234,
"learning_rate": 1.09494297815e-07,
"logits/chosen": -0.8617357611656189,
"logits/rejected": -0.86259526014328,
"logps/chosen": -510.9992980957031,
"logps/ref_chosen": -283.0184326171875,
"logps/ref_rejected": -266.8457336425781,
"logps/rejected": -578.082275390625,
"loss": 4.772,
"margin_dpo/margin_mean": 83.25572967529297,
"margin_dpo/margin_std": 138.2306365966797,
"step": 345
},
{
"KL/chosen_KL_mean": -242.52352905273438,
"KL/mean": -294.254638671875,
"KL/rejected_KL_mean": -345.9857177734375,
"KL/std": 155.80648803710938,
"epoch": 0.724607329842932,
"fcm_dpo/beta": 0.0030984547920525074,
"fcm_dpo/delta": -0.06594164669513702,
"fcm_dpo/margin": 103.4621810913086,
"fcm_dpo/q_t": 0.4260784685611725,
"grad_norm": 74.91835021972656,
"learning_rate": 1.0798381331721107e-07,
"logits/chosen": -0.9356507062911987,
"logits/rejected": -0.883700966835022,
"logps/chosen": -510.96478271484375,
"logps/ref_chosen": -268.44122314453125,
"logps/ref_rejected": -227.8225860595703,
"logps/rejected": -573.8082885742188,
"loss": 4.6604,
"margin_dpo/margin_mean": 103.4621810913086,
"margin_dpo/margin_std": 165.5200958251953,
"step": 346
},
{
"KL/chosen_KL_mean": -224.31581115722656,
"KL/mean": -266.2325439453125,
"KL/rejected_KL_mean": -308.1493225097656,
"KL/std": 134.7196044921875,
"epoch": 0.7267015706806282,
"fcm_dpo/beta": 0.003096605185419321,
"fcm_dpo/delta": 0.03434152901172638,
"fcm_dpo/margin": 83.83348846435547,
"fcm_dpo/q_t": 0.43899720907211304,
"grad_norm": 97.60405731201172,
"learning_rate": 1.0648094471651722e-07,
"logits/chosen": -0.7862703800201416,
"logits/rejected": -0.8122860193252563,
"logps/chosen": -498.0193786621094,
"logps/ref_chosen": -273.70355224609375,
"logps/ref_rejected": -243.65521240234375,
"logps/rejected": -551.8045043945312,
"loss": 4.805,
"margin_dpo/margin_mean": 83.83348846435547,
"margin_dpo/margin_std": 149.16348266601562,
"step": 347
},
{
"KL/chosen_KL_mean": -224.01858520507812,
"KL/mean": -263.1257019042969,
"KL/rejected_KL_mean": -302.2328186035156,
"KL/std": 143.7584686279297,
"epoch": 0.7287958115183246,
"fcm_dpo/beta": 0.0031398944556713104,
"fcm_dpo/delta": 0.013123849406838417,
"fcm_dpo/margin": 78.21422576904297,
"fcm_dpo/q_t": 0.44496262073516846,
"grad_norm": 60.390602111816406,
"learning_rate": 1.0498577260720048e-07,
"logits/chosen": -0.8914543390274048,
"logits/rejected": -0.8751494884490967,
"logps/chosen": -509.6600036621094,
"logps/ref_chosen": -285.64141845703125,
"logps/ref_rejected": -265.6270446777344,
"logps/rejected": -567.85986328125,
"loss": 4.8846,
"margin_dpo/margin_mean": 78.2142333984375,
"margin_dpo/margin_std": 162.25692749023438,
"step": 348
},
{
"KL/chosen_KL_mean": -223.03997802734375,
"KL/mean": -281.030517578125,
"KL/rejected_KL_mean": -339.02105712890625,
"KL/std": 154.15811157226562,
"epoch": 0.7308900523560209,
"fcm_dpo/beta": 0.003127102041617036,
"fcm_dpo/delta": -0.03881394863128662,
"fcm_dpo/margin": 115.98108673095703,
"fcm_dpo/q_t": 0.41518571972846985,
"grad_norm": 71.31893157958984,
"learning_rate": 1.0349837717080347e-07,
"logits/chosen": -0.8437027335166931,
"logits/rejected": -0.8291211724281311,
"logps/chosen": -551.3574829101562,
"logps/ref_chosen": -328.3175048828125,
"logps/ref_rejected": -292.37872314453125,
"logps/rejected": -631.3997802734375,
"loss": 4.5358,
"margin_dpo/margin_mean": 115.98108673095703,
"margin_dpo/margin_std": 175.9039306640625,
"step": 349
},
{
"KL/chosen_KL_mean": -236.86434936523438,
"KL/mean": -285.975830078125,
"KL/rejected_KL_mean": -335.08734130859375,
"KL/std": 149.2736053466797,
"epoch": 0.7329842931937173,
"fcm_dpo/beta": 0.003120012581348419,
"fcm_dpo/delta": 0.012096697464585304,
"fcm_dpo/margin": 98.22299194335938,
"fcm_dpo/q_t": 0.42966753244400024,
"grad_norm": 92.44560241699219,
"learning_rate": 1.0201883817182949e-07,
"logits/chosen": -0.8361937999725342,
"logits/rejected": -0.8472439646720886,
"logps/chosen": -529.6690063476562,
"logps/ref_chosen": -292.8046569824219,
"logps/ref_rejected": -250.35504150390625,
"logps/rejected": -585.4423217773438,
"loss": 4.6988,
"margin_dpo/margin_mean": 98.2229995727539,
"margin_dpo/margin_std": 171.5211944580078,
"step": 350
},
{
"KL/chosen_KL_mean": -232.821044921875,
"KL/mean": -262.088134765625,
"KL/rejected_KL_mean": -291.3552551269531,
"KL/std": 137.2724609375,
"epoch": 0.7350785340314137,
"fcm_dpo/beta": 0.0031031551770865917,
"fcm_dpo/delta": 0.0010303156450390816,
"fcm_dpo/margin": 58.534210205078125,
"fcm_dpo/q_t": 0.45783162117004395,
"grad_norm": 53.25884246826172,
"learning_rate": 1.0054723495346482e-07,
"logits/chosen": -0.8983533382415771,
"logits/rejected": -0.8776514530181885,
"logps/chosen": -544.7100219726562,
"logps/ref_chosen": -311.8890380859375,
"logps/ref_rejected": -263.59033203125,
"logps/rejected": -554.945556640625,
"loss": 5.1347,
"margin_dpo/margin_mean": 58.534210205078125,
"margin_dpo/margin_std": 169.1748046875,
"step": 351
},
{
"KL/chosen_KL_mean": -216.96815490722656,
"KL/mean": -275.911865234375,
"KL/rejected_KL_mean": -334.8555603027344,
"KL/std": 151.17433166503906,
"epoch": 0.7371727748691099,
"fcm_dpo/beta": 0.0030314219184219837,
"fcm_dpo/delta": -0.031058115884661674,
"fcm_dpo/margin": 117.88742065429688,
"fcm_dpo/q_t": 0.4165811240673065,
"grad_norm": 51.545772552490234,
"learning_rate": 9.908364643332398e-08,
"logits/chosen": -0.8226331472396851,
"logits/rejected": -0.7918906211853027,
"logps/chosen": -471.87603759765625,
"logps/ref_chosen": -254.9078826904297,
"logps/ref_rejected": -257.1688232421875,
"logps/rejected": -592.0244140625,
"loss": 4.5028,
"margin_dpo/margin_mean": 117.88742065429688,
"margin_dpo/margin_std": 162.4287872314453,
"step": 352
},
{
"KL/chosen_KL_mean": -233.33163452148438,
"KL/mean": -281.92626953125,
"KL/rejected_KL_mean": -330.5209045410156,
"KL/std": 144.54786682128906,
"epoch": 0.7392670157068063,
"fcm_dpo/beta": 0.002987043932080269,
"fcm_dpo/delta": -0.027638321742415428,
"fcm_dpo/margin": 97.18927764892578,
"fcm_dpo/q_t": 0.43258699774742126,
"grad_norm": 134.96693420410156,
"learning_rate": 9.76281510992176e-08,
"logits/chosen": -0.8499235510826111,
"logits/rejected": -0.8441857695579529,
"logps/chosen": -503.70770263671875,
"logps/ref_chosen": -270.3760681152344,
"logps/ref_rejected": -264.65234375,
"logps/rejected": -595.1732177734375,
"loss": 4.721,
"margin_dpo/margin_mean": 97.18927001953125,
"margin_dpo/margin_std": 162.46739196777344,
"step": 353
},
{
"KL/chosen_KL_mean": -230.3787841796875,
"KL/mean": -269.59588623046875,
"KL/rejected_KL_mean": -308.8129577636719,
"KL/std": 154.2929229736328,
"epoch": 0.7413612565445026,
"fcm_dpo/beta": 0.003026010002940893,
"fcm_dpo/delta": 0.03974125534296036,
"fcm_dpo/margin": 78.43419647216797,
"fcm_dpo/q_t": 0.4459616541862488,
"grad_norm": 68.59556579589844,
"learning_rate": 9.618082700494318e-08,
"logits/chosen": -0.838015079498291,
"logits/rejected": -0.8697560429573059,
"logps/chosen": -488.02734375,
"logps/ref_chosen": -257.6485595703125,
"logps/ref_rejected": -246.94203186035156,
"logps/rejected": -555.7550048828125,
"loss": 4.9336,
"margin_dpo/margin_mean": 78.43421173095703,
"margin_dpo/margin_std": 173.0082550048828,
"step": 354
},
{
"KL/chosen_KL_mean": -236.45423889160156,
"KL/mean": -288.6083984375,
"KL/rejected_KL_mean": -340.7625732421875,
"KL/std": 159.89952087402344,
"epoch": 0.743455497382199,
"fcm_dpo/beta": 0.0030326605774462223,
"fcm_dpo/delta": 0.006990807130932808,
"fcm_dpo/margin": 104.30831909179688,
"fcm_dpo/q_t": 0.42733824253082275,
"grad_norm": 103.6706771850586,
"learning_rate": 9.474175176609956e-08,
"logits/chosen": -0.8978556394577026,
"logits/rejected": -0.8931483030319214,
"logps/chosen": -529.8076171875,
"logps/ref_chosen": -293.35333251953125,
"logps/ref_rejected": -275.6051940917969,
"logps/rejected": -616.3677368164062,
"loss": 4.7378,
"margin_dpo/margin_mean": 104.30831909179688,
"margin_dpo/margin_std": 191.57334899902344,
"step": 355
},
{
"KL/chosen_KL_mean": -218.8563690185547,
"KL/mean": -256.1012268066406,
"KL/rejected_KL_mean": -293.34613037109375,
"KL/std": 133.4541473388672,
"epoch": 0.7455497382198953,
"fcm_dpo/beta": 0.0030662843491882086,
"fcm_dpo/delta": 0.013114407658576965,
"fcm_dpo/margin": 74.48975372314453,
"fcm_dpo/q_t": 0.4463121294975281,
"grad_norm": 82.64666748046875,
"learning_rate": 9.331100255592436e-08,
"logits/chosen": -0.8024091124534607,
"logits/rejected": -0.8281019926071167,
"logps/chosen": -423.11187744140625,
"logps/ref_chosen": -204.25550842285156,
"logps/ref_rejected": -213.467529296875,
"logps/rejected": -506.8136291503906,
"loss": 4.8785,
"margin_dpo/margin_mean": 74.48975372314453,
"margin_dpo/margin_std": 138.68128967285156,
"step": 356
},
{
"KL/chosen_KL_mean": -243.69976806640625,
"KL/mean": -292.585205078125,
"KL/rejected_KL_mean": -341.47064208984375,
"KL/std": 151.95843505859375,
"epoch": 0.7476439790575916,
"fcm_dpo/beta": 0.0030811114702373743,
"fcm_dpo/delta": -0.00847620703279972,
"fcm_dpo/margin": 97.77091979980469,
"fcm_dpo/q_t": 0.4313168227672577,
"grad_norm": 77.11827850341797,
"learning_rate": 9.18886561011557e-08,
"logits/chosen": -0.7848935723304749,
"logits/rejected": -0.7811943888664246,
"logps/chosen": -510.0703125,
"logps/ref_chosen": -266.3705749511719,
"logps/ref_rejected": -239.04490661621094,
"logps/rejected": -580.5155639648438,
"loss": 4.7273,
"margin_dpo/margin_mean": 97.77091979980469,
"margin_dpo/margin_std": 172.3494873046875,
"step": 357
},
{
"KL/chosen_KL_mean": -218.75701904296875,
"KL/mean": -275.1843566894531,
"KL/rejected_KL_mean": -331.6116943359375,
"KL/std": 139.84754943847656,
"epoch": 0.749738219895288,
"fcm_dpo/beta": 0.0030688412953168154,
"fcm_dpo/delta": -0.01982707343995571,
"fcm_dpo/margin": 112.85469818115234,
"fcm_dpo/q_t": 0.41943082213401794,
"grad_norm": 91.5672836303711,
"learning_rate": 9.047478867791731e-08,
"logits/chosen": -0.8832120895385742,
"logits/rejected": -0.8587401509284973,
"logps/chosen": -517.9044799804688,
"logps/ref_chosen": -299.1474609375,
"logps/ref_rejected": -257.2531433105469,
"logps/rejected": -588.8648071289062,
"loss": 4.5521,
"margin_dpo/margin_mean": 112.85469818115234,
"margin_dpo/margin_std": 167.56106567382812,
"step": 358
},
{
"KL/chosen_KL_mean": -230.01956176757812,
"KL/mean": -279.5022888183594,
"KL/rejected_KL_mean": -328.9850158691406,
"KL/std": 144.20054626464844,
"epoch": 0.7518324607329843,
"fcm_dpo/beta": 0.0030328300781548023,
"fcm_dpo/delta": -0.007977485656738281,
"fcm_dpo/margin": 98.9654312133789,
"fcm_dpo/q_t": 0.4301950931549072,
"grad_norm": 80.08256530761719,
"learning_rate": 8.906947610762825e-08,
"logits/chosen": -0.8496133089065552,
"logits/rejected": -0.8573009967803955,
"logps/chosen": -533.0174560546875,
"logps/ref_chosen": -302.99786376953125,
"logps/ref_rejected": -260.4137268066406,
"logps/rejected": -589.3987426757812,
"loss": 4.7148,
"margin_dpo/margin_mean": 98.9654312133789,
"margin_dpo/margin_std": 172.8538055419922,
"step": 359
},
{
"KL/chosen_KL_mean": -234.53375244140625,
"KL/mean": -281.4641418457031,
"KL/rejected_KL_mean": -328.39453125,
"KL/std": 157.81326293945312,
"epoch": 0.7539267015706806,
"fcm_dpo/beta": 0.0029506399296224117,
"fcm_dpo/delta": -0.040261730551719666,
"fcm_dpo/margin": 93.8608169555664,
"fcm_dpo/q_t": 0.4368816614151001,
"grad_norm": 72.47200775146484,
"learning_rate": 8.76727937529367e-08,
"logits/chosen": -0.8603860139846802,
"logits/rejected": -0.8375499844551086,
"logps/chosen": -544.1452026367188,
"logps/ref_chosen": -309.6114501953125,
"logps/ref_rejected": -256.64031982421875,
"logps/rejected": -585.0348510742188,
"loss": 4.8678,
"margin_dpo/margin_mean": 93.8608169555664,
"margin_dpo/margin_std": 188.05604553222656,
"step": 360
},
{
"KL/chosen_KL_mean": -199.42823791503906,
"KL/mean": -260.2237548828125,
"KL/rejected_KL_mean": -321.0193176269531,
"KL/std": 154.24195861816406,
"epoch": 0.7560209424083769,
"fcm_dpo/beta": 0.0029223968740552664,
"fcm_dpo/delta": -0.005785183981060982,
"fcm_dpo/margin": 121.59111022949219,
"fcm_dpo/q_t": 0.4192415475845337,
"grad_norm": 47.21493911743164,
"learning_rate": 8.628481651367875e-08,
"logits/chosen": -0.7954362630844116,
"logits/rejected": -0.7759599089622498,
"logps/chosen": -462.8079833984375,
"logps/ref_chosen": -263.3797607421875,
"logps/ref_rejected": -271.18157958984375,
"logps/rejected": -592.2008666992188,
"loss": 4.5488,
"margin_dpo/margin_mean": 121.59109497070312,
"margin_dpo/margin_std": 189.3097381591797,
"step": 361
},
{
"KL/chosen_KL_mean": -222.83444213867188,
"KL/mean": -270.36297607421875,
"KL/rejected_KL_mean": -317.89154052734375,
"KL/std": 126.81389617919922,
"epoch": 0.7581151832460733,
"fcm_dpo/beta": 0.0029204203747212887,
"fcm_dpo/delta": 0.0328575074672699,
"fcm_dpo/margin": 95.0571060180664,
"fcm_dpo/q_t": 0.43568283319473267,
"grad_norm": 72.48114776611328,
"learning_rate": 8.490561882286135e-08,
"logits/chosen": -0.8405733704566956,
"logits/rejected": -0.822902262210846,
"logps/chosen": -526.0927734375,
"logps/ref_chosen": -303.2583923339844,
"logps/ref_rejected": -243.22891235351562,
"logps/rejected": -561.1204223632812,
"loss": 4.7016,
"margin_dpo/margin_mean": 95.05711364746094,
"margin_dpo/margin_std": 146.4384765625,
"step": 362
},
{
"KL/chosen_KL_mean": -226.85987854003906,
"KL/mean": -280.93182373046875,
"KL/rejected_KL_mean": -335.0037536621094,
"KL/std": 155.6280517578125,
"epoch": 0.7602094240837697,
"fcm_dpo/beta": 0.0029316996224224567,
"fcm_dpo/delta": -0.0422758013010025,
"fcm_dpo/margin": 108.14390563964844,
"fcm_dpo/q_t": 0.427703857421875,
"grad_norm": 81.47772216796875,
"learning_rate": 8.353527464267104e-08,
"logits/chosen": -0.8603069186210632,
"logits/rejected": -0.8154016137123108,
"logps/chosen": -530.2070922851562,
"logps/ref_chosen": -303.34722900390625,
"logps/ref_rejected": -262.05419921875,
"logps/rejected": -597.0579223632812,
"loss": 4.6611,
"margin_dpo/margin_mean": 108.14390563964844,
"margin_dpo/margin_std": 177.33998107910156,
"step": 363
},
{
"KL/chosen_KL_mean": -257.1740417480469,
"KL/mean": -305.13641357421875,
"KL/rejected_KL_mean": -353.0987243652344,
"KL/std": 165.08631896972656,
"epoch": 0.762303664921466,
"fcm_dpo/beta": 0.0028783483430743217,
"fcm_dpo/delta": 0.0030604167841374874,
"fcm_dpo/margin": 95.92469787597656,
"fcm_dpo/q_t": 0.43887603282928467,
"grad_norm": 71.76043701171875,
"learning_rate": 8.217385746050742e-08,
"logits/chosen": -0.8362113833427429,
"logits/rejected": -0.8350046873092651,
"logps/chosen": -542.7178344726562,
"logps/ref_chosen": -285.54376220703125,
"logps/ref_rejected": -284.84619140625,
"logps/rejected": -637.9449462890625,
"loss": 4.8453,
"margin_dpo/margin_mean": 95.92469024658203,
"margin_dpo/margin_std": 200.49720764160156,
"step": 364
},
{
"KL/chosen_KL_mean": -238.0579376220703,
"KL/mean": -284.226806640625,
"KL/rejected_KL_mean": -330.3956604003906,
"KL/std": 158.41650390625,
"epoch": 0.7643979057591623,
"fcm_dpo/beta": 0.0029075169004499912,
"fcm_dpo/delta": 0.017943935468792915,
"fcm_dpo/margin": 92.33773803710938,
"fcm_dpo/q_t": 0.43834882974624634,
"grad_norm": 86.59075164794922,
"learning_rate": 8.082144028504231e-08,
"logits/chosen": -0.8370366096496582,
"logits/rejected": -0.835037112236023,
"logps/chosen": -512.8457641601562,
"logps/ref_chosen": -274.7878112792969,
"logps/ref_rejected": -256.5738220214844,
"logps/rejected": -586.9694213867188,
"loss": 4.8049,
"margin_dpo/margin_mean": 92.33773803710938,
"margin_dpo/margin_std": 173.68531799316406,
"step": 365
},
{
"KL/chosen_KL_mean": -224.99862670898438,
"KL/mean": -273.92193603515625,
"KL/rejected_KL_mean": -322.84521484375,
"KL/std": 138.87074279785156,
"epoch": 0.7664921465968586,
"fcm_dpo/beta": 0.002992146648466587,
"fcm_dpo/delta": 0.0403238870203495,
"fcm_dpo/margin": 97.8465576171875,
"fcm_dpo/q_t": 0.43188464641571045,
"grad_norm": 69.0442886352539,
"learning_rate": 7.947809564230445e-08,
"logits/chosen": -0.8111105561256409,
"logits/rejected": -0.8164764046669006,
"logps/chosen": -511.64825439453125,
"logps/ref_chosen": -286.6496276855469,
"logps/ref_rejected": -251.97140502929688,
"logps/rejected": -574.816650390625,
"loss": 4.7061,
"margin_dpo/margin_mean": 97.8465576171875,
"margin_dpo/margin_std": 166.59800720214844,
"step": 366
},
{
"KL/chosen_KL_mean": -225.3688201904297,
"KL/mean": -279.73223876953125,
"KL/rejected_KL_mean": -334.09564208984375,
"KL/std": 145.0337371826172,
"epoch": 0.768586387434555,
"fcm_dpo/beta": 0.002969947410747409,
"fcm_dpo/delta": -0.022818509489297867,
"fcm_dpo/margin": 108.726806640625,
"fcm_dpo/q_t": 0.42565882205963135,
"grad_norm": 117.63641357421875,
"learning_rate": 7.814389557179016e-08,
"logits/chosen": -0.834640383720398,
"logits/rejected": -0.8060700297355652,
"logps/chosen": -527.3137817382812,
"logps/ref_chosen": -301.9449768066406,
"logps/ref_rejected": -265.5677185058594,
"logps/rejected": -599.663330078125,
"loss": 4.6168,
"margin_dpo/margin_mean": 108.72679901123047,
"margin_dpo/margin_std": 170.5776824951172,
"step": 367
},
{
"KL/chosen_KL_mean": -209.96340942382812,
"KL/mean": -275.3994445800781,
"KL/rejected_KL_mean": -340.83544921875,
"KL/std": 145.23342895507812,
"epoch": 0.7706806282722513,
"fcm_dpo/beta": 0.0029076621867716312,
"fcm_dpo/delta": -0.05346748232841492,
"fcm_dpo/margin": 130.87203979492188,
"fcm_dpo/q_t": 0.4109519422054291,
"grad_norm": 84.33660888671875,
"learning_rate": 7.681891162260015e-08,
"logits/chosen": -0.8131657838821411,
"logits/rejected": -0.8158013820648193,
"logps/chosen": -504.5899353027344,
"logps/ref_chosen": -294.62652587890625,
"logps/ref_rejected": -258.7628479003906,
"logps/rejected": -599.5983276367188,
"loss": 4.3871,
"margin_dpo/margin_mean": 130.87203979492188,
"margin_dpo/margin_std": 156.81239318847656,
"step": 368
},
{
"KL/chosen_KL_mean": -225.32662963867188,
"KL/mean": -272.04248046875,
"KL/rejected_KL_mean": -318.75836181640625,
"KL/std": 135.3495635986328,
"epoch": 0.7727748691099476,
"fcm_dpo/beta": 0.002850665245205164,
"fcm_dpo/delta": 0.019876670092344284,
"fcm_dpo/margin": 93.43172454833984,
"fcm_dpo/q_t": 0.43696144223213196,
"grad_norm": 69.02677154541016,
"learning_rate": 7.550321484960251e-08,
"logits/chosen": -0.8901256322860718,
"logits/rejected": -0.8731386065483093,
"logps/chosen": -507.8323974609375,
"logps/ref_chosen": -282.5057373046875,
"logps/ref_rejected": -266.41607666015625,
"logps/rejected": -585.1744384765625,
"loss": 4.7213,
"margin_dpo/margin_mean": 93.43172454833984,
"margin_dpo/margin_std": 143.53280639648438,
"step": 369
},
{
"KL/chosen_KL_mean": -217.40830993652344,
"KL/mean": -273.412109375,
"KL/rejected_KL_mean": -329.4158630371094,
"KL/std": 149.15206909179688,
"epoch": 0.774869109947644,
"fcm_dpo/beta": 0.002886255271732807,
"fcm_dpo/delta": -0.009045472368597984,
"fcm_dpo/margin": 112.00758361816406,
"fcm_dpo/q_t": 0.4255586564540863,
"grad_norm": 58.84016799926758,
"learning_rate": 7.419687580962222e-08,
"logits/chosen": -0.8674653172492981,
"logits/rejected": -0.8908781409263611,
"logps/chosen": -468.41473388671875,
"logps/ref_chosen": -251.00640869140625,
"logps/ref_rejected": -238.12542724609375,
"logps/rejected": -567.541259765625,
"loss": 4.6418,
"margin_dpo/margin_mean": 112.00758361816406,
"margin_dpo/margin_std": 187.31283569335938,
"step": 370
},
{
"KL/chosen_KL_mean": -239.92498779296875,
"KL/mean": -274.255126953125,
"KL/rejected_KL_mean": -308.5852966308594,
"KL/std": 141.81640625,
"epoch": 0.7769633507853403,
"fcm_dpo/beta": 0.0028966227546334267,
"fcm_dpo/delta": 0.007331667002290487,
"fcm_dpo/margin": 68.66031646728516,
"fcm_dpo/q_t": 0.4531916677951813,
"grad_norm": 86.75391387939453,
"learning_rate": 7.289996455765748e-08,
"logits/chosen": -0.8269072771072388,
"logits/rejected": -0.8060893416404724,
"logps/chosen": -536.5841064453125,
"logps/ref_chosen": -296.6591491699219,
"logps/ref_rejected": -251.14675903320312,
"logps/rejected": -559.7320556640625,
"loss": 4.9955,
"margin_dpo/margin_mean": 68.66031646728516,
"margin_dpo/margin_std": 151.32308959960938,
"step": 371
},
{
"KL/chosen_KL_mean": -232.09654235839844,
"KL/mean": -288.6462097167969,
"KL/rejected_KL_mean": -345.19586181640625,
"KL/std": 156.81259155273438,
"epoch": 0.7790575916230367,
"fcm_dpo/beta": 0.0028610192239284515,
"fcm_dpo/delta": 0.002475677989423275,
"fcm_dpo/margin": 113.09928894042969,
"fcm_dpo/q_t": 0.4260843098163605,
"grad_norm": 66.6779556274414,
"learning_rate": 7.161255064312283e-08,
"logits/chosen": -0.7979535460472107,
"logits/rejected": -0.783908486366272,
"logps/chosen": -563.4679565429688,
"logps/ref_chosen": -331.3714599609375,
"logps/ref_rejected": -285.56805419921875,
"logps/rejected": -630.763916015625,
"loss": 4.6371,
"margin_dpo/margin_mean": 113.09928894042969,
"margin_dpo/margin_std": 179.6610107421875,
"step": 372
},
{
"KL/chosen_KL_mean": -219.7714385986328,
"KL/mean": -270.5410461425781,
"KL/rejected_KL_mean": -321.31060791015625,
"KL/std": 135.90286254882812,
"epoch": 0.7811518324607329,
"fcm_dpo/beta": 0.002902068430557847,
"fcm_dpo/delta": 0.01581491529941559,
"fcm_dpo/margin": 101.53917694091797,
"fcm_dpo/q_t": 0.4303041994571686,
"grad_norm": 62.56334686279297,
"learning_rate": 7.033470310611945e-08,
"logits/chosen": -0.8850827217102051,
"logits/rejected": -0.8500010371208191,
"logps/chosen": -541.7144165039062,
"logps/ref_chosen": -321.9429931640625,
"logps/ref_rejected": -271.2288513183594,
"logps/rejected": -592.53955078125,
"loss": 4.6699,
"margin_dpo/margin_mean": 101.53917694091797,
"margin_dpo/margin_std": 156.63763427734375,
"step": 373
},
{
"KL/chosen_KL_mean": -232.57269287109375,
"KL/mean": -277.5435791015625,
"KL/rejected_KL_mean": -322.514404296875,
"KL/std": 155.8538818359375,
"epoch": 0.7832460732984293,
"fcm_dpo/beta": 0.0028754910454154015,
"fcm_dpo/delta": -0.02850567176938057,
"fcm_dpo/margin": 89.94171142578125,
"fcm_dpo/q_t": 0.43950414657592773,
"grad_norm": 53.21077346801758,
"learning_rate": 6.906649047373245e-08,
"logits/chosen": -0.883882462978363,
"logits/rejected": -0.8781501650810242,
"logps/chosen": -551.7412719726562,
"logps/ref_chosen": -319.1685485839844,
"logps/ref_rejected": -284.6263732910156,
"logps/rejected": -607.1407470703125,
"loss": 4.8022,
"margin_dpo/margin_mean": 89.94171142578125,
"margin_dpo/margin_std": 160.33763122558594,
"step": 374
},
{
"KL/chosen_KL_mean": -242.3306121826172,
"KL/mean": -279.80706787109375,
"KL/rejected_KL_mean": -317.2834777832031,
"KL/std": 129.10975646972656,
"epoch": 0.7853403141361257,
"fcm_dpo/beta": 0.0028963014483451843,
"fcm_dpo/delta": 0.03359980508685112,
"fcm_dpo/margin": 74.95287322998047,
"fcm_dpo/q_t": 0.4491427540779114,
"grad_norm": 106.22050476074219,
"learning_rate": 6.780798075635675e-08,
"logits/chosen": -0.8753329515457153,
"logits/rejected": -0.8530557155609131,
"logps/chosen": -557.2063598632812,
"logps/ref_chosen": -314.87579345703125,
"logps/ref_rejected": -259.1965026855469,
"logps/rejected": -576.47998046875,
"loss": 4.9298,
"margin_dpo/margin_mean": 74.95287322998047,
"margin_dpo/margin_std": 153.65232849121094,
"step": 375
},
{
"KL/chosen_KL_mean": -236.53941345214844,
"KL/mean": -286.677734375,
"KL/rejected_KL_mean": -336.8160400390625,
"KL/std": 152.2532958984375,
"epoch": 0.787434554973822,
"fcm_dpo/beta": 0.003028448671102524,
"fcm_dpo/delta": 0.047010768204927444,
"fcm_dpo/margin": 100.27667236328125,
"fcm_dpo/q_t": 0.4310030937194824,
"grad_norm": 92.208251953125,
"learning_rate": 6.655924144404906e-08,
"logits/chosen": -0.8538745045661926,
"logits/rejected": -0.8539252281188965,
"logps/chosen": -524.212646484375,
"logps/ref_chosen": -287.6732482910156,
"logps/ref_rejected": -256.6697082519531,
"logps/rejected": -593.4857788085938,
"loss": 4.7153,
"margin_dpo/margin_mean": 100.27667236328125,
"margin_dpo/margin_std": 179.34959411621094,
"step": 376
},
{
"KL/chosen_KL_mean": -250.41348266601562,
"KL/mean": -287.26507568359375,
"KL/rejected_KL_mean": -324.11663818359375,
"KL/std": 149.1571807861328,
"epoch": 0.7895287958115184,
"fcm_dpo/beta": 0.003050284693017602,
"fcm_dpo/delta": 0.007031728979200125,
"fcm_dpo/margin": 73.703125,
"fcm_dpo/q_t": 0.4477493166923523,
"grad_norm": 58.59090042114258,
"learning_rate": 6.532033950290885e-08,
"logits/chosen": -0.8329107761383057,
"logits/rejected": -0.8246560096740723,
"logps/chosen": -555.6749267578125,
"logps/ref_chosen": -305.261474609375,
"logps/ref_rejected": -271.8887023925781,
"logps/rejected": -596.00537109375,
"loss": 4.9961,
"margin_dpo/margin_mean": 73.703125,
"margin_dpo/margin_std": 176.01364135742188,
"step": 377
},
{
"KL/chosen_KL_mean": -239.36642456054688,
"KL/mean": -284.56353759765625,
"KL/rejected_KL_mean": -329.7606506347656,
"KL/std": 154.51390075683594,
"epoch": 0.7916230366492146,
"fcm_dpo/beta": 0.0031418083235621452,
"fcm_dpo/delta": 0.04664887487888336,
"fcm_dpo/margin": 90.39424896240234,
"fcm_dpo/q_t": 0.43551376461982727,
"grad_norm": 92.76583862304688,
"learning_rate": 6.409134137148736e-08,
"logits/chosen": -0.8306005001068115,
"logits/rejected": -0.8188484311103821,
"logps/chosen": -520.89599609375,
"logps/ref_chosen": -281.5295715332031,
"logps/ref_rejected": -296.980224609375,
"logps/rejected": -626.7409057617188,
"loss": 4.7842,
"margin_dpo/margin_mean": 90.39423370361328,
"margin_dpo/margin_std": 172.24591064453125,
"step": 378
},
{
"KL/chosen_KL_mean": -224.11880493164062,
"KL/mean": -268.5871276855469,
"KL/rejected_KL_mean": -313.055419921875,
"KL/std": 126.2266616821289,
"epoch": 0.793717277486911,
"fcm_dpo/beta": 0.003183132503181696,
"fcm_dpo/delta": 0.004846029914915562,
"fcm_dpo/margin": 88.9366226196289,
"fcm_dpo/q_t": 0.4342249631881714,
"grad_norm": 70.53761291503906,
"learning_rate": 6.28723129572247e-08,
"logits/chosen": -0.8911904096603394,
"logits/rejected": -0.8721175789833069,
"logps/chosen": -489.1995849609375,
"logps/ref_chosen": -265.0807800292969,
"logps/ref_rejected": -230.58932495117188,
"logps/rejected": -543.6447143554688,
"loss": 4.7555,
"margin_dpo/margin_mean": 88.93663024902344,
"margin_dpo/margin_std": 160.14614868164062,
"step": 379
},
{
"KL/chosen_KL_mean": -241.3148193359375,
"KL/mean": -293.77069091796875,
"KL/rejected_KL_mean": -346.2265319824219,
"KL/std": 157.62762451171875,
"epoch": 0.7958115183246073,
"fcm_dpo/beta": 0.003145367605611682,
"fcm_dpo/delta": -0.042871177196502686,
"fcm_dpo/margin": 104.91173553466797,
"fcm_dpo/q_t": 0.4266766607761383,
"grad_norm": 98.90926361083984,
"learning_rate": 6.166331963291519e-08,
"logits/chosen": -0.8594827651977539,
"logits/rejected": -0.834078848361969,
"logps/chosen": -547.2232055664062,
"logps/ref_chosen": -305.90838623046875,
"logps/ref_rejected": -286.5906677246094,
"logps/rejected": -632.8171997070312,
"loss": 4.696,
"margin_dpo/margin_mean": 104.91173553466797,
"margin_dpo/margin_std": 186.09226989746094,
"step": 380
},
{
"KL/chosen_KL_mean": -238.97413635253906,
"KL/mean": -295.09765625,
"KL/rejected_KL_mean": -351.2211608886719,
"KL/std": 172.82876586914062,
"epoch": 0.7979057591623037,
"fcm_dpo/beta": 0.0030606756918132305,
"fcm_dpo/delta": -0.03527439385652542,
"fcm_dpo/margin": 112.24700164794922,
"fcm_dpo/q_t": 0.42362287640571594,
"grad_norm": 90.26060485839844,
"learning_rate": 6.046442623320145e-08,
"logits/chosen": -0.8346625566482544,
"logits/rejected": -0.8009909391403198,
"logps/chosen": -491.8448181152344,
"logps/ref_chosen": -252.87066650390625,
"logps/ref_rejected": -261.1927490234375,
"logps/rejected": -612.4138793945312,
"loss": 4.5969,
"margin_dpo/margin_mean": 112.24700927734375,
"margin_dpo/margin_std": 181.87994384765625,
"step": 381
},
{
"KL/chosen_KL_mean": -227.16079711914062,
"KL/mean": -288.4083251953125,
"KL/rejected_KL_mean": -349.6558837890625,
"KL/std": 143.6719970703125,
"epoch": 0.8,
"fcm_dpo/beta": 0.002987783635035157,
"fcm_dpo/delta": -0.017105957493185997,
"fcm_dpo/margin": 122.49507141113281,
"fcm_dpo/q_t": 0.4151462912559509,
"grad_norm": 70.07193756103516,
"learning_rate": 5.9275697051098275e-08,
"logits/chosen": -0.8694351315498352,
"logits/rejected": -0.8645322322845459,
"logps/chosen": -516.3722534179688,
"logps/ref_chosen": -289.2114562988281,
"logps/ref_rejected": -278.45751953125,
"logps/rejected": -628.1133422851562,
"loss": 4.4638,
"margin_dpo/margin_mean": 122.49507141113281,
"margin_dpo/margin_std": 168.80638122558594,
"step": 382
},
{
"KL/chosen_KL_mean": -224.1322784423828,
"KL/mean": -280.5713195800781,
"KL/rejected_KL_mean": -337.0103759765625,
"KL/std": 159.5373992919922,
"epoch": 0.8020942408376963,
"fcm_dpo/beta": 0.0029694954864680767,
"fcm_dpo/delta": -0.028972981497645378,
"fcm_dpo/margin": 112.87809753417969,
"fcm_dpo/q_t": 0.4229624271392822,
"grad_norm": 93.5083999633789,
"learning_rate": 5.809719583454414e-08,
"logits/chosen": -0.8512469530105591,
"logits/rejected": -0.8341723680496216,
"logps/chosen": -497.76312255859375,
"logps/ref_chosen": -273.630859375,
"logps/ref_rejected": -261.44024658203125,
"logps/rejected": -598.4506225585938,
"loss": 4.6078,
"margin_dpo/margin_mean": 112.87809753417969,
"margin_dpo/margin_std": 181.7798309326172,
"step": 383
},
{
"KL/chosen_KL_mean": -233.4850616455078,
"KL/mean": -281.0430603027344,
"KL/rejected_KL_mean": -328.6011047363281,
"KL/std": 128.45787048339844,
"epoch": 0.8041884816753927,
"fcm_dpo/beta": 0.002953949151560664,
"fcm_dpo/delta": 0.040556252002716064,
"fcm_dpo/margin": 95.11605834960938,
"fcm_dpo/q_t": 0.4346542954444885,
"grad_norm": 70.20569610595703,
"learning_rate": 5.6928985782982524e-08,
"logits/chosen": -0.8760598301887512,
"logits/rejected": -0.8713008165359497,
"logps/chosen": -508.0550537109375,
"logps/ref_chosen": -274.5699462890625,
"logps/ref_rejected": -285.8253479003906,
"logps/rejected": -614.4264526367188,
"loss": 4.7248,
"margin_dpo/margin_mean": 95.11605834960938,
"margin_dpo/margin_std": 159.0487060546875,
"step": 384
},
{
"KL/chosen_KL_mean": -238.326416015625,
"KL/mean": -279.248779296875,
"KL/rejected_KL_mean": -320.1711730957031,
"KL/std": 137.06533813476562,
"epoch": 0.806282722513089,
"fcm_dpo/beta": 0.0030650359112769365,
"fcm_dpo/delta": 0.028704283758997917,
"fcm_dpo/margin": 81.8447494506836,
"fcm_dpo/q_t": 0.44181060791015625,
"grad_norm": 69.25728607177734,
"learning_rate": 5.57711295439732e-08,
"logits/chosen": -0.8149707913398743,
"logits/rejected": -0.8097137212753296,
"logps/chosen": -522.47705078125,
"logps/ref_chosen": -284.150634765625,
"logps/ref_rejected": -244.87921142578125,
"logps/rejected": -565.0503540039062,
"loss": 4.8258,
"margin_dpo/margin_mean": 81.84474182128906,
"margin_dpo/margin_std": 149.96339416503906,
"step": 385
},
{
"KL/chosen_KL_mean": -229.14710998535156,
"KL/mean": -288.1037292480469,
"KL/rejected_KL_mean": -347.060302734375,
"KL/std": 148.16629028320312,
"epoch": 0.8083769633507853,
"fcm_dpo/beta": 0.002984508639201522,
"fcm_dpo/delta": -0.04910660907626152,
"fcm_dpo/margin": 117.91322326660156,
"fcm_dpo/q_t": 0.4191738963127136,
"grad_norm": 86.30573272705078,
"learning_rate": 5.4623689209832484e-08,
"logits/chosen": -0.7966516613960266,
"logits/rejected": -0.795097291469574,
"logps/chosen": -549.3233642578125,
"logps/ref_chosen": -320.1762390136719,
"logps/ref_rejected": -302.05023193359375,
"logps/rejected": -649.1105346679688,
"loss": 4.5273,
"margin_dpo/margin_mean": 117.91322326660156,
"margin_dpo/margin_std": 162.8836212158203,
"step": 386
},
{
"KL/chosen_KL_mean": -247.12374877929688,
"KL/mean": -293.9572448730469,
"KL/rejected_KL_mean": -340.79071044921875,
"KL/std": 143.6245880126953,
"epoch": 0.8104712041884817,
"fcm_dpo/beta": 0.0029312213882803917,
"fcm_dpo/delta": -0.015069102868437767,
"fcm_dpo/margin": 93.66694641113281,
"fcm_dpo/q_t": 0.43505552411079407,
"grad_norm": 69.55741882324219,
"learning_rate": 5.3486726314303175e-08,
"logits/chosen": -0.8296703696250916,
"logits/rejected": -0.8330702185630798,
"logps/chosen": -519.4038696289062,
"logps/ref_chosen": -272.2801513671875,
"logps/ref_rejected": -265.1615905761719,
"logps/rejected": -605.9522705078125,
"loss": 4.8091,
"margin_dpo/margin_mean": 93.66694641113281,
"margin_dpo/margin_std": 173.94976806640625,
"step": 387
},
{
"KL/chosen_KL_mean": -262.6239318847656,
"KL/mean": -296.395263671875,
"KL/rejected_KL_mean": -330.1665954589844,
"KL/std": 146.9426727294922,
"epoch": 0.812565445026178,
"fcm_dpo/beta": 0.002879001200199127,
"fcm_dpo/delta": -0.010795612819492817,
"fcm_dpo/margin": 67.54266357421875,
"fcm_dpo/q_t": 0.45520469546318054,
"grad_norm": 92.43110656738281,
"learning_rate": 5.2360301829254745e-08,
"logits/chosen": -0.8211464285850525,
"logits/rejected": -0.8075721859931946,
"logps/chosen": -535.1552734375,
"logps/ref_chosen": -272.5313415527344,
"logps/ref_rejected": -239.55735778808594,
"logps/rejected": -569.7239379882812,
"loss": 5.0687,
"margin_dpo/margin_mean": 67.54266357421875,
"margin_dpo/margin_std": 172.599853515625,
"step": 388
},
{
"KL/chosen_KL_mean": -257.0192565917969,
"KL/mean": -309.4371643066406,
"KL/rejected_KL_mean": -361.8551025390625,
"KL/std": 156.31715393066406,
"epoch": 0.8146596858638744,
"fcm_dpo/beta": 0.0028533104341477156,
"fcm_dpo/delta": -0.017784729599952698,
"fcm_dpo/margin": 104.83583068847656,
"fcm_dpo/q_t": 0.43158379197120667,
"grad_norm": 70.29999542236328,
"learning_rate": 5.1244476161413806e-08,
"logits/chosen": -0.8539035320281982,
"logits/rejected": -0.8511427640914917,
"logps/chosen": -538.1085205078125,
"logps/ref_chosen": -281.0892639160156,
"logps/ref_rejected": -246.50045776367188,
"logps/rejected": -608.3555908203125,
"loss": 4.7168,
"margin_dpo/margin_mean": 104.83582305908203,
"margin_dpo/margin_std": 178.91343688964844,
"step": 389
},
{
"KL/chosen_KL_mean": -247.18812561035156,
"KL/mean": -299.47222900390625,
"KL/rejected_KL_mean": -351.75634765625,
"KL/std": 147.55088806152344,
"epoch": 0.8167539267015707,
"fcm_dpo/beta": 0.0028385238256305456,
"fcm_dpo/delta": -0.008637780323624611,
"fcm_dpo/margin": 104.5682373046875,
"fcm_dpo/q_t": 0.43057528138160706,
"grad_norm": 55.768348693847656,
"learning_rate": 5.013930914912476e-08,
"logits/chosen": -0.8642117977142334,
"logits/rejected": -0.8738707900047302,
"logps/chosen": -531.1755981445312,
"logps/ref_chosen": -283.98748779296875,
"logps/ref_rejected": -283.465087890625,
"logps/rejected": -635.221435546875,
"loss": 4.6778,
"margin_dpo/margin_mean": 104.5682373046875,
"margin_dpo/margin_std": 167.7999725341797,
"step": 390
},
{
"KL/chosen_KL_mean": -262.1589660644531,
"KL/mean": -318.81219482421875,
"KL/rejected_KL_mean": -375.4654541015625,
"KL/std": 163.68447875976562,
"epoch": 0.818848167539267,
"fcm_dpo/beta": 0.002770882099866867,
"fcm_dpo/delta": -0.04480758681893349,
"fcm_dpo/margin": 113.30651092529297,
"fcm_dpo/q_t": 0.4273582696914673,
"grad_norm": 74.78135681152344,
"learning_rate": 4.904486005914027e-08,
"logits/chosen": -0.8259506821632385,
"logits/rejected": -0.8088208436965942,
"logps/chosen": -546.0203857421875,
"logps/ref_chosen": -283.86138916015625,
"logps/ref_rejected": -263.5093688964844,
"logps/rejected": -638.974853515625,
"loss": 4.6416,
"margin_dpo/margin_mean": 113.3065185546875,
"margin_dpo/margin_std": 179.85752868652344,
"step": 391
},
{
"KL/chosen_KL_mean": -242.8234100341797,
"KL/mean": -304.9492492675781,
"KL/rejected_KL_mean": -367.0750732421875,
"KL/std": 145.44244384765625,
"epoch": 0.8209424083769633,
"fcm_dpo/beta": 0.002657483099028468,
"fcm_dpo/delta": -0.030804261565208435,
"fcm_dpo/margin": 124.25167083740234,
"fcm_dpo/q_t": 0.4213445782661438,
"grad_norm": 74.31720733642578,
"learning_rate": 4.796118758344353e-08,
"logits/chosen": -0.8100302219390869,
"logits/rejected": -0.8226779103279114,
"logps/chosen": -552.8934326171875,
"logps/ref_chosen": -310.070068359375,
"logps/ref_rejected": -252.89817810058594,
"logps/rejected": -619.9732055664062,
"loss": 4.5373,
"margin_dpo/margin_mean": 124.25165557861328,
"margin_dpo/margin_std": 161.54566955566406,
"step": 392
},
{
"KL/chosen_KL_mean": -244.78565979003906,
"KL/mean": -289.96575927734375,
"KL/rejected_KL_mean": -335.14581298828125,
"KL/std": 140.73870849609375,
"epoch": 0.8230366492146597,
"fcm_dpo/beta": 0.0027394951321184635,
"fcm_dpo/delta": 0.06814486533403397,
"fcm_dpo/margin": 90.36016082763672,
"fcm_dpo/q_t": 0.4422205984592438,
"grad_norm": 57.734832763671875,
"learning_rate": 4.688834983610082e-08,
"logits/chosen": -0.8401621580123901,
"logits/rejected": -0.820095419883728,
"logps/chosen": -531.5012817382812,
"logps/ref_chosen": -286.7156677246094,
"logps/ref_rejected": -230.00357055664062,
"logps/rejected": -565.1493530273438,
"loss": 4.8246,
"margin_dpo/margin_mean": 90.36016845703125,
"margin_dpo/margin_std": 166.87240600585938,
"step": 393
},
{
"KL/chosen_KL_mean": -249.3653564453125,
"KL/mean": -294.98590087890625,
"KL/rejected_KL_mean": -340.6064453125,
"KL/std": 166.85951232910156,
"epoch": 0.8251308900523561,
"fcm_dpo/beta": 0.002803670009598136,
"fcm_dpo/delta": 0.0016157319769263268,
"fcm_dpo/margin": 91.24108123779297,
"fcm_dpo/q_t": 0.44337010383605957,
"grad_norm": 77.8697280883789,
"learning_rate": 4.582640435014459e-08,
"logits/chosen": -0.8874049782752991,
"logits/rejected": -0.8782488107681274,
"logps/chosen": -575.3588256835938,
"logps/ref_chosen": -325.9934387207031,
"logps/ref_rejected": -317.42706298828125,
"logps/rejected": -658.0335083007812,
"loss": 4.8929,
"margin_dpo/margin_mean": 91.24108123779297,
"margin_dpo/margin_std": 193.69656372070312,
"step": 394
},
{
"KL/chosen_KL_mean": -237.0347442626953,
"KL/mean": -299.3041687011719,
"KL/rejected_KL_mean": -361.5736083984375,
"KL/std": 156.9813232421875,
"epoch": 0.8272251308900523,
"fcm_dpo/beta": 0.0028244995046406984,
"fcm_dpo/delta": -0.0022739041596651077,
"fcm_dpo/margin": 124.53888702392578,
"fcm_dpo/q_t": 0.42142555117607117,
"grad_norm": 63.480865478515625,
"learning_rate": 4.477540807448832e-08,
"logits/chosen": -0.8299747109413147,
"logits/rejected": -0.8384482860565186,
"logps/chosen": -505.9355773925781,
"logps/ref_chosen": -268.90081787109375,
"logps/ref_rejected": -272.85809326171875,
"logps/rejected": -634.4317626953125,
"loss": 4.545,
"margin_dpo/margin_mean": 124.53888702392578,
"margin_dpo/margin_std": 189.89027404785156,
"step": 395
},
{
"KL/chosen_KL_mean": -254.85581970214844,
"KL/mean": -299.94439697265625,
"KL/rejected_KL_mean": -345.032958984375,
"KL/std": 150.55859375,
"epoch": 0.8293193717277487,
"fcm_dpo/beta": 0.0028044539503753185,
"fcm_dpo/delta": 0.0007072770968079567,
"fcm_dpo/margin": 90.17716217041016,
"fcm_dpo/q_t": 0.44121721386909485,
"grad_norm": 89.61280822753906,
"learning_rate": 4.373541737087263e-08,
"logits/chosen": -0.846572995185852,
"logits/rejected": -0.8284200429916382,
"logps/chosen": -546.0541381835938,
"logps/ref_chosen": -291.19830322265625,
"logps/ref_rejected": -253.2803955078125,
"logps/rejected": -598.3133544921875,
"loss": 4.8408,
"margin_dpo/margin_mean": 90.17715454101562,
"margin_dpo/margin_std": 173.43312072753906,
"step": 396
},
{
"KL/chosen_KL_mean": -244.94935607910156,
"KL/mean": -288.22979736328125,
"KL/rejected_KL_mean": -331.51025390625,
"KL/std": 147.65113830566406,
"epoch": 0.831413612565445,
"fcm_dpo/beta": 0.0027613083366304636,
"fcm_dpo/delta": -0.03639310598373413,
"fcm_dpo/margin": 86.56092071533203,
"fcm_dpo/q_t": 0.4450097680091858,
"grad_norm": 56.71054458618164,
"learning_rate": 4.270648801084295e-08,
"logits/chosen": -0.8588390946388245,
"logits/rejected": -0.8406790494918823,
"logps/chosen": -554.7717895507812,
"logps/ref_chosen": -309.8224182128906,
"logps/ref_rejected": -291.9057922363281,
"logps/rejected": -623.4160766601562,
"loss": 4.9138,
"margin_dpo/margin_mean": 86.5609130859375,
"margin_dpo/margin_std": 175.44119262695312,
"step": 397
},
{
"KL/chosen_KL_mean": -269.9547424316406,
"KL/mean": -307.1197509765625,
"KL/rejected_KL_mean": -344.2847595214844,
"KL/std": 160.36767578125,
"epoch": 0.8335078534031414,
"fcm_dpo/beta": 0.0027223476208746433,
"fcm_dpo/delta": 0.005399319343268871,
"fcm_dpo/margin": 74.3299789428711,
"fcm_dpo/q_t": 0.4516116678714752,
"grad_norm": 67.60769653320312,
"learning_rate": 4.168867517275806e-08,
"logits/chosen": -0.7625999450683594,
"logits/rejected": -0.8003882169723511,
"logps/chosen": -567.768310546875,
"logps/ref_chosen": -297.8135070800781,
"logps/ref_rejected": -270.5025634765625,
"logps/rejected": -614.7872924804688,
"loss": 5.1248,
"margin_dpo/margin_mean": 74.32998657226562,
"margin_dpo/margin_std": 205.70623779296875,
"step": 398
},
{
"KL/chosen_KL_mean": -258.96551513671875,
"KL/mean": -308.9136047363281,
"KL/rejected_KL_mean": -358.8616027832031,
"KL/std": 152.3387451171875,
"epoch": 0.8356020942408376,
"fcm_dpo/beta": 0.0027614845894277096,
"fcm_dpo/delta": 0.038348905742168427,
"fcm_dpo/margin": 99.89608764648438,
"fcm_dpo/q_t": 0.43669405579566956,
"grad_norm": 90.15144348144531,
"learning_rate": 4.0682033438831584e-08,
"logits/chosen": -0.8555898070335388,
"logits/rejected": -0.8154149651527405,
"logps/chosen": -551.8123168945312,
"logps/ref_chosen": -292.8467712402344,
"logps/ref_rejected": -268.3638916015625,
"logps/rejected": -627.2255249023438,
"loss": 4.7632,
"margin_dpo/margin_mean": 99.89608764648438,
"margin_dpo/margin_std": 176.52777099609375,
"step": 399
},
{
"KL/chosen_KL_mean": -247.63442993164062,
"KL/mean": -294.90032958984375,
"KL/rejected_KL_mean": -342.166259765625,
"KL/std": 147.7308349609375,
"epoch": 0.837696335078534,
"fcm_dpo/beta": 0.002795381471514702,
"fcm_dpo/delta": -0.007865451276302338,
"fcm_dpo/margin": 94.53181457519531,
"fcm_dpo/q_t": 0.4383159875869751,
"grad_norm": 59.41498947143555,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": -0.890003502368927,
"logits/rejected": -0.8898519277572632,
"logps/chosen": -511.31085205078125,
"logps/ref_chosen": -263.6763916015625,
"logps/ref_rejected": -258.67266845703125,
"logps/rejected": -600.8389282226562,
"loss": 4.8212,
"margin_dpo/margin_mean": 94.53180694580078,
"margin_dpo/margin_std": 177.65081787109375,
"step": 400
},
{
"epoch": 0.837696335078534,
"eval_KL/chosen_KL_mean": -251.34669494628906,
"eval_KL/mean": -301.0162048339844,
"eval_KL/rejected_KL_mean": -350.6857604980469,
"eval_KL/std": 151.94033813476562,
"eval_fcm_dpo/beta": 0.002804369665682316,
"eval_logits/chosen": -0.8488364815711975,
"eval_logits/rejected": -0.8314180374145508,
"eval_logps/chosen": -539.1735229492188,
"eval_logps/ref_chosen": -287.8268127441406,
"eval_logps/ref_rejected": -266.9300231933594,
"eval_logps/rejected": -617.61572265625,
"eval_loss": 0.5985068678855896,
"eval_margin_dpo/margin_mean": 99.33906555175781,
"eval_margin_dpo/margin_std": 172.5637664794922,
"eval_runtime": 78.0551,
"eval_samples_per_second": 25.623,
"eval_steps_per_second": 3.203,
"step": 400
},
{
"KL/chosen_KL_mean": -244.6216583251953,
"KL/mean": -295.5447082519531,
"KL/rejected_KL_mean": -346.4677734375,
"KL/std": 154.69151306152344,
"epoch": 0.8397905759162304,
"fcm_dpo/beta": 0.002830990357324481,
"fcm_dpo/delta": 0.01728871278464794,
"fcm_dpo/margin": 101.84611511230469,
"fcm_dpo/q_t": 0.4332505464553833,
"grad_norm": 111.5696792602539,
"learning_rate": 3.8702478614051345e-08,
"logits/chosen": -0.8225564956665039,
"logits/rejected": -0.8207831382751465,
"logps/chosen": -562.9069213867188,
"logps/ref_chosen": -318.2853088378906,
"logps/ref_rejected": -293.75225830078125,
"logps/rejected": -640.2200317382812,
"loss": 4.7265,
"margin_dpo/margin_mean": 101.84611511230469,
"margin_dpo/margin_std": 175.7081298828125,
"step": 401
},
{
"KL/chosen_KL_mean": -242.03074645996094,
"KL/mean": -288.13262939453125,
"KL/rejected_KL_mean": -334.2344665527344,
"KL/std": 158.20025634765625,
"epoch": 0.8418848167539267,
"fcm_dpo/beta": 0.002913672011345625,
"fcm_dpo/delta": 0.06168531998991966,
"fcm_dpo/margin": 92.20372772216797,
"fcm_dpo/q_t": 0.4407009482383728,
"grad_norm": 75.40254974365234,
"learning_rate": 3.772967168071517e-08,
"logits/chosen": -0.8898206949234009,
"logits/rejected": -0.8673676252365112,
"logps/chosen": -551.4586181640625,
"logps/ref_chosen": -309.4278564453125,
"logps/ref_rejected": -282.0279846191406,
"logps/rejected": -616.262451171875,
"loss": 4.8418,
"margin_dpo/margin_mean": 92.20372772216797,
"margin_dpo/margin_std": 192.0637969970703,
"step": 402
},
{
"KL/chosen_KL_mean": -232.39500427246094,
"KL/mean": -302.0286560058594,
"KL/rejected_KL_mean": -371.66229248046875,
"KL/std": 162.0281982421875,
"epoch": 0.8439790575916231,
"fcm_dpo/beta": 0.002955038333311677,
"fcm_dpo/delta": -0.06371486186981201,
"fcm_dpo/margin": 139.26730346679688,
"fcm_dpo/q_t": 0.4056738615036011,
"grad_norm": 72.25170135498047,
"learning_rate": 3.676824816087978e-08,
"logits/chosen": -0.8628606200218201,
"logits/rejected": -0.8384885787963867,
"logps/chosen": -541.4235229492188,
"logps/ref_chosen": -309.0284729003906,
"logps/ref_rejected": -272.9622497558594,
"logps/rejected": -644.62451171875,
"loss": 4.3167,
"margin_dpo/margin_mean": 139.26730346679688,
"margin_dpo/margin_std": 168.85223388671875,
"step": 403
},
{
"KL/chosen_KL_mean": -252.7679443359375,
"KL/mean": -304.2862548828125,
"KL/rejected_KL_mean": -355.80450439453125,
"KL/std": 164.53274536132812,
"epoch": 0.8460732984293193,
"fcm_dpo/beta": 0.0028354106470942497,
"fcm_dpo/delta": -0.011737764813005924,
"fcm_dpo/margin": 103.03657531738281,
"fcm_dpo/q_t": 0.4333052933216095,
"grad_norm": 85.11771392822266,
"learning_rate": 3.581825961277074e-08,
"logits/chosen": -0.8758622407913208,
"logits/rejected": -0.8507161140441895,
"logps/chosen": -550.0516357421875,
"logps/ref_chosen": -297.2837219238281,
"logps/ref_rejected": -256.99041748046875,
"logps/rejected": -612.7949829101562,
"loss": 4.7764,
"margin_dpo/margin_mean": 103.03657531738281,
"margin_dpo/margin_std": 195.86697387695312,
"step": 404
},
{
"KL/chosen_KL_mean": -246.62994384765625,
"KL/mean": -303.05877685546875,
"KL/rejected_KL_mean": -359.48760986328125,
"KL/std": 159.86688232421875,
"epoch": 0.8481675392670157,
"fcm_dpo/beta": 0.002817755099385977,
"fcm_dpo/delta": 0.008052084594964981,
"fcm_dpo/margin": 112.85768127441406,
"fcm_dpo/q_t": 0.4265621304512024,
"grad_norm": 53.79868698120117,
"learning_rate": 3.487975698139084e-08,
"logits/chosen": -0.8158195614814758,
"logits/rejected": -0.8253852128982544,
"logps/chosen": -504.5953063964844,
"logps/ref_chosen": -257.96533203125,
"logps/ref_rejected": -255.811279296875,
"logps/rejected": -615.2988891601562,
"loss": 4.6274,
"margin_dpo/margin_mean": 112.85768127441406,
"margin_dpo/margin_std": 179.63681030273438,
"step": 405
},
{
"KL/chosen_KL_mean": -264.41668701171875,
"KL/mean": -302.35516357421875,
"KL/rejected_KL_mean": -340.2936706542969,
"KL/std": 148.21942138671875,
"epoch": 0.8502617801047121,
"fcm_dpo/beta": 0.0028648902662098408,
"fcm_dpo/delta": 0.0225369930267334,
"fcm_dpo/margin": 75.87698364257812,
"fcm_dpo/q_t": 0.4482402503490448,
"grad_norm": 95.1996078491211,
"learning_rate": 3.3952790595787986e-08,
"logits/chosen": -0.8537810444831848,
"logits/rejected": -0.8314708471298218,
"logps/chosen": -549.5977172851562,
"logps/ref_chosen": -285.1810607910156,
"logps/ref_rejected": -264.41351318359375,
"logps/rejected": -604.7071533203125,
"loss": 4.9276,
"margin_dpo/margin_mean": 75.87698364257812,
"margin_dpo/margin_std": 154.45828247070312,
"step": 406
},
{
"KL/chosen_KL_mean": -265.21063232421875,
"KL/mean": -317.1808776855469,
"KL/rejected_KL_mean": -369.151123046875,
"KL/std": 160.95401000976562,
"epoch": 0.8523560209424084,
"fcm_dpo/beta": 0.002887902082875371,
"fcm_dpo/delta": -0.0053447941318154335,
"fcm_dpo/margin": 103.94047546386719,
"fcm_dpo/q_t": 0.43184512853622437,
"grad_norm": 55.63980484008789,
"learning_rate": 3.303741016635614e-08,
"logits/chosen": -0.8277978897094727,
"logits/rejected": -0.8441427946090698,
"logps/chosen": -530.44873046875,
"logps/ref_chosen": -265.23809814453125,
"logps/ref_rejected": -219.0631561279297,
"logps/rejected": -588.2142944335938,
"loss": 4.7298,
"margin_dpo/margin_mean": 103.94047546386719,
"margin_dpo/margin_std": 186.37350463867188,
"step": 407
},
{
"KL/chosen_KL_mean": -241.85614013671875,
"KL/mean": -296.291748046875,
"KL/rejected_KL_mean": -350.7274169921875,
"KL/std": 152.62454223632812,
"epoch": 0.8544502617801047,
"fcm_dpo/beta": 0.0028567167464643717,
"fcm_dpo/delta": -0.04666806012392044,
"fcm_dpo/margin": 108.8712158203125,
"fcm_dpo/q_t": 0.4278115928173065,
"grad_norm": 70.93869018554688,
"learning_rate": 3.2133664782169944e-08,
"logits/chosen": -0.8690704703330994,
"logits/rejected": -0.8604199886322021,
"logps/chosen": -538.8287353515625,
"logps/ref_chosen": -296.9726257324219,
"logps/ref_rejected": -295.4786376953125,
"logps/rejected": -646.2060546875,
"loss": 4.6472,
"margin_dpo/margin_mean": 108.8712158203125,
"margin_dpo/margin_std": 166.20672607421875,
"step": 408
},
{
"KL/chosen_KL_mean": -249.7004852294922,
"KL/mean": -310.64984130859375,
"KL/rejected_KL_mean": -371.59918212890625,
"KL/std": 195.6245574951172,
"epoch": 0.856544502617801,
"fcm_dpo/beta": 0.0027475841343402863,
"fcm_dpo/delta": -0.02165559120476246,
"fcm_dpo/margin": 121.8986587524414,
"fcm_dpo/q_t": 0.43092209100723267,
"grad_norm": 49.7144889831543,
"learning_rate": 3.12416029083514e-08,
"logits/chosen": -0.8461386561393738,
"logits/rejected": -0.8305491209030151,
"logps/chosen": -537.079833984375,
"logps/ref_chosen": -287.37933349609375,
"logps/ref_rejected": -275.80291748046875,
"logps/rejected": -647.402099609375,
"loss": 4.7179,
"margin_dpo/margin_mean": 121.8986587524414,
"margin_dpo/margin_std": 237.97662353515625,
"step": 409
},
{
"KL/chosen_KL_mean": -243.9736785888672,
"KL/mean": -292.99066162109375,
"KL/rejected_KL_mean": -342.0075988769531,
"KL/std": 148.37387084960938,
"epoch": 0.8586387434554974,
"fcm_dpo/beta": 0.0027260768692940474,
"fcm_dpo/delta": 0.008642399683594704,
"fcm_dpo/margin": 98.03392791748047,
"fcm_dpo/q_t": 0.43822577595710754,
"grad_norm": 100.64157104492188,
"learning_rate": 3.036127238347164e-08,
"logits/chosen": -0.8542839288711548,
"logits/rejected": -0.8603944778442383,
"logps/chosen": -525.7538452148438,
"logps/ref_chosen": -281.7801818847656,
"logps/ref_rejected": -266.7550354003906,
"logps/rejected": -608.7626342773438,
"loss": 4.7851,
"margin_dpo/margin_mean": 98.03392791748047,
"margin_dpo/margin_std": 175.8046417236328,
"step": 410
},
{
"KL/chosen_KL_mean": -243.66639709472656,
"KL/mean": -300.5707702636719,
"KL/rejected_KL_mean": -357.4750671386719,
"KL/std": 162.13427734375,
"epoch": 0.8607329842931937,
"fcm_dpo/beta": 0.0027524821925908327,
"fcm_dpo/delta": -0.009361675009131432,
"fcm_dpo/margin": 113.80868530273438,
"fcm_dpo/q_t": 0.4285432696342468,
"grad_norm": 46.32007598876953,
"learning_rate": 2.9492720416985e-08,
"logits/chosen": -0.8611247539520264,
"logits/rejected": -0.8141701817512512,
"logps/chosen": -525.2536010742188,
"logps/ref_chosen": -281.5872497558594,
"logps/ref_rejected": -254.78916931152344,
"logps/rejected": -612.2642822265625,
"loss": 4.6081,
"margin_dpo/margin_mean": 113.8086929321289,
"margin_dpo/margin_std": 163.32958984375,
"step": 411
},
{
"KL/chosen_KL_mean": -265.3537292480469,
"KL/mean": -312.93505859375,
"KL/rejected_KL_mean": -360.51641845703125,
"KL/std": 155.24221801757812,
"epoch": 0.86282722513089,
"fcm_dpo/beta": 0.002621360355988145,
"fcm_dpo/delta": -0.052118320018053055,
"fcm_dpo/margin": 95.16270446777344,
"fcm_dpo/q_t": 0.44208434224128723,
"grad_norm": 89.77355194091797,
"learning_rate": 2.863599358669755e-08,
"logits/chosen": -0.8341819643974304,
"logits/rejected": -0.8465805649757385,
"logps/chosen": -541.8878784179688,
"logps/ref_chosen": -276.5341796875,
"logps/ref_rejected": -273.8751220703125,
"logps/rejected": -634.3915405273438,
"loss": 4.8408,
"margin_dpo/margin_mean": 95.16270446777344,
"margin_dpo/margin_std": 173.06126403808594,
"step": 412
},
{
"KL/chosen_KL_mean": -248.24635314941406,
"KL/mean": -292.7178649902344,
"KL/rejected_KL_mean": -337.1893005371094,
"KL/std": 152.73260498046875,
"epoch": 0.8649214659685864,
"fcm_dpo/beta": 0.0026765938382595778,
"fcm_dpo/delta": 0.06977185606956482,
"fcm_dpo/margin": 88.94297790527344,
"fcm_dpo/q_t": 0.44494855403900146,
"grad_norm": 95.87439727783203,
"learning_rate": 2.7791137836269158e-08,
"logits/chosen": -0.8483795523643494,
"logits/rejected": -0.8444381952285767,
"logps/chosen": -519.5209350585938,
"logps/ref_chosen": -271.2745666503906,
"logps/ref_rejected": -270.16912841796875,
"logps/rejected": -607.3585205078125,
"loss": 4.8712,
"margin_dpo/margin_mean": 88.94297790527344,
"margin_dpo/margin_std": 177.28707885742188,
"step": 413
},
{
"KL/chosen_KL_mean": -251.517578125,
"KL/mean": -309.97723388671875,
"KL/rejected_KL_mean": -368.4368591308594,
"KL/std": 155.47898864746094,
"epoch": 0.8670157068062827,
"fcm_dpo/beta": 0.0027606477960944176,
"fcm_dpo/delta": -0.01383502408862114,
"fcm_dpo/margin": 116.91925811767578,
"fcm_dpo/q_t": 0.4259103536605835,
"grad_norm": 60.382442474365234,
"learning_rate": 2.6958198472749717e-08,
"logits/chosen": -0.889758288860321,
"logits/rejected": -0.8836604952812195,
"logps/chosen": -548.6326293945312,
"logps/ref_chosen": -297.11505126953125,
"logps/ref_rejected": -271.7034606933594,
"logps/rejected": -640.1403198242188,
"loss": 4.6044,
"margin_dpo/margin_mean": 116.91926574707031,
"margin_dpo/margin_std": 176.09750366210938,
"step": 414
},
{
"KL/chosen_KL_mean": -243.7269744873047,
"KL/mean": -295.453125,
"KL/rejected_KL_mean": -347.1793212890625,
"KL/std": 143.5938720703125,
"epoch": 0.8691099476439791,
"fcm_dpo/beta": 0.002773047424852848,
"fcm_dpo/delta": 0.0341414213180542,
"fcm_dpo/margin": 103.45230865478516,
"fcm_dpo/q_t": 0.4327102601528168,
"grad_norm": 61.61573028564453,
"learning_rate": 2.613722016414943e-08,
"logits/chosen": -0.8744388222694397,
"logits/rejected": -0.8600894808769226,
"logps/chosen": -541.419677734375,
"logps/ref_chosen": -297.6926574707031,
"logps/ref_rejected": -279.0503234863281,
"logps/rejected": -626.2296752929688,
"loss": 4.6806,
"margin_dpo/margin_mean": 103.45231628417969,
"margin_dpo/margin_std": 161.2438507080078,
"step": 415
},
{
"KL/chosen_KL_mean": -235.6446075439453,
"KL/mean": -296.1462097167969,
"KL/rejected_KL_mean": -356.6477966308594,
"KL/std": 144.9048614501953,
"epoch": 0.8712041884816754,
"fcm_dpo/beta": 0.0027621614281088114,
"fcm_dpo/delta": -0.010026758536696434,
"fcm_dpo/margin": 121.0031967163086,
"fcm_dpo/q_t": 0.42156314849853516,
"grad_norm": 54.852943420410156,
"learning_rate": 2.5328246937043525e-08,
"logits/chosen": -0.8805437088012695,
"logits/rejected": -0.8778148293495178,
"logps/chosen": -547.47021484375,
"logps/ref_chosen": -311.8255615234375,
"logps/ref_rejected": -268.6170654296875,
"logps/rejected": -625.2648315429688,
"loss": 4.5416,
"margin_dpo/margin_mean": 121.0031967163086,
"margin_dpo/margin_std": 169.40794372558594,
"step": 416
},
{
"KL/chosen_KL_mean": -257.8272705078125,
"KL/mean": -309.54876708984375,
"KL/rejected_KL_mean": -361.27032470703125,
"KL/std": 162.28265380859375,
"epoch": 0.8732984293193717,
"fcm_dpo/beta": 0.0027372105978429317,
"fcm_dpo/delta": -0.038760267198085785,
"fcm_dpo/margin": 103.44303894042969,
"fcm_dpo/q_t": 0.4357483386993408,
"grad_norm": 67.29402160644531,
"learning_rate": 2.4531322174210973e-08,
"logits/chosen": -0.8284046649932861,
"logits/rejected": -0.816261887550354,
"logps/chosen": -568.2640991210938,
"logps/ref_chosen": -310.43682861328125,
"logps/ref_rejected": -277.15283203125,
"logps/rejected": -638.4231567382812,
"loss": 4.7596,
"margin_dpo/margin_mean": 103.44303894042969,
"margin_dpo/margin_std": 180.10482788085938,
"step": 417
},
{
"KL/chosen_KL_mean": -267.19671630859375,
"KL/mean": -322.150634765625,
"KL/rejected_KL_mean": -377.1046142578125,
"KL/std": 174.22171020507812,
"epoch": 0.875392670157068,
"fcm_dpo/beta": 0.0026559322141110897,
"fcm_dpo/delta": -0.0309746153652668,
"fcm_dpo/margin": 109.90789794921875,
"fcm_dpo/q_t": 0.4333530366420746,
"grad_norm": 83.46846008300781,
"learning_rate": 2.3746488612308295e-08,
"logits/chosen": -0.8226134181022644,
"logits/rejected": -0.8000130653381348,
"logps/chosen": -545.6926879882812,
"logps/ref_chosen": -278.49591064453125,
"logps/ref_rejected": -276.56671142578125,
"logps/rejected": -653.67138671875,
"loss": 4.7553,
"margin_dpo/margin_mean": 109.90789794921875,
"margin_dpo/margin_std": 201.9710235595703,
"step": 418
},
{
"KL/chosen_KL_mean": -270.344482421875,
"KL/mean": -320.5840148925781,
"KL/rejected_KL_mean": -370.82354736328125,
"KL/std": 162.86825561523438,
"epoch": 0.8774869109947644,
"fcm_dpo/beta": 0.0026421635411679745,
"fcm_dpo/delta": 0.00882997177541256,
"fcm_dpo/margin": 100.47907257080078,
"fcm_dpo/q_t": 0.43709927797317505,
"grad_norm": 75.29345703125,
"learning_rate": 2.297378833957761e-08,
"logits/chosen": -0.8669901490211487,
"logits/rejected": -0.8375420570373535,
"logps/chosen": -569.2447509765625,
"logps/ref_chosen": -298.9002380371094,
"logps/ref_rejected": -246.1540985107422,
"logps/rejected": -616.9776611328125,
"loss": 4.8279,
"margin_dpo/margin_mean": 100.47908020019531,
"margin_dpo/margin_std": 196.37474060058594,
"step": 419
},
{
"KL/chosen_KL_mean": -257.9861145019531,
"KL/mean": -322.0567626953125,
"KL/rejected_KL_mean": -386.12738037109375,
"KL/std": 174.96910095214844,
"epoch": 0.8795811518324608,
"fcm_dpo/beta": 0.002635931596159935,
"fcm_dpo/delta": -0.010491513647139072,
"fcm_dpo/margin": 128.1412811279297,
"fcm_dpo/q_t": 0.4231594204902649,
"grad_norm": 73.04295349121094,
"learning_rate": 2.2213262793589482e-08,
"logits/chosen": -0.8123749494552612,
"logits/rejected": -0.7837444543838501,
"logps/chosen": -522.5469970703125,
"logps/ref_chosen": -264.5608825683594,
"logps/ref_rejected": -245.67031860351562,
"logps/rejected": -631.7977294921875,
"loss": 4.6445,
"margin_dpo/margin_mean": 128.14129638671875,
"margin_dpo/margin_std": 220.80238342285156,
"step": 420
},
{
"KL/chosen_KL_mean": -239.97381591796875,
"KL/mean": -294.1003723144531,
"KL/rejected_KL_mean": -348.2269592285156,
"KL/std": 146.87315368652344,
"epoch": 0.881675392670157,
"fcm_dpo/beta": 0.0026289948727935553,
"fcm_dpo/delta": 0.016483336687088013,
"fcm_dpo/margin": 108.2531509399414,
"fcm_dpo/q_t": 0.4323281943798065,
"grad_norm": 102.15831756591797,
"learning_rate": 2.1464952759020856e-08,
"logits/chosen": -0.8939631581306458,
"logits/rejected": -0.8694461584091187,
"logps/chosen": -537.6788330078125,
"logps/ref_chosen": -297.70501708984375,
"logps/ref_rejected": -243.74771118164062,
"logps/rejected": -591.9746704101562,
"loss": 4.6963,
"margin_dpo/margin_mean": 108.2531509399414,
"margin_dpo/margin_std": 168.7926788330078,
"step": 421
},
{
"KL/chosen_KL_mean": -253.19100952148438,
"KL/mean": -304.7960205078125,
"KL/rejected_KL_mean": -356.4010925292969,
"KL/std": 160.6505889892578,
"epoch": 0.8837696335078534,
"fcm_dpo/beta": 0.002700560726225376,
"fcm_dpo/delta": 0.02611323446035385,
"fcm_dpo/margin": 103.21012878417969,
"fcm_dpo/q_t": 0.43617942929267883,
"grad_norm": 58.64923858642578,
"learning_rate": 2.07288983654679e-08,
"logits/chosen": -0.7524704933166504,
"logits/rejected": -0.7914985418319702,
"logps/chosen": -541.5497436523438,
"logps/ref_chosen": -288.3587646484375,
"logps/ref_rejected": -256.4377746582031,
"logps/rejected": -612.8389282226562,
"loss": 4.7559,
"margin_dpo/margin_mean": 103.21012878417969,
"margin_dpo/margin_std": 184.8786163330078,
"step": 422
},
{
"KL/chosen_KL_mean": -259.6864318847656,
"KL/mean": -316.55615234375,
"KL/rejected_KL_mean": -373.4259033203125,
"KL/std": 150.36436462402344,
"epoch": 0.8858638743455497,
"fcm_dpo/beta": 0.0027262200601398945,
"fcm_dpo/delta": 0.01529497466981411,
"fcm_dpo/margin": 113.73950958251953,
"fcm_dpo/q_t": 0.42795345187187195,
"grad_norm": 71.0254898071289,
"learning_rate": 2.0005139085293942e-08,
"logits/chosen": -0.8736763000488281,
"logits/rejected": -0.850985050201416,
"logps/chosen": -555.6934814453125,
"logps/ref_chosen": -296.00701904296875,
"logps/ref_rejected": -261.3480529785156,
"logps/rejected": -634.7738647460938,
"loss": 4.6501,
"margin_dpo/margin_mean": 113.73950958251953,
"margin_dpo/margin_std": 184.51678466796875,
"step": 423
},
{
"KL/chosen_KL_mean": -236.1890411376953,
"KL/mean": -291.9098205566406,
"KL/rejected_KL_mean": -347.6305847167969,
"KL/std": 145.9111328125,
"epoch": 0.8879581151832461,
"fcm_dpo/beta": 0.002728747669607401,
"fcm_dpo/delta": 0.0019058811012655497,
"fcm_dpo/margin": 111.44156646728516,
"fcm_dpo/q_t": 0.4285033643245697,
"grad_norm": 88.48895263671875,
"learning_rate": 1.9293713731512673e-08,
"logits/chosen": -0.8554025292396545,
"logits/rejected": -0.8399917483329773,
"logps/chosen": -545.6109008789062,
"logps/ref_chosen": -309.421875,
"logps/ref_rejected": -249.14886474609375,
"logps/rejected": -596.7794799804688,
"loss": 4.6199,
"margin_dpo/margin_mean": 111.44155883789062,
"margin_dpo/margin_std": 159.28919982910156,
"step": 424
},
{
"KL/chosen_KL_mean": -249.6146240234375,
"KL/mean": -301.1488342285156,
"KL/rejected_KL_mean": -352.6830139160156,
"KL/std": 154.60667419433594,
"epoch": 0.8900523560209425,
"fcm_dpo/beta": 0.002732752589508891,
"fcm_dpo/delta": -0.018169663846492767,
"fcm_dpo/margin": 103.06843566894531,
"fcm_dpo/q_t": 0.43511763215065,
"grad_norm": 70.74845886230469,
"learning_rate": 1.8594660455706763e-08,
"logits/chosen": -0.837730884552002,
"logits/rejected": -0.8395782709121704,
"logps/chosen": -530.1237182617188,
"logps/ref_chosen": -280.50909423828125,
"logps/ref_rejected": -276.8252258300781,
"logps/rejected": -629.5082397460938,
"loss": 4.7381,
"margin_dpo/margin_mean": 103.06842803955078,
"margin_dpo/margin_std": 173.00927734375,
"step": 425
},
{
"KL/chosen_KL_mean": -250.49110412597656,
"KL/mean": -305.4483947753906,
"KL/rejected_KL_mean": -360.4056701660156,
"KL/std": 157.17343139648438,
"epoch": 0.8921465968586387,
"fcm_dpo/beta": 0.002728844992816448,
"fcm_dpo/delta": -0.007583186496049166,
"fcm_dpo/margin": 109.91456604003906,
"fcm_dpo/q_t": 0.42983826994895935,
"grad_norm": 78.41447448730469,
"learning_rate": 1.7908016745981856e-08,
"logits/chosen": -0.8673391342163086,
"logits/rejected": -0.8460459113121033,
"logps/chosen": -543.2763061523438,
"logps/ref_chosen": -292.78521728515625,
"logps/ref_rejected": -255.62698364257812,
"logps/rejected": -616.0326538085938,
"loss": 4.6647,
"margin_dpo/margin_mean": 109.91456604003906,
"margin_dpo/margin_std": 170.11758422851562,
"step": 426
},
{
"KL/chosen_KL_mean": -247.78399658203125,
"KL/mean": -326.1251220703125,
"KL/rejected_KL_mean": -404.4662170410156,
"KL/std": 178.78623962402344,
"epoch": 0.8942408376963351,
"fcm_dpo/beta": 0.002634434960782528,
"fcm_dpo/delta": -0.09540139138698578,
"fcm_dpo/margin": 156.68223571777344,
"fcm_dpo/q_t": 0.40496763586997986,
"grad_norm": 97.31462860107422,
"learning_rate": 1.7233819424956247e-08,
"logits/chosen": -0.8368352055549622,
"logits/rejected": -0.7985715866088867,
"logps/chosen": -536.552734375,
"logps/ref_chosen": -288.7687072753906,
"logps/ref_rejected": -268.4986572265625,
"logps/rejected": -672.9649658203125,
"loss": 4.3996,
"margin_dpo/margin_mean": 156.68223571777344,
"margin_dpo/margin_std": 214.5924072265625,
"step": 427
},
{
"KL/chosen_KL_mean": -279.4367370605469,
"KL/mean": -345.8340759277344,
"KL/rejected_KL_mean": -412.2314453125,
"KL/std": 175.0027313232422,
"epoch": 0.8963350785340314,
"fcm_dpo/beta": 0.002493727719411254,
"fcm_dpo/delta": -0.009016228839755058,
"fcm_dpo/margin": 132.79470825195312,
"fcm_dpo/q_t": 0.4228391945362091,
"grad_norm": 58.75352096557617,
"learning_rate": 1.6572104647786245e-08,
"logits/chosen": -0.803855836391449,
"logits/rejected": -0.834683895111084,
"logps/chosen": -574.9577026367188,
"logps/ref_chosen": -295.5209655761719,
"logps/ref_rejected": -275.71026611328125,
"logps/rejected": -687.9417724609375,
"loss": 4.5616,
"margin_dpo/margin_mean": 132.79470825195312,
"margin_dpo/margin_std": 189.15499877929688,
"step": 428
},
{
"KL/chosen_KL_mean": -259.98162841796875,
"KL/mean": -314.2020263671875,
"KL/rejected_KL_mean": -368.4223937988281,
"KL/std": 167.9196319580078,
"epoch": 0.8984293193717278,
"fcm_dpo/beta": 0.0024902529548853636,
"fcm_dpo/delta": 0.018422594293951988,
"fcm_dpo/margin": 108.44074249267578,
"fcm_dpo/q_t": 0.4369828701019287,
"grad_norm": 66.33638000488281,
"learning_rate": 1.5922907900227017e-08,
"logits/chosen": -0.8072247505187988,
"logits/rejected": -0.8141688108444214,
"logps/chosen": -534.3739624023438,
"logps/ref_chosen": -274.392333984375,
"logps/ref_rejected": -258.574462890625,
"logps/rejected": -626.9968872070312,
"loss": 4.8132,
"margin_dpo/margin_mean": 108.44074249267578,
"margin_dpo/margin_std": 207.27586364746094,
"step": 429
},
{
"KL/chosen_KL_mean": -254.06158447265625,
"KL/mean": -306.5641174316406,
"KL/rejected_KL_mean": -359.066650390625,
"KL/std": 159.74868774414062,
"epoch": 0.900523560209424,
"fcm_dpo/beta": 0.0025652057956904173,
"fcm_dpo/delta": 0.017119091004133224,
"fcm_dpo/margin": 105.00508117675781,
"fcm_dpo/q_t": 0.4378116726875305,
"grad_norm": 76.95693969726562,
"learning_rate": 1.5286263996730026e-08,
"logits/chosen": -0.8991668820381165,
"logits/rejected": -0.8685553073883057,
"logps/chosen": -542.8007202148438,
"logps/ref_chosen": -288.7391357421875,
"logps/ref_rejected": -268.6106262207031,
"logps/rejected": -627.6773071289062,
"loss": 4.7653,
"margin_dpo/margin_mean": 105.00508880615234,
"margin_dpo/margin_std": 181.61851501464844,
"step": 430
},
{
"KL/chosen_KL_mean": -256.83135986328125,
"KL/mean": -298.0772399902344,
"KL/rejected_KL_mean": -339.3231201171875,
"KL/std": 148.50230407714844,
"epoch": 0.9026178010471204,
"fcm_dpo/beta": 0.0025556902401149273,
"fcm_dpo/delta": -0.009694522246718407,
"fcm_dpo/margin": 82.49180603027344,
"fcm_dpo/q_t": 0.45022091269493103,
"grad_norm": 64.07176971435547,
"learning_rate": 1.4662207078575684e-08,
"logits/chosen": -0.8577299118041992,
"logits/rejected": -0.8271062970161438,
"logps/chosen": -532.5560913085938,
"logps/ref_chosen": -275.7247314453125,
"logps/ref_rejected": -268.91729736328125,
"logps/rejected": -608.2404174804688,
"loss": 4.9432,
"margin_dpo/margin_mean": 82.4917984008789,
"margin_dpo/margin_std": 168.22735595703125,
"step": 431
},
{
"KL/chosen_KL_mean": -235.85765075683594,
"KL/mean": -293.31103515625,
"KL/rejected_KL_mean": -350.76446533203125,
"KL/std": 162.96881103515625,
"epoch": 0.9047120418848168,
"fcm_dpo/beta": 0.0025669136084616184,
"fcm_dpo/delta": 0.02326844446361065,
"fcm_dpo/margin": 114.90679931640625,
"fcm_dpo/q_t": 0.4326931834220886,
"grad_norm": 106.33002471923828,
"learning_rate": 1.40507706120426e-08,
"logits/chosen": -0.8967832922935486,
"logits/rejected": -0.8778641819953918,
"logps/chosen": -527.2777709960938,
"logps/ref_chosen": -291.42010498046875,
"logps/ref_rejected": -255.48202514648438,
"logps/rejected": -606.2464599609375,
"loss": 4.7029,
"margin_dpo/margin_mean": 114.90681457519531,
"margin_dpo/margin_std": 193.0474090576172,
"step": 432
},
{
"KL/chosen_KL_mean": -263.96636962890625,
"KL/mean": -314.3001403808594,
"KL/rejected_KL_mean": -364.6339111328125,
"KL/std": 153.60552978515625,
"epoch": 0.9068062827225131,
"fcm_dpo/beta": 0.0026351790875196457,
"fcm_dpo/delta": 0.05364841967821121,
"fcm_dpo/margin": 100.6675796508789,
"fcm_dpo/q_t": 0.43835511803627014,
"grad_norm": 52.755271911621094,
"learning_rate": 1.345198738661285e-08,
"logits/chosen": -0.8388685584068298,
"logits/rejected": -0.8329813480377197,
"logps/chosen": -510.1932373046875,
"logps/ref_chosen": -246.2268829345703,
"logps/ref_rejected": -253.65924072265625,
"logps/rejected": -618.2931518554688,
"loss": 4.7584,
"margin_dpo/margin_mean": 100.6675796508789,
"margin_dpo/margin_std": 172.32330322265625,
"step": 433
},
{
"KL/chosen_KL_mean": -281.077880859375,
"KL/mean": -328.9410400390625,
"KL/rejected_KL_mean": -376.80419921875,
"KL/std": 159.2793731689453,
"epoch": 0.9089005235602095,
"fcm_dpo/beta": 0.002758192829787731,
"fcm_dpo/delta": 0.04489844664931297,
"fcm_dpo/margin": 95.72633361816406,
"fcm_dpo/q_t": 0.4392809271812439,
"grad_norm": 82.58972930908203,
"learning_rate": 1.2865889513213628e-08,
"logits/chosen": -0.8396630883216858,
"logits/rejected": -0.8432857394218445,
"logps/chosen": -576.5396728515625,
"logps/ref_chosen": -295.4618225097656,
"logps/ref_rejected": -256.2254333496094,
"logps/rejected": -633.0296630859375,
"loss": 4.7972,
"margin_dpo/margin_mean": 95.72633361816406,
"margin_dpo/margin_std": 174.9141387939453,
"step": 434
},
{
"KL/chosen_KL_mean": -246.79896545410156,
"KL/mean": -306.7959899902344,
"KL/rejected_KL_mean": -366.79302978515625,
"KL/std": 164.5160369873047,
"epoch": 0.9109947643979057,
"fcm_dpo/beta": 0.0027714024763554335,
"fcm_dpo/delta": -0.030311204493045807,
"fcm_dpo/margin": 119.99405670166016,
"fcm_dpo/q_t": 0.425067663192749,
"grad_norm": 90.30838775634766,
"learning_rate": 1.2292508422495157e-08,
"logits/chosen": -0.8386386632919312,
"logits/rejected": -0.8267545104026794,
"logps/chosen": -507.537353515625,
"logps/ref_chosen": -260.7384033203125,
"logps/ref_rejected": -248.5688018798828,
"logps/rejected": -615.36181640625,
"loss": 4.5755,
"margin_dpo/margin_mean": 119.99406433105469,
"margin_dpo/margin_std": 176.0413055419922,
"step": 435
},
{
"KL/chosen_KL_mean": -276.3723449707031,
"KL/mean": -319.248046875,
"KL/rejected_KL_mean": -362.1237487792969,
"KL/std": 180.16555786132812,
"epoch": 0.9130890052356021,
"fcm_dpo/beta": 0.0027568363584578037,
"fcm_dpo/delta": 0.03259842097759247,
"fcm_dpo/margin": 85.7514419555664,
"fcm_dpo/q_t": 0.4493623971939087,
"grad_norm": 97.92943572998047,
"learning_rate": 1.1731874863145142e-08,
"logits/chosen": -0.8305177092552185,
"logits/rejected": -0.8246312141418457,
"logps/chosen": -595.69482421875,
"logps/ref_chosen": -319.3224792480469,
"logps/ref_rejected": -299.30322265625,
"logps/rejected": -661.4270629882812,
"loss": 4.9529,
"margin_dpo/margin_mean": 85.7514419555664,
"margin_dpo/margin_std": 200.18887329101562,
"step": 436
},
{
"KL/chosen_KL_mean": -256.4522399902344,
"KL/mean": -318.58221435546875,
"KL/rejected_KL_mean": -380.7121887207031,
"KL/std": 179.59681701660156,
"epoch": 0.9151832460732985,
"fcm_dpo/beta": 0.002750533865764737,
"fcm_dpo/delta": -0.027415964752435684,
"fcm_dpo/margin": 124.25997924804688,
"fcm_dpo/q_t": 0.42281562089920044,
"grad_norm": 56.36848831176758,
"learning_rate": 1.118401890024001e-08,
"logits/chosen": -0.8621474504470825,
"logits/rejected": -0.8434783220291138,
"logps/chosen": -535.281005859375,
"logps/ref_chosen": -278.82879638671875,
"logps/ref_rejected": -272.55303955078125,
"logps/rejected": -653.2651977539062,
"loss": 4.6259,
"margin_dpo/margin_mean": 124.25997924804688,
"margin_dpo/margin_std": 206.07656860351562,
"step": 437
},
{
"KL/chosen_KL_mean": -268.6119384765625,
"KL/mean": -298.52435302734375,
"KL/rejected_KL_mean": -328.4367370605469,
"KL/std": 162.5165557861328,
"epoch": 0.9172774869109948,
"fcm_dpo/beta": 0.0027467007748782635,
"fcm_dpo/delta": -0.0029242374002933502,
"fcm_dpo/margin": 59.82483673095703,
"fcm_dpo/q_t": 0.461994469165802,
"grad_norm": 66.8003158569336,
"learning_rate": 1.06489699136324e-08,
"logits/chosen": -0.8481613993644714,
"logits/rejected": -0.8591020107269287,
"logps/chosen": -527.9309692382812,
"logps/ref_chosen": -259.31903076171875,
"logps/ref_rejected": -240.99581909179688,
"logps/rejected": -569.4325561523438,
"loss": 5.1956,
"margin_dpo/margin_mean": 59.82484817504883,
"margin_dpo/margin_std": 185.36160278320312,
"step": 438
},
{
"KL/chosen_KL_mean": -255.05923461914062,
"KL/mean": -310.136474609375,
"KL/rejected_KL_mean": -365.21368408203125,
"KL/std": 151.9537353515625,
"epoch": 0.9193717277486911,
"fcm_dpo/beta": 0.002724123653024435,
"fcm_dpo/delta": 0.012698237784206867,
"fcm_dpo/margin": 110.15446472167969,
"fcm_dpo/q_t": 0.4297390878200531,
"grad_norm": 113.9437255859375,
"learning_rate": 1.0126756596375685e-08,
"logits/chosen": -0.8277400135993958,
"logits/rejected": -0.824100136756897,
"logps/chosen": -512.18359375,
"logps/ref_chosen": -257.1243896484375,
"logps/ref_rejected": -243.20416259765625,
"logps/rejected": -608.4178466796875,
"loss": 4.6466,
"margin_dpo/margin_mean": 110.15446472167969,
"margin_dpo/margin_std": 167.56851196289062,
"step": 439
},
{
"KL/chosen_KL_mean": -277.35302734375,
"KL/mean": -317.02935791015625,
"KL/rejected_KL_mean": -356.7057189941406,
"KL/std": 143.4852294921875,
"epoch": 0.9214659685863874,
"fcm_dpo/beta": 0.0028076376765966415,
"fcm_dpo/delta": 0.030055997893214226,
"fcm_dpo/margin": 79.3526840209961,
"fcm_dpo/q_t": 0.44725722074508667,
"grad_norm": 59.82633590698242,
"learning_rate": 9.617406953185136e-09,
"logits/chosen": -0.8821266889572144,
"logits/rejected": -0.873234748840332,
"logps/chosen": -584.884521484375,
"logps/ref_chosen": -307.5315246582031,
"logps/ref_rejected": -264.3540954589844,
"logps/rejected": -621.059814453125,
"loss": 4.9115,
"margin_dpo/margin_mean": 79.3526840209961,
"margin_dpo/margin_std": 156.04428100585938,
"step": 440
},
{
"KL/chosen_KL_mean": -263.2364807128906,
"KL/mean": -323.9188537597656,
"KL/rejected_KL_mean": -384.6011962890625,
"KL/std": 157.65415954589844,
"epoch": 0.9235602094240838,
"fcm_dpo/beta": 0.0027934680692851543,
"fcm_dpo/delta": -0.028223905712366104,
"fcm_dpo/margin": 121.36473083496094,
"fcm_dpo/q_t": 0.42069217562675476,
"grad_norm": 88.51628112792969,
"learning_rate": 9.12094829893642e-09,
"logits/chosen": -0.8281688094139099,
"logits/rejected": -0.821442723274231,
"logps/chosen": -573.218505859375,
"logps/ref_chosen": -309.9819641113281,
"logps/ref_rejected": -297.4968566894531,
"logps/rejected": -682.0980224609375,
"loss": 4.5291,
"margin_dpo/margin_mean": 121.36473083496094,
"margin_dpo/margin_std": 165.20242309570312,
"step": 441
},
{
"KL/chosen_KL_mean": -285.0911560058594,
"KL/mean": -330.148681640625,
"KL/rejected_KL_mean": -375.20623779296875,
"KL/std": 159.1475067138672,
"epoch": 0.9256544502617801,
"fcm_dpo/beta": 0.0027609432581812143,
"fcm_dpo/delta": 0.0063867755234241486,
"fcm_dpo/margin": 90.11505126953125,
"fcm_dpo/q_t": 0.4421730935573578,
"grad_norm": 53.26344680786133,
"learning_rate": 8.637407257200496e-09,
"logits/chosen": -0.9114494323730469,
"logits/rejected": -0.8700456023216248,
"logps/chosen": -564.0703125,
"logps/ref_chosen": -278.9791564941406,
"logps/ref_rejected": -242.87310791015625,
"logps/rejected": -618.079345703125,
"loss": 4.8762,
"margin_dpo/margin_mean": 90.11505126953125,
"margin_dpo/margin_std": 186.69032287597656,
"step": 442
},
{
"KL/chosen_KL_mean": -268.91424560546875,
"KL/mean": -314.4470520019531,
"KL/rejected_KL_mean": -359.9798889160156,
"KL/std": 142.87657165527344,
"epoch": 0.9277486910994764,
"fcm_dpo/beta": 0.0027999957092106342,
"fcm_dpo/delta": -0.00435899943113327,
"fcm_dpo/margin": 91.06564331054688,
"fcm_dpo/q_t": 0.4397067725658417,
"grad_norm": 64.01789855957031,
"learning_rate": 8.166809758815895e-09,
"logits/chosen": -0.801997721195221,
"logits/rejected": -0.8227972388267517,
"logps/chosen": -542.4733276367188,
"logps/ref_chosen": -273.5590515136719,
"logps/ref_rejected": -264.0199279785156,
"logps/rejected": -623.9998779296875,
"loss": 4.8378,
"margin_dpo/margin_mean": 91.0656509399414,
"margin_dpo/margin_std": 170.69918823242188,
"step": 443
},
{
"KL/chosen_KL_mean": -255.1349334716797,
"KL/mean": -311.1309509277344,
"KL/rejected_KL_mean": -367.12701416015625,
"KL/std": 171.04638671875,
"epoch": 0.9298429319371728,
"fcm_dpo/beta": 0.0027402995619922876,
"fcm_dpo/delta": -0.02012881450355053,
"fcm_dpo/margin": 111.99202728271484,
"fcm_dpo/q_t": 0.42987489700317383,
"grad_norm": 78.47362518310547,
"learning_rate": 7.709181040498253e-09,
"logits/chosen": -0.817521333694458,
"logits/rejected": -0.8048192858695984,
"logps/chosen": -553.2791137695312,
"logps/ref_chosen": -298.1441955566406,
"logps/ref_rejected": -268.0572814941406,
"logps/rejected": -635.1842041015625,
"loss": 4.6927,
"margin_dpo/margin_mean": 111.99202728271484,
"margin_dpo/margin_std": 193.68740844726562,
"step": 444
},
{
"KL/chosen_KL_mean": -263.73199462890625,
"KL/mean": -311.5459289550781,
"KL/rejected_KL_mean": -359.3598327636719,
"KL/std": 162.5752716064453,
"epoch": 0.9319371727748691,
"fcm_dpo/beta": 0.0027151252143085003,
"fcm_dpo/delta": -0.023878220468759537,
"fcm_dpo/margin": 95.62787628173828,
"fcm_dpo/q_t": 0.4410240650177002,
"grad_norm": 60.90433883666992,
"learning_rate": 7.2645456434869965e-09,
"logits/chosen": -0.8886433243751526,
"logits/rejected": -0.8941999673843384,
"logps/chosen": -518.2726440429688,
"logps/ref_chosen": -254.54067993164062,
"logps/ref_rejected": -264.2445983886719,
"logps/rejected": -623.6044311523438,
"loss": 4.8625,
"margin_dpo/margin_mean": 95.62787628173828,
"margin_dpo/margin_std": 188.66397094726562,
"step": 445
},
{
"KL/chosen_KL_mean": -256.6874084472656,
"KL/mean": -311.60101318359375,
"KL/rejected_KL_mean": -366.5146179199219,
"KL/std": 157.3093719482422,
"epoch": 0.9340314136125655,
"fcm_dpo/beta": 0.002662414452061057,
"fcm_dpo/delta": 0.01298682950437069,
"fcm_dpo/margin": 109.82720947265625,
"fcm_dpo/q_t": 0.4332526922225952,
"grad_norm": 72.25860595703125,
"learning_rate": 6.832927412229017e-09,
"logits/chosen": -0.8085878491401672,
"logits/rejected": -0.8123000860214233,
"logps/chosen": -563.409912109375,
"logps/ref_chosen": -306.72247314453125,
"logps/ref_rejected": -266.3735656738281,
"logps/rejected": -632.88818359375,
"loss": 4.7131,
"margin_dpo/margin_mean": 109.82720947265625,
"margin_dpo/margin_std": 187.44256591796875,
"step": 446
},
{
"KL/chosen_KL_mean": -251.7252960205078,
"KL/mean": -314.72784423828125,
"KL/rejected_KL_mean": -377.7303771972656,
"KL/std": 160.4945068359375,
"epoch": 0.9361256544502617,
"fcm_dpo/beta": 0.0026090561877936125,
"fcm_dpo/delta": -0.04460753872990608,
"fcm_dpo/margin": 126.00508880615234,
"fcm_dpo/q_t": 0.42350420355796814,
"grad_norm": 80.54020690917969,
"learning_rate": 6.414349493100129e-09,
"logits/chosen": -0.8133091330528259,
"logits/rejected": -0.8063684105873108,
"logps/chosen": -512.2425537109375,
"logps/ref_chosen": -260.51727294921875,
"logps/ref_rejected": -236.47061157226562,
"logps/rejected": -614.2009887695312,
"loss": 4.5824,
"margin_dpo/margin_mean": 126.00509643554688,
"margin_dpo/margin_std": 186.56752014160156,
"step": 447
},
{
"KL/chosen_KL_mean": -270.5550842285156,
"KL/mean": -327.3743896484375,
"KL/rejected_KL_mean": -384.19366455078125,
"KL/std": 155.24166870117188,
"epoch": 0.9382198952879581,
"fcm_dpo/beta": 0.0025703529827296734,
"fcm_dpo/delta": -0.01508631743490696,
"fcm_dpo/margin": 113.63859558105469,
"fcm_dpo/q_t": 0.43178319931030273,
"grad_norm": 54.260047912597656,
"learning_rate": 6.0088343331638756e-09,
"logits/chosen": -0.8144208192825317,
"logits/rejected": -0.8058769702911377,
"logps/chosen": -539.3421020507812,
"logps/ref_chosen": -268.78704833984375,
"logps/ref_rejected": -262.1703796386719,
"logps/rejected": -646.364013671875,
"loss": 4.6702,
"margin_dpo/margin_mean": 113.63859558105469,
"margin_dpo/margin_std": 171.42291259765625,
"step": 448
},
{
"KL/chosen_KL_mean": -256.49884033203125,
"KL/mean": -310.0642395019531,
"KL/rejected_KL_mean": -363.62969970703125,
"KL/std": 157.3622589111328,
"epoch": 0.9403141361256544,
"fcm_dpo/beta": 0.002572092227637768,
"fcm_dpo/delta": -0.0028328632470220327,
"fcm_dpo/margin": 107.13084411621094,
"fcm_dpo/q_t": 0.433704674243927,
"grad_norm": 97.39542388916016,
"learning_rate": 5.616403678967624e-09,
"logits/chosen": -0.9088627696037292,
"logits/rejected": -0.8921771049499512,
"logps/chosen": -587.4502563476562,
"logps/ref_chosen": -330.9514465332031,
"logps/ref_rejected": -239.76974487304688,
"logps/rejected": -603.3994140625,
"loss": 4.77,
"margin_dpo/margin_mean": 107.13084411621094,
"margin_dpo/margin_std": 188.76866149902344,
"step": 449
},
{
"KL/chosen_KL_mean": -279.85650634765625,
"KL/mean": -327.69305419921875,
"KL/rejected_KL_mean": -375.5296325683594,
"KL/std": 164.77511596679688,
"epoch": 0.9424083769633508,
"fcm_dpo/beta": 0.0025807241909205914,
"fcm_dpo/delta": 0.017238600179553032,
"fcm_dpo/margin": 95.67314147949219,
"fcm_dpo/q_t": 0.44215500354766846,
"grad_norm": 72.74471282958984,
"learning_rate": 5.2370785753763356e-09,
"logits/chosen": -0.8057252764701843,
"logits/rejected": -0.7975594997406006,
"logps/chosen": -564.1218872070312,
"logps/ref_chosen": -284.26544189453125,
"logps/ref_rejected": -250.5401611328125,
"logps/rejected": -626.06982421875,
"loss": 4.8131,
"margin_dpo/margin_mean": 95.67314147949219,
"margin_dpo/margin_std": 164.05947875976562,
"step": 450
},
{
"KL/chosen_KL_mean": -282.51611328125,
"KL/mean": -326.5624084472656,
"KL/rejected_KL_mean": -370.60870361328125,
"KL/std": 158.82203674316406,
"epoch": 0.9445026178010472,
"fcm_dpo/beta": 0.002585146576166153,
"fcm_dpo/delta": 0.0006589877884835005,
"fcm_dpo/margin": 88.09262084960938,
"fcm_dpo/q_t": 0.44617974758148193,
"grad_norm": 76.11978912353516,
"learning_rate": 4.8708793644441086e-09,
"logits/chosen": -0.8257752656936646,
"logits/rejected": -0.7940558791160583,
"logps/chosen": -584.8370361328125,
"logps/ref_chosen": -302.3209228515625,
"logps/ref_rejected": -254.09747314453125,
"logps/rejected": -624.7062377929688,
"loss": 4.8982,
"margin_dpo/margin_mean": 88.09262084960938,
"margin_dpo/margin_std": 171.1018524169922,
"step": 451
},
{
"KL/chosen_KL_mean": -255.13052368164062,
"KL/mean": -322.3986511230469,
"KL/rejected_KL_mean": -389.6668395996094,
"KL/std": 191.00897216796875,
"epoch": 0.9465968586387434,
"fcm_dpo/beta": 0.002510292222723365,
"fcm_dpo/delta": -0.04761781170964241,
"fcm_dpo/margin": 134.53628540039062,
"fcm_dpo/q_t": 0.425809383392334,
"grad_norm": 56.017059326171875,
"learning_rate": 4.517825684323323e-09,
"logits/chosen": -0.8782578706741333,
"logits/rejected": -0.8507087826728821,
"logps/chosen": -554.522705078125,
"logps/ref_chosen": -299.39215087890625,
"logps/ref_rejected": -284.3475036621094,
"logps/rejected": -674.0143432617188,
"loss": 4.6199,
"margin_dpo/margin_mean": 134.53628540039062,
"margin_dpo/margin_std": 222.55630493164062,
"step": 452
},
{
"KL/chosen_KL_mean": -276.0933837890625,
"KL/mean": -331.9483642578125,
"KL/rejected_KL_mean": -387.80328369140625,
"KL/std": 161.8867950439453,
"epoch": 0.9486910994764398,
"fcm_dpo/beta": 0.002537027932703495,
"fcm_dpo/delta": 0.02657966874539852,
"fcm_dpo/margin": 111.70990753173828,
"fcm_dpo/q_t": 0.43421390652656555,
"grad_norm": 85.80929565429688,
"learning_rate": 4.1779364682113794e-09,
"logits/chosen": -0.8239843249320984,
"logits/rejected": -0.814884603023529,
"logps/chosen": -600.7450561523438,
"logps/ref_chosen": -324.6517028808594,
"logps/ref_rejected": -304.1527099609375,
"logps/rejected": -691.9559936523438,
"loss": 4.7206,
"margin_dpo/margin_mean": 111.70991516113281,
"margin_dpo/margin_std": 191.12188720703125,
"step": 453
},
{
"KL/chosen_KL_mean": -268.03387451171875,
"KL/mean": -332.931396484375,
"KL/rejected_KL_mean": -397.8288269042969,
"KL/std": 183.07398986816406,
"epoch": 0.9507853403141361,
"fcm_dpo/beta": 0.0025168166030198336,
"fcm_dpo/delta": -0.019286856055259705,
"fcm_dpo/margin": 129.79495239257812,
"fcm_dpo/q_t": 0.4242181181907654,
"grad_norm": 61.91704177856445,
"learning_rate": 3.851229943335393e-09,
"logits/chosen": -0.8545857667922974,
"logits/rejected": -0.8613591194152832,
"logps/chosen": -567.6456298828125,
"logps/ref_chosen": -299.6117248535156,
"logps/ref_rejected": -303.74224853515625,
"logps/rejected": -701.5711059570312,
"loss": 4.6043,
"margin_dpo/margin_mean": 129.7949676513672,
"margin_dpo/margin_std": 201.04190063476562,
"step": 454
},
{
"KL/chosen_KL_mean": -260.20513916015625,
"KL/mean": -300.1604919433594,
"KL/rejected_KL_mean": -340.1158447265625,
"KL/std": 160.56234741210938,
"epoch": 0.9528795811518325,
"fcm_dpo/beta": 0.002559874439612031,
"fcm_dpo/delta": 0.018644150346517563,
"fcm_dpo/margin": 79.91071319580078,
"fcm_dpo/q_t": 0.4540133774280548,
"grad_norm": 80.64703369140625,
"learning_rate": 3.5377236299748147e-09,
"logits/chosen": -0.8247178196907043,
"logits/rejected": -0.8339167833328247,
"logps/chosen": -533.8168334960938,
"logps/ref_chosen": -273.6116943359375,
"logps/ref_rejected": -274.4293518066406,
"logps/rejected": -614.545166015625,
"loss": 5.0062,
"margin_dpo/margin_mean": 79.91071319580078,
"margin_dpo/margin_std": 183.53799438476562,
"step": 455
},
{
"KL/chosen_KL_mean": -286.6998291015625,
"KL/mean": -344.54241943359375,
"KL/rejected_KL_mean": -402.385009765625,
"KL/std": 184.26705932617188,
"epoch": 0.9549738219895288,
"fcm_dpo/beta": 0.0026082973927259445,
"fcm_dpo/delta": 0.026392972096800804,
"fcm_dpo/margin": 115.68516540527344,
"fcm_dpo/q_t": 0.4336429834365845,
"grad_norm": 63.203765869140625,
"learning_rate": 3.2374343405217884e-09,
"logits/chosen": -0.7551023364067078,
"logits/rejected": -0.7697523236274719,
"logps/chosen": -608.8717651367188,
"logps/ref_chosen": -322.17193603515625,
"logps/ref_rejected": -294.54461669921875,
"logps/rejected": -696.9296264648438,
"loss": 4.7445,
"margin_dpo/margin_mean": 115.68516540527344,
"margin_dpo/margin_std": 216.65805053710938,
"step": 456
},
{
"KL/chosen_KL_mean": -280.3325500488281,
"KL/mean": -341.10992431640625,
"KL/rejected_KL_mean": -401.88720703125,
"KL/std": 178.5426483154297,
"epoch": 0.9570680628272251,
"fcm_dpo/beta": 0.002582404762506485,
"fcm_dpo/delta": -0.02019866183400154,
"fcm_dpo/margin": 121.55464172363281,
"fcm_dpo/q_t": 0.42459535598754883,
"grad_norm": 68.02951049804688,
"learning_rate": 2.9503781785795713e-09,
"logits/chosen": -0.8246585130691528,
"logits/rejected": -0.826050341129303,
"logps/chosen": -588.1287841796875,
"logps/ref_chosen": -307.7962341308594,
"logps/ref_rejected": -274.5501403808594,
"logps/rejected": -676.4373168945312,
"loss": 4.6975,
"margin_dpo/margin_mean": 121.55464172363281,
"margin_dpo/margin_std": 210.71910095214844,
"step": 457
},
{
"KL/chosen_KL_mean": -275.7753601074219,
"KL/mean": -325.453857421875,
"KL/rejected_KL_mean": -375.1323547363281,
"KL/std": 178.79847717285156,
"epoch": 0.9591623036649215,
"fcm_dpo/beta": 0.002581921638920903,
"fcm_dpo/delta": 0.01107083261013031,
"fcm_dpo/margin": 99.35697937011719,
"fcm_dpo/q_t": 0.44110581278800964,
"grad_norm": 55.92534637451172,
"learning_rate": 2.6765705380989432e-09,
"logits/chosen": -0.8420109152793884,
"logits/rejected": -0.8187612891197205,
"logps/chosen": -572.8070068359375,
"logps/ref_chosen": -297.0316467285156,
"logps/ref_rejected": -276.1112365722656,
"logps/rejected": -651.2435913085938,
"loss": 4.8562,
"margin_dpo/margin_mean": 99.35697937011719,
"margin_dpo/margin_std": 197.4419708251953,
"step": 458
},
{
"KL/chosen_KL_mean": -262.0546569824219,
"KL/mean": -305.3924865722656,
"KL/rejected_KL_mean": -348.73028564453125,
"KL/std": 151.888916015625,
"epoch": 0.9612565445026178,
"fcm_dpo/beta": 0.002643053652718663,
"fcm_dpo/delta": 0.04129417985677719,
"fcm_dpo/margin": 86.67559814453125,
"fcm_dpo/q_t": 0.4476607143878937,
"grad_norm": 63.60211944580078,
"learning_rate": 2.416026102552732e-09,
"logits/chosen": -0.9004296064376831,
"logits/rejected": -0.8917874097824097,
"logps/chosen": -555.5799560546875,
"logps/ref_chosen": -293.5252990722656,
"logps/ref_rejected": -289.30126953125,
"logps/rejected": -638.031494140625,
"loss": 4.924,
"margin_dpo/margin_mean": 86.67559814453125,
"margin_dpo/margin_std": 182.57568359375,
"step": 459
},
{
"KL/chosen_KL_mean": -271.2803039550781,
"KL/mean": -317.36767578125,
"KL/rejected_KL_mean": -363.455078125,
"KL/std": 146.47784423828125,
"epoch": 0.9633507853403142,
"fcm_dpo/beta": 0.0027003977447748184,
"fcm_dpo/delta": 0.012249761261045933,
"fcm_dpo/margin": 92.17478942871094,
"fcm_dpo/q_t": 0.4414074122905731,
"grad_norm": 90.7199478149414,
"learning_rate": 2.168758844148272e-09,
"logits/chosen": -0.8628847002983093,
"logits/rejected": -0.8590608835220337,
"logps/chosen": -590.0607299804688,
"logps/ref_chosen": -318.7803649902344,
"logps/ref_rejected": -258.7906799316406,
"logps/rejected": -622.2457885742188,
"loss": 4.8383,
"margin_dpo/margin_mean": 92.17478942871094,
"margin_dpo/margin_std": 174.8225555419922,
"step": 460
},
{
"KL/chosen_KL_mean": -266.7901306152344,
"KL/mean": -318.44488525390625,
"KL/rejected_KL_mean": -370.0997009277344,
"KL/std": 165.7501220703125,
"epoch": 0.9654450261780104,
"fcm_dpo/beta": 0.002741393866017461,
"fcm_dpo/delta": 0.0116231394931674,
"fcm_dpo/margin": 103.30960083007812,
"fcm_dpo/q_t": 0.4361855983734131,
"grad_norm": 88.08728790283203,
"learning_rate": 1.9347820230782295e-09,
"logits/chosen": -0.8326213955879211,
"logits/rejected": -0.8544647097587585,
"logps/chosen": -510.7001037597656,
"logps/ref_chosen": -243.9099884033203,
"logps/ref_rejected": -232.6382293701172,
"logps/rejected": -602.7379760742188,
"loss": 4.8029,
"margin_dpo/margin_mean": 103.30960083007812,
"margin_dpo/margin_std": 200.97842407226562,
"step": 461
},
{
"KL/chosen_KL_mean": -260.3146057128906,
"KL/mean": -323.3621826171875,
"KL/rejected_KL_mean": -386.40972900390625,
"KL/std": 158.8644561767578,
"epoch": 0.9675392670157068,
"fcm_dpo/beta": 0.002706458792090416,
"fcm_dpo/delta": -0.016491197049617767,
"fcm_dpo/margin": 126.09507751464844,
"fcm_dpo/q_t": 0.42121651768684387,
"grad_norm": 65.64117431640625,
"learning_rate": 1.7141081868094209e-09,
"logits/chosen": -0.8658933043479919,
"logits/rejected": -0.8126181364059448,
"logps/chosen": -604.4056396484375,
"logps/ref_chosen": -344.09100341796875,
"logps/ref_rejected": -252.45037841796875,
"logps/rejected": -638.8601684570312,
"loss": 4.5488,
"margin_dpo/margin_mean": 126.09507751464844,
"margin_dpo/margin_std": 184.6616668701172,
"step": 462
},
{
"KL/chosen_KL_mean": -263.3692321777344,
"KL/mean": -310.8293151855469,
"KL/rejected_KL_mean": -358.28936767578125,
"KL/std": 168.63275146484375,
"epoch": 0.9696335078534032,
"fcm_dpo/beta": 0.0027042459696531296,
"fcm_dpo/delta": -0.0022338179405778646,
"fcm_dpo/margin": 94.9201431274414,
"fcm_dpo/q_t": 0.4414103031158447,
"grad_norm": 89.20877075195312,
"learning_rate": 1.5067491694100153e-09,
"logits/chosen": -0.8476633429527283,
"logits/rejected": -0.8041394352912903,
"logps/chosen": -560.5116577148438,
"logps/ref_chosen": -297.1424560546875,
"logps/ref_rejected": -234.0208282470703,
"logps/rejected": -592.3101806640625,
"loss": 4.8758,
"margin_dpo/margin_mean": 94.92015075683594,
"margin_dpo/margin_std": 198.87655639648438,
"step": 463
},
{
"KL/chosen_KL_mean": -274.56988525390625,
"KL/mean": -323.9346618652344,
"KL/rejected_KL_mean": -373.29937744140625,
"KL/std": 161.66122436523438,
"epoch": 0.9717277486910995,
"fcm_dpo/beta": 0.002703792415559292,
"fcm_dpo/delta": 0.016853541135787964,
"fcm_dpo/margin": 98.72955322265625,
"fcm_dpo/q_t": 0.43772199749946594,
"grad_norm": 65.47233581542969,
"learning_rate": 1.3127160909147672e-09,
"logits/chosen": -0.8353570699691772,
"logits/rejected": -0.8507075905799866,
"logps/chosen": -540.2806396484375,
"logps/ref_chosen": -265.71075439453125,
"logps/ref_rejected": -256.4108581542969,
"logps/rejected": -629.7102661132812,
"loss": 4.7798,
"margin_dpo/margin_mean": 98.72955322265625,
"margin_dpo/margin_std": 175.61636352539062,
"step": 464
},
{
"KL/chosen_KL_mean": -252.164306640625,
"KL/mean": -317.9529724121094,
"KL/rejected_KL_mean": -383.7416076660156,
"KL/std": 157.08651733398438,
"epoch": 0.9738219895287958,
"fcm_dpo/beta": 0.002617460675537586,
"fcm_dpo/delta": -0.06499166041612625,
"fcm_dpo/margin": 131.57730102539062,
"fcm_dpo/q_t": 0.4203506112098694,
"grad_norm": 61.7449951171875,
"learning_rate": 1.1320193567288527e-09,
"logits/chosen": -0.9151032567024231,
"logits/rejected": -0.8762094974517822,
"logps/chosen": -545.3170166015625,
"logps/ref_chosen": -293.1527404785156,
"logps/ref_rejected": -293.70947265625,
"logps/rejected": -677.4511108398438,
"loss": 4.5364,
"margin_dpo/margin_mean": 131.57730102539062,
"margin_dpo/margin_std": 182.81796264648438,
"step": 465
},
{
"KL/chosen_KL_mean": -238.81729125976562,
"KL/mean": -307.2685546875,
"KL/rejected_KL_mean": -375.71978759765625,
"KL/std": 156.15011596679688,
"epoch": 0.9759162303664921,
"fcm_dpo/beta": 0.002551088575273752,
"fcm_dpo/delta": 0.0003731958568096161,
"fcm_dpo/margin": 136.90252685546875,
"fcm_dpo/q_t": 0.4190123379230499,
"grad_norm": 84.08358764648438,
"learning_rate": 9.64668657069706e-10,
"logits/chosen": -0.8227401375770569,
"logits/rejected": -0.7646010518074036,
"logps/chosen": -500.29486083984375,
"logps/ref_chosen": -261.4775695800781,
"logps/ref_rejected": -248.36282348632812,
"logps/rejected": -624.0826416015625,
"loss": 4.4914,
"margin_dpo/margin_mean": 136.90252685546875,
"margin_dpo/margin_std": 183.17254638671875,
"step": 466
},
{
"KL/chosen_KL_mean": -265.2427978515625,
"KL/mean": -315.06036376953125,
"KL/rejected_KL_mean": -364.8779602050781,
"KL/std": 160.9639892578125,
"epoch": 0.9780104712041885,
"fcm_dpo/beta": 0.002554756123572588,
"fcm_dpo/delta": -0.012596940621733665,
"fcm_dpo/margin": 99.6351547241211,
"fcm_dpo/q_t": 0.4416211247444153,
"grad_norm": 81.79702758789062,
"learning_rate": 8.106729664475176e-10,
"logits/chosen": -0.8072253465652466,
"logits/rejected": -0.7998838424682617,
"logps/chosen": -531.5970458984375,
"logps/ref_chosen": -266.354248046875,
"logps/ref_rejected": -277.76324462890625,
"logps/rejected": -642.6411743164062,
"loss": 4.8475,
"margin_dpo/margin_mean": 99.6351547241211,
"margin_dpo/margin_std": 196.27394104003906,
"step": 467
},
{
"KL/chosen_KL_mean": -268.8735046386719,
"KL/mean": -305.91766357421875,
"KL/rejected_KL_mean": -342.9618225097656,
"KL/std": 154.4098358154297,
"epoch": 0.9801047120418848,
"fcm_dpo/beta": 0.002624538727104664,
"fcm_dpo/delta": 0.048930246382951736,
"fcm_dpo/margin": 74.08831024169922,
"fcm_dpo/q_t": 0.45356136560440063,
"grad_norm": 70.1420669555664,
"learning_rate": 6.700405431837585e-10,
"logits/chosen": -0.8905247449874878,
"logits/rejected": -0.8599618673324585,
"logps/chosen": -586.8367309570312,
"logps/ref_chosen": -317.9631652832031,
"logps/ref_rejected": -261.8744201660156,
"logps/rejected": -604.8362426757812,
"loss": 5.0869,
"margin_dpo/margin_mean": 74.08831024169922,
"margin_dpo/margin_std": 198.05606079101562,
"step": 468
},
{
"KL/chosen_KL_mean": -249.63702392578125,
"KL/mean": -310.2472839355469,
"KL/rejected_KL_mean": -370.8575744628906,
"KL/std": 164.26007080078125,
"epoch": 0.9821989528795811,
"fcm_dpo/beta": 0.0026474245823919773,
"fcm_dpo/delta": -0.009328970685601234,
"fcm_dpo/margin": 121.22054290771484,
"fcm_dpo/q_t": 0.4254051744937897,
"grad_norm": 38.51364517211914,
"learning_rate": 5.427789289685347e-10,
"logits/chosen": -0.8271424174308777,
"logits/rejected": -0.8110153079032898,
"logps/chosen": -574.5238647460938,
"logps/ref_chosen": -324.8868103027344,
"logps/ref_rejected": -264.0421447753906,
"logps/rejected": -634.8997802734375,
"loss": 4.6194,
"margin_dpo/margin_mean": 121.22053527832031,
"margin_dpo/margin_std": 189.99745178222656,
"step": 469
},
{
"KL/chosen_KL_mean": -251.842529296875,
"KL/mean": -312.7073974609375,
"KL/rejected_KL_mean": -373.57232666015625,
"KL/std": 160.9497833251953,
"epoch": 0.9842931937172775,
"fcm_dpo/beta": 0.0025979301426559687,
"fcm_dpo/delta": -0.04329398646950722,
"fcm_dpo/margin": 121.72976684570312,
"fcm_dpo/q_t": 0.4263451397418976,
"grad_norm": 64.70094299316406,
"learning_rate": 4.288949484559934e-10,
"logits/chosen": -0.8176441788673401,
"logits/rejected": -0.8153640031814575,
"logps/chosen": -566.5467529296875,
"logps/ref_chosen": -314.7042541503906,
"logps/ref_rejected": -259.2276611328125,
"logps/rejected": -632.7999877929688,
"loss": 4.619,
"margin_dpo/margin_mean": 121.72976684570312,
"margin_dpo/margin_std": 182.20755004882812,
"step": 470
},
{
"KL/chosen_KL_mean": -273.6257629394531,
"KL/mean": -332.0756530761719,
"KL/rejected_KL_mean": -390.52557373046875,
"KL/std": 192.36666870117188,
"epoch": 0.9863874345549738,
"fcm_dpo/beta": 0.0025639727246016264,
"fcm_dpo/delta": -0.009167149662971497,
"fcm_dpo/margin": 116.89979553222656,
"fcm_dpo/q_t": 0.4352128505706787,
"grad_norm": 86.51072692871094,
"learning_rate": 3.2839470889836627e-10,
"logits/chosen": -0.8735991716384888,
"logits/rejected": -0.8589140176773071,
"logps/chosen": -566.20068359375,
"logps/ref_chosen": -292.5748291015625,
"logps/ref_rejected": -298.7525329589844,
"logps/rejected": -689.278076171875,
"loss": 4.7466,
"margin_dpo/margin_mean": 116.89979553222656,
"margin_dpo/margin_std": 217.3658905029297,
"step": 471
},
{
"KL/chosen_KL_mean": -249.77090454101562,
"KL/mean": -314.00897216796875,
"KL/rejected_KL_mean": -378.2470397949219,
"KL/std": 159.54998779296875,
"epoch": 0.9884816753926702,
"fcm_dpo/beta": 0.0025038199964910746,
"fcm_dpo/delta": -0.012427094392478466,
"fcm_dpo/margin": 128.47613525390625,
"fcm_dpo/q_t": 0.4258285164833069,
"grad_norm": 79.63658905029297,
"learning_rate": 2.412835998185092e-10,
"logits/chosen": -0.8494762778282166,
"logits/rejected": -0.8620040416717529,
"logps/chosen": -493.144775390625,
"logps/ref_chosen": -243.37380981445312,
"logps/ref_rejected": -251.12109375,
"logps/rejected": -629.3681640625,
"loss": 4.5936,
"margin_dpo/margin_mean": 128.47613525390625,
"margin_dpo/margin_std": 189.0301971435547,
"step": 472
},
{
"KL/chosen_KL_mean": -261.2845153808594,
"KL/mean": -321.57684326171875,
"KL/rejected_KL_mean": -381.86920166015625,
"KL/std": 147.27406311035156,
"epoch": 0.9905759162303664,
"fcm_dpo/beta": 0.0025835472624748945,
"fcm_dpo/delta": 0.03919682651758194,
"fcm_dpo/margin": 120.58467102050781,
"fcm_dpo/q_t": 0.4267313778400421,
"grad_norm": 50.619747161865234,
"learning_rate": 1.6756629272085544e-10,
"logits/chosen": -0.8187922835350037,
"logits/rejected": -0.816412091255188,
"logps/chosen": -547.6131591796875,
"logps/ref_chosen": -286.3286437988281,
"logps/ref_rejected": -258.6535339355469,
"logps/rejected": -640.5227661132812,
"loss": 4.583,
"margin_dpo/margin_mean": 120.58467102050781,
"margin_dpo/margin_std": 162.5080108642578,
"step": 473
},
{
"KL/chosen_KL_mean": -267.7467041015625,
"KL/mean": -321.2333984375,
"KL/rejected_KL_mean": -374.72015380859375,
"KL/std": 164.40090942382812,
"epoch": 0.9926701570680628,
"fcm_dpo/beta": 0.002585589187219739,
"fcm_dpo/delta": 0.016463520005345345,
"fcm_dpo/margin": 106.97345733642578,
"fcm_dpo/q_t": 0.43555474281311035,
"grad_norm": 92.50374603271484,
"learning_rate": 1.072467408408384e-10,
"logits/chosen": -0.8593537211418152,
"logits/rejected": -0.8538248538970947,
"logps/chosen": -555.8363037109375,
"logps/ref_chosen": -288.08966064453125,
"logps/ref_rejected": -266.69696044921875,
"logps/rejected": -641.4171142578125,
"loss": 4.7239,
"margin_dpo/margin_mean": 106.97344970703125,
"margin_dpo/margin_std": 171.1471405029297,
"step": 474
},
{
"KL/chosen_KL_mean": -246.10379028320312,
"KL/mean": -295.0997314453125,
"KL/rejected_KL_mean": -344.09564208984375,
"KL/std": 147.87771606445312,
"epoch": 0.9947643979057592,
"fcm_dpo/beta": 0.0025957191828638315,
"fcm_dpo/delta": -0.01816142536699772,
"fcm_dpo/margin": 97.99187469482422,
"fcm_dpo/q_t": 0.4415433406829834,
"grad_norm": 66.68531799316406,
"learning_rate": 6.032817893297793e-11,
"logits/chosen": -0.8354889750480652,
"logits/rejected": -0.8441330790519714,
"logps/chosen": -502.10687255859375,
"logps/ref_chosen": -256.0030517578125,
"logps/ref_rejected": -244.50660705566406,
"logps/rejected": -588.6022338867188,
"loss": 4.806,
"margin_dpo/margin_mean": 97.99187469482422,
"margin_dpo/margin_std": 172.44944763183594,
"step": 475
},
{
"KL/chosen_KL_mean": -248.5246124267578,
"KL/mean": -301.4661865234375,
"KL/rejected_KL_mean": -354.40777587890625,
"KL/std": 148.5124969482422,
"epoch": 0.9968586387434555,
"fcm_dpo/beta": 0.002539725974202156,
"fcm_dpo/delta": -0.031470753252506256,
"fcm_dpo/margin": 105.8831558227539,
"fcm_dpo/q_t": 0.43887490034103394,
"grad_norm": 69.0814208984375,
"learning_rate": 2.6813123097352287e-11,
"logits/chosen": -0.9057525396347046,
"logits/rejected": -0.8594703674316406,
"logps/chosen": -569.9921264648438,
"logps/ref_chosen": -321.467529296875,
"logps/ref_rejected": -295.0592956542969,
"logps/rejected": -649.4671630859375,
"loss": 4.7909,
"margin_dpo/margin_mean": 105.8831558227539,
"margin_dpo/margin_std": 185.74710083007812,
"step": 476
},
{
"KL/chosen_KL_mean": -270.9020690917969,
"KL/mean": -323.1831359863281,
"KL/rejected_KL_mean": -375.4642333984375,
"KL/std": 173.52398681640625,
"epoch": 0.9989528795811519,
"fcm_dpo/beta": 0.002541655208915472,
"fcm_dpo/delta": -0.0096766147762537,
"fcm_dpo/margin": 104.5621566772461,
"fcm_dpo/q_t": 0.4383348822593689,
"grad_norm": 120.1622085571289,
"learning_rate": 6.7033706447061635e-12,
"logits/chosen": -0.7790302634239197,
"logits/rejected": -0.7840569019317627,
"logps/chosen": -547.696044921875,
"logps/ref_chosen": -276.7939758300781,
"logps/ref_rejected": -244.82919311523438,
"logps/rejected": -620.2933959960938,
"loss": 4.8684,
"margin_dpo/margin_mean": 104.5621566772461,
"margin_dpo/margin_std": 218.2042236328125,
"step": 477
},
{
"epoch": 0.9989528795811519,
"step": 477,
"total_flos": 0.0,
"train_loss": 4.871757778231703,
"train_runtime": 6533.4358,
"train_samples_per_second": 9.357,
"train_steps_per_second": 0.073
}
],
"logging_steps": 1,
"max_steps": 477,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 200,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink