jackf857/qwen-3-8b-base-r-dpo-ultrafeedback-4xH200-batch-128-rerun-2-runpod
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
qwen-3-8b-base-r-dpo-ultraf…/trainer_state.json
ModelHub XC a1e826d8db 初始化项目，由ModelHub XC社区提供模型 
Model: jackf857/qwen-3-8b-base-r-dpo-ultrafeedback-4xH200-batch-128-rerun-2-runpod
Source: Original Platform
2026-05-09 21:14:25 +08:00

896 lines
32 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9989528795811519,
"eval_steps": 200,
"global_step": 477,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0020942408376963353,
"grad_norm": 19.463237762451172,
"learning_rate": 0.0,
"logits/chosen": 2.203179359436035,
"logits/rejected": 2.035616397857666,
"logps/chosen": -257.4821472167969,
"logps/ref_chosen": -257.3987731933594,
"logps/ref_rejected": -199.99972534179688,
"logps/rejected": -199.93338012695312,
"loss": 45.6501,
"r_dpo/chosen_len": 260.25,
"r_dpo/length_delta": 46.875,
"r_dpo/regularization_term": 4.6875,
"r_dpo/rejected_len": 213.375,
"step": 1
},
{
"epoch": 0.020942408376963352,
"grad_norm": 18.84970474243164,
"learning_rate": 9.375e-08,
"logits/chosen": 1.9343596696853638,
"logits/rejected": 1.8566161394119263,
"logps/chosen": -290.1725158691406,
"logps/ref_chosen": -290.0645751953125,
"logps/ref_rejected": -255.95347595214844,
"logps/rejected": -256.05364990234375,
"loss": 43.0812,
"r_dpo/chosen_len": 295.9757080078125,
"r_dpo/length_delta": 49.59722137451172,
"r_dpo/regularization_term": 4.959722518920898,
"r_dpo/rejected_len": 246.37847900390625,
"step": 10
},
{
"epoch": 0.041884816753926704,
"grad_norm": 19.003461837768555,
"learning_rate": 1.9791666666666664e-07,
"logits/chosen": 1.8805112838745117,
"logits/rejected": 1.8091768026351929,
"logps/chosen": -289.0137023925781,
"logps/ref_chosen": -288.9098815917969,
"logps/ref_rejected": -254.6271514892578,
"logps/rejected": -254.85696411132812,
"loss": 41.2955,
"r_dpo/chosen_len": 295.1187438964844,
"r_dpo/length_delta": 54.203125,
"r_dpo/regularization_term": 5.42031192779541,
"r_dpo/rejected_len": 240.91561889648438,
"step": 20
},
{
"epoch": 0.06282722513089005,
"grad_norm": 21.310501098632812,
"learning_rate": 3.020833333333333e-07,
"logits/chosen": 1.8696081638336182,
"logits/rejected": 1.818481683731079,
"logps/chosen": -269.52618408203125,
"logps/ref_chosen": -270.3978576660156,
"logps/ref_rejected": -247.89431762695312,
"logps/rejected": -247.4498291015625,
"loss": 43.6841,
"r_dpo/chosen_len": 272.8062438964844,
"r_dpo/length_delta": 25.693750381469727,
"r_dpo/regularization_term": 2.5693750381469727,
"r_dpo/rejected_len": 247.1125030517578,
"step": 30
},
{
"epoch": 0.08376963350785341,
"grad_norm": 19.77940559387207,
"learning_rate": 4.0625e-07,
"logits/chosen": 1.8189456462860107,
"logits/rejected": 1.8397687673568726,
"logps/chosen": -275.187744140625,
"logps/ref_chosen": -274.24859619140625,
"logps/ref_rejected": -261.91693115234375,
"logps/rejected": -264.21368408203125,
"loss": 41.7583,
"r_dpo/chosen_len": 284.7875061035156,
"r_dpo/length_delta": 33.97187423706055,
"r_dpo/regularization_term": 3.3971877098083496,
"r_dpo/rejected_len": 250.8156280517578,
"step": 40
},
{
"epoch": 0.10471204188481675,
"grad_norm": 23.84255027770996,
"learning_rate": 4.999932966293553e-07,
"logits/chosen": 1.8647778034210205,
"logits/rejected": 1.9068819284439087,
"logps/chosen": -280.23028564453125,
"logps/ref_chosen": -270.4697570800781,
"logps/ref_rejected": -271.0193176269531,
"logps/rejected": -282.69647216796875,
"loss": 43.553,
"r_dpo/chosen_len": 294.27813720703125,
"r_dpo/length_delta": 37.4375,
"r_dpo/regularization_term": 3.7437500953674316,
"r_dpo/rejected_len": 256.84063720703125,
"step": 50
},
{
"epoch": 0.1256544502617801,
"grad_norm": 34.24372482299805,
"learning_rate": 4.991893270335525e-07,
"logits/chosen": 1.659531831741333,
"logits/rejected": 1.6509793996810913,
"logps/chosen": -291.05303955078125,
"logps/ref_chosen": -273.5165710449219,
"logps/ref_rejected": -257.33099365234375,
"logps/rejected": -275.82696533203125,
"loss": 46.1736,
"r_dpo/chosen_len": 277.1875,
"r_dpo/length_delta": 29.518749237060547,
"r_dpo/regularization_term": 2.9518749713897705,
"r_dpo/rejected_len": 247.6687469482422,
"step": 60
},
{
"epoch": 0.14659685863874344,
"grad_norm": 31.267230987548828,
"learning_rate": 4.970496218214204e-07,
"logits/chosen": 1.6715799570083618,
"logits/rejected": 1.7152721881866455,
"logps/chosen": -303.29656982421875,
"logps/ref_chosen": -279.4599914550781,
"logps/ref_rejected": -274.3787536621094,
"logps/rejected": -300.0,
"loss": 41.8889,
"r_dpo/chosen_len": 269.234375,
"r_dpo/length_delta": 14.634374618530273,
"r_dpo/regularization_term": 1.4634376764297485,
"r_dpo/rejected_len": 254.60000610351562,
"step": 70
},
{
"epoch": 0.16753926701570682,
"grad_norm": 86.9355697631836,
"learning_rate": 4.935856505068998e-07,
"logits/chosen": 1.5558438301086426,
"logits/rejected": 1.589551568031311,
"logps/chosen": -319.4896545410156,
"logps/ref_chosen": -272.08538818359375,
"logps/ref_rejected": -255.73300170898438,
"logps/rejected": -304.3583984375,
"loss": 44.2864,
"r_dpo/chosen_len": 270.97186279296875,
"r_dpo/length_delta": 32.712501525878906,
"r_dpo/regularization_term": 3.2712502479553223,
"r_dpo/rejected_len": 238.25936889648438,
"step": 80
},
{
"epoch": 0.18848167539267016,
"grad_norm": 53.2010498046875,
"learning_rate": 4.8881598109976e-07,
"logits/chosen": 1.4091914892196655,
"logits/rejected": 1.4223301410675049,
"logps/chosen": -342.06500244140625,
"logps/ref_chosen": -269.1466369628906,
"logps/ref_rejected": -247.70956420898438,
"logps/rejected": -320.5975341796875,
"loss": 43.205,
"r_dpo/chosen_len": 278.11248779296875,
"r_dpo/length_delta": 47.27812576293945,
"r_dpo/regularization_term": 4.727812767028809,
"r_dpo/rejected_len": 230.83438110351562,
"step": 90
},
{
"epoch": 0.2094240837696335,
"grad_norm": 221.70379638671875,
"learning_rate": 4.827661805750437e-07,
"logits/chosen": 1.171535849571228,
"logits/rejected": 1.1951229572296143,
"logps/chosen": -339.74676513671875,
"logps/ref_chosen": -265.9335632324219,
"logps/ref_rejected": -270.045654296875,
"logps/rejected": -358.4281921386719,
"loss": 41.9093,
"r_dpo/chosen_len": 278.3656311035156,
"r_dpo/length_delta": 22.96875,
"r_dpo/regularization_term": 2.296875,
"r_dpo/rejected_len": 255.39688110351562,
"step": 100
},
{
"epoch": 0.23036649214659685,
"grad_norm": 206.1243438720703,
"learning_rate": 4.75468677825789e-07,
"logits/chosen": 1.2150897979736328,
"logits/rejected": 1.3317502737045288,
"logps/chosen": -387.6827087402344,
"logps/ref_chosen": -265.02532958984375,
"logps/ref_rejected": -259.968017578125,
"logps/rejected": -389.2899475097656,
"loss": 36.8666,
"r_dpo/chosen_len": 287.140625,
"r_dpo/length_delta": 50.765625,
"r_dpo/regularization_term": 5.076562881469727,
"r_dpo/rejected_len": 236.375,
"step": 110
},
{
"epoch": 0.2513089005235602,
"grad_norm": 155.615966796875,
"learning_rate": 4.669625898336438e-07,
"logits/chosen": 1.069676399230957,
"logits/rejected": 1.0872914791107178,
"logps/chosen": -421.203125,
"logps/ref_chosen": -262.50323486328125,
"logps/ref_rejected": -268.14996337890625,
"logps/rejected": -438.32208251953125,
"loss": 40.1069,
"r_dpo/chosen_len": 266.70001220703125,
"r_dpo/length_delta": 16.190624237060547,
"r_dpo/regularization_term": 1.6190624237060547,
"r_dpo/rejected_len": 250.50936889648438,
"step": 120
},
{
"epoch": 0.27225130890052357,
"grad_norm": 440.5760498046875,
"learning_rate": 4.5729351198915705e-07,
"logits/chosen": 0.7952426671981812,
"logits/rejected": 0.9414566159248352,
"logps/chosen": -471.2674865722656,
"logps/ref_chosen": -267.5928039550781,
"logps/ref_rejected": -272.86810302734375,
"logps/rejected": -491.12835693359375,
"loss": 41.9211,
"r_dpo/chosen_len": 268.75,
"r_dpo/length_delta": 19.178125381469727,
"r_dpo/regularization_term": 1.9178125858306885,
"r_dpo/rejected_len": 249.57186889648438,
"step": 130
},
{
"epoch": 0.2931937172774869,
"grad_norm": 194.5912322998047,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": 0.6943808794021606,
"logits/rejected": 0.750653088092804,
"logps/chosen": -524.4630126953125,
"logps/ref_chosen": -268.4759521484375,
"logps/ref_rejected": -254.6080780029297,
"logps/rejected": -514.9945678710938,
"loss": 40.9788,
"r_dpo/chosen_len": 261.46875,
"r_dpo/length_delta": 21.350000381469727,
"r_dpo/regularization_term": 2.135000228881836,
"r_dpo/rejected_len": 240.11874389648438,
"step": 140
},
{
"epoch": 0.31413612565445026,
"grad_norm": 271.27703857421875,
"learning_rate": 4.346796604970912e-07,
"logits/chosen": 0.33445867896080017,
"logits/rejected": 0.435607373714447,
"logps/chosen": -838.4150390625,
"logps/ref_chosen": -290.3619384765625,
"logps/ref_rejected": -252.4545135498047,
"logps/rejected": -750.5610961914062,
"loss": 32.2283,
"r_dpo/chosen_len": 286.29998779296875,
"r_dpo/length_delta": 48.62187576293945,
"r_dpo/regularization_term": 4.862187385559082,
"r_dpo/rejected_len": 237.67813110351562,
"step": 150
},
{
"epoch": 0.33507853403141363,
"grad_norm": 1304.436279296875,
"learning_rate": 4.218561044282098e-07,
"logits/chosen": -0.5341374278068542,
"logits/rejected": -0.3209038972854614,
"logps/chosen": -1746.661376953125,
"logps/ref_chosen": -272.7589111328125,
"logps/ref_rejected": -252.6876678466797,
"logps/rejected": -1500.4178466796875,
"loss": 23.2094,
"r_dpo/chosen_len": 271.48748779296875,
"r_dpo/length_delta": 41.578125,
"r_dpo/regularization_term": 4.157812118530273,
"r_dpo/rejected_len": 229.9093780517578,
"step": 160
},
{
"epoch": 0.35602094240837695,
"grad_norm": 672.2740478515625,
"learning_rate": 4.081113438988443e-07,
"logits/chosen": -0.30694156885147095,
"logits/rejected": -0.16359911859035492,
"logps/chosen": -2336.08984375,
"logps/ref_chosen": -281.5549621582031,
"logps/ref_rejected": -238.20858764648438,
"logps/rejected": -2039.7515869140625,
"loss": 10.9533,
"r_dpo/chosen_len": 288.2718811035156,
"r_dpo/length_delta": 47.34375,
"r_dpo/regularization_term": 4.734375476837158,
"r_dpo/rejected_len": 240.92813110351562,
"step": 170
},
{
"epoch": 0.3769633507853403,
"grad_norm": 437.60687255859375,
"learning_rate": 3.935190552834828e-07,
"logits/chosen": -0.3067488968372345,
"logits/rejected": -0.20810024440288544,
"logps/chosen": -2306.261474609375,
"logps/ref_chosen": -278.9187927246094,
"logps/ref_rejected": -248.9375457763672,
"logps/rejected": -2002.85546875,
"loss": 9.0173,
"r_dpo/chosen_len": 269.1656188964844,
"r_dpo/length_delta": 41.087501525878906,
"r_dpo/regularization_term": 4.108750343322754,
"r_dpo/rejected_len": 228.078125,
"step": 180
},
{
"epoch": 0.39790575916230364,
"grad_norm": 321.61346435546875,
"learning_rate": 3.781574579820464e-07,
"logits/chosen": 0.40285807847976685,
"logits/rejected": 0.36570462584495544,
"logps/chosen": -2637.16845703125,
"logps/ref_chosen": -272.6444091796875,
"logps/ref_rejected": -251.23965454101562,
"logps/rejected": -2281.21337890625,
"loss": 7.2881,
"r_dpo/chosen_len": 280.03436279296875,
"r_dpo/length_delta": 43.209373474121094,
"r_dpo/regularization_term": 4.320937156677246,
"r_dpo/rejected_len": 236.8249969482422,
"step": 190
},
{
"epoch": 0.418848167539267,
"grad_norm": 389.5361022949219,
"learning_rate": 3.621088951385353e-07,
"logits/chosen": 0.8133895993232727,
"logits/rejected": 0.7692806720733643,
"logps/chosen": -2436.361083984375,
"logps/ref_chosen": -247.26144409179688,
"logps/ref_rejected": -240.1345977783203,
"logps/rejected": -2211.449951171875,
"loss": 6.6608,
"r_dpo/chosen_len": 249.71249389648438,
"r_dpo/length_delta": 27.921875,
"r_dpo/regularization_term": 2.7921876907348633,
"r_dpo/rejected_len": 221.79061889648438,
"step": 200
},
{
"epoch": 0.418848167539267,
"eval_logits/chosen": 0.6076958775520325,
"eval_logits/rejected": 0.5855662822723389,
"eval_logps/chosen": -2819.177978515625,
"eval_logps/ref_chosen": -281.4849548339844,
"eval_logps/ref_rejected": -261.800537109375,
"eval_logps/rejected": -2438.321044921875,
"eval_loss": 0.7287593483924866,
"eval_r_dpo/chosen_len": 294.4800109863281,
"eval_r_dpo/length_delta": 44.61000061035156,
"eval_r_dpo/regularization_term": 4.461000442504883,
"eval_r_dpo/rejected_len": 249.8699951171875,
"eval_runtime": 97.4324,
"eval_samples_per_second": 20.527,
"eval_steps_per_second": 1.283,
"step": 200
},
{
"epoch": 0.4397905759162304,
"grad_norm": 347.1392517089844,
"learning_rate": 3.454593922550693e-07,
"logits/chosen": 0.6368786692619324,
"logits/rejected": 0.5997772216796875,
"logps/chosen": -2782.99755859375,
"logps/ref_chosen": -280.40606689453125,
"logps/ref_rejected": -259.3473205566406,
"logps/rejected": -2457.272705078125,
"loss": 6.155,
"r_dpo/chosen_len": 282.6156311035156,
"r_dpo/length_delta": 36.171875,
"r_dpo/regularization_term": 3.6171875,
"r_dpo/rejected_len": 246.44375610351562,
"step": 210
},
{
"epoch": 0.4607329842931937,
"grad_norm": 168.1587371826172,
"learning_rate": 3.2829819606729477e-07,
"logits/chosen": 0.771518886089325,
"logits/rejected": 0.7589208483695984,
"logps/chosen": -2625.2099609375,
"logps/ref_chosen": -276.30694580078125,
"logps/ref_rejected": -268.7990417480469,
"logps/rejected": -2493.98193359375,
"loss": 5.5739,
"r_dpo/chosen_len": 265.546875,
"r_dpo/length_delta": 18.515625,
"r_dpo/regularization_term": 1.8515625,
"r_dpo/rejected_len": 247.03125,
"step": 220
},
{
"epoch": 0.4816753926701571,
"grad_norm": 290.4512939453125,
"learning_rate": 3.1071729615293424e-07,
"logits/chosen": 0.2807890474796295,
"logits/rejected": 0.27411508560180664,
"logps/chosen": -2629.66943359375,
"logps/ref_chosen": -265.4647216796875,
"logps/ref_rejected": -251.6260223388672,
"logps/rejected": -2396.610107421875,
"loss": 5.3014,
"r_dpo/chosen_len": 268.1812438964844,
"r_dpo/length_delta": 31.868749618530273,
"r_dpo/regularization_term": 3.1868748664855957,
"r_dpo/rejected_len": 236.3125,
"step": 230
},
{
"epoch": 0.5026178010471204,
"grad_norm": 200.7110137939453,
"learning_rate": 2.9281093183781403e-07,
"logits/chosen": 0.6274879574775696,
"logits/rejected": 0.589483916759491,
"logps/chosen": -2683.999267578125,
"logps/ref_chosen": -262.2581481933594,
"logps/ref_rejected": -251.36373901367188,
"logps/rejected": -2375.33642578125,
"loss": 5.475,
"r_dpo/chosen_len": 275.421875,
"r_dpo/length_delta": 37.65937423706055,
"r_dpo/regularization_term": 3.765937328338623,
"r_dpo/rejected_len": 237.7624969482422,
"step": 240
},
{
"epoch": 0.5235602094240838,
"grad_norm": 328.9794921875,
"learning_rate": 2.7467508704251135e-07,
"logits/chosen": 0.4434322416782379,
"logits/rejected": 0.4112902581691742,
"logps/chosen": -2788.759765625,
"logps/ref_chosen": -282.21917724609375,
"logps/ref_rejected": -262.7974548339844,
"logps/rejected": -2449.12939453125,
"loss": 4.8484,
"r_dpo/chosen_len": 280.37811279296875,
"r_dpo/length_delta": 41.196876525878906,
"r_dpo/regularization_term": 4.119687557220459,
"r_dpo/rejected_len": 239.18124389648438,
"step": 250
},
{
"epoch": 0.5445026178010471,
"grad_norm": 174.2191925048828,
"learning_rate": 2.5640697577740815e-07,
"logits/chosen": 0.510331392288208,
"logits/rejected": 0.46630868315696716,
"logps/chosen": -2726.13232421875,
"logps/ref_chosen": -280.39166259765625,
"logps/ref_rejected": -263.56573486328125,
"logps/rejected": -2545.450439453125,
"loss": 5.0081,
"r_dpo/chosen_len": 273.4468688964844,
"r_dpo/length_delta": 25.353124618530273,
"r_dpo/regularization_term": 2.5353128910064697,
"r_dpo/rejected_len": 248.09375,
"step": 260
},
{
"epoch": 0.5654450261780105,
"grad_norm": 183.47354125976562,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": 0.6359630823135376,
"logits/rejected": 0.6186918020248413,
"logps/chosen": -2736.114501953125,
"logps/ref_chosen": -271.36041259765625,
"logps/ref_rejected": -262.174072265625,
"logps/rejected": -2623.23974609375,
"loss": 5.0971,
"r_dpo/chosen_len": 274.98126220703125,
"r_dpo/length_delta": 19.128124237060547,
"r_dpo/regularization_term": 1.9128128290176392,
"r_dpo/rejected_len": 255.85311889648438,
"step": 270
},
{
"epoch": 0.5863874345549738,
"grad_norm": 193.64608764648438,
"learning_rate": 2.1986582993616925e-07,
"logits/chosen": 0.2083236277103424,
"logits/rejected": 0.16970780491828918,
"logps/chosen": -2850.24462890625,
"logps/ref_chosen": -293.512939453125,
"logps/ref_rejected": -253.37246704101562,
"logps/rejected": -2385.664794921875,
"loss": 5.0361,
"r_dpo/chosen_len": 287.9437561035156,
"r_dpo/length_delta": 52.75312423706055,
"r_dpo/regularization_term": 5.275312423706055,
"r_dpo/rejected_len": 235.1906280517578,
"step": 280
},
{
"epoch": 0.6073298429319371,
"grad_norm": 193.0575408935547,
"learning_rate": 2.0178866775369774e-07,
"logits/chosen": 0.7391489148139954,
"logits/rejected": 0.714061975479126,
"logps/chosen": -2967.319091796875,
"logps/ref_chosen": -294.9578857421875,
"logps/ref_rejected": -288.91802978515625,
"logps/rejected": -2823.838623046875,
"loss": 4.9628,
"r_dpo/chosen_len": 298.48748779296875,
"r_dpo/length_delta": 22.706249237060547,
"r_dpo/regularization_term": 2.270624876022339,
"r_dpo/rejected_len": 275.78125,
"step": 290
},
{
"epoch": 0.6282722513089005,
"grad_norm": 167.63522338867188,
"learning_rate": 1.839699339491937e-07,
"logits/chosen": 0.733379065990448,
"logits/rejected": 0.6921027898788452,
"logps/chosen": -2751.304931640625,
"logps/ref_chosen": -267.9673767089844,
"logps/ref_rejected": -262.1747131347656,
"logps/rejected": -2606.73583984375,
"loss": 4.8425,
"r_dpo/chosen_len": 269.3999938964844,
"r_dpo/length_delta": 20.831249237060547,
"r_dpo/regularization_term": 2.083124876022339,
"r_dpo/rejected_len": 248.56875610351562,
"step": 300
},
{
"epoch": 0.6492146596858639,
"grad_norm": 373.60211181640625,
"learning_rate": 1.6650514271527465e-07,
"logits/chosen": 0.473613440990448,
"logits/rejected": 0.41956576704978943,
"logps/chosen": -2961.7822265625,
"logps/ref_chosen": -286.36895751953125,
"logps/ref_rejected": -274.4711608886719,
"logps/rejected": -2710.29736328125,
"loss": 4.8166,
"r_dpo/chosen_len": 296.7749938964844,
"r_dpo/length_delta": 32.959373474121094,
"r_dpo/regularization_term": 3.2959372997283936,
"r_dpo/rejected_len": 263.81561279296875,
"step": 310
},
{
"epoch": 0.6701570680628273,
"grad_norm": 178.9756317138672,
"learning_rate": 1.4948791099758052e-07,
"logits/chosen": 0.6881324648857117,
"logits/rejected": 0.6306595802307129,
"logps/chosen": -2834.637451171875,
"logps/ref_chosen": -276.7845764160156,
"logps/ref_rejected": -258.2370300292969,
"logps/rejected": -2472.187744140625,
"loss": 4.5986,
"r_dpo/chosen_len": 282.578125,
"r_dpo/length_delta": 45.78437423706055,
"r_dpo/regularization_term": 4.578437805175781,
"r_dpo/rejected_len": 236.7937469482422,
"step": 320
},
{
"epoch": 0.6910994764397905,
"grad_norm": 184.68521118164062,
"learning_rate": 1.3300945667758012e-07,
"logits/chosen": 0.6964876651763916,
"logits/rejected": 0.6566891670227051,
"logps/chosen": -2763.50390625,
"logps/ref_chosen": -275.9217529296875,
"logps/ref_rejected": -267.70782470703125,
"logps/rejected": -2724.64453125,
"loss": 4.854,
"r_dpo/chosen_len": 270.0531311035156,
"r_dpo/length_delta": 12.415624618530273,
"r_dpo/regularization_term": 1.2415623664855957,
"r_dpo/rejected_len": 257.63751220703125,
"step": 330
},
{
"epoch": 0.7120418848167539,
"grad_norm": 233.2672882080078,
"learning_rate": 1.1715810961514072e-07,
"logits/chosen": 0.45854058861732483,
"logits/rejected": 0.4010751247406006,
"logps/chosen": -2628.388671875,
"logps/ref_chosen": -251.37319946289062,
"logps/ref_rejected": -240.65988159179688,
"logps/rejected": -2376.85595703125,
"loss": 4.7891,
"r_dpo/chosen_len": 258.17498779296875,
"r_dpo/length_delta": 32.990623474121094,
"r_dpo/regularization_term": 3.2990622520446777,
"r_dpo/rejected_len": 225.1843719482422,
"step": 340
},
{
"epoch": 0.7329842931937173,
"grad_norm": 246.9112548828125,
"learning_rate": 1.0201883817182949e-07,
"logits/chosen": 0.6637452840805054,
"logits/rejected": 0.5826420187950134,
"logps/chosen": -2949.142333984375,
"logps/ref_chosen": -290.9896240234375,
"logps/ref_rejected": -269.58502197265625,
"logps/rejected": -2537.853271484375,
"loss": 4.6654,
"r_dpo/chosen_len": 284.6187438964844,
"r_dpo/length_delta": 46.57500076293945,
"r_dpo/regularization_term": 4.657500267028809,
"r_dpo/rejected_len": 238.0437469482422,
"step": 350
},
{
"epoch": 0.7539267015706806,
"grad_norm": 221.2776336669922,
"learning_rate": 8.76727937529367e-08,
"logits/chosen": 0.47227534651756287,
"logits/rejected": 0.4251709580421448,
"logps/chosen": -2778.8525390625,
"logps/ref_chosen": -272.4156799316406,
"logps/ref_rejected": -261.71087646484375,
"logps/rejected": -2545.5712890625,
"loss": 4.7021,
"r_dpo/chosen_len": 275.23748779296875,
"r_dpo/length_delta": 30.956249237060547,
"r_dpo/regularization_term": 3.095625162124634,
"r_dpo/rejected_len": 244.28125,
"step": 360
},
{
"epoch": 0.774869109947644,
"grad_norm": 179.33848571777344,
"learning_rate": 7.419687580962222e-08,
"logits/chosen": 0.4994059205055237,
"logits/rejected": 0.43345919251441956,
"logps/chosen": -2812.973876953125,
"logps/ref_chosen": -295.822998046875,
"logps/ref_rejected": -263.49639892578125,
"logps/rejected": -2578.302734375,
"loss": 4.31,
"r_dpo/chosen_len": 277.0718688964844,
"r_dpo/length_delta": 33.87187576293945,
"r_dpo/regularization_term": 3.3871874809265137,
"r_dpo/rejected_len": 243.1999969482422,
"step": 370
},
{
"epoch": 0.7958115183246073,
"grad_norm": 127.59620666503906,
"learning_rate": 6.166331963291519e-08,
"logits/chosen": 0.36374586820602417,
"logits/rejected": 0.3168891966342926,
"logps/chosen": -2917.382568359375,
"logps/ref_chosen": -292.1449890136719,
"logps/ref_rejected": -262.7100524902344,
"logps/rejected": -2657.446533203125,
"loss": 4.4736,
"r_dpo/chosen_len": 289.8687438964844,
"r_dpo/length_delta": 33.578125,
"r_dpo/regularization_term": 3.3578124046325684,
"r_dpo/rejected_len": 256.2906188964844,
"step": 380
},
{
"epoch": 0.8167539267015707,
"grad_norm": 180.08554077148438,
"learning_rate": 5.013930914912476e-08,
"logits/chosen": 0.6072987914085388,
"logits/rejected": 0.5558676719665527,
"logps/chosen": -2953.486083984375,
"logps/ref_chosen": -286.49505615234375,
"logps/ref_rejected": -274.98065185546875,
"logps/rejected": -2727.49951171875,
"loss": 4.5136,
"r_dpo/chosen_len": 290.92498779296875,
"r_dpo/length_delta": 31.337499618530273,
"r_dpo/regularization_term": 3.1337499618530273,
"r_dpo/rejected_len": 259.5874938964844,
"step": 390
},
{
"epoch": 0.837696335078534,
"grad_norm": 181.38357543945312,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": 0.6835483312606812,
"logits/rejected": 0.6234606504440308,
"logps/chosen": -2877.609130859375,
"logps/ref_chosen": -289.52215576171875,
"logps/ref_rejected": -254.75961303710938,
"logps/rejected": -2525.186279296875,
"loss": 4.5388,
"r_dpo/chosen_len": 283.69061279296875,
"r_dpo/length_delta": 40.421875,
"r_dpo/regularization_term": 4.042187690734863,
"r_dpo/rejected_len": 243.2687530517578,
"step": 400
},
{
"epoch": 0.837696335078534,
"eval_logits/chosen": 0.6320263743400574,
"eval_logits/rejected": 0.5333738327026367,
"eval_logps/chosen": -2936.774658203125,
"eval_logps/ref_chosen": -281.4849548339844,
"eval_logps/ref_rejected": -261.800537109375,
"eval_logps/rejected": -2579.16845703125,
"eval_loss": 0.5469310879707336,
"eval_r_dpo/chosen_len": 294.4800109863281,
"eval_r_dpo/length_delta": 44.61000061035156,
"eval_r_dpo/regularization_term": 4.461000442504883,
"eval_r_dpo/rejected_len": 249.8699951171875,
"eval_runtime": 96.769,
"eval_samples_per_second": 20.668,
"eval_steps_per_second": 1.292,
"step": 400
},
{
"epoch": 0.8586387434554974,
"grad_norm": 192.98953247070312,
"learning_rate": 3.036127238347164e-08,
"logits/chosen": 0.8459997177124023,
"logits/rejected": 0.8034231066703796,
"logps/chosen": -2868.36279296875,
"logps/ref_chosen": -282.5110778808594,
"logps/ref_rejected": -269.2117004394531,
"logps/rejected": -2660.260009765625,
"loss": 4.6056,
"r_dpo/chosen_len": 283.82501220703125,
"r_dpo/length_delta": 27.762500762939453,
"r_dpo/regularization_term": 2.7762503623962402,
"r_dpo/rejected_len": 256.0625,
"step": 410
},
{
"epoch": 0.8795811518324608,
"grad_norm": 174.4208526611328,
"learning_rate": 2.2213262793589482e-08,
"logits/chosen": 0.7089523077011108,
"logits/rejected": 0.6533206105232239,
"logps/chosen": -3067.21728515625,
"logps/ref_chosen": -300.7202453613281,
"logps/ref_rejected": -276.92340087890625,
"logps/rejected": -2775.168701171875,
"loss": 4.254,
"r_dpo/chosen_len": 298.9937438964844,
"r_dpo/length_delta": 37.178123474121094,
"r_dpo/regularization_term": 3.7178127765655518,
"r_dpo/rejected_len": 261.81561279296875,
"step": 420
},
{
"epoch": 0.900523560209424,
"grad_norm": 184.81227111816406,
"learning_rate": 1.5286263996730026e-08,
"logits/chosen": 0.6772060990333557,
"logits/rejected": 0.5796979665756226,
"logps/chosen": -2999.145263671875,
"logps/ref_chosen": -287.7877502441406,
"logps/ref_rejected": -262.2414245605469,
"logps/rejected": -2588.61669921875,
"loss": 4.3751,
"r_dpo/chosen_len": 292.76251220703125,
"r_dpo/length_delta": 49.0,
"r_dpo/regularization_term": 4.900000095367432,
"r_dpo/rejected_len": 243.7624969482422,
"step": 430
},
{
"epoch": 0.9214659685863874,
"grad_norm": 175.2359619140625,
"learning_rate": 9.617406953185136e-09,
"logits/chosen": 0.6800681352615356,
"logits/rejected": 0.6796324849128723,
"logps/chosen": -2927.792236328125,
"logps/ref_chosen": -286.9573669433594,
"logps/ref_rejected": -285.1015319824219,
"logps/rejected": -2881.3701171875,
"loss": 4.664,
"r_dpo/chosen_len": 289.5218811035156,
"r_dpo/length_delta": 13.934374809265137,
"r_dpo/regularization_term": 1.3934376239776611,
"r_dpo/rejected_len": 275.5874938964844,
"step": 440
},
{
"epoch": 0.9424083769633508,
"grad_norm": 213.96315002441406,
"learning_rate": 5.2370785753763356e-09,
"logits/chosen": 0.702254056930542,
"logits/rejected": 0.6395606994628906,
"logps/chosen": -2904.09423828125,
"logps/ref_chosen": -288.1899719238281,
"logps/ref_rejected": -258.13909912109375,
"logps/rejected": -2574.60986328125,
"loss": 4.4079,
"r_dpo/chosen_len": 285.5,
"r_dpo/length_delta": 39.22187423706055,
"r_dpo/regularization_term": 3.9221878051757812,
"r_dpo/rejected_len": 246.2781219482422,
"step": 450
},
{
"epoch": 0.9633507853403142,
"grad_norm": 308.9233703613281,
"learning_rate": 2.168758844148272e-09,
"logits/chosen": 0.7860502600669861,
"logits/rejected": 0.6886175870895386,
"logps/chosen": -2969.99609375,
"logps/ref_chosen": -304.67333984375,
"logps/ref_rejected": -275.60491943359375,
"logps/rejected": -2588.6171875,
"loss": 4.4801,
"r_dpo/chosen_len": 291.12188720703125,
"r_dpo/length_delta": 44.39374923706055,
"r_dpo/regularization_term": 4.439374923706055,
"r_dpo/rejected_len": 246.72811889648438,
"step": 460
},
{
"epoch": 0.9842931937172775,
"grad_norm": 133.6565704345703,
"learning_rate": 4.288949484559934e-10,
"logits/chosen": 0.6798645853996277,
"logits/rejected": 0.6197268962860107,
"logps/chosen": -2756.23876953125,
"logps/ref_chosen": -270.48248291015625,
"logps/ref_rejected": -253.36062622070312,
"logps/rejected": -2583.438232421875,
"loss": 4.3438,
"r_dpo/chosen_len": 271.95623779296875,
"r_dpo/length_delta": 25.784374237060547,
"r_dpo/regularization_term": 2.578437566757202,
"r_dpo/rejected_len": 246.171875,
"step": 470
},
{
"epoch": 0.9989528795811519,
"step": 477,
"total_flos": 0.0,
"train_loss": 17.04746322311945,
"train_runtime": 7166.7787,
"train_samples_per_second": 8.53,
"train_steps_per_second": 0.067
}
],
"logging_steps": 10,
"max_steps": 477,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink