7da87889a0
Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-s_star0.85-4xh200-batch-64-20260421-213851 Source: Original Platform
2631 lines
96 KiB
JSON
2631 lines
96 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.999244142101285,
|
|
"eval_steps": 200,
|
|
"global_step": 661,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0015117157974300832,
|
|
"fcm_dpo/beta": 0.10000000149011612,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.0013532638549804688,
|
|
"fcm_dpo/q_t": 0.5000336766242981,
|
|
"grad_norm": 28.219520568847656,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": 0.13337239623069763,
|
|
"logits/rejected": 0.12492948770523071,
|
|
"logps/chosen": -64.5841293334961,
|
|
"logps/ref_chosen": -64.61280822753906,
|
|
"logps/ref_rejected": -64.17195129394531,
|
|
"logps/rejected": -64.14192199707031,
|
|
"loss": 1.3866,
|
|
"margin_dpo/margin_mean": -0.0013527870178222656,
|
|
"margin_dpo/margin_std": 0.2561596930027008,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.007558578987150416,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.01627078652381897,
|
|
"fcm_dpo/q_t": 0.49959343671798706,
|
|
"grad_norm": 29.57394790649414,
|
|
"learning_rate": 2.9850746268656714e-08,
|
|
"logits/chosen": 0.09715696424245834,
|
|
"logits/rejected": 0.06874927133321762,
|
|
"logps/chosen": -65.34068298339844,
|
|
"logps/ref_chosen": -65.34695434570312,
|
|
"logps/ref_rejected": -79.315673828125,
|
|
"logps/rejected": -79.32568359375,
|
|
"loss": 1.3849,
|
|
"margin_dpo/margin_mean": 0.016270801424980164,
|
|
"margin_dpo/margin_std": 0.3146827518939972,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.015117157974300832,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.006588774733245373,
|
|
"fcm_dpo/q_t": 0.49983543157577515,
|
|
"grad_norm": 29.71192741394043,
|
|
"learning_rate": 6.71641791044776e-08,
|
|
"logits/chosen": 0.11220695823431015,
|
|
"logits/rejected": 0.0727226510643959,
|
|
"logps/chosen": -56.66105270385742,
|
|
"logps/ref_chosen": -56.65692901611328,
|
|
"logps/ref_rejected": -80.12786865234375,
|
|
"logps/rejected": -80.13858795166016,
|
|
"loss": 1.3859,
|
|
"margin_dpo/margin_mean": 0.006588757038116455,
|
|
"margin_dpo/margin_std": 0.3392222821712494,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.022675736961451247,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.0029546499717980623,
|
|
"fcm_dpo/q_t": 0.5000736713409424,
|
|
"grad_norm": 33.624202728271484,
|
|
"learning_rate": 1.044776119402985e-07,
|
|
"logits/chosen": 0.08530768007040024,
|
|
"logits/rejected": 0.05562921613454819,
|
|
"logps/chosen": -60.097312927246094,
|
|
"logps/ref_chosen": -60.09392166137695,
|
|
"logps/ref_rejected": -78.99056243896484,
|
|
"logps/rejected": -78.99102020263672,
|
|
"loss": 1.3869,
|
|
"margin_dpo/margin_mean": -0.0029546141158789396,
|
|
"margin_dpo/margin_std": 0.3288004994392395,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.030234315948601664,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.022632252424955368,
|
|
"fcm_dpo/q_t": 0.499434232711792,
|
|
"grad_norm": 29.050533294677734,
|
|
"learning_rate": 1.4179104477611938e-07,
|
|
"logits/chosen": 0.09454663097858429,
|
|
"logits/rejected": 0.06706688553094864,
|
|
"logps/chosen": -55.46174240112305,
|
|
"logps/ref_chosen": -55.464561462402344,
|
|
"logps/ref_rejected": -77.40013122558594,
|
|
"logps/rejected": -77.41993713378906,
|
|
"loss": 1.3843,
|
|
"margin_dpo/margin_mean": 0.02263212762773037,
|
|
"margin_dpo/margin_std": 0.3095693290233612,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.03779289493575208,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.007454699371010065,
|
|
"fcm_dpo/q_t": 0.4998142123222351,
|
|
"grad_norm": 29.52878761291504,
|
|
"learning_rate": 1.7910447761194027e-07,
|
|
"logits/chosen": 0.09918306022882462,
|
|
"logits/rejected": 0.06782061606645584,
|
|
"logps/chosen": -60.728736877441406,
|
|
"logps/ref_chosen": -60.711814880371094,
|
|
"logps/ref_rejected": -82.71756744384766,
|
|
"logps/rejected": -82.74192810058594,
|
|
"loss": 1.3858,
|
|
"margin_dpo/margin_mean": 0.007454258389770985,
|
|
"margin_dpo/margin_std": 0.3171737492084503,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.045351473922902494,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.03235532343387604,
|
|
"fcm_dpo/q_t": 0.49919143319129944,
|
|
"grad_norm": 30.365568161010742,
|
|
"learning_rate": 2.1641791044776117e-07,
|
|
"logits/chosen": 0.10435309261083603,
|
|
"logits/rejected": 0.0797884613275528,
|
|
"logps/chosen": -60.894203186035156,
|
|
"logps/ref_chosen": -60.880210876464844,
|
|
"logps/ref_rejected": -78.44148254394531,
|
|
"logps/rejected": -78.48783111572266,
|
|
"loss": 1.3833,
|
|
"margin_dpo/margin_mean": 0.03235520049929619,
|
|
"margin_dpo/margin_std": 0.2916334867477417,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.05291005291005291,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.027074579149484634,
|
|
"fcm_dpo/q_t": 0.4993230402469635,
|
|
"grad_norm": 27.81580924987793,
|
|
"learning_rate": 2.537313432835821e-07,
|
|
"logits/chosen": 0.06959636509418488,
|
|
"logits/rejected": 0.0434596873819828,
|
|
"logps/chosen": -62.28264617919922,
|
|
"logps/ref_chosen": -62.248138427734375,
|
|
"logps/ref_rejected": -79.56475830078125,
|
|
"logps/rejected": -79.62632751464844,
|
|
"loss": 1.3839,
|
|
"margin_dpo/margin_mean": 0.02707470953464508,
|
|
"margin_dpo/margin_std": 0.34622931480407715,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.06046863189720333,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.04390318691730499,
|
|
"fcm_dpo/q_t": 0.4989030957221985,
|
|
"grad_norm": 31.27858543395996,
|
|
"learning_rate": 2.9104477611940296e-07,
|
|
"logits/chosen": 0.10496401786804199,
|
|
"logits/rejected": 0.06094109266996384,
|
|
"logps/chosen": -58.95783615112305,
|
|
"logps/ref_chosen": -58.87812423706055,
|
|
"logps/ref_rejected": -84.22982025146484,
|
|
"logps/rejected": -84.35343933105469,
|
|
"loss": 1.3823,
|
|
"margin_dpo/margin_mean": 0.04390326514840126,
|
|
"margin_dpo/margin_std": 0.36435216665267944,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.06802721088435375,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.024266783148050308,
|
|
"fcm_dpo/q_t": 0.4993937015533447,
|
|
"grad_norm": 31.89389419555664,
|
|
"learning_rate": 3.2835820895522385e-07,
|
|
"logits/chosen": 0.07427901774644852,
|
|
"logits/rejected": 0.047025423496961594,
|
|
"logps/chosen": -66.02748107910156,
|
|
"logps/ref_chosen": -65.88298034667969,
|
|
"logps/ref_rejected": -83.87881469726562,
|
|
"logps/rejected": -84.0475845336914,
|
|
"loss": 1.3843,
|
|
"margin_dpo/margin_mean": 0.024267155677080154,
|
|
"margin_dpo/margin_std": 0.4141690135002136,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.07558578987150416,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.09846670925617218,
|
|
"fcm_dpo/q_t": 0.4975399971008301,
|
|
"grad_norm": 27.96340560913086,
|
|
"learning_rate": 3.6567164179104475e-07,
|
|
"logits/chosen": 0.07577279210090637,
|
|
"logits/rejected": 0.04236779734492302,
|
|
"logps/chosen": -55.37511444091797,
|
|
"logps/ref_chosen": -55.172386169433594,
|
|
"logps/ref_rejected": -69.63300323486328,
|
|
"logps/rejected": -69.93418884277344,
|
|
"loss": 1.377,
|
|
"margin_dpo/margin_mean": 0.09846682846546173,
|
|
"margin_dpo/margin_std": 0.43755125999450684,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.08314436885865457,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.2034468650817871,
|
|
"fcm_dpo/q_t": 0.49491971731185913,
|
|
"grad_norm": 31.36685562133789,
|
|
"learning_rate": 4.0298507462686564e-07,
|
|
"logits/chosen": 0.07709817588329315,
|
|
"logits/rejected": 0.04077417403459549,
|
|
"logps/chosen": -57.53302764892578,
|
|
"logps/ref_chosen": -57.193580627441406,
|
|
"logps/ref_rejected": -79.69940948486328,
|
|
"logps/rejected": -80.24231719970703,
|
|
"loss": 1.3668,
|
|
"margin_dpo/margin_mean": 0.20344702899456024,
|
|
"margin_dpo/margin_std": 0.53203284740448,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.09070294784580499,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.21591635048389435,
|
|
"fcm_dpo/q_t": 0.4946247935295105,
|
|
"grad_norm": 29.535348892211914,
|
|
"learning_rate": 4.4029850746268654e-07,
|
|
"logits/chosen": 0.11357074975967407,
|
|
"logits/rejected": 0.07937975972890854,
|
|
"logps/chosen": -60.59405517578125,
|
|
"logps/ref_chosen": -60.068870544433594,
|
|
"logps/ref_rejected": -74.41178894042969,
|
|
"logps/rejected": -75.15288543701172,
|
|
"loss": 1.3666,
|
|
"margin_dpo/margin_mean": 0.21591635048389435,
|
|
"margin_dpo/margin_std": 0.7985161542892456,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.0982615268329554,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.29146575927734375,
|
|
"fcm_dpo/q_t": 0.49273061752319336,
|
|
"grad_norm": 30.81733512878418,
|
|
"learning_rate": 4.776119402985074e-07,
|
|
"logits/chosen": 0.1470279097557068,
|
|
"logits/rejected": 0.11669997125864029,
|
|
"logps/chosen": -58.90019607543945,
|
|
"logps/ref_chosen": -58.1558952331543,
|
|
"logps/ref_rejected": -76.06512451171875,
|
|
"logps/rejected": -77.10090637207031,
|
|
"loss": 1.3598,
|
|
"margin_dpo/margin_mean": 0.2914656102657318,
|
|
"margin_dpo/margin_std": 0.9535614848136902,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.10582010582010581,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.3953782618045807,
|
|
"fcm_dpo/q_t": 0.4901546835899353,
|
|
"grad_norm": 28.811260223388672,
|
|
"learning_rate": 4.999860140229787e-07,
|
|
"logits/chosen": 0.10076414048671722,
|
|
"logits/rejected": 0.06563268601894379,
|
|
"logps/chosen": -68.42546844482422,
|
|
"logps/ref_chosen": -67.35506439208984,
|
|
"logps/ref_rejected": -82.24962615966797,
|
|
"logps/rejected": -83.71540832519531,
|
|
"loss": 1.3506,
|
|
"margin_dpo/margin_mean": 0.3953781723976135,
|
|
"margin_dpo/margin_std": 1.153785228729248,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.11337868480725624,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.4928799569606781,
|
|
"fcm_dpo/q_t": 0.4877691864967346,
|
|
"grad_norm": 26.195096969604492,
|
|
"learning_rate": 4.998286897523808e-07,
|
|
"logits/chosen": 0.11780299246311188,
|
|
"logits/rejected": 0.08228245377540588,
|
|
"logps/chosen": -58.29374313354492,
|
|
"logps/ref_chosen": -56.86763381958008,
|
|
"logps/ref_rejected": -72.56938934326172,
|
|
"logps/rejected": -74.4883804321289,
|
|
"loss": 1.3431,
|
|
"margin_dpo/margin_mean": 0.4928799569606781,
|
|
"margin_dpo/margin_std": 1.4089515209197998,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.12093726379440665,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.6249088048934937,
|
|
"fcm_dpo/q_t": 0.4845584034919739,
|
|
"grad_norm": 28.388460159301758,
|
|
"learning_rate": 4.994966691179711e-07,
|
|
"logits/chosen": 0.134972482919693,
|
|
"logits/rejected": 0.09020257741212845,
|
|
"logps/chosen": -59.4559440612793,
|
|
"logps/ref_chosen": -57.687095642089844,
|
|
"logps/ref_rejected": -78.06813049316406,
|
|
"logps/rejected": -80.46187591552734,
|
|
"loss": 1.3321,
|
|
"margin_dpo/margin_mean": 0.6249088048934937,
|
|
"margin_dpo/margin_std": 1.652012586593628,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12849584278155707,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.039618968963623,
|
|
"fcm_dpo/q_t": 0.4743143618106842,
|
|
"grad_norm": 26.78014373779297,
|
|
"learning_rate": 4.989901842900325e-07,
|
|
"logits/chosen": 0.1461409032344818,
|
|
"logits/rejected": 0.10004488378763199,
|
|
"logps/chosen": -59.10978317260742,
|
|
"logps/ref_chosen": -56.96040725708008,
|
|
"logps/ref_rejected": -75.22166442871094,
|
|
"logps/rejected": -78.41065979003906,
|
|
"loss": 1.2937,
|
|
"margin_dpo/margin_mean": 1.039618730545044,
|
|
"margin_dpo/margin_std": 1.8173624277114868,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.1360544217687075,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.0965173244476318,
|
|
"fcm_dpo/q_t": 0.4731336236000061,
|
|
"grad_norm": 29.045164108276367,
|
|
"learning_rate": 4.983095894354857e-07,
|
|
"logits/chosen": 0.19696776568889618,
|
|
"logits/rejected": 0.14858455955982208,
|
|
"logps/chosen": -60.32122802734375,
|
|
"logps/ref_chosen": -57.41730499267578,
|
|
"logps/ref_rejected": -80.87986755371094,
|
|
"logps/rejected": -84.88032531738281,
|
|
"loss": 1.2959,
|
|
"margin_dpo/margin_mean": 1.0965173244476318,
|
|
"margin_dpo/margin_std": 2.4852540493011475,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.1436130007558579,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.4192250967025757,
|
|
"fcm_dpo/q_t": 0.4657936990261078,
|
|
"grad_norm": 29.357852935791016,
|
|
"learning_rate": 4.974553604702332e-07,
|
|
"logits/chosen": 0.18920177221298218,
|
|
"logits/rejected": 0.14621102809906006,
|
|
"logps/chosen": -57.5679931640625,
|
|
"logps/ref_chosen": -54.08087158203125,
|
|
"logps/ref_rejected": -76.15860748291016,
|
|
"logps/rejected": -81.06494903564453,
|
|
"loss": 1.2767,
|
|
"margin_dpo/margin_mean": 1.4192249774932861,
|
|
"margin_dpo/margin_std": 3.294682025909424,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.15117157974300832,
|
|
"fcm_dpo/beta": 0.10000000894069672,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.2000526189804077,
|
|
"fcm_dpo/q_t": 0.47190871834754944,
|
|
"grad_norm": 32.946434020996094,
|
|
"learning_rate": 4.964280947263676e-07,
|
|
"logits/chosen": 0.19950337707996368,
|
|
"logits/rejected": 0.1676480770111084,
|
|
"logps/chosen": -68.6276626586914,
|
|
"logps/ref_chosen": -63.875038146972656,
|
|
"logps/ref_rejected": -82.077880859375,
|
|
"logps/rejected": -88.03055572509766,
|
|
"loss": 1.3245,
|
|
"margin_dpo/margin_mean": 1.2000528573989868,
|
|
"margin_dpo/margin_std": 4.610485076904297,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15873015873015872,
|
|
"fcm_dpo/beta": 0.11480066925287247,
|
|
"fcm_dpo/delta": 0.30130642652511597,
|
|
"fcm_dpo/margin": 1.7428162097930908,
|
|
"fcm_dpo/q_t": 0.45676541328430176,
|
|
"grad_norm": 40.23371887207031,
|
|
"learning_rate": 4.952285105344791e-07,
|
|
"logits/chosen": 0.2236131876707077,
|
|
"logits/rejected": 0.17266307771205902,
|
|
"logps/chosen": -67.44294738769531,
|
|
"logps/ref_chosen": -62.572479248046875,
|
|
"logps/ref_rejected": -80.93415069580078,
|
|
"logps/rejected": -87.54743194580078,
|
|
"loss": 1.2749,
|
|
"margin_dpo/margin_mean": 1.7428159713745117,
|
|
"margin_dpo/margin_std": 4.91826868057251,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.16628873771730915,
|
|
"fcm_dpo/beta": 0.1559300720691681,
|
|
"fcm_dpo/delta": 0.24887438118457794,
|
|
"fcm_dpo/margin": 1.5040204524993896,
|
|
"fcm_dpo/q_t": 0.4498668611049652,
|
|
"grad_norm": 84.47704315185547,
|
|
"learning_rate": 4.938574467213517e-07,
|
|
"logits/chosen": 0.20236167311668396,
|
|
"logits/rejected": 0.17875656485557556,
|
|
"logps/chosen": -74.19175720214844,
|
|
"logps/ref_chosen": -68.67534637451172,
|
|
"logps/ref_rejected": -78.82028198242188,
|
|
"logps/rejected": -85.84071350097656,
|
|
"loss": 1.3248,
|
|
"margin_dpo/margin_mean": 1.5040203332901,
|
|
"margin_dpo/margin_std": 5.396058559417725,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.17384731670445955,
|
|
"fcm_dpo/beta": 0.2165088653564453,
|
|
"fcm_dpo/delta": 0.38581734895706177,
|
|
"fcm_dpo/margin": 1.90188467502594,
|
|
"fcm_dpo/q_t": 0.41888388991355896,
|
|
"grad_norm": 73.28482818603516,
|
|
"learning_rate": 4.923158620234019e-07,
|
|
"logits/chosen": 0.22939562797546387,
|
|
"logits/rejected": 0.1761929988861084,
|
|
"logps/chosen": -63.686767578125,
|
|
"logps/ref_chosen": -58.65370559692383,
|
|
"logps/ref_rejected": -81.89688873291016,
|
|
"logps/rejected": -88.83183288574219,
|
|
"loss": 1.2058,
|
|
"margin_dpo/margin_mean": 1.90188467502594,
|
|
"margin_dpo/margin_std": 4.321715354919434,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.18140589569160998,
|
|
"fcm_dpo/beta": 0.28120937943458557,
|
|
"fcm_dpo/delta": 0.1904822289943695,
|
|
"fcm_dpo/margin": 2.1323933601379395,
|
|
"fcm_dpo/q_t": 0.38111627101898193,
|
|
"grad_norm": 71.18403625488281,
|
|
"learning_rate": 4.906048344162676e-07,
|
|
"logits/chosen": 0.2239595204591751,
|
|
"logits/rejected": 0.17475782334804535,
|
|
"logps/chosen": -60.79193878173828,
|
|
"logps/ref_chosen": -56.16423797607422,
|
|
"logps/ref_rejected": -75.87689971923828,
|
|
"logps/rejected": -82.63700103759766,
|
|
"loss": 1.0965,
|
|
"margin_dpo/margin_mean": 2.1323935985565186,
|
|
"margin_dpo/margin_std": 3.5653164386749268,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.1889644746787604,
|
|
"fcm_dpo/beta": 0.3244084417819977,
|
|
"fcm_dpo/delta": 0.06813462823629379,
|
|
"fcm_dpo/margin": 2.4218626022338867,
|
|
"fcm_dpo/q_t": 0.35703063011169434,
|
|
"grad_norm": 79.45623016357422,
|
|
"learning_rate": 4.887255603610184e-07,
|
|
"logits/chosen": 0.2398073375225067,
|
|
"logits/rejected": 0.18310315907001495,
|
|
"logps/chosen": -64.57884216308594,
|
|
"logps/ref_chosen": -59.744285583496094,
|
|
"logps/ref_rejected": -86.77314758300781,
|
|
"logps/rejected": -94.0295639038086,
|
|
"loss": 1.001,
|
|
"margin_dpo/margin_mean": 2.4218626022338867,
|
|
"margin_dpo/margin_std": 3.614306926727295,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.1965230536659108,
|
|
"fcm_dpo/beta": 0.3527252972126007,
|
|
"fcm_dpo/delta": 0.11244706064462662,
|
|
"fcm_dpo/margin": 2.107297897338867,
|
|
"fcm_dpo/q_t": 0.364054411649704,
|
|
"grad_norm": 102.10346984863281,
|
|
"learning_rate": 4.866793539675126e-07,
|
|
"logits/chosen": 0.15565961599349976,
|
|
"logits/rejected": 0.12577588856220245,
|
|
"logps/chosen": -69.22906494140625,
|
|
"logps/ref_chosen": -64.15296936035156,
|
|
"logps/ref_rejected": -75.17271423339844,
|
|
"logps/rejected": -82.3561019897461,
|
|
"loss": 1.0858,
|
|
"margin_dpo/margin_mean": 2.1072981357574463,
|
|
"margin_dpo/margin_std": 3.5260016918182373,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.20408163265306123,
|
|
"fcm_dpo/beta": 0.3580014705657959,
|
|
"fcm_dpo/delta": -0.032318420708179474,
|
|
"fcm_dpo/margin": 2.452709197998047,
|
|
"fcm_dpo/q_t": 0.34046944975852966,
|
|
"grad_norm": 78.44767761230469,
|
|
"learning_rate": 4.844676460754862e-07,
|
|
"logits/chosen": 0.2094561755657196,
|
|
"logits/rejected": 0.1701136827468872,
|
|
"logps/chosen": -61.97556686401367,
|
|
"logps/ref_chosen": -57.006690979003906,
|
|
"logps/ref_rejected": -73.71768188476562,
|
|
"logps/rejected": -81.13926696777344,
|
|
"loss": 1.0182,
|
|
"margin_dpo/margin_mean": 2.452709197998047,
|
|
"margin_dpo/margin_std": 3.6691651344299316,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.21164021164021163,
|
|
"fcm_dpo/beta": 0.35650908946990967,
|
|
"fcm_dpo/delta": -0.04635324329137802,
|
|
"fcm_dpo/margin": 2.5026161670684814,
|
|
"fcm_dpo/q_t": 0.3460763394832611,
|
|
"grad_norm": 101.91556549072266,
|
|
"learning_rate": 4.820919832540181e-07,
|
|
"logits/chosen": 0.17463022470474243,
|
|
"logits/rejected": 0.13293002545833588,
|
|
"logps/chosen": -68.91121673583984,
|
|
"logps/ref_chosen": -63.36246871948242,
|
|
"logps/ref_rejected": -79.62621307373047,
|
|
"logps/rejected": -87.67756652832031,
|
|
"loss": 1.0808,
|
|
"margin_dpo/margin_mean": 2.5026164054870605,
|
|
"margin_dpo/margin_std": 4.126684665679932,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.21919879062736206,
|
|
"fcm_dpo/beta": 0.34031468629837036,
|
|
"fcm_dpo/delta": 0.007221451494842768,
|
|
"fcm_dpo/margin": 2.475119113922119,
|
|
"fcm_dpo/q_t": 0.3505968153476715,
|
|
"grad_norm": 101.12755584716797,
|
|
"learning_rate": 4.795540267200686e-07,
|
|
"logits/chosen": 0.18211673200130463,
|
|
"logits/rejected": 0.1625284105539322,
|
|
"logps/chosen": -70.48210906982422,
|
|
"logps/ref_chosen": -65.01470184326172,
|
|
"logps/ref_rejected": -80.49073791503906,
|
|
"logps/rejected": -88.43326568603516,
|
|
"loss": 1.0887,
|
|
"margin_dpo/margin_mean": 2.4751193523406982,
|
|
"margin_dpo/margin_std": 4.171751022338867,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.22675736961451248,
|
|
"fcm_dpo/beta": 0.34030789136886597,
|
|
"fcm_dpo/delta": -0.09494070708751678,
|
|
"fcm_dpo/margin": 2.759521484375,
|
|
"fcm_dpo/q_t": 0.3320358395576477,
|
|
"grad_norm": 70.31517791748047,
|
|
"learning_rate": 4.768555511768486e-07,
|
|
"logits/chosen": 0.17095252871513367,
|
|
"logits/rejected": 0.12804347276687622,
|
|
"logps/chosen": -64.57518768310547,
|
|
"logps/ref_chosen": -59.19135284423828,
|
|
"logps/ref_rejected": -74.0339126586914,
|
|
"logps/rejected": -82.17726135253906,
|
|
"loss": 0.9915,
|
|
"margin_dpo/margin_mean": 2.759521961212158,
|
|
"margin_dpo/margin_std": 3.9055495262145996,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.23431594860166288,
|
|
"fcm_dpo/beta": 0.30031818151474,
|
|
"fcm_dpo/delta": -0.01693885400891304,
|
|
"fcm_dpo/margin": 2.873197078704834,
|
|
"fcm_dpo/q_t": 0.3435518741607666,
|
|
"grad_norm": 81.77188873291016,
|
|
"learning_rate": 4.7399844357283393e-07,
|
|
"logits/chosen": 0.18896853923797607,
|
|
"logits/rejected": 0.15728981792926788,
|
|
"logps/chosen": -66.36070251464844,
|
|
"logps/ref_chosen": -60.93949508666992,
|
|
"logps/ref_rejected": -74.51151275634766,
|
|
"logps/rejected": -82.80590057373047,
|
|
"loss": 1.0175,
|
|
"margin_dpo/margin_mean": 2.873196840286255,
|
|
"margin_dpo/margin_std": 4.220882415771484,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.2418745275888133,
|
|
"fcm_dpo/beta": 0.30263930559158325,
|
|
"fcm_dpo/delta": -0.013651391491293907,
|
|
"fcm_dpo/margin": 2.843362808227539,
|
|
"fcm_dpo/q_t": 0.3426353335380554,
|
|
"grad_norm": 73.85985565185547,
|
|
"learning_rate": 4.7098470178228755e-07,
|
|
"logits/chosen": 0.1567428857088089,
|
|
"logits/rejected": 0.12234382331371307,
|
|
"logps/chosen": -64.45149993896484,
|
|
"logps/ref_chosen": -58.763816833496094,
|
|
"logps/ref_rejected": -74.94743347167969,
|
|
"logps/rejected": -83.47847747802734,
|
|
"loss": 1.0002,
|
|
"margin_dpo/margin_mean": 2.843362808227539,
|
|
"margin_dpo/margin_std": 4.1234354972839355,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.2494331065759637,
|
|
"fcm_dpo/beta": 0.2814979553222656,
|
|
"fcm_dpo/delta": -0.13840754330158234,
|
|
"fcm_dpo/margin": 3.4627203941345215,
|
|
"fcm_dpo/q_t": 0.32190772891044617,
|
|
"grad_norm": 61.125823974609375,
|
|
"learning_rate": 4.678164332082175e-07,
|
|
"logits/chosen": 0.19466723501682281,
|
|
"logits/rejected": 0.14662370085716248,
|
|
"logps/chosen": -61.967384338378906,
|
|
"logps/ref_chosen": -55.70417022705078,
|
|
"logps/ref_rejected": -76.59439849853516,
|
|
"logps/rejected": -86.32032775878906,
|
|
"loss": 0.9311,
|
|
"margin_dpo/margin_mean": 3.4627203941345215,
|
|
"margin_dpo/margin_std": 4.364522457122803,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.25699168556311414,
|
|
"fcm_dpo/beta": 0.2759741246700287,
|
|
"fcm_dpo/delta": 0.055651336908340454,
|
|
"fcm_dpo/margin": 2.887462615966797,
|
|
"fcm_dpo/q_t": 0.35355013608932495,
|
|
"grad_norm": 61.50341796875,
|
|
"learning_rate": 4.6449585330874425e-07,
|
|
"logits/chosen": 0.1680757850408554,
|
|
"logits/rejected": 0.13121065497398376,
|
|
"logps/chosen": -67.40464782714844,
|
|
"logps/ref_chosen": -61.169105529785156,
|
|
"logps/ref_rejected": -77.21674346923828,
|
|
"logps/rejected": -86.33975219726562,
|
|
"loss": 1.0136,
|
|
"margin_dpo/margin_mean": 2.887462615966797,
|
|
"margin_dpo/margin_std": 4.266605854034424,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.26455026455026454,
|
|
"fcm_dpo/beta": 0.26409897208213806,
|
|
"fcm_dpo/delta": -0.11066864430904388,
|
|
"fcm_dpo/margin": 3.591811418533325,
|
|
"fcm_dpo/q_t": 0.3342060148715973,
|
|
"grad_norm": 67.46173095703125,
|
|
"learning_rate": 4.6102528404790965e-07,
|
|
"logits/chosen": 0.21134984493255615,
|
|
"logits/rejected": 0.15731702744960785,
|
|
"logps/chosen": -66.0682144165039,
|
|
"logps/ref_chosen": -59.24176788330078,
|
|
"logps/ref_rejected": -81.80384826660156,
|
|
"logps/rejected": -92.22210693359375,
|
|
"loss": 1.0169,
|
|
"margin_dpo/margin_mean": 3.591811418533325,
|
|
"margin_dpo/margin_std": 5.276784420013428,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.272108843537415,
|
|
"fcm_dpo/beta": 0.2461521178483963,
|
|
"fcm_dpo/delta": -0.0015753842890262604,
|
|
"fcm_dpo/margin": 3.4451968669891357,
|
|
"fcm_dpo/q_t": 0.34774264693260193,
|
|
"grad_norm": 69.0323486328125,
|
|
"learning_rate": 4.5740715227200897e-07,
|
|
"logits/chosen": 0.18618321418762207,
|
|
"logits/rejected": 0.1469568908214569,
|
|
"logps/chosen": -70.56135559082031,
|
|
"logps/ref_chosen": -63.24883270263672,
|
|
"logps/ref_rejected": -79.00736236572266,
|
|
"logps/rejected": -89.76508331298828,
|
|
"loss": 1.0378,
|
|
"margin_dpo/margin_mean": 3.445197582244873,
|
|
"margin_dpo/margin_std": 5.239095211029053,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.2796674225245654,
|
|
"fcm_dpo/beta": 0.2409546822309494,
|
|
"fcm_dpo/delta": 0.000774085521697998,
|
|
"fcm_dpo/margin": 3.5118987560272217,
|
|
"fcm_dpo/q_t": 0.34693118929862976,
|
|
"grad_norm": 62.61862564086914,
|
|
"learning_rate": 4.5364398801258394e-07,
|
|
"logits/chosen": 0.22854506969451904,
|
|
"logits/rejected": 0.18141530454158783,
|
|
"logps/chosen": -63.767189025878906,
|
|
"logps/ref_chosen": -56.390625,
|
|
"logps/ref_rejected": -76.81001281738281,
|
|
"logps/rejected": -87.69847106933594,
|
|
"loss": 1.0261,
|
|
"margin_dpo/margin_mean": 3.5118987560272217,
|
|
"margin_dpo/margin_std": 5.162919044494629,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.2872260015117158,
|
|
"fcm_dpo/beta": 0.2379799336194992,
|
|
"fcm_dpo/delta": -0.09538835287094116,
|
|
"fcm_dpo/margin": 3.932285785675049,
|
|
"fcm_dpo/q_t": 0.3300981819629669,
|
|
"grad_norm": 61.39169692993164,
|
|
"learning_rate": 4.4973842271726024e-07,
|
|
"logits/chosen": 0.21160800755023956,
|
|
"logits/rejected": 0.16087199747562408,
|
|
"logps/chosen": -75.3604736328125,
|
|
"logps/ref_chosen": -68.25389099121094,
|
|
"logps/ref_rejected": -86.461181640625,
|
|
"logps/rejected": -97.50004577636719,
|
|
"loss": 0.984,
|
|
"margin_dpo/margin_mean": 3.932285785675049,
|
|
"margin_dpo/margin_std": 5.494342803955078,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.2947845804988662,
|
|
"fcm_dpo/beta": 0.23223213851451874,
|
|
"fcm_dpo/delta": 0.02907167747616768,
|
|
"fcm_dpo/margin": 3.536024808883667,
|
|
"fcm_dpo/q_t": 0.3476114869117737,
|
|
"grad_norm": 62.45728302001953,
|
|
"learning_rate": 4.4569318740967043e-07,
|
|
"logits/chosen": 0.22130648791790009,
|
|
"logits/rejected": 0.20145495235919952,
|
|
"logps/chosen": -69.86652374267578,
|
|
"logps/ref_chosen": -62.1484260559082,
|
|
"logps/ref_rejected": -71.33458709716797,
|
|
"logps/rejected": -82.58869934082031,
|
|
"loss": 1.0216,
|
|
"margin_dpo/margin_mean": 3.536024570465088,
|
|
"margin_dpo/margin_std": 5.259249210357666,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.30234315948601664,
|
|
"fcm_dpo/beta": 0.22571632266044617,
|
|
"fcm_dpo/delta": -0.10032586753368378,
|
|
"fcm_dpo/margin": 4.166001319885254,
|
|
"fcm_dpo/q_t": 0.32745927572250366,
|
|
"grad_norm": 52.517520904541016,
|
|
"learning_rate": 4.415111107797445e-07,
|
|
"logits/chosen": 0.2718667685985565,
|
|
"logits/rejected": 0.21812739968299866,
|
|
"logps/chosen": -64.17549133300781,
|
|
"logps/ref_chosen": -56.950096130371094,
|
|
"logps/ref_rejected": -78.66989135742188,
|
|
"logps/rejected": -90.06129455566406,
|
|
"loss": 0.9883,
|
|
"margin_dpo/margin_mean": 4.166001319885254,
|
|
"margin_dpo/margin_std": 5.721753120422363,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.30234315948601664,
|
|
"eval_fcm_dpo/beta": 0.24067771434783936,
|
|
"eval_fcm_dpo/delta": 0.013510049320757389,
|
|
"eval_fcm_dpo/margin": 3.4733290672302246,
|
|
"eval_fcm_dpo/q_t": 0.3572106659412384,
|
|
"eval_logits/chosen": 0.27457359433174133,
|
|
"eval_logits/rejected": 0.23094330728054047,
|
|
"eval_logps/chosen": -82.1817626953125,
|
|
"eval_logps/ref_chosen": -74.85946655273438,
|
|
"eval_logps/ref_rejected": -79.54898834228516,
|
|
"eval_logps/rejected": -90.34461212158203,
|
|
"eval_loss": 0.5558829307556152,
|
|
"eval_margin_dpo/margin_mean": 3.4733293056488037,
|
|
"eval_margin_dpo/margin_std": 5.973160743713379,
|
|
"eval_runtime": 38.7469,
|
|
"eval_samples_per_second": 59.437,
|
|
"eval_steps_per_second": 1.858,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.30990173847316704,
|
|
"fcm_dpo/beta": 0.22565153241157532,
|
|
"fcm_dpo/delta": -0.03439360111951828,
|
|
"fcm_dpo/margin": 3.891812562942505,
|
|
"fcm_dpo/q_t": 0.34396880865097046,
|
|
"grad_norm": 78.97357177734375,
|
|
"learning_rate": 4.3719511720570814e-07,
|
|
"logits/chosen": 0.2548893094062805,
|
|
"logits/rejected": 0.1971297264099121,
|
|
"logps/chosen": -65.59779357910156,
|
|
"logps/ref_chosen": -57.99428176879883,
|
|
"logps/ref_rejected": -83.5367431640625,
|
|
"logps/rejected": -95.03206634521484,
|
|
"loss": 1.0303,
|
|
"margin_dpo/margin_mean": 3.891812562942505,
|
|
"margin_dpo/margin_std": 5.916425704956055,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.31746031746031744,
|
|
"fcm_dpo/beta": 0.2281331717967987,
|
|
"fcm_dpo/delta": -0.03499128669500351,
|
|
"fcm_dpo/margin": 3.858633041381836,
|
|
"fcm_dpo/q_t": 0.3482638895511627,
|
|
"grad_norm": 51.546024322509766,
|
|
"learning_rate": 4.327482247091679e-07,
|
|
"logits/chosen": 0.2419578731060028,
|
|
"logits/rejected": 0.19466528296470642,
|
|
"logps/chosen": -71.57328796386719,
|
|
"logps/ref_chosen": -63.77195358276367,
|
|
"logps/ref_rejected": -82.56491088867188,
|
|
"logps/rejected": -94.22489929199219,
|
|
"loss": 1.0706,
|
|
"margin_dpo/margin_mean": 3.8586325645446777,
|
|
"margin_dpo/margin_std": 6.176082134246826,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.3250188964474679,
|
|
"fcm_dpo/beta": 0.2101043164730072,
|
|
"fcm_dpo/delta": -0.09397365897893906,
|
|
"fcm_dpo/margin": 4.455512523651123,
|
|
"fcm_dpo/q_t": 0.33203133940696716,
|
|
"grad_norm": 52.70698165893555,
|
|
"learning_rate": 4.281735428447157e-07,
|
|
"logits/chosen": 0.2374175786972046,
|
|
"logits/rejected": 0.18057873845100403,
|
|
"logps/chosen": -67.35072326660156,
|
|
"logps/ref_chosen": -60.27800750732422,
|
|
"logps/ref_rejected": -83.91607666015625,
|
|
"logps/rejected": -95.4443130493164,
|
|
"loss": 0.9609,
|
|
"margin_dpo/margin_mean": 4.455513000488281,
|
|
"margin_dpo/margin_std": 6.042638778686523,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.3325774754346183,
|
|
"fcm_dpo/beta": 0.19473975896835327,
|
|
"fcm_dpo/delta": 0.03177637979388237,
|
|
"fcm_dpo/margin": 4.209561347961426,
|
|
"fcm_dpo/q_t": 0.35216838121414185,
|
|
"grad_norm": 45.12868881225586,
|
|
"learning_rate": 4.234742705255272e-07,
|
|
"logits/chosen": 0.2297150194644928,
|
|
"logits/rejected": 0.1750687062740326,
|
|
"logps/chosen": -68.48099517822266,
|
|
"logps/ref_chosen": -60.88572311401367,
|
|
"logps/ref_rejected": -80.1805191040039,
|
|
"logps/rejected": -91.9853515625,
|
|
"loss": 1.0252,
|
|
"margin_dpo/margin_mean": 4.209561824798584,
|
|
"margin_dpo/margin_std": 6.372952461242676,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.3401360544217687,
|
|
"fcm_dpo/beta": 0.19399337470531464,
|
|
"fcm_dpo/delta": -0.06365469843149185,
|
|
"fcm_dpo/margin": 4.6687235832214355,
|
|
"fcm_dpo/q_t": 0.33533960580825806,
|
|
"grad_norm": 63.21636199951172,
|
|
"learning_rate": 4.186536937864752e-07,
|
|
"logits/chosen": 0.28859376907348633,
|
|
"logits/rejected": 0.22636933624744415,
|
|
"logps/chosen": -68.21263122558594,
|
|
"logps/ref_chosen": -61.02507781982422,
|
|
"logps/ref_rejected": -91.92439270019531,
|
|
"logps/rejected": -103.7806625366211,
|
|
"loss": 0.9378,
|
|
"margin_dpo/margin_mean": 4.668723106384277,
|
|
"margin_dpo/margin_std": 6.053750038146973,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.3476946334089191,
|
|
"fcm_dpo/beta": 0.19620037078857422,
|
|
"fcm_dpo/delta": 0.027376512065529823,
|
|
"fcm_dpo/margin": 4.197124481201172,
|
|
"fcm_dpo/q_t": 0.3492209017276764,
|
|
"grad_norm": 53.633026123046875,
|
|
"learning_rate": 4.137151834863213e-07,
|
|
"logits/chosen": 0.2996854782104492,
|
|
"logits/rejected": 0.2581411302089691,
|
|
"logps/chosen": -61.68690872192383,
|
|
"logps/ref_chosen": -54.49797821044922,
|
|
"logps/ref_rejected": -71.96363830566406,
|
|
"logps/rejected": -83.34968566894531,
|
|
"loss": 1.0406,
|
|
"margin_dpo/margin_mean": 4.197124004364014,
|
|
"margin_dpo/margin_std": 6.458170413970947,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.35525321239606955,
|
|
"fcm_dpo/beta": 0.18717727065086365,
|
|
"fcm_dpo/delta": -0.05658251792192459,
|
|
"fcm_dpo/margin": 4.787070274353027,
|
|
"fcm_dpo/q_t": 0.33542177081108093,
|
|
"grad_norm": 44.37601089477539,
|
|
"learning_rate": 4.08662192950594e-07,
|
|
"logits/chosen": 0.2540052533149719,
|
|
"logits/rejected": 0.22796034812927246,
|
|
"logps/chosen": -70.99009704589844,
|
|
"logps/ref_chosen": -63.250282287597656,
|
|
"logps/ref_rejected": -73.09049987792969,
|
|
"logps/rejected": -85.61738586425781,
|
|
"loss": 0.9708,
|
|
"margin_dpo/margin_mean": 4.787070274353027,
|
|
"margin_dpo/margin_std": 6.3684468269348145,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.36281179138321995,
|
|
"fcm_dpo/beta": 0.1794944554567337,
|
|
"fcm_dpo/delta": -0.10686789453029633,
|
|
"fcm_dpo/margin": 5.282033443450928,
|
|
"fcm_dpo/q_t": 0.3281763195991516,
|
|
"grad_norm": 36.0201416015625,
|
|
"learning_rate": 4.0349825555680045e-07,
|
|
"logits/chosen": 0.2764965891838074,
|
|
"logits/rejected": 0.2302006185054779,
|
|
"logps/chosen": -73.965576171875,
|
|
"logps/ref_chosen": -65.26150512695312,
|
|
"logps/ref_rejected": -87.60311126708984,
|
|
"logps/rejected": -101.58921813964844,
|
|
"loss": 0.9687,
|
|
"margin_dpo/margin_mean": 5.282033443450928,
|
|
"margin_dpo/margin_std": 7.131825923919678,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.37037037037037035,
|
|
"fcm_dpo/beta": 0.16662733256816864,
|
|
"fcm_dpo/delta": -0.09280146658420563,
|
|
"fcm_dpo/margin": 5.621182918548584,
|
|
"fcm_dpo/q_t": 0.3262787461280823,
|
|
"grad_norm": 118.54082489013672,
|
|
"learning_rate": 3.982269822636601e-07,
|
|
"logits/chosen": 0.2676697075366974,
|
|
"logits/rejected": 0.23526987433433533,
|
|
"logps/chosen": -74.21736145019531,
|
|
"logps/ref_chosen": -65.73170471191406,
|
|
"logps/ref_rejected": -75.19642639160156,
|
|
"logps/rejected": -89.3032455444336,
|
|
"loss": 0.9184,
|
|
"margin_dpo/margin_mean": 5.621182918548584,
|
|
"margin_dpo/margin_std": 6.879896640777588,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.3779289493575208,
|
|
"fcm_dpo/beta": 0.1570025235414505,
|
|
"fcm_dpo/delta": 0.021422909572720528,
|
|
"fcm_dpo/margin": 4.839144229888916,
|
|
"fcm_dpo/q_t": 0.35960644483566284,
|
|
"grad_norm": 40.19977569580078,
|
|
"learning_rate": 3.9285205908608934e-07,
|
|
"logits/chosen": 0.30098170042037964,
|
|
"logits/rejected": 0.2913910746574402,
|
|
"logps/chosen": -81.00502014160156,
|
|
"logps/ref_chosen": -70.71224212646484,
|
|
"logps/ref_rejected": -76.12723541259766,
|
|
"logps/rejected": -91.25916290283203,
|
|
"loss": 1.0905,
|
|
"margin_dpo/margin_mean": 4.839144229888916,
|
|
"margin_dpo/margin_std": 7.96373987197876,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.3854875283446712,
|
|
"fcm_dpo/beta": 0.16586363315582275,
|
|
"fcm_dpo/delta": 0.05151579529047012,
|
|
"fcm_dpo/margin": 4.809377193450928,
|
|
"fcm_dpo/q_t": 0.3572459816932678,
|
|
"grad_norm": 40.74630355834961,
|
|
"learning_rate": 3.873772445177015e-07,
|
|
"logits/chosen": 0.280979186296463,
|
|
"logits/rejected": 0.24082987010478973,
|
|
"logps/chosen": -71.6010513305664,
|
|
"logps/ref_chosen": -61.767662048339844,
|
|
"logps/ref_rejected": -77.38813018798828,
|
|
"logps/rejected": -92.03089904785156,
|
|
"loss": 1.0752,
|
|
"margin_dpo/margin_mean": 4.8093767166137695,
|
|
"margin_dpo/margin_std": 7.841381072998047,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.3930461073318216,
|
|
"fcm_dpo/beta": 0.15231892466545105,
|
|
"fcm_dpo/delta": -0.14338229596614838,
|
|
"fcm_dpo/margin": 6.435052394866943,
|
|
"fcm_dpo/q_t": 0.32444363832473755,
|
|
"grad_norm": 36.22502899169922,
|
|
"learning_rate": 3.818063669026256e-07,
|
|
"logits/chosen": 0.2699643671512604,
|
|
"logits/rejected": 0.20497925579547882,
|
|
"logps/chosen": -71.86540222167969,
|
|
"logps/ref_chosen": -61.57584762573242,
|
|
"logps/ref_rejected": -91.87513732910156,
|
|
"logps/rejected": -108.5997314453125,
|
|
"loss": 0.9336,
|
|
"margin_dpo/margin_mean": 6.435051918029785,
|
|
"margin_dpo/margin_std": 8.262994766235352,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.40060468631897206,
|
|
"fcm_dpo/beta": 0.1468384563922882,
|
|
"fcm_dpo/delta": 0.02609989419579506,
|
|
"fcm_dpo/margin": 5.617129325866699,
|
|
"fcm_dpo/q_t": 0.34967631101608276,
|
|
"grad_norm": 49.42305374145508,
|
|
"learning_rate": 3.7614332175848027e-07,
|
|
"logits/chosen": 0.2545235753059387,
|
|
"logits/rejected": 0.20748016238212585,
|
|
"logps/chosen": -75.91340637207031,
|
|
"logps/ref_chosen": -65.75422668457031,
|
|
"logps/ref_rejected": -77.9569320678711,
|
|
"logps/rejected": -93.73323059082031,
|
|
"loss": 1.0298,
|
|
"margin_dpo/margin_mean": 5.617129325866699,
|
|
"margin_dpo/margin_std": 8.462356567382812,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.40816326530612246,
|
|
"fcm_dpo/beta": 0.1501874178647995,
|
|
"fcm_dpo/delta": 0.04239519312977791,
|
|
"fcm_dpo/margin": 5.382081031799316,
|
|
"fcm_dpo/q_t": 0.3489643931388855,
|
|
"grad_norm": 45.916934967041016,
|
|
"learning_rate": 3.7039206905237656e-07,
|
|
"logits/chosen": 0.29415538907051086,
|
|
"logits/rejected": 0.2512832283973694,
|
|
"logps/chosen": -71.48078155517578,
|
|
"logps/ref_chosen": -62.27649688720703,
|
|
"logps/ref_rejected": -76.56950378417969,
|
|
"logps/rejected": -91.15586853027344,
|
|
"loss": 1.0162,
|
|
"margin_dpo/margin_mean": 5.382081031799316,
|
|
"margin_dpo/margin_std": 7.868254661560059,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.41572184429327286,
|
|
"fcm_dpo/beta": 0.1482977569103241,
|
|
"fcm_dpo/delta": -0.0749291256070137,
|
|
"fcm_dpo/margin": 6.190282344818115,
|
|
"fcm_dpo/q_t": 0.33805128931999207,
|
|
"grad_norm": 35.60213851928711,
|
|
"learning_rate": 3.645566304318526e-07,
|
|
"logits/chosen": 0.2803798019886017,
|
|
"logits/rejected": 0.23834796249866486,
|
|
"logps/chosen": -70.77764892578125,
|
|
"logps/ref_chosen": -61.854393005371094,
|
|
"logps/ref_rejected": -77.22246551513672,
|
|
"logps/rejected": -92.33599853515625,
|
|
"loss": 0.9918,
|
|
"margin_dpo/margin_mean": 6.190281867980957,
|
|
"margin_dpo/margin_std": 8.802213668823242,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.42328042328042326,
|
|
"fcm_dpo/beta": 0.1402733027935028,
|
|
"fcm_dpo/delta": -0.0842902883887291,
|
|
"fcm_dpo/margin": 6.6146416664123535,
|
|
"fcm_dpo/q_t": 0.32419517636299133,
|
|
"grad_norm": 29.430072784423828,
|
|
"learning_rate": 3.586410864126781e-07,
|
|
"logits/chosen": 0.26457709074020386,
|
|
"logits/rejected": 0.23606328666210175,
|
|
"logps/chosen": -69.50015258789062,
|
|
"logps/ref_chosen": -61.29896926879883,
|
|
"logps/ref_rejected": -73.35762023925781,
|
|
"logps/rejected": -88.17344665527344,
|
|
"loss": 0.9108,
|
|
"margin_dpo/margin_mean": 6.6146416664123535,
|
|
"margin_dpo/margin_std": 7.892550468444824,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.4308390022675737,
|
|
"fcm_dpo/beta": 0.12960846722126007,
|
|
"fcm_dpo/delta": 0.017505940049886703,
|
|
"fcm_dpo/margin": 6.419048309326172,
|
|
"fcm_dpo/q_t": 0.3450102210044861,
|
|
"grad_norm": 40.250282287597656,
|
|
"learning_rate": 3.5264957352549375e-07,
|
|
"logits/chosen": 0.2826913893222809,
|
|
"logits/rejected": 0.23917248845100403,
|
|
"logps/chosen": -72.90406799316406,
|
|
"logps/ref_chosen": -63.435462951660156,
|
|
"logps/ref_rejected": -79.73661804199219,
|
|
"logps/rejected": -95.6242904663086,
|
|
"loss": 0.9727,
|
|
"margin_dpo/margin_mean": 6.419049263000488,
|
|
"margin_dpo/margin_std": 8.68272590637207,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.4383975812547241,
|
|
"fcm_dpo/beta": 0.12238939106464386,
|
|
"fcm_dpo/delta": -0.08919016271829605,
|
|
"fcm_dpo/margin": 7.575901985168457,
|
|
"fcm_dpo/q_t": 0.3298376500606537,
|
|
"grad_norm": 39.561485290527344,
|
|
"learning_rate": 3.465862814232821e-07,
|
|
"logits/chosen": 0.2843787670135498,
|
|
"logits/rejected": 0.22869081795215607,
|
|
"logps/chosen": -68.79210662841797,
|
|
"logps/ref_chosen": -57.696876525878906,
|
|
"logps/ref_rejected": -79.78132629394531,
|
|
"logps/rejected": -98.45246124267578,
|
|
"loss": 0.9391,
|
|
"margin_dpo/margin_mean": 7.575902462005615,
|
|
"margin_dpo/margin_std": 9.794181823730469,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.4459561602418745,
|
|
"fcm_dpo/beta": 0.11619777977466583,
|
|
"fcm_dpo/delta": -0.05573344975709915,
|
|
"fcm_dpo/margin": 7.736564636230469,
|
|
"fcm_dpo/q_t": 0.33567458391189575,
|
|
"grad_norm": 35.2142333984375,
|
|
"learning_rate": 3.4045544995169125e-07,
|
|
"logits/chosen": 0.357764333486557,
|
|
"logits/rejected": 0.2943243086338043,
|
|
"logps/chosen": -67.75780487060547,
|
|
"logps/ref_chosen": -55.430633544921875,
|
|
"logps/ref_rejected": -78.1390151977539,
|
|
"logps/rejected": -98.20275115966797,
|
|
"loss": 0.9584,
|
|
"margin_dpo/margin_mean": 7.736564636230469,
|
|
"margin_dpo/margin_std": 10.325262069702148,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.45351473922902497,
|
|
"fcm_dpo/beta": 0.1122034341096878,
|
|
"fcm_dpo/delta": 0.0012714609038084745,
|
|
"fcm_dpo/margin": 7.55871057510376,
|
|
"fcm_dpo/q_t": 0.34325939416885376,
|
|
"grad_norm": 35.152587890625,
|
|
"learning_rate": 3.3426136618426043e-07,
|
|
"logits/chosen": 0.3110581338405609,
|
|
"logits/rejected": 0.26223868131637573,
|
|
"logps/chosen": -74.74542999267578,
|
|
"logps/ref_chosen": -61.207069396972656,
|
|
"logps/ref_rejected": -75.23294067382812,
|
|
"logps/rejected": -96.33000946044922,
|
|
"loss": 0.9948,
|
|
"margin_dpo/margin_mean": 7.55871057510376,
|
|
"margin_dpo/margin_std": 10.6420259475708,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.46107331821617537,
|
|
"fcm_dpo/beta": 0.11439672857522964,
|
|
"fcm_dpo/delta": -0.020849300548434258,
|
|
"fcm_dpo/margin": 7.582640171051025,
|
|
"fcm_dpo/q_t": 0.34288135170936584,
|
|
"grad_norm": 37.6644287109375,
|
|
"learning_rate": 3.280083614246217e-07,
|
|
"logits/chosen": 0.29315268993377686,
|
|
"logits/rejected": 0.2572418749332428,
|
|
"logps/chosen": -75.87992858886719,
|
|
"logps/ref_chosen": -63.06663131713867,
|
|
"logps/ref_rejected": -78.45845031738281,
|
|
"logps/rejected": -98.85440826416016,
|
|
"loss": 1.0103,
|
|
"margin_dpo/margin_mean": 7.582640171051025,
|
|
"margin_dpo/margin_std": 10.94432258605957,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.46863189720332576,
|
|
"fcm_dpo/beta": 0.11963458359241486,
|
|
"fcm_dpo/delta": 0.13357484340667725,
|
|
"fcm_dpo/margin": 6.066429615020752,
|
|
"fcm_dpo/q_t": 0.3705330193042755,
|
|
"grad_norm": 44.41813278198242,
|
|
"learning_rate": 3.2170080817777257e-07,
|
|
"logits/chosen": 0.3520604968070984,
|
|
"logits/rejected": 0.3058929741382599,
|
|
"logps/chosen": -75.83242797851562,
|
|
"logps/ref_chosen": -63.60908889770508,
|
|
"logps/ref_rejected": -74.06394958496094,
|
|
"logps/rejected": -92.3537368774414,
|
|
"loss": 1.1056,
|
|
"margin_dpo/margin_mean": 6.06643009185791,
|
|
"margin_dpo/margin_std": 10.367918968200684,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.47619047619047616,
|
|
"fcm_dpo/beta": 0.12420061975717545,
|
|
"fcm_dpo/delta": -0.0700756087899208,
|
|
"fcm_dpo/margin": 7.3467817306518555,
|
|
"fcm_dpo/q_t": 0.3341953158378601,
|
|
"grad_norm": 32.38040542602539,
|
|
"learning_rate": 3.1534311709253723e-07,
|
|
"logits/chosen": 0.31929245591163635,
|
|
"logits/rejected": 0.26849374175071716,
|
|
"logps/chosen": -72.7978744506836,
|
|
"logps/ref_chosen": -62.31493377685547,
|
|
"logps/ref_rejected": -75.07472229003906,
|
|
"logps/rejected": -92.90444946289062,
|
|
"loss": 0.9722,
|
|
"margin_dpo/margin_mean": 7.3467817306518555,
|
|
"margin_dpo/margin_std": 10.090209007263184,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.4837490551776266,
|
|
"fcm_dpo/beta": 0.11273982375860214,
|
|
"fcm_dpo/delta": -0.05503999441862106,
|
|
"fcm_dpo/margin": 7.940859317779541,
|
|
"fcm_dpo/q_t": 0.3381638824939728,
|
|
"grad_norm": 32.69892883300781,
|
|
"learning_rate": 3.0893973387735683e-07,
|
|
"logits/chosen": 0.3051146864891052,
|
|
"logits/rejected": 0.24970924854278564,
|
|
"logps/chosen": -64.74360656738281,
|
|
"logps/ref_chosen": -55.336036682128906,
|
|
"logps/ref_rejected": -80.05536651611328,
|
|
"logps/rejected": -97.40379333496094,
|
|
"loss": 0.9832,
|
|
"margin_dpo/margin_mean": 7.940859317779541,
|
|
"margin_dpo/margin_std": 10.87829875946045,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.491307634164777,
|
|
"fcm_dpo/beta": 0.1108032613992691,
|
|
"fcm_dpo/delta": -0.02011686936020851,
|
|
"fcm_dpo/margin": 7.813392639160156,
|
|
"fcm_dpo/q_t": 0.33666709065437317,
|
|
"grad_norm": 31.099613189697266,
|
|
"learning_rate": 3.0249513619156206e-07,
|
|
"logits/chosen": 0.30844759941101074,
|
|
"logits/rejected": 0.253657728433609,
|
|
"logps/chosen": -68.05531311035156,
|
|
"logps/ref_chosen": -57.90629959106445,
|
|
"logps/ref_rejected": -74.2243881225586,
|
|
"logps/rejected": -92.1867904663086,
|
|
"loss": 0.9669,
|
|
"margin_dpo/margin_mean": 7.81339168548584,
|
|
"margin_dpo/margin_std": 10.380613327026367,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.4988662131519274,
|
|
"fcm_dpo/beta": 0.11853732913732529,
|
|
"fcm_dpo/delta": 0.04765941947698593,
|
|
"fcm_dpo/margin": 6.770005702972412,
|
|
"fcm_dpo/q_t": 0.3563086986541748,
|
|
"grad_norm": 31.464454650878906,
|
|
"learning_rate": 2.9601383051430505e-07,
|
|
"logits/chosen": 0.2874017357826233,
|
|
"logits/rejected": 0.24997815489768982,
|
|
"logps/chosen": -76.50579071044922,
|
|
"logps/ref_chosen": -65.17555236816406,
|
|
"logps/ref_rejected": -78.53681182861328,
|
|
"logps/rejected": -96.63706970214844,
|
|
"loss": 1.0658,
|
|
"margin_dpo/margin_mean": 6.770005226135254,
|
|
"margin_dpo/margin_std": 10.903402328491211,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.5064247921390779,
|
|
"fcm_dpo/beta": 0.10601578652858734,
|
|
"fcm_dpo/delta": -0.10322023928165436,
|
|
"fcm_dpo/margin": 8.851563453674316,
|
|
"fcm_dpo/q_t": 0.3293716013431549,
|
|
"grad_norm": 30.588729858398438,
|
|
"learning_rate": 2.895003489933375e-07,
|
|
"logits/chosen": 0.3414556086063385,
|
|
"logits/rejected": 0.2969668507575989,
|
|
"logps/chosen": -74.1660385131836,
|
|
"logps/ref_chosen": -62.62797927856445,
|
|
"logps/ref_rejected": -79.9095458984375,
|
|
"logps/rejected": -100.2991714477539,
|
|
"loss": 0.946,
|
|
"margin_dpo/margin_mean": 8.8515625,
|
|
"margin_dpo/margin_std": 11.624635696411133,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.5139833711262283,
|
|
"fcm_dpo/beta": 0.10893759876489639,
|
|
"fcm_dpo/delta": 0.029664043337106705,
|
|
"fcm_dpo/margin": 7.534571647644043,
|
|
"fcm_dpo/q_t": 0.35428622364997864,
|
|
"grad_norm": 29.96834945678711,
|
|
"learning_rate": 2.8295924627584004e-07,
|
|
"logits/chosen": 0.29519161581993103,
|
|
"logits/rejected": 0.2586953043937683,
|
|
"logps/chosen": -75.3774642944336,
|
|
"logps/ref_chosen": -61.1064567565918,
|
|
"logps/ref_rejected": -76.71846008300781,
|
|
"logps/rejected": -98.52404022216797,
|
|
"loss": 1.0284,
|
|
"margin_dpo/margin_mean": 7.534572601318359,
|
|
"margin_dpo/margin_std": 11.482037544250488,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.5215419501133787,
|
|
"fcm_dpo/beta": 0.10208149254322052,
|
|
"fcm_dpo/delta": -0.058262355625629425,
|
|
"fcm_dpo/margin": 8.224496841430664,
|
|
"fcm_dpo/q_t": 0.34580913186073303,
|
|
"grad_norm": 27.487586975097656,
|
|
"learning_rate": 2.7639509632351927e-07,
|
|
"logits/chosen": 0.33861398696899414,
|
|
"logits/rejected": 0.29883503913879395,
|
|
"logps/chosen": -73.38694763183594,
|
|
"logps/ref_chosen": -60.12370681762695,
|
|
"logps/ref_rejected": -78.58574676513672,
|
|
"logps/rejected": -100.073486328125,
|
|
"loss": 1.0059,
|
|
"margin_dpo/margin_mean": 8.224496841430664,
|
|
"margin_dpo/margin_std": 11.686861038208008,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.5291005291005291,
|
|
"fcm_dpo/beta": 0.09699536859989166,
|
|
"fcm_dpo/delta": -0.10742504894733429,
|
|
"fcm_dpo/margin": 9.773272514343262,
|
|
"fcm_dpo/q_t": 0.33077019453048706,
|
|
"grad_norm": 28.703025817871094,
|
|
"learning_rate": 2.698124892141971e-07,
|
|
"logits/chosen": 0.3923797905445099,
|
|
"logits/rejected": 0.3320349454879761,
|
|
"logps/chosen": -67.83167266845703,
|
|
"logps/ref_chosen": -55.104461669921875,
|
|
"logps/ref_rejected": -80.63292694091797,
|
|
"logps/rejected": -103.13340759277344,
|
|
"loss": 0.9368,
|
|
"margin_dpo/margin_mean": 9.773272514343262,
|
|
"margin_dpo/margin_std": 12.738961219787598,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.5366591080876795,
|
|
"fcm_dpo/beta": 0.09114541113376617,
|
|
"fcm_dpo/delta": -0.022746775299310684,
|
|
"fcm_dpo/margin": 9.547518730163574,
|
|
"fcm_dpo/q_t": 0.3351755142211914,
|
|
"grad_norm": 27.59998321533203,
|
|
"learning_rate": 2.632160279321328e-07,
|
|
"logits/chosen": 0.35097748041152954,
|
|
"logits/rejected": 0.2783544659614563,
|
|
"logps/chosen": -67.34779357910156,
|
|
"logps/ref_chosen": -54.87224197387695,
|
|
"logps/ref_rejected": -77.01316833496094,
|
|
"logps/rejected": -99.03623962402344,
|
|
"loss": 0.9697,
|
|
"margin_dpo/margin_mean": 9.54751968383789,
|
|
"margin_dpo/margin_std": 12.819206237792969,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.54421768707483,
|
|
"fcm_dpo/beta": 0.09261602908372879,
|
|
"fcm_dpo/delta": 0.0572737380862236,
|
|
"fcm_dpo/margin": 8.571386337280273,
|
|
"fcm_dpo/q_t": 0.35587573051452637,
|
|
"grad_norm": 29.345165252685547,
|
|
"learning_rate": 2.5661032514931834e-07,
|
|
"logits/chosen": 0.3156466484069824,
|
|
"logits/rejected": 0.27303022146224976,
|
|
"logps/chosen": -74.54544830322266,
|
|
"logps/ref_chosen": -60.75285720825195,
|
|
"logps/ref_rejected": -75.21507263183594,
|
|
"logps/rejected": -97.57904815673828,
|
|
"loss": 1.055,
|
|
"margin_dpo/margin_mean": 8.571386337280273,
|
|
"margin_dpo/margin_std": 13.336027145385742,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.5517762660619804,
|
|
"fcm_dpo/beta": 0.08920048177242279,
|
|
"fcm_dpo/delta": -0.097532719373703,
|
|
"fcm_dpo/margin": 10.514504432678223,
|
|
"fcm_dpo/q_t": 0.3252837061882019,
|
|
"grad_norm": 22.68549919128418,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": 0.3721466362476349,
|
|
"logits/rejected": 0.29883819818496704,
|
|
"logps/chosen": -71.89385986328125,
|
|
"logps/ref_chosen": -58.56513595581055,
|
|
"logps/ref_rejected": -84.06403350830078,
|
|
"logps/rejected": -107.9072494506836,
|
|
"loss": 0.925,
|
|
"margin_dpo/margin_mean": 10.514504432678223,
|
|
"margin_dpo/margin_std": 13.245100021362305,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.5593348450491308,
|
|
"fcm_dpo/beta": 0.08988513052463531,
|
|
"fcm_dpo/delta": 0.04898020997643471,
|
|
"fcm_dpo/margin": 8.933305740356445,
|
|
"fcm_dpo/q_t": 0.35057497024536133,
|
|
"grad_norm": 27.776588439941406,
|
|
"learning_rate": 2.4338967485068164e-07,
|
|
"logits/chosen": 0.31535226106643677,
|
|
"logits/rejected": 0.26749491691589355,
|
|
"logps/chosen": -72.20616149902344,
|
|
"logps/ref_chosen": -59.443138122558594,
|
|
"logps/ref_rejected": -75.80937194824219,
|
|
"logps/rejected": -97.5057144165039,
|
|
"loss": 1.0391,
|
|
"margin_dpo/margin_mean": 8.933306694030762,
|
|
"margin_dpo/margin_std": 13.573244094848633,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.5668934240362812,
|
|
"fcm_dpo/beta": 0.09227562695741653,
|
|
"fcm_dpo/delta": 0.025557735934853554,
|
|
"fcm_dpo/margin": 8.922365188598633,
|
|
"fcm_dpo/q_t": 0.3516390919685364,
|
|
"grad_norm": 32.859962463378906,
|
|
"learning_rate": 2.3678397206786715e-07,
|
|
"logits/chosen": 0.3718469738960266,
|
|
"logits/rejected": 0.3221621513366699,
|
|
"logps/chosen": -72.0213394165039,
|
|
"logps/ref_chosen": -58.59185028076172,
|
|
"logps/ref_rejected": -73.7529525756836,
|
|
"logps/rejected": -96.10481262207031,
|
|
"loss": 1.0756,
|
|
"margin_dpo/margin_mean": 8.922365188598633,
|
|
"margin_dpo/margin_std": 14.205751419067383,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.5744520030234316,
|
|
"fcm_dpo/beta": 0.0887007862329483,
|
|
"fcm_dpo/delta": -0.08066672086715698,
|
|
"fcm_dpo/margin": 10.392019271850586,
|
|
"fcm_dpo/q_t": 0.3319475054740906,
|
|
"grad_norm": 25.409692764282227,
|
|
"learning_rate": 2.3018751078580283e-07,
|
|
"logits/chosen": 0.38309699296951294,
|
|
"logits/rejected": 0.3357131779193878,
|
|
"logps/chosen": -71.56037902832031,
|
|
"logps/ref_chosen": -58.93424606323242,
|
|
"logps/ref_rejected": -76.27055358886719,
|
|
"logps/rejected": -99.28871154785156,
|
|
"loss": 0.9694,
|
|
"margin_dpo/margin_mean": 10.392019271850586,
|
|
"margin_dpo/margin_std": 14.05096435546875,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.582010582010582,
|
|
"fcm_dpo/beta": 0.08751741051673889,
|
|
"fcm_dpo/delta": 0.00886045303195715,
|
|
"fcm_dpo/margin": 8.799856185913086,
|
|
"fcm_dpo/q_t": 0.354455828666687,
|
|
"grad_norm": 27.636934280395508,
|
|
"learning_rate": 2.2360490367648084e-07,
|
|
"logits/chosen": 0.32640478014945984,
|
|
"logits/rejected": 0.29895031452178955,
|
|
"logps/chosen": -79.97639465332031,
|
|
"logps/ref_chosen": -66.42684173583984,
|
|
"logps/ref_rejected": -76.96304321289062,
|
|
"logps/rejected": -99.31244659423828,
|
|
"loss": 1.0173,
|
|
"margin_dpo/margin_mean": 8.799858093261719,
|
|
"margin_dpo/margin_std": 12.752384185791016,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.5895691609977324,
|
|
"fcm_dpo/beta": 0.0878995880484581,
|
|
"fcm_dpo/delta": 0.0270698219537735,
|
|
"fcm_dpo/margin": 9.374975204467773,
|
|
"fcm_dpo/q_t": 0.3447960913181305,
|
|
"grad_norm": 23.937883377075195,
|
|
"learning_rate": 2.170407537241599e-07,
|
|
"logits/chosen": 0.36849507689476013,
|
|
"logits/rejected": 0.3158005177974701,
|
|
"logps/chosen": -74.28120422363281,
|
|
"logps/ref_chosen": -60.984214782714844,
|
|
"logps/ref_rejected": -79.54056549072266,
|
|
"logps/rejected": -102.21253967285156,
|
|
"loss": 0.9833,
|
|
"margin_dpo/margin_mean": 9.374975204467773,
|
|
"margin_dpo/margin_std": 12.953145980834961,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.5971277399848829,
|
|
"fcm_dpo/beta": 0.08353351801633835,
|
|
"fcm_dpo/delta": -0.054874349385499954,
|
|
"fcm_dpo/margin": 10.738783836364746,
|
|
"fcm_dpo/q_t": 0.33437737822532654,
|
|
"grad_norm": 27.47764015197754,
|
|
"learning_rate": 2.104996510066625e-07,
|
|
"logits/chosen": 0.3972620368003845,
|
|
"logits/rejected": 0.3417403995990753,
|
|
"logps/chosen": -70.49010467529297,
|
|
"logps/ref_chosen": -58.30937957763672,
|
|
"logps/ref_rejected": -80.09587097167969,
|
|
"logps/rejected": -103.015380859375,
|
|
"loss": 0.9473,
|
|
"margin_dpo/margin_mean": 10.738783836364746,
|
|
"margin_dpo/margin_std": 13.871736526489258,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.6046863189720333,
|
|
"fcm_dpo/beta": 0.08445827662944794,
|
|
"fcm_dpo/delta": 0.01629134640097618,
|
|
"fcm_dpo/margin": 9.836727142333984,
|
|
"fcm_dpo/q_t": 0.34674686193466187,
|
|
"grad_norm": 32.77301788330078,
|
|
"learning_rate": 2.0398616948569493e-07,
|
|
"logits/chosen": 0.3434901833534241,
|
|
"logits/rejected": 0.2678333818912506,
|
|
"logps/chosen": -75.10372924804688,
|
|
"logps/ref_chosen": -61.39867401123047,
|
|
"logps/ref_rejected": -89.0177993774414,
|
|
"logps/rejected": -112.5595703125,
|
|
"loss": 0.999,
|
|
"margin_dpo/margin_mean": 9.8367280960083,
|
|
"margin_dpo/margin_std": 13.822813034057617,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.6046863189720333,
|
|
"eval_fcm_dpo/beta": 0.09531117975711823,
|
|
"eval_fcm_dpo/delta": 0.010658747516572475,
|
|
"eval_fcm_dpo/margin": 8.804010391235352,
|
|
"eval_fcm_dpo/q_t": 0.3533555567264557,
|
|
"eval_logits/chosen": 0.3666928708553314,
|
|
"eval_logits/rejected": 0.31859561800956726,
|
|
"eval_logps/chosen": -88.04875183105469,
|
|
"eval_logps/ref_chosen": -74.85946655273438,
|
|
"eval_logps/ref_rejected": -79.54898834228516,
|
|
"eval_logps/rejected": -101.54228210449219,
|
|
"eval_loss": 0.5335844159126282,
|
|
"eval_margin_dpo/margin_mean": 8.804010391235352,
|
|
"eval_margin_dpo/margin_std": 14.164993286132812,
|
|
"eval_runtime": 38.3222,
|
|
"eval_samples_per_second": 60.096,
|
|
"eval_steps_per_second": 1.879,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.6122448979591837,
|
|
"fcm_dpo/beta": 0.08217521011829376,
|
|
"fcm_dpo/delta": -0.1325281262397766,
|
|
"fcm_dpo/margin": 11.717107772827148,
|
|
"fcm_dpo/q_t": 0.32436299324035645,
|
|
"grad_norm": 22.69822120666504,
|
|
"learning_rate": 1.975048638084379e-07,
|
|
"logits/chosen": 0.39365020394325256,
|
|
"logits/rejected": 0.3269892930984497,
|
|
"logps/chosen": -68.60611724853516,
|
|
"logps/ref_chosen": -55.953521728515625,
|
|
"logps/ref_rejected": -77.67539978027344,
|
|
"logps/rejected": -102.04509735107422,
|
|
"loss": 0.9174,
|
|
"margin_dpo/margin_mean": 11.717107772827148,
|
|
"margin_dpo/margin_std": 14.406232833862305,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.6198034769463341,
|
|
"fcm_dpo/beta": 0.07945375144481659,
|
|
"fcm_dpo/delta": 0.00813866127282381,
|
|
"fcm_dpo/margin": 10.570515632629395,
|
|
"fcm_dpo/q_t": 0.3427334427833557,
|
|
"grad_norm": 25.729167938232422,
|
|
"learning_rate": 1.9106026612264315e-07,
|
|
"logits/chosen": 0.3750688135623932,
|
|
"logits/rejected": 0.3115871548652649,
|
|
"logps/chosen": -76.83714294433594,
|
|
"logps/ref_chosen": -63.40419387817383,
|
|
"logps/ref_rejected": -80.85710144042969,
|
|
"logps/rejected": -104.8605728149414,
|
|
"loss": 0.9674,
|
|
"margin_dpo/margin_mean": 10.570514678955078,
|
|
"margin_dpo/margin_std": 13.993307113647461,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.6273620559334845,
|
|
"fcm_dpo/beta": 0.07837341725826263,
|
|
"fcm_dpo/delta": -0.051430594176054,
|
|
"fcm_dpo/margin": 11.435976028442383,
|
|
"fcm_dpo/q_t": 0.33827075362205505,
|
|
"grad_norm": 25.88274383544922,
|
|
"learning_rate": 1.846568829074628e-07,
|
|
"logits/chosen": 0.3385092616081238,
|
|
"logits/rejected": 0.29142823815345764,
|
|
"logps/chosen": -73.53083801269531,
|
|
"logps/ref_chosen": -57.6942024230957,
|
|
"logps/ref_rejected": -71.74036407470703,
|
|
"logps/rejected": -99.01299285888672,
|
|
"loss": 0.9808,
|
|
"margin_dpo/margin_mean": 11.435976028442383,
|
|
"margin_dpo/margin_std": 16.124309539794922,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.6349206349206349,
|
|
"fcm_dpo/beta": 0.0756881982088089,
|
|
"fcm_dpo/delta": 0.03068714216351509,
|
|
"fcm_dpo/margin": 9.99744987487793,
|
|
"fcm_dpo/q_t": 0.36168405413627625,
|
|
"grad_norm": 32.788272857666016,
|
|
"learning_rate": 1.782991918222275e-07,
|
|
"logits/chosen": 0.3768512010574341,
|
|
"logits/rejected": 0.33426493406295776,
|
|
"logps/chosen": -76.74162292480469,
|
|
"logps/ref_chosen": -59.169517517089844,
|
|
"logps/ref_rejected": -69.47721099853516,
|
|
"logps/rejected": -97.0467758178711,
|
|
"loss": 1.0833,
|
|
"margin_dpo/margin_mean": 9.99744987487793,
|
|
"margin_dpo/margin_std": 16.1351261138916,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.6424792139077853,
|
|
"fcm_dpo/beta": 0.07661359012126923,
|
|
"fcm_dpo/delta": -0.07890407741069794,
|
|
"fcm_dpo/margin": 12.04144287109375,
|
|
"fcm_dpo/q_t": 0.3369203507900238,
|
|
"grad_norm": 22.537609100341797,
|
|
"learning_rate": 1.7199163857537824e-07,
|
|
"logits/chosen": 0.3781582713127136,
|
|
"logits/rejected": 0.3327697813510895,
|
|
"logps/chosen": -74.77796936035156,
|
|
"logps/ref_chosen": -58.09320831298828,
|
|
"logps/ref_rejected": -73.98226165771484,
|
|
"logps/rejected": -102.7084732055664,
|
|
"loss": 0.985,
|
|
"margin_dpo/margin_mean": 12.041443824768066,
|
|
"margin_dpo/margin_std": 16.98557472229004,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.6500377928949358,
|
|
"fcm_dpo/beta": 0.07582681626081467,
|
|
"fcm_dpo/delta": -0.017875388264656067,
|
|
"fcm_dpo/margin": 11.410634994506836,
|
|
"fcm_dpo/q_t": 0.3470071256160736,
|
|
"grad_norm": 23.696964263916016,
|
|
"learning_rate": 1.6573863381573954e-07,
|
|
"logits/chosen": 0.3497718274593353,
|
|
"logits/rejected": 0.3285056948661804,
|
|
"logps/chosen": -82.10186767578125,
|
|
"logps/ref_chosen": -62.7039909362793,
|
|
"logps/ref_rejected": -74.52284240722656,
|
|
"logps/rejected": -105.33135986328125,
|
|
"loss": 1.0312,
|
|
"margin_dpo/margin_mean": 11.410634994506836,
|
|
"margin_dpo/margin_std": 17.32351303100586,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.6575963718820862,
|
|
"fcm_dpo/beta": 0.07011202722787857,
|
|
"fcm_dpo/delta": -0.05792248249053955,
|
|
"fcm_dpo/margin": 12.86561393737793,
|
|
"fcm_dpo/q_t": 0.3342890739440918,
|
|
"grad_norm": 23.711318969726562,
|
|
"learning_rate": 1.5954455004830878e-07,
|
|
"logits/chosen": 0.43730515241622925,
|
|
"logits/rejected": 0.39576274156570435,
|
|
"logps/chosen": -73.4112777709961,
|
|
"logps/ref_chosen": -56.12516403198242,
|
|
"logps/ref_rejected": -74.36073303222656,
|
|
"logps/rejected": -104.51246643066406,
|
|
"loss": 0.9666,
|
|
"margin_dpo/margin_mean": 12.86561393737793,
|
|
"margin_dpo/margin_std": 17.37973403930664,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.6651549508692366,
|
|
"fcm_dpo/beta": 0.06788729131221771,
|
|
"fcm_dpo/delta": -0.03912908583879471,
|
|
"fcm_dpo/margin": 13.019098281860352,
|
|
"fcm_dpo/q_t": 0.33916863799095154,
|
|
"grad_norm": 28.951744079589844,
|
|
"learning_rate": 1.534137185767178e-07,
|
|
"logits/chosen": 0.3755987584590912,
|
|
"logits/rejected": 0.30046412348747253,
|
|
"logps/chosen": -73.85144805908203,
|
|
"logps/ref_chosen": -55.67548751831055,
|
|
"logps/ref_rejected": -76.62055206298828,
|
|
"logps/rejected": -107.81561279296875,
|
|
"loss": 0.9807,
|
|
"margin_dpo/margin_mean": 13.019099235534668,
|
|
"margin_dpo/margin_std": 17.854541778564453,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.672713529856387,
|
|
"fcm_dpo/beta": 0.06287918984889984,
|
|
"fcm_dpo/delta": -0.04617983475327492,
|
|
"fcm_dpo/margin": 13.171060562133789,
|
|
"fcm_dpo/q_t": 0.342844158411026,
|
|
"grad_norm": 23.502403259277344,
|
|
"learning_rate": 1.473504264745062e-07,
|
|
"logits/chosen": 0.39106225967407227,
|
|
"logits/rejected": 0.3300808370113373,
|
|
"logps/chosen": -79.40790557861328,
|
|
"logps/ref_chosen": -59.903411865234375,
|
|
"logps/ref_rejected": -82.02873229980469,
|
|
"logps/rejected": -114.70428466796875,
|
|
"loss": 0.9757,
|
|
"margin_dpo/margin_mean": 13.171060562133789,
|
|
"margin_dpo/margin_std": 17.581602096557617,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.6802721088435374,
|
|
"fcm_dpo/beta": 0.059750013053417206,
|
|
"fcm_dpo/delta": 0.028130989521741867,
|
|
"fcm_dpo/margin": 13.773831367492676,
|
|
"fcm_dpo/q_t": 0.34102991223335266,
|
|
"grad_norm": 25.237123489379883,
|
|
"learning_rate": 1.4135891358732205e-07,
|
|
"logits/chosen": 0.3934227228164673,
|
|
"logits/rejected": 0.32492750883102417,
|
|
"logps/chosen": -75.26545715332031,
|
|
"logps/ref_chosen": -55.83526611328125,
|
|
"logps/ref_rejected": -79.63658142089844,
|
|
"logps/rejected": -112.8405990600586,
|
|
"loss": 0.9533,
|
|
"margin_dpo/margin_mean": 13.773831367492676,
|
|
"margin_dpo/margin_std": 17.184816360473633,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.6878306878306878,
|
|
"fcm_dpo/beta": 0.06479842960834503,
|
|
"fcm_dpo/delta": -0.0254224743694067,
|
|
"fcm_dpo/margin": 13.480517387390137,
|
|
"fcm_dpo/q_t": 0.34084728360176086,
|
|
"grad_norm": 22.608943939208984,
|
|
"learning_rate": 1.354433695681474e-07,
|
|
"logits/chosen": 0.36722415685653687,
|
|
"logits/rejected": 0.3191031515598297,
|
|
"logps/chosen": -79.89408111572266,
|
|
"logps/ref_chosen": -60.59226608276367,
|
|
"logps/ref_rejected": -73.37936401367188,
|
|
"logps/rejected": -106.16169738769531,
|
|
"loss": 0.996,
|
|
"margin_dpo/margin_mean": 13.480517387390137,
|
|
"margin_dpo/margin_std": 19.260950088500977,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.6953892668178382,
|
|
"fcm_dpo/beta": 0.06176671385765076,
|
|
"fcm_dpo/delta": -0.08011804521083832,
|
|
"fcm_dpo/margin": 14.938470840454102,
|
|
"fcm_dpo/q_t": 0.3288452625274658,
|
|
"grad_norm": 21.24822235107422,
|
|
"learning_rate": 1.2960793094762345e-07,
|
|
"logits/chosen": 0.41707643866539,
|
|
"logits/rejected": 0.3374462127685547,
|
|
"logps/chosen": -76.28910827636719,
|
|
"logps/ref_chosen": -56.21283721923828,
|
|
"logps/ref_rejected": -83.02075958251953,
|
|
"logps/rejected": -118.0354995727539,
|
|
"loss": 0.9189,
|
|
"margin_dpo/margin_mean": 14.938468933105469,
|
|
"margin_dpo/margin_std": 18.22463607788086,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.7029478458049887,
|
|
"fcm_dpo/beta": 0.05575148016214371,
|
|
"fcm_dpo/delta": -0.01975960284471512,
|
|
"fcm_dpo/margin": 15.527587890625,
|
|
"fcm_dpo/q_t": 0.33378711342811584,
|
|
"grad_norm": 26.985567092895508,
|
|
"learning_rate": 1.238566782415197e-07,
|
|
"logits/chosen": 0.44119149446487427,
|
|
"logits/rejected": 0.3906787633895874,
|
|
"logps/chosen": -79.55015563964844,
|
|
"logps/ref_chosen": -59.0674934387207,
|
|
"logps/ref_rejected": -74.53498840332031,
|
|
"logps/rejected": -110.54522705078125,
|
|
"loss": 0.9361,
|
|
"margin_dpo/margin_mean": 15.527587890625,
|
|
"margin_dpo/margin_std": 19.12790298461914,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.7105064247921391,
|
|
"fcm_dpo/beta": 0.05737541988492012,
|
|
"fcm_dpo/delta": -0.026664074510335922,
|
|
"fcm_dpo/margin": 15.2103853225708,
|
|
"fcm_dpo/q_t": 0.33609598875045776,
|
|
"grad_norm": 19.195602416992188,
|
|
"learning_rate": 1.1819363309737438e-07,
|
|
"logits/chosen": 0.43744245171546936,
|
|
"logits/rejected": 0.39032047986984253,
|
|
"logps/chosen": -79.61660766601562,
|
|
"logps/ref_chosen": -58.3397331237793,
|
|
"logps/ref_rejected": -74.33660125732422,
|
|
"logps/rejected": -110.82386779785156,
|
|
"loss": 0.9356,
|
|
"margin_dpo/margin_mean": 15.210382461547852,
|
|
"margin_dpo/margin_std": 19.02154541015625,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.7180650037792895,
|
|
"fcm_dpo/beta": 0.053016453981399536,
|
|
"fcm_dpo/delta": -0.08009858429431915,
|
|
"fcm_dpo/margin": 17.379627227783203,
|
|
"fcm_dpo/q_t": 0.3324171006679535,
|
|
"grad_norm": 18.154373168945312,
|
|
"learning_rate": 1.126227554822985e-07,
|
|
"logits/chosen": 0.45518144965171814,
|
|
"logits/rejected": 0.3864469826221466,
|
|
"logps/chosen": -74.70694732666016,
|
|
"logps/ref_chosen": -54.60407638549805,
|
|
"logps/ref_rejected": -79.94635009765625,
|
|
"logps/rejected": -117.42884826660156,
|
|
"loss": 0.9517,
|
|
"margin_dpo/margin_mean": 17.379627227783203,
|
|
"margin_dpo/margin_std": 22.713855743408203,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.7256235827664399,
|
|
"fcm_dpo/beta": 0.0540558323264122,
|
|
"fcm_dpo/delta": 0.08679202944040298,
|
|
"fcm_dpo/margin": 14.206324577331543,
|
|
"fcm_dpo/q_t": 0.35919860005378723,
|
|
"grad_norm": 26.6975040435791,
|
|
"learning_rate": 1.0714794091391072e-07,
|
|
"logits/chosen": 0.3859735131263733,
|
|
"logits/rejected": 0.36156952381134033,
|
|
"logps/chosen": -87.61283874511719,
|
|
"logps/ref_chosen": -63.0672492980957,
|
|
"logps/ref_rejected": -68.59602355957031,
|
|
"logps/rejected": -107.34793853759766,
|
|
"loss": 1.0886,
|
|
"margin_dpo/margin_mean": 14.206324577331543,
|
|
"margin_dpo/margin_std": 23.62371063232422,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.7331821617535903,
|
|
"fcm_dpo/beta": 0.05409102514386177,
|
|
"fcm_dpo/delta": -0.02745746448636055,
|
|
"fcm_dpo/margin": 16.139617919921875,
|
|
"fcm_dpo/q_t": 0.3403390049934387,
|
|
"grad_norm": 21.20849609375,
|
|
"learning_rate": 1.0177301773633992e-07,
|
|
"logits/chosen": 0.4326368272304535,
|
|
"logits/rejected": 0.37018251419067383,
|
|
"logps/chosen": -81.90067291259766,
|
|
"logps/ref_chosen": -58.75799560546875,
|
|
"logps/ref_rejected": -79.72233581542969,
|
|
"logps/rejected": -119.00462341308594,
|
|
"loss": 0.9903,
|
|
"margin_dpo/margin_mean": 16.139617919921875,
|
|
"margin_dpo/margin_std": 22.64340591430664,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.7407407407407407,
|
|
"fcm_dpo/beta": 0.05500262230634689,
|
|
"fcm_dpo/delta": 0.05548369884490967,
|
|
"fcm_dpo/margin": 14.508279800415039,
|
|
"fcm_dpo/q_t": 0.3582765758037567,
|
|
"grad_norm": 29.062875747680664,
|
|
"learning_rate": 9.650174444319956e-08,
|
|
"logits/chosen": 0.48578470945358276,
|
|
"logits/rejected": 0.4218401312828064,
|
|
"logps/chosen": -86.71459197998047,
|
|
"logps/ref_chosen": -61.394195556640625,
|
|
"logps/ref_rejected": -81.1914291381836,
|
|
"logps/rejected": -121.02009582519531,
|
|
"loss": 1.0744,
|
|
"margin_dpo/margin_mean": 14.508279800415039,
|
|
"margin_dpo/margin_std": 23.701549530029297,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.7482993197278912,
|
|
"fcm_dpo/beta": 0.05768350884318352,
|
|
"fcm_dpo/delta": 0.0378982238471508,
|
|
"fcm_dpo/margin": 14.095187187194824,
|
|
"fcm_dpo/q_t": 0.35380709171295166,
|
|
"grad_norm": 28.836990356445312,
|
|
"learning_rate": 9.133780704940594e-08,
|
|
"logits/chosen": 0.39025309681892395,
|
|
"logits/rejected": 0.3245175778865814,
|
|
"logps/chosen": -84.3089828491211,
|
|
"logps/ref_chosen": -59.85382843017578,
|
|
"logps/ref_rejected": -80.63748931884766,
|
|
"logps/rejected": -119.18782806396484,
|
|
"loss": 1.0793,
|
|
"margin_dpo/margin_mean": 14.095187187194824,
|
|
"margin_dpo/margin_std": 22.910062789916992,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.7558578987150416,
|
|
"fcm_dpo/beta": 0.05563545227050781,
|
|
"fcm_dpo/delta": -0.0920797735452652,
|
|
"fcm_dpo/margin": 16.756519317626953,
|
|
"fcm_dpo/q_t": 0.3351852297782898,
|
|
"grad_norm": 25.257465362548828,
|
|
"learning_rate": 8.628481651367875e-08,
|
|
"logits/chosen": 0.4320411682128906,
|
|
"logits/rejected": 0.38041049242019653,
|
|
"logps/chosen": -90.02726745605469,
|
|
"logps/ref_chosen": -66.17753601074219,
|
|
"logps/ref_rejected": -83.75955200195312,
|
|
"logps/rejected": -124.36579895019531,
|
|
"loss": 0.9875,
|
|
"margin_dpo/margin_mean": 16.756519317626953,
|
|
"margin_dpo/margin_std": 23.864498138427734,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.763416477702192,
|
|
"fcm_dpo/beta": 0.0546867772936821,
|
|
"fcm_dpo/delta": 0.0059572369791567326,
|
|
"fcm_dpo/margin": 15.405729293823242,
|
|
"fcm_dpo/q_t": 0.3424641489982605,
|
|
"grad_norm": 25.822641372680664,
|
|
"learning_rate": 8.134630621352483e-08,
|
|
"logits/chosen": 0.4177117347717285,
|
|
"logits/rejected": 0.3903140127658844,
|
|
"logps/chosen": -84.20030975341797,
|
|
"logps/ref_chosen": -62.11005401611328,
|
|
"logps/ref_rejected": -74.64705657958984,
|
|
"logps/rejected": -112.14305114746094,
|
|
"loss": 0.9686,
|
|
"margin_dpo/margin_mean": 15.405729293823242,
|
|
"margin_dpo/margin_std": 20.559818267822266,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.7709750566893424,
|
|
"fcm_dpo/beta": 0.05428300425410271,
|
|
"fcm_dpo/delta": -0.01792770065367222,
|
|
"fcm_dpo/margin": 15.9463472366333,
|
|
"fcm_dpo/q_t": 0.34202420711517334,
|
|
"grad_norm": 21.753067016601562,
|
|
"learning_rate": 7.652572947447272e-08,
|
|
"logits/chosen": 0.4249315857887268,
|
|
"logits/rejected": 0.36216017603874207,
|
|
"logps/chosen": -86.62996673583984,
|
|
"logps/ref_chosen": -64.42265319824219,
|
|
"logps/ref_rejected": -87.00096130371094,
|
|
"logps/rejected": -125.15462493896484,
|
|
"loss": 0.9914,
|
|
"margin_dpo/margin_mean": 15.946345329284668,
|
|
"margin_dpo/margin_std": 22.66326904296875,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.7785336356764928,
|
|
"fcm_dpo/beta": 0.05105254799127579,
|
|
"fcm_dpo/delta": -0.04923310875892639,
|
|
"fcm_dpo/margin": 17.49344253540039,
|
|
"fcm_dpo/q_t": 0.33522480726242065,
|
|
"grad_norm": 22.888206481933594,
|
|
"learning_rate": 7.182645715528435e-08,
|
|
"logits/chosen": 0.445891797542572,
|
|
"logits/rejected": 0.3968077301979065,
|
|
"logps/chosen": -81.5649185180664,
|
|
"logps/ref_chosen": -58.284393310546875,
|
|
"logps/ref_rejected": -79.09356689453125,
|
|
"logps/rejected": -119.8675308227539,
|
|
"loss": 0.9487,
|
|
"margin_dpo/margin_mean": 17.49344253540039,
|
|
"margin_dpo/margin_std": 23.09518814086914,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.7860922146636432,
|
|
"fcm_dpo/beta": 0.05502038449048996,
|
|
"fcm_dpo/delta": 0.14013263583183289,
|
|
"fcm_dpo/margin": 13.08886432647705,
|
|
"fcm_dpo/q_t": 0.36814695596694946,
|
|
"grad_norm": 26.21905517578125,
|
|
"learning_rate": 6.725177529083209e-08,
|
|
"logits/chosen": 0.45919299125671387,
|
|
"logits/rejected": 0.4132995009422302,
|
|
"logps/chosen": -85.0035629272461,
|
|
"logps/ref_chosen": -61.03638458251953,
|
|
"logps/ref_rejected": -72.15824890136719,
|
|
"logps/rejected": -109.21429443359375,
|
|
"loss": 1.0952,
|
|
"margin_dpo/margin_mean": 13.08886432647705,
|
|
"margin_dpo/margin_std": 22.34012794494629,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.7936507936507936,
|
|
"fcm_dpo/beta": 0.0579003170132637,
|
|
"fcm_dpo/delta": 0.026444396004080772,
|
|
"fcm_dpo/margin": 14.259921073913574,
|
|
"fcm_dpo/q_t": 0.3475073277950287,
|
|
"grad_norm": 28.941152572631836,
|
|
"learning_rate": 6.280488279429185e-08,
|
|
"logits/chosen": 0.3702975809574127,
|
|
"logits/rejected": 0.3222340941429138,
|
|
"logps/chosen": -91.05569458007812,
|
|
"logps/ref_chosen": -68.02732849121094,
|
|
"logps/ref_rejected": -85.41429901123047,
|
|
"logps/rejected": -122.70259094238281,
|
|
"loss": 1.0429,
|
|
"margin_dpo/margin_mean": 14.259921073913574,
|
|
"margin_dpo/margin_std": 21.997106552124023,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.8012093726379441,
|
|
"fcm_dpo/beta": 0.06148042529821396,
|
|
"fcm_dpo/delta": -0.02484763227403164,
|
|
"fcm_dpo/margin": 14.103360176086426,
|
|
"fcm_dpo/q_t": 0.34585410356521606,
|
|
"grad_norm": 25.038785934448242,
|
|
"learning_rate": 5.848888922025552e-08,
|
|
"logits/chosen": 0.4189159870147705,
|
|
"logits/rejected": 0.3552294671535492,
|
|
"logps/chosen": -82.51399993896484,
|
|
"logps/ref_chosen": -58.67436599731445,
|
|
"logps/ref_rejected": -79.38807678222656,
|
|
"logps/rejected": -117.33106994628906,
|
|
"loss": 1.0419,
|
|
"margin_dpo/margin_mean": 14.103360176086426,
|
|
"margin_dpo/margin_std": 20.779891967773438,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.8087679516250945,
|
|
"fcm_dpo/beta": 0.05881527066230774,
|
|
"fcm_dpo/delta": -0.001515106880106032,
|
|
"fcm_dpo/margin": 14.447545051574707,
|
|
"fcm_dpo/q_t": 0.34865468740463257,
|
|
"grad_norm": 26.478731155395508,
|
|
"learning_rate": 5.430681259032957e-08,
|
|
"logits/chosen": 0.4039006233215332,
|
|
"logits/rejected": 0.34653300046920776,
|
|
"logps/chosen": -80.86461639404297,
|
|
"logps/ref_chosen": -57.640098571777344,
|
|
"logps/ref_rejected": -77.25399780273438,
|
|
"logps/rejected": -114.92607116699219,
|
|
"loss": 1.0428,
|
|
"margin_dpo/margin_mean": 14.447545051574707,
|
|
"margin_dpo/margin_std": 22.40571403503418,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.8163265306122449,
|
|
"fcm_dpo/beta": 0.05530095100402832,
|
|
"fcm_dpo/delta": -0.10176970809698105,
|
|
"fcm_dpo/margin": 17.00299072265625,
|
|
"fcm_dpo/q_t": 0.32453209161758423,
|
|
"grad_norm": 22.490652084350586,
|
|
"learning_rate": 5.026157728273966e-08,
|
|
"logits/chosen": 0.44853654503822327,
|
|
"logits/rejected": 0.37307029962539673,
|
|
"logps/chosen": -84.00617218017578,
|
|
"logps/ref_chosen": -60.17341995239258,
|
|
"logps/ref_rejected": -85.50316619873047,
|
|
"logps/rejected": -126.33892822265625,
|
|
"loss": 0.9067,
|
|
"margin_dpo/margin_mean": 17.002992630004883,
|
|
"margin_dpo/margin_std": 20.63648796081543,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.8238851095993953,
|
|
"fcm_dpo/beta": 0.05035700276494026,
|
|
"fcm_dpo/delta": -0.04968580603599548,
|
|
"fcm_dpo/margin": 17.73804473876953,
|
|
"fcm_dpo/q_t": 0.33170104026794434,
|
|
"grad_norm": 22.115388870239258,
|
|
"learning_rate": 4.635601198741607e-08,
|
|
"logits/chosen": 0.43089842796325684,
|
|
"logits/rejected": 0.37691324949264526,
|
|
"logps/chosen": -79.18077087402344,
|
|
"logps/ref_chosen": -56.985809326171875,
|
|
"logps/ref_rejected": -73.21353912353516,
|
|
"logps/rejected": -113.14653015136719,
|
|
"loss": 0.9448,
|
|
"margin_dpo/margin_mean": 17.73804473876953,
|
|
"margin_dpo/margin_std": 22.97998046875,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.8314436885865457,
|
|
"fcm_dpo/beta": 0.052602656185626984,
|
|
"fcm_dpo/delta": 0.12176811695098877,
|
|
"fcm_dpo/margin": 14.053430557250977,
|
|
"fcm_dpo/q_t": 0.359437495470047,
|
|
"grad_norm": 23.821199417114258,
|
|
"learning_rate": 4.259284772799099e-08,
|
|
"logits/chosen": 0.42888736724853516,
|
|
"logits/rejected": 0.3785988390445709,
|
|
"logps/chosen": -83.29499816894531,
|
|
"logps/ref_chosen": -59.600929260253906,
|
|
"logps/ref_rejected": -75.24870300292969,
|
|
"logps/rejected": -112.99620056152344,
|
|
"loss": 1.0383,
|
|
"margin_dpo/margin_mean": 14.053430557250977,
|
|
"margin_dpo/margin_std": 21.19103240966797,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.8390022675736961,
|
|
"fcm_dpo/beta": 0.05704032629728317,
|
|
"fcm_dpo/delta": -0.019035974517464638,
|
|
"fcm_dpo/margin": 15.177105903625488,
|
|
"fcm_dpo/q_t": 0.34175577759742737,
|
|
"grad_norm": 26.89251708984375,
|
|
"learning_rate": 3.89747159520904e-08,
|
|
"logits/chosen": 0.4168682098388672,
|
|
"logits/rejected": 0.36572036147117615,
|
|
"logps/chosen": -88.90975189208984,
|
|
"logps/ref_chosen": -63.578895568847656,
|
|
"logps/ref_rejected": -78.87867736816406,
|
|
"logps/rejected": -119.38664245605469,
|
|
"loss": 1.0105,
|
|
"margin_dpo/margin_mean": 15.177105903625488,
|
|
"margin_dpo/margin_std": 22.151811599731445,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.8465608465608465,
|
|
"fcm_dpo/beta": 0.05392115190625191,
|
|
"fcm_dpo/delta": -0.05340118333697319,
|
|
"fcm_dpo/margin": 16.63024139404297,
|
|
"fcm_dpo/q_t": 0.3411148190498352,
|
|
"grad_norm": 22.571895599365234,
|
|
"learning_rate": 3.550414669125573e-08,
|
|
"logits/chosen": 0.4584956169128418,
|
|
"logits/rejected": 0.3978951573371887,
|
|
"logps/chosen": -84.6823501586914,
|
|
"logps/ref_chosen": -58.651512145996094,
|
|
"logps/ref_rejected": -78.67181396484375,
|
|
"logps/rejected": -121.3328857421875,
|
|
"loss": 0.9717,
|
|
"margin_dpo/margin_mean": 16.63024139404297,
|
|
"margin_dpo/margin_std": 22.779064178466797,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.854119425547997,
|
|
"fcm_dpo/beta": 0.05112370103597641,
|
|
"fcm_dpo/delta": -0.035866815596818924,
|
|
"fcm_dpo/margin": 17.239295959472656,
|
|
"fcm_dpo/q_t": 0.33712780475616455,
|
|
"grad_norm": 23.28145980834961,
|
|
"learning_rate": 3.218356679178252e-08,
|
|
"logits/chosen": 0.4415108561515808,
|
|
"logits/rejected": 0.3982042968273163,
|
|
"logps/chosen": -84.83526611328125,
|
|
"logps/ref_chosen": -60.3114128112793,
|
|
"logps/ref_rejected": -78.25270080566406,
|
|
"logps/rejected": -120.0158462524414,
|
|
"loss": 0.9636,
|
|
"margin_dpo/margin_mean": 17.239295959472656,
|
|
"margin_dpo/margin_std": 23.117795944213867,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.8616780045351474,
|
|
"fcm_dpo/beta": 0.05089680105447769,
|
|
"fcm_dpo/delta": -0.05341871827840805,
|
|
"fcm_dpo/margin": 17.616899490356445,
|
|
"fcm_dpo/q_t": 0.3324287533760071,
|
|
"grad_norm": 18.90673828125,
|
|
"learning_rate": 2.9015298217712453e-08,
|
|
"logits/chosen": 0.41903671622276306,
|
|
"logits/rejected": 0.35510531067848206,
|
|
"logps/chosen": -82.22904205322266,
|
|
"logps/ref_chosen": -57.752410888671875,
|
|
"logps/ref_rejected": -76.99858093261719,
|
|
"logps/rejected": -119.09210205078125,
|
|
"loss": 0.9449,
|
|
"margin_dpo/margin_mean": 17.616899490356445,
|
|
"margin_dpo/margin_std": 22.199068069458008,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.8692365835222978,
|
|
"fcm_dpo/beta": 0.051069360226392746,
|
|
"fcm_dpo/delta": 0.08544424921274185,
|
|
"fcm_dpo/margin": 15.04365348815918,
|
|
"fcm_dpo/q_t": 0.35535532236099243,
|
|
"grad_norm": 24.414873123168945,
|
|
"learning_rate": 2.600155642716606e-08,
|
|
"logits/chosen": 0.45879822969436646,
|
|
"logits/rejected": 0.4188869595527649,
|
|
"logps/chosen": -89.65702819824219,
|
|
"logps/ref_chosen": -63.61958694458008,
|
|
"logps/ref_rejected": -79.51353454589844,
|
|
"logps/rejected": -120.5946273803711,
|
|
"loss": 1.0306,
|
|
"margin_dpo/margin_mean": 15.043652534484863,
|
|
"margin_dpo/margin_std": 22.42341423034668,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.8767951625094482,
|
|
"fcm_dpo/beta": 0.05287238955497742,
|
|
"fcm_dpo/delta": -0.0031969905830919743,
|
|
"fcm_dpo/margin": 16.094629287719727,
|
|
"fcm_dpo/q_t": 0.3464842438697815,
|
|
"grad_norm": 24.391796112060547,
|
|
"learning_rate": 2.3144448823151392e-08,
|
|
"logits/chosen": 0.43840184807777405,
|
|
"logits/rejected": 0.39051514863967896,
|
|
"logps/chosen": -81.55989074707031,
|
|
"logps/ref_chosen": -57.3541145324707,
|
|
"logps/ref_rejected": -73.14434051513672,
|
|
"logps/rejected": -113.44474792480469,
|
|
"loss": 1.0045,
|
|
"margin_dpo/margin_mean": 16.094629287719727,
|
|
"margin_dpo/margin_std": 23.192731857299805,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.8843537414965986,
|
|
"fcm_dpo/beta": 0.053858768194913864,
|
|
"fcm_dpo/delta": 0.050987910479307175,
|
|
"fcm_dpo/margin": 14.902254104614258,
|
|
"fcm_dpo/q_t": 0.35386043787002563,
|
|
"grad_norm": 23.28061294555664,
|
|
"learning_rate": 2.044597327993153e-08,
|
|
"logits/chosen": 0.48447608947753906,
|
|
"logits/rejected": 0.4253684878349304,
|
|
"logps/chosen": -81.11854553222656,
|
|
"logps/ref_chosen": -56.0127067565918,
|
|
"logps/ref_rejected": -77.16522216796875,
|
|
"logps/rejected": -117.1733169555664,
|
|
"loss": 1.0578,
|
|
"margin_dpo/margin_mean": 14.902254104614258,
|
|
"margin_dpo/margin_std": 23.740413665771484,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.891912320483749,
|
|
"fcm_dpo/beta": 0.05255385488271713,
|
|
"fcm_dpo/delta": -0.10637931525707245,
|
|
"fcm_dpo/margin": 18.024349212646484,
|
|
"fcm_dpo/q_t": 0.32385098934173584,
|
|
"grad_norm": 21.337730407714844,
|
|
"learning_rate": 1.7908016745981856e-08,
|
|
"logits/chosen": 0.4491957128047943,
|
|
"logits/rejected": 0.4123144745826721,
|
|
"logps/chosen": -85.22700500488281,
|
|
"logps/ref_chosen": -60.5894660949707,
|
|
"logps/ref_rejected": -74.34771728515625,
|
|
"logps/rejected": -117.0096206665039,
|
|
"loss": 0.9336,
|
|
"margin_dpo/margin_mean": 18.024349212646484,
|
|
"margin_dpo/margin_std": 23.227336883544922,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.8994708994708994,
|
|
"fcm_dpo/beta": 0.04942792281508446,
|
|
"fcm_dpo/delta": -0.011591309681534767,
|
|
"fcm_dpo/margin": 17.376907348632812,
|
|
"fcm_dpo/q_t": 0.3416403830051422,
|
|
"grad_norm": 20.537391662597656,
|
|
"learning_rate": 1.553235392451377e-08,
|
|
"logits/chosen": 0.4560145437717438,
|
|
"logits/rejected": 0.3797822594642639,
|
|
"logps/chosen": -77.99388122558594,
|
|
"logps/ref_chosen": -54.77838897705078,
|
|
"logps/ref_rejected": -78.102783203125,
|
|
"logps/rejected": -118.69517517089844,
|
|
"loss": 0.9909,
|
|
"margin_dpo/margin_mean": 17.376907348632812,
|
|
"margin_dpo/margin_std": 24.222240447998047,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.9070294784580499,
|
|
"fcm_dpo/beta": 0.05428706854581833,
|
|
"fcm_dpo/delta": 0.1492157131433487,
|
|
"fcm_dpo/margin": 13.049295425415039,
|
|
"fcm_dpo/q_t": 0.37635013461112976,
|
|
"grad_norm": 27.10298728942871,
|
|
"learning_rate": 1.3320646032487393e-08,
|
|
"logits/chosen": 0.4530029296875,
|
|
"logits/rejected": 0.4146535396575928,
|
|
"logps/chosen": -84.59120178222656,
|
|
"logps/ref_chosen": -58.45500564575195,
|
|
"logps/ref_rejected": -70.7367172241211,
|
|
"logps/rejected": -109.92220306396484,
|
|
"loss": 1.1106,
|
|
"margin_dpo/margin_mean": 13.049296379089355,
|
|
"margin_dpo/margin_std": 23.154495239257812,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.9070294784580499,
|
|
"eval_fcm_dpo/beta": 0.05690416321158409,
|
|
"eval_fcm_dpo/delta": -0.009421434253454208,
|
|
"eval_fcm_dpo/margin": 14.929535865783691,
|
|
"eval_fcm_dpo/q_t": 0.3486931324005127,
|
|
"eval_logits/chosen": 0.4598844349384308,
|
|
"eval_logits/rejected": 0.41061776876449585,
|
|
"eval_logps/chosen": -98.01021575927734,
|
|
"eval_logps/ref_chosen": -74.85946655273438,
|
|
"eval_logps/ref_rejected": -79.54898834228516,
|
|
"eval_logps/rejected": -117.62928009033203,
|
|
"eval_loss": 0.5165370106697083,
|
|
"eval_margin_dpo/margin_mean": 14.929535865783691,
|
|
"eval_margin_dpo/margin_std": 22.652507781982422,
|
|
"eval_runtime": 38.3435,
|
|
"eval_samples_per_second": 60.062,
|
|
"eval_steps_per_second": 1.878,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.9145880574452003,
|
|
"fcm_dpo/beta": 0.051525335758924484,
|
|
"fcm_dpo/delta": -0.01808151602745056,
|
|
"fcm_dpo/margin": 16.793476104736328,
|
|
"fcm_dpo/q_t": 0.341521680355072,
|
|
"grad_norm": 23.69171714782715,
|
|
"learning_rate": 1.1274439638981532e-08,
|
|
"logits/chosen": 0.4442858099937439,
|
|
"logits/rejected": 0.39685678482055664,
|
|
"logps/chosen": -81.65498352050781,
|
|
"logps/ref_chosen": -59.87483596801758,
|
|
"logps/ref_rejected": -75.75318908691406,
|
|
"logps/rejected": -114.32682800292969,
|
|
"loss": 0.9813,
|
|
"margin_dpo/margin_mean": 16.79347801208496,
|
|
"margin_dpo/margin_std": 23.46561622619629,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.9221466364323507,
|
|
"fcm_dpo/beta": 0.052094198763370514,
|
|
"fcm_dpo/delta": -0.02113976515829563,
|
|
"fcm_dpo/margin": 16.645000457763672,
|
|
"fcm_dpo/q_t": 0.3394031822681427,
|
|
"grad_norm": 24.6243839263916,
|
|
"learning_rate": 9.395165583732379e-09,
|
|
"logits/chosen": 0.44218477606773376,
|
|
"logits/rejected": 0.40106362104415894,
|
|
"logps/chosen": -84.5372085571289,
|
|
"logps/ref_chosen": -60.35883712768555,
|
|
"logps/ref_rejected": -81.3543930053711,
|
|
"logps/rejected": -122.17777252197266,
|
|
"loss": 0.9567,
|
|
"margin_dpo/margin_mean": 16.645000457763672,
|
|
"margin_dpo/margin_std": 21.96414566040039,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.9297052154195011,
|
|
"fcm_dpo/beta": 0.050758205354213715,
|
|
"fcm_dpo/delta": -0.04426204413175583,
|
|
"fcm_dpo/margin": 17.528545379638672,
|
|
"fcm_dpo/q_t": 0.3327665627002716,
|
|
"grad_norm": 24.149803161621094,
|
|
"learning_rate": 7.684137976598088e-09,
|
|
"logits/chosen": 0.42691120505332947,
|
|
"logits/rejected": 0.3889518976211548,
|
|
"logps/chosen": -83.37763977050781,
|
|
"logps/ref_chosen": -59.17219161987305,
|
|
"logps/ref_rejected": -79.92167663574219,
|
|
"logps/rejected": -121.65567779541016,
|
|
"loss": 0.9518,
|
|
"margin_dpo/margin_mean": 17.528545379638672,
|
|
"margin_dpo/margin_std": 22.845678329467773,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.9372637944066515,
|
|
"fcm_dpo/beta": 0.05097709968686104,
|
|
"fcm_dpo/delta": 0.06250528991222382,
|
|
"fcm_dpo/margin": 15.492701530456543,
|
|
"fcm_dpo/q_t": 0.3534691631793976,
|
|
"grad_norm": 26.220901489257812,
|
|
"learning_rate": 6.142553278648238e-09,
|
|
"logits/chosen": 0.46585145592689514,
|
|
"logits/rejected": 0.4045208990573883,
|
|
"logps/chosen": -82.3936767578125,
|
|
"logps/ref_chosen": -58.052696228027344,
|
|
"logps/ref_rejected": -78.37252807617188,
|
|
"logps/rejected": -118.2061996459961,
|
|
"loss": 0.9912,
|
|
"margin_dpo/margin_mean": 15.492703437805176,
|
|
"margin_dpo/margin_std": 21.617921829223633,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.9448223733938019,
|
|
"fcm_dpo/beta": 0.05353456735610962,
|
|
"fcm_dpo/delta": 0.03296828269958496,
|
|
"fcm_dpo/margin": 15.291746139526367,
|
|
"fcm_dpo/q_t": 0.34970173239707947,
|
|
"grad_norm": 23.437349319458008,
|
|
"learning_rate": 4.7714894655209174e-09,
|
|
"logits/chosen": 0.4618695378303528,
|
|
"logits/rejected": 0.3831492066383362,
|
|
"logps/chosen": -82.37210845947266,
|
|
"logps/ref_chosen": -56.957862854003906,
|
|
"logps/ref_rejected": -82.68255615234375,
|
|
"logps/rejected": -123.38853454589844,
|
|
"loss": 0.9953,
|
|
"margin_dpo/margin_mean": 15.291746139526367,
|
|
"margin_dpo/margin_std": 21.720396041870117,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.9523809523809523,
|
|
"fcm_dpo/beta": 0.04969333857297897,
|
|
"fcm_dpo/delta": -0.11904563009738922,
|
|
"fcm_dpo/margin": 19.20981216430664,
|
|
"fcm_dpo/q_t": 0.324840247631073,
|
|
"grad_norm": 18.3225040435791,
|
|
"learning_rate": 3.5719052736323806e-09,
|
|
"logits/chosen": 0.452108770608902,
|
|
"logits/rejected": 0.38503187894821167,
|
|
"logps/chosen": -79.83077239990234,
|
|
"logps/ref_chosen": -56.71510696411133,
|
|
"logps/ref_rejected": -82.94544219970703,
|
|
"logps/rejected": -125.27091979980469,
|
|
"loss": 0.9319,
|
|
"margin_dpo/margin_mean": 19.20981216430664,
|
|
"margin_dpo/margin_std": 24.018808364868164,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.9599395313681028,
|
|
"fcm_dpo/beta": 0.04755784571170807,
|
|
"fcm_dpo/delta": 0.07769346237182617,
|
|
"fcm_dpo/margin": 16.40091323852539,
|
|
"fcm_dpo/q_t": 0.35872796177864075,
|
|
"grad_norm": 25.841398239135742,
|
|
"learning_rate": 2.5446395297668287e-09,
|
|
"logits/chosen": 0.46524158120155334,
|
|
"logits/rejected": 0.4145492613315582,
|
|
"logps/chosen": -83.50657653808594,
|
|
"logps/ref_chosen": -59.33793258666992,
|
|
"logps/ref_rejected": -75.01703643798828,
|
|
"logps/rejected": -115.58660888671875,
|
|
"loss": 1.0358,
|
|
"margin_dpo/margin_mean": 16.40091323852539,
|
|
"margin_dpo/margin_std": 24.275365829467773,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.9674981103552532,
|
|
"fcm_dpo/beta": 0.05097305774688721,
|
|
"fcm_dpo/delta": -0.007989849895238876,
|
|
"fcm_dpo/margin": 16.800077438354492,
|
|
"fcm_dpo/q_t": 0.3421487808227539,
|
|
"grad_norm": 23.426517486572266,
|
|
"learning_rate": 1.690410564514244e-09,
|
|
"logits/chosen": 0.4413645267486572,
|
|
"logits/rejected": 0.3641572892665863,
|
|
"logps/chosen": -84.63844299316406,
|
|
"logps/ref_chosen": -58.1605339050293,
|
|
"logps/ref_rejected": -79.85365295410156,
|
|
"logps/rejected": -123.13163757324219,
|
|
"loss": 0.9848,
|
|
"margin_dpo/margin_mean": 16.800077438354492,
|
|
"margin_dpo/margin_std": 23.6998348236084,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.9750566893424036,
|
|
"fcm_dpo/beta": 0.05383114144206047,
|
|
"fcm_dpo/delta": 0.07376549392938614,
|
|
"fcm_dpo/margin": 14.506462097167969,
|
|
"fcm_dpo/q_t": 0.3585270643234253,
|
|
"grad_norm": 24.570833206176758,
|
|
"learning_rate": 1.0098157099674987e-09,
|
|
"logits/chosen": 0.42010536789894104,
|
|
"logits/rejected": 0.3857712149620056,
|
|
"logps/chosen": -89.77040100097656,
|
|
"logps/ref_chosen": -63.45180130004883,
|
|
"logps/ref_rejected": -74.18285369873047,
|
|
"logps/rejected": -115.0079116821289,
|
|
"loss": 1.0557,
|
|
"margin_dpo/margin_mean": 14.506462097167969,
|
|
"margin_dpo/margin_std": 23.072330474853516,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.982615268329554,
|
|
"fcm_dpo/beta": 0.0525779128074646,
|
|
"fcm_dpo/delta": -0.06335767358541489,
|
|
"fcm_dpo/margin": 17.234725952148438,
|
|
"fcm_dpo/q_t": 0.3357749879360199,
|
|
"grad_norm": 26.537277221679688,
|
|
"learning_rate": 5.033308820289184e-10,
|
|
"logits/chosen": 0.46734586358070374,
|
|
"logits/rejected": 0.4007229804992676,
|
|
"logps/chosen": -86.1024169921875,
|
|
"logps/ref_chosen": -59.75496292114258,
|
|
"logps/ref_rejected": -84.31481170654297,
|
|
"logps/rejected": -127.89698791503906,
|
|
"loss": 0.984,
|
|
"margin_dpo/margin_mean": 17.234725952148438,
|
|
"margin_dpo/margin_std": 24.128082275390625,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.9901738473167044,
|
|
"fcm_dpo/beta": 0.05265795439481735,
|
|
"fcm_dpo/delta": 0.012200763449072838,
|
|
"fcm_dpo/margin": 15.872207641601562,
|
|
"fcm_dpo/q_t": 0.3481621742248535,
|
|
"grad_norm": 20.39609718322754,
|
|
"learning_rate": 1.7131024761923852e-10,
|
|
"logits/chosen": 0.45558881759643555,
|
|
"logits/rejected": 0.38255399465560913,
|
|
"logps/chosen": -82.5105209350586,
|
|
"logps/ref_chosen": -57.817848205566406,
|
|
"logps/ref_rejected": -79.81755065917969,
|
|
"logps/rejected": -120.38243103027344,
|
|
"loss": 0.9862,
|
|
"margin_dpo/margin_mean": 15.872209548950195,
|
|
"margin_dpo/margin_std": 22.03376579284668,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.9977324263038548,
|
|
"fcm_dpo/beta": 0.05069095641374588,
|
|
"fcm_dpo/delta": -0.023842424154281616,
|
|
"fcm_dpo/margin": 17.15415382385254,
|
|
"fcm_dpo/q_t": 0.33922240138053894,
|
|
"grad_norm": 21.749004364013672,
|
|
"learning_rate": 1.3985977021235829e-11,
|
|
"logits/chosen": 0.5243078470230103,
|
|
"logits/rejected": 0.4590482711791992,
|
|
"logps/chosen": -85.16634368896484,
|
|
"logps/ref_chosen": -59.12651443481445,
|
|
"logps/ref_rejected": -79.42085266113281,
|
|
"logps/rejected": -122.6148452758789,
|
|
"loss": 0.9785,
|
|
"margin_dpo/margin_mean": 17.15415382385254,
|
|
"margin_dpo/margin_std": 23.605112075805664,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.999244142101285,
|
|
"step": 661,
|
|
"total_flos": 0.0,
|
|
"train_loss": 1.06011358295735,
|
|
"train_runtime": 1750.6093,
|
|
"train_samples_per_second": 24.184,
|
|
"train_steps_per_second": 0.378
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 661,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|