1145 lines
49 KiB
JSON
1145 lines
49 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.05504587155963303,
|
|
"eval_steps": 50,
|
|
"global_step": 300,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 58.4,
|
|
"completions/max_terminated_length": 58.4,
|
|
"completions/mean_length": 26.00937557220459,
|
|
"completions/mean_terminated_length": 26.00937557220459,
|
|
"completions/min_length": 9.7,
|
|
"completions/min_terminated_length": 9.7,
|
|
"entropy": 0.6222207842394709,
|
|
"epoch": 0.001834862385321101,
|
|
"frac_reward_zero_std": 0.00416666679084301,
|
|
"grad_norm": 1.048504114151001,
|
|
"kl": 2.7334264186151813e-05,
|
|
"learning_rate": 4.5e-07,
|
|
"loss": -0.0019,
|
|
"num_tokens": 95625.0,
|
|
"reward": 0.32962159216403963,
|
|
"reward_std": 0.04940153956413269,
|
|
"rewards/bleu_reward_func/mean": 0.3562518090009689,
|
|
"rewards/bleu_reward_func/std": 0.2016133487224579,
|
|
"rewards/chrf_reward_func/mean": 0.6361060380935669,
|
|
"rewards/chrf_reward_func/std": 0.1444593071937561,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2545995950698853,
|
|
"rewards/cometkiwi_reward_func/std": 0.08734418824315071,
|
|
"rewards/format_reward_func/mean": -0.002083333395421505,
|
|
"rewards/format_reward_func/std": 0.02041241228580475,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.019695587921887636,
|
|
"rewards/length_ratio_reward_func/std": 0.04623677581548691,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 67.2,
|
|
"completions/max_terminated_length": 67.2,
|
|
"completions/mean_length": 27.778125953674316,
|
|
"completions/mean_terminated_length": 27.778125953674316,
|
|
"completions/min_length": 8.7,
|
|
"completions/min_terminated_length": 8.7,
|
|
"entropy": 0.658123054727912,
|
|
"epoch": 0.003669724770642202,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.32069429755210876,
|
|
"kl": 6.983184273252618e-05,
|
|
"learning_rate": 9.499999999999999e-07,
|
|
"loss": -0.0061,
|
|
"num_tokens": 195408.0,
|
|
"reward": 0.3221228033304214,
|
|
"reward_std": 0.05070964209735394,
|
|
"rewards/bleu_reward_func/mean": 0.34737427830696105,
|
|
"rewards/bleu_reward_func/std": 0.19081887900829314,
|
|
"rewards/chrf_reward_func/mean": 0.6202116131782531,
|
|
"rewards/chrf_reward_func/std": 0.14867648631334304,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2565790504217148,
|
|
"rewards/cometkiwi_reward_func/std": 0.08886389508843422,
|
|
"rewards/format_reward_func/mean": -0.004687500093132258,
|
|
"rewards/format_reward_func/std": 0.028995268791913987,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.03262660829350352,
|
|
"rewards/length_ratio_reward_func/std": 0.0901365652680397,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 68.6,
|
|
"completions/max_terminated_length": 68.6,
|
|
"completions/mean_length": 27.58645877838135,
|
|
"completions/mean_terminated_length": 27.58645877838135,
|
|
"completions/min_length": 8.7,
|
|
"completions/min_terminated_length": 8.7,
|
|
"entropy": 0.6231416624039412,
|
|
"epoch": 0.005504587155963303,
|
|
"frac_reward_zero_std": 0.00833333358168602,
|
|
"grad_norm": 0.9816285967826843,
|
|
"kl": 9.183844148346054e-05,
|
|
"learning_rate": 9.678571428571428e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 294839.0,
|
|
"reward": 0.3247929662466049,
|
|
"reward_std": 0.047703846171498296,
|
|
"rewards/bleu_reward_func/mean": 0.3441034257411957,
|
|
"rewards/bleu_reward_func/std": 0.1984228879213333,
|
|
"rewards/chrf_reward_func/mean": 0.6319877922534942,
|
|
"rewards/chrf_reward_func/std": 0.1485195867717266,
|
|
"rewards/cometkiwi_reward_func/mean": 0.256802462041378,
|
|
"rewards/cometkiwi_reward_func/std": 0.08940923810005189,
|
|
"rewards/format_reward_func/mean": -0.003125,
|
|
"rewards/format_reward_func/std": 0.017490598559379577,
|
|
"rewards/language_consistency_reward_func/mean": -0.00416666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.020087526738643648,
|
|
"rewards/length_ratio_reward_func/mean": -0.02921138172969222,
|
|
"rewards/length_ratio_reward_func/std": 0.07287587337195874,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 59.3,
|
|
"completions/max_terminated_length": 59.3,
|
|
"completions/mean_length": 26.26250057220459,
|
|
"completions/mean_terminated_length": 26.26250057220459,
|
|
"completions/min_length": 8.8,
|
|
"completions/min_terminated_length": 8.8,
|
|
"entropy": 0.6190456669777632,
|
|
"epoch": 0.007339449541284404,
|
|
"frac_reward_zero_std": 0.00833333358168602,
|
|
"grad_norm": 0.519716203212738,
|
|
"kl": 0.0002696820651216569,
|
|
"learning_rate": 9.321428571428572e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 391335.0,
|
|
"reward": 0.3305883288383484,
|
|
"reward_std": 0.05123380608856678,
|
|
"rewards/bleu_reward_func/mean": 0.36549694538116456,
|
|
"rewards/bleu_reward_func/std": 0.20037643611431122,
|
|
"rewards/chrf_reward_func/mean": 0.6372063994407654,
|
|
"rewards/chrf_reward_func/std": 0.1415681518614292,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2466452181339264,
|
|
"rewards/cometkiwi_reward_func/std": 0.08479956649243832,
|
|
"rewards/format_reward_func/mean": -0.0026041667442768812,
|
|
"rewards/format_reward_func/std": 0.01946069225668907,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.020329439267516135,
|
|
"rewards/length_ratio_reward_func/std": 0.04152334425598383,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 63.6,
|
|
"completions/max_terminated_length": 63.6,
|
|
"completions/mean_length": 26.019792366027833,
|
|
"completions/mean_terminated_length": 26.019792366027833,
|
|
"completions/min_length": 9.4,
|
|
"completions/min_terminated_length": 9.4,
|
|
"entropy": 0.5883081251755357,
|
|
"epoch": 0.009174311926605505,
|
|
"frac_reward_zero_std": 0.00833333358168602,
|
|
"grad_norm": 0.8595839142799377,
|
|
"kl": 0.0005263936727867957,
|
|
"learning_rate": 8.964285714285715e-07,
|
|
"loss": -0.0023,
|
|
"num_tokens": 488426.0,
|
|
"reward": 0.3323968887329102,
|
|
"reward_std": 0.04607201106846333,
|
|
"rewards/bleu_reward_func/mean": 0.3590943604707718,
|
|
"rewards/bleu_reward_func/std": 0.19223689585924147,
|
|
"rewards/chrf_reward_func/mean": 0.6436836957931519,
|
|
"rewards/chrf_reward_func/std": 0.14234987199306487,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25749468356370925,
|
|
"rewards/cometkiwi_reward_func/std": 0.0893651120364666,
|
|
"rewards/format_reward_func/mean": -0.0026041667442768812,
|
|
"rewards/format_reward_func/std": 0.01514686644077301,
|
|
"rewards/language_consistency_reward_func/mean": -0.00416666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.020087526738643648,
|
|
"rewards/length_ratio_reward_func/mean": -0.02607359541580081,
|
|
"rewards/length_ratio_reward_func/std": 0.06909670382738113,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 55.5,
|
|
"completions/max_terminated_length": 55.5,
|
|
"completions/mean_length": 26.47500057220459,
|
|
"completions/mean_terminated_length": 26.47500057220459,
|
|
"completions/min_length": 8.1,
|
|
"completions/min_terminated_length": 8.1,
|
|
"entropy": 0.577550302259624,
|
|
"epoch": 0.011009174311926606,
|
|
"frac_reward_zero_std": 0.00833333358168602,
|
|
"grad_norm": 0.5159716606140137,
|
|
"kl": 0.0008936724035265798,
|
|
"learning_rate": 8.607142857142857e-07,
|
|
"loss": -0.0022,
|
|
"num_tokens": 584542.0,
|
|
"reward": 0.33943485021591185,
|
|
"reward_std": 0.05077244155108929,
|
|
"rewards/bleu_reward_func/mean": 0.380869859457016,
|
|
"rewards/bleu_reward_func/std": 0.2181686282157898,
|
|
"rewards/chrf_reward_func/mean": 0.6479440569877625,
|
|
"rewards/chrf_reward_func/std": 0.15388925224542618,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2559502810239792,
|
|
"rewards/cometkiwi_reward_func/std": 0.09293145835399627,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.007178794592618942,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.025034761056303978,
|
|
"rewards/length_ratio_reward_func/std": 0.06962351836264133,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 61.9,
|
|
"completions/max_terminated_length": 61.9,
|
|
"completions/mean_length": 25.890625762939454,
|
|
"completions/mean_terminated_length": 25.890625762939454,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.5623728793114424,
|
|
"epoch": 0.012844036697247707,
|
|
"frac_reward_zero_std": 0.00416666679084301,
|
|
"grad_norm": 0.38885268568992615,
|
|
"kl": 0.0013928474750173336,
|
|
"learning_rate": 8.249999999999999e-07,
|
|
"loss": 0.0026,
|
|
"num_tokens": 680053.0,
|
|
"reward": 0.3369089126586914,
|
|
"reward_std": 0.047715747356414796,
|
|
"rewards/bleu_reward_func/mean": 0.38009184002876284,
|
|
"rewards/bleu_reward_func/std": 0.21366384327411653,
|
|
"rewards/chrf_reward_func/mean": 0.6426849246025086,
|
|
"rewards/chrf_reward_func/std": 0.15480013191699982,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2530184432864189,
|
|
"rewards/cometkiwi_reward_func/std": 0.08466955572366715,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.00416666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.020087526738643648,
|
|
"rewards/length_ratio_reward_func/mean": -0.02213119938969612,
|
|
"rewards/length_ratio_reward_func/std": 0.05329653732478619,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 60.2,
|
|
"completions/max_terminated_length": 60.2,
|
|
"completions/mean_length": 26.68645896911621,
|
|
"completions/mean_terminated_length": 26.68645896911621,
|
|
"completions/min_length": 8.7,
|
|
"completions/min_terminated_length": 8.7,
|
|
"entropy": 0.5620506957173348,
|
|
"epoch": 0.014678899082568808,
|
|
"frac_reward_zero_std": 0.01666666716337204,
|
|
"grad_norm": 0.45386120676994324,
|
|
"kl": 0.0019803574601610307,
|
|
"learning_rate": 7.892857142857142e-07,
|
|
"loss": 0.0025,
|
|
"num_tokens": 777048.0,
|
|
"reward": 0.33512409031391144,
|
|
"reward_std": 0.04799444749951363,
|
|
"rewards/bleu_reward_func/mean": 0.3715683728456497,
|
|
"rewards/bleu_reward_func/std": 0.20637690275907516,
|
|
"rewards/chrf_reward_func/mean": 0.6423242092132568,
|
|
"rewards/chrf_reward_func/std": 0.14877257198095323,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25558496117591856,
|
|
"rewards/cometkiwi_reward_func/std": 0.09067081809043884,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.007178794592618942,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.029562136624008417,
|
|
"rewards/length_ratio_reward_func/std": 0.08073766008019448,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 56.3,
|
|
"completions/max_terminated_length": 56.3,
|
|
"completions/mean_length": 26.50520877838135,
|
|
"completions/mean_terminated_length": 26.50520877838135,
|
|
"completions/min_length": 8.3,
|
|
"completions/min_terminated_length": 8.3,
|
|
"entropy": 0.5507419442757964,
|
|
"epoch": 0.01651376146788991,
|
|
"frac_reward_zero_std": 0.00833333358168602,
|
|
"grad_norm": 0.3904930353164673,
|
|
"kl": 0.002531985064888431,
|
|
"learning_rate": 7.535714285714285e-07,
|
|
"loss": -0.0021,
|
|
"num_tokens": 873781.0,
|
|
"reward": 0.33635487854480745,
|
|
"reward_std": 0.04732672348618507,
|
|
"rewards/bleu_reward_func/mean": 0.3720450133085251,
|
|
"rewards/bleu_reward_func/std": 0.1961376890540123,
|
|
"rewards/chrf_reward_func/mean": 0.6445945739746094,
|
|
"rewards/chrf_reward_func/std": 0.15143516063690185,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2600402444601059,
|
|
"rewards/cometkiwi_reward_func/std": 0.09315923526883126,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.007178794592618942,
|
|
"rewards/language_consistency_reward_func/mean": -0.00833333358168602,
|
|
"rewards/language_consistency_reward_func/std": 0.040175053477287295,
|
|
"rewards/length_ratio_reward_func/mean": -0.022106224577873944,
|
|
"rewards/length_ratio_reward_func/std": 0.05355628468096256,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 51.3,
|
|
"completions/max_terminated_length": 51.3,
|
|
"completions/mean_length": 24.51770896911621,
|
|
"completions/mean_terminated_length": 24.51770896911621,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 0.5281606610864401,
|
|
"epoch": 0.01834862385321101,
|
|
"frac_reward_zero_std": 0.01250000037252903,
|
|
"grad_norm": 1.379321813583374,
|
|
"kl": 0.0030472023292531956,
|
|
"learning_rate": 7.178571428571428e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 965082.0,
|
|
"reward": 0.3303297787904739,
|
|
"reward_std": 0.04582682959735394,
|
|
"rewards/bleu_reward_func/mean": 0.36906697750091555,
|
|
"rewards/bleu_reward_func/std": 0.20976329743862152,
|
|
"rewards/chrf_reward_func/mean": 0.6352234184741974,
|
|
"rewards/chrf_reward_func/std": 0.16477679163217546,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2451363429427147,
|
|
"rewards/cometkiwi_reward_func/std": 0.09033353254199028,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.00416666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.020087526738643648,
|
|
"rewards/length_ratio_reward_func/mean": -0.022292132023721933,
|
|
"rewards/length_ratio_reward_func/std": 0.0563433988019824,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 60.9,
|
|
"completions/max_terminated_length": 60.9,
|
|
"completions/mean_length": 26.343750762939454,
|
|
"completions/mean_terminated_length": 26.343750762939454,
|
|
"completions/min_length": 8.6,
|
|
"completions/min_terminated_length": 8.6,
|
|
"entropy": 0.5302650343626738,
|
|
"epoch": 0.02018348623853211,
|
|
"frac_reward_zero_std": 0.01666666716337204,
|
|
"grad_norm": 0.5465396046638489,
|
|
"kl": 0.004301348544686334,
|
|
"learning_rate": 6.821428571428571e-07,
|
|
"loss": 0.0044,
|
|
"num_tokens": 1061800.0,
|
|
"reward": 0.3459406852722168,
|
|
"reward_std": 0.04332970455288887,
|
|
"rewards/bleu_reward_func/mean": 0.391244700551033,
|
|
"rewards/bleu_reward_func/std": 0.21132230311632155,
|
|
"rewards/chrf_reward_func/mean": 0.6589273929595947,
|
|
"rewards/chrf_reward_func/std": 0.14488666355609894,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25874917954206467,
|
|
"rewards/cometkiwi_reward_func/std": 0.08399482890963554,
|
|
"rewards/format_reward_func/mean": -0.0015625000465661286,
|
|
"rewards/format_reward_func/std": 0.012281897664070129,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.022846315056085587,
|
|
"rewards/length_ratio_reward_func/std": 0.057243801094591616,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 60.8,
|
|
"completions/max_terminated_length": 60.8,
|
|
"completions/mean_length": 27.026042556762697,
|
|
"completions/mean_terminated_length": 27.026042556762697,
|
|
"completions/min_length": 8.5,
|
|
"completions/min_terminated_length": 8.5,
|
|
"entropy": 0.5148698446340859,
|
|
"epoch": 0.022018348623853212,
|
|
"frac_reward_zero_std": 0.00833333358168602,
|
|
"grad_norm": 0.38539135456085205,
|
|
"kl": 0.004865404597876477,
|
|
"learning_rate": 6.464285714285714e-07,
|
|
"loss": 0.0008,
|
|
"num_tokens": 1159457.0,
|
|
"reward": 0.3414710432291031,
|
|
"reward_std": 0.04265272133052349,
|
|
"rewards/bleu_reward_func/mean": 0.3898064374923706,
|
|
"rewards/bleu_reward_func/std": 0.20412424206733704,
|
|
"rewards/chrf_reward_func/mean": 0.6483352482318878,
|
|
"rewards/chrf_reward_func/std": 0.15241166949272156,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25361969918012617,
|
|
"rewards/cometkiwi_reward_func/std": 0.08811011388897896,
|
|
"rewards/format_reward_func/mean": -0.0005208333488553763,
|
|
"rewards/format_reward_func/std": 0.005103103071451187,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.023060283996164797,
|
|
"rewards/length_ratio_reward_func/std": 0.06016158610582352,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 51.2,
|
|
"completions/max_terminated_length": 51.2,
|
|
"completions/mean_length": 25.717709159851076,
|
|
"completions/mean_terminated_length": 25.717709159851076,
|
|
"completions/min_length": 9.5,
|
|
"completions/min_terminated_length": 9.5,
|
|
"entropy": 0.49568773806095123,
|
|
"epoch": 0.023853211009174313,
|
|
"frac_reward_zero_std": 0.00416666679084301,
|
|
"grad_norm": 0.4856732487678528,
|
|
"kl": 0.005321522797385114,
|
|
"learning_rate": 6.107142857142857e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 1254522.0,
|
|
"reward": 0.344367054104805,
|
|
"reward_std": 0.0459334347397089,
|
|
"rewards/bleu_reward_func/mean": 0.38874934911727904,
|
|
"rewards/bleu_reward_func/std": 0.21183241307735443,
|
|
"rewards/chrf_reward_func/mean": 0.6565169870853425,
|
|
"rewards/chrf_reward_func/std": 0.14695971310138703,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2602541774511337,
|
|
"rewards/cometkiwi_reward_func/std": 0.08299052193760872,
|
|
"rewards/format_reward_func/mean": -0.004687500139698386,
|
|
"rewards/format_reward_func/std": 0.029504455626010895,
|
|
"rewards/language_consistency_reward_func/mean": -0.00416666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.020087526738643648,
|
|
"rewards/length_ratio_reward_func/mean": -0.018816507142037153,
|
|
"rewards/length_ratio_reward_func/std": 0.042659093253314496,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 64.6,
|
|
"completions/max_terminated_length": 64.6,
|
|
"completions/mean_length": 26.62187557220459,
|
|
"completions/mean_terminated_length": 26.62187557220459,
|
|
"completions/min_length": 10.8,
|
|
"completions/min_terminated_length": 10.8,
|
|
"entropy": 0.5190794890746474,
|
|
"epoch": 0.025688073394495414,
|
|
"frac_reward_zero_std": 0.00833333358168602,
|
|
"grad_norm": 0.5719404220581055,
|
|
"kl": 0.006120507846208057,
|
|
"learning_rate": 5.749999999999999e-07,
|
|
"loss": -0.0013,
|
|
"num_tokens": 1351919.0,
|
|
"reward": 0.33380680680274966,
|
|
"reward_std": 0.04730495139956474,
|
|
"rewards/bleu_reward_func/mean": 0.3641845941543579,
|
|
"rewards/bleu_reward_func/std": 0.18840066939592362,
|
|
"rewards/chrf_reward_func/mean": 0.6409313976764679,
|
|
"rewards/chrf_reward_func/std": 0.13473675549030303,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2602712482213974,
|
|
"rewards/cometkiwi_reward_func/std": 0.07914783954620361,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.03146066842600703,
|
|
"rewards/length_ratio_reward_func/std": 0.08479252448305488,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 62.9,
|
|
"completions/max_terminated_length": 62.9,
|
|
"completions/mean_length": 25.56250057220459,
|
|
"completions/mean_terminated_length": 25.56250057220459,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.5087966058403254,
|
|
"epoch": 0.027522935779816515,
|
|
"frac_reward_zero_std": 0.00416666679084301,
|
|
"grad_norm": 0.2896914482116699,
|
|
"kl": 0.007184029257041402,
|
|
"learning_rate": 5.392857142857142e-07,
|
|
"loss": 0.002,
|
|
"num_tokens": 1446735.0,
|
|
"reward": 0.3324300765991211,
|
|
"reward_std": 0.04533998966217041,
|
|
"rewards/bleu_reward_func/mean": 0.3695583015680313,
|
|
"rewards/bleu_reward_func/std": 0.2065991461277008,
|
|
"rewards/chrf_reward_func/mean": 0.6346977472305297,
|
|
"rewards/chrf_reward_func/std": 0.14786104559898378,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25382005572319033,
|
|
"rewards/cometkiwi_reward_func/std": 0.08584653735160827,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.007178794592618942,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.020573680661618708,
|
|
"rewards/length_ratio_reward_func/std": 0.04955196138471365,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 57.4,
|
|
"completions/max_terminated_length": 57.4,
|
|
"completions/mean_length": 26.107291984558106,
|
|
"completions/mean_terminated_length": 26.107291984558106,
|
|
"completions/min_length": 8.1,
|
|
"completions/min_terminated_length": 8.1,
|
|
"entropy": 0.4888286115601659,
|
|
"epoch": 0.029357798165137616,
|
|
"frac_reward_zero_std": 0.01250000037252903,
|
|
"grad_norm": 0.6822677850723267,
|
|
"kl": 0.00796789695014013,
|
|
"learning_rate": 5.035714285714285e-07,
|
|
"loss": 0.0012,
|
|
"num_tokens": 1542486.0,
|
|
"reward": 0.33759790360927583,
|
|
"reward_std": 0.043225977942347525,
|
|
"rewards/bleu_reward_func/mean": 0.3754344075918198,
|
|
"rewards/bleu_reward_func/std": 0.18359038829803467,
|
|
"rewards/chrf_reward_func/mean": 0.6393754661083222,
|
|
"rewards/chrf_reward_func/std": 0.14539056569337844,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2666306748986244,
|
|
"rewards/cometkiwi_reward_func/std": 0.08358251601457596,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.00416666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.020087526738643648,
|
|
"rewards/length_ratio_reward_func/mean": -0.019656529650092125,
|
|
"rewards/length_ratio_reward_func/std": 0.05039132870733738,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 57.8,
|
|
"completions/max_terminated_length": 57.8,
|
|
"completions/mean_length": 26.693750953674318,
|
|
"completions/mean_terminated_length": 26.693750953674318,
|
|
"completions/min_length": 9.2,
|
|
"completions/min_terminated_length": 9.2,
|
|
"entropy": 0.4615952713415027,
|
|
"epoch": 0.031192660550458717,
|
|
"frac_reward_zero_std": 0.02500000074505806,
|
|
"grad_norm": 0.6145430207252502,
|
|
"kl": 0.008257978129404365,
|
|
"learning_rate": 4.6785714285714283e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 1639640.0,
|
|
"reward": 0.3480651468038559,
|
|
"reward_std": 0.04329443201422691,
|
|
"rewards/bleu_reward_func/mean": 0.3926641434431076,
|
|
"rewards/bleu_reward_func/std": 0.19998904913663865,
|
|
"rewards/chrf_reward_func/mean": 0.6598639070987702,
|
|
"rewards/chrf_reward_func/std": 0.14076659679412842,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2674468278884888,
|
|
"rewards/cometkiwi_reward_func/std": 0.08690465092658997,
|
|
"rewards/format_reward_func/mean": -0.0031250000931322573,
|
|
"rewards/format_reward_func/std": 0.017222557961940766,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.02473886264488101,
|
|
"rewards/length_ratio_reward_func/std": 0.06855531064793467,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 57.8,
|
|
"completions/max_terminated_length": 57.8,
|
|
"completions/mean_length": 26.677084159851074,
|
|
"completions/mean_terminated_length": 26.677084159851074,
|
|
"completions/min_length": 8.8,
|
|
"completions/min_terminated_length": 8.8,
|
|
"entropy": 0.48688437789678574,
|
|
"epoch": 0.03302752293577982,
|
|
"frac_reward_zero_std": 0.01250000037252903,
|
|
"grad_norm": 0.5329998731613159,
|
|
"kl": 0.009095941767736804,
|
|
"learning_rate": 4.3214285714285713e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 1736898.0,
|
|
"reward": 0.33813450038433074,
|
|
"reward_std": 0.043333154171705246,
|
|
"rewards/bleu_reward_func/mean": 0.37101092040538786,
|
|
"rewards/bleu_reward_func/std": 0.19802427738904954,
|
|
"rewards/chrf_reward_func/mean": 0.6441381931304931,
|
|
"rewards/chrf_reward_func/std": 0.14031176045536994,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2662051394581795,
|
|
"rewards/cometkiwi_reward_func/std": 0.0872592743486166,
|
|
"rewards/format_reward_func/mean": -0.0015625,
|
|
"rewards/format_reward_func/std": 0.008745299279689788,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.01888986174017191,
|
|
"rewards/length_ratio_reward_func/std": 0.038116248324513435,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 61.5,
|
|
"completions/max_terminated_length": 61.5,
|
|
"completions/mean_length": 26.494792556762697,
|
|
"completions/mean_terminated_length": 26.494792556762697,
|
|
"completions/min_length": 8.3,
|
|
"completions/min_terminated_length": 8.3,
|
|
"entropy": 0.4804433848708868,
|
|
"epoch": 0.03486238532110092,
|
|
"frac_reward_zero_std": 0.00833333358168602,
|
|
"grad_norm": 0.7416422963142395,
|
|
"kl": 0.010487396054668353,
|
|
"learning_rate": 3.9642857142857137e-07,
|
|
"loss": 0.0041,
|
|
"num_tokens": 1833605.0,
|
|
"reward": 0.3384909749031067,
|
|
"reward_std": 0.042207219451665876,
|
|
"rewards/bleu_reward_func/mean": 0.3789520889520645,
|
|
"rewards/bleu_reward_func/std": 0.1956760123372078,
|
|
"rewards/chrf_reward_func/mean": 0.6447181522846221,
|
|
"rewards/chrf_reward_func/std": 0.14536231830716134,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25804260522127154,
|
|
"rewards/cometkiwi_reward_func/std": 0.09187940880656242,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.023337508924305438,
|
|
"rewards/length_ratio_reward_func/std": 0.062202737852931025,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 59.3,
|
|
"completions/max_terminated_length": 59.3,
|
|
"completions/mean_length": 26.517709159851073,
|
|
"completions/mean_terminated_length": 26.517709159851073,
|
|
"completions/min_length": 9.2,
|
|
"completions/min_terminated_length": 9.2,
|
|
"entropy": 0.45145587548613547,
|
|
"epoch": 0.03669724770642202,
|
|
"frac_reward_zero_std": 0.02083333395421505,
|
|
"grad_norm": 0.7497228384017944,
|
|
"kl": 0.00926748447236605,
|
|
"learning_rate": 3.607142857142857e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 1930338.0,
|
|
"reward": 0.34477376341819765,
|
|
"reward_std": 0.040812935680150986,
|
|
"rewards/bleu_reward_func/mean": 0.3824522137641907,
|
|
"rewards/bleu_reward_func/std": 0.21309778988361358,
|
|
"rewards/chrf_reward_func/mean": 0.6564513206481933,
|
|
"rewards/chrf_reward_func/std": 0.14746373593807222,
|
|
"rewards/cometkiwi_reward_func/mean": 0.26673379391431806,
|
|
"rewards/cometkiwi_reward_func/std": 0.0806577555835247,
|
|
"rewards/format_reward_func/mean": -0.0015625,
|
|
"rewards/format_reward_func/std": 0.008745299279689788,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.01930427164770663,
|
|
"rewards/length_ratio_reward_func/std": 0.04468219336122274,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 62.9,
|
|
"completions/max_terminated_length": 62.9,
|
|
"completions/mean_length": 26.56666717529297,
|
|
"completions/mean_terminated_length": 26.56666717529297,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 0.4824184008873999,
|
|
"epoch": 0.03853211009174312,
|
|
"frac_reward_zero_std": 0.01250000037252903,
|
|
"grad_norm": 0.33352646231651306,
|
|
"kl": 0.009850474193081027,
|
|
"learning_rate": 3.25e-07,
|
|
"loss": 0.0042,
|
|
"num_tokens": 2027350.0,
|
|
"reward": 0.33611359596252444,
|
|
"reward_std": 0.044886444509029386,
|
|
"rewards/bleu_reward_func/mean": 0.3742979884147644,
|
|
"rewards/bleu_reward_func/std": 0.20413557440042496,
|
|
"rewards/chrf_reward_func/mean": 0.6429576098918914,
|
|
"rewards/chrf_reward_func/std": 0.1520675465464592,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25396435111761095,
|
|
"rewards/cometkiwi_reward_func/std": 0.0930557519197464,
|
|
"rewards/format_reward_func/mean": -0.0015625,
|
|
"rewards/format_reward_func/std": 0.008745299279689788,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.019696515519171953,
|
|
"rewards/length_ratio_reward_func/std": 0.050847092363983395,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 50.7,
|
|
"completions/max_terminated_length": 50.7,
|
|
"completions/mean_length": 26.220833778381348,
|
|
"completions/mean_terminated_length": 26.220833778381348,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.46546676820144056,
|
|
"epoch": 0.04036697247706422,
|
|
"frac_reward_zero_std": 0.01250000037252903,
|
|
"grad_norm": 0.7209844589233398,
|
|
"kl": 0.010255527343542781,
|
|
"learning_rate": 2.892857142857143e-07,
|
|
"loss": -0.0009,
|
|
"num_tokens": 2123270.0,
|
|
"reward": 0.3395238608121872,
|
|
"reward_std": 0.0435000155121088,
|
|
"rewards/bleu_reward_func/mean": 0.38385539054870604,
|
|
"rewards/bleu_reward_func/std": 0.21344026029109955,
|
|
"rewards/chrf_reward_func/mean": 0.6466961205005646,
|
|
"rewards/chrf_reward_func/std": 0.1567239746451378,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25794110894203187,
|
|
"rewards/cometkiwi_reward_func/std": 0.09034003391861915,
|
|
"rewards/format_reward_func/mean": -0.0005208333488553763,
|
|
"rewards/format_reward_func/std": 0.005103103071451187,
|
|
"rewards/language_consistency_reward_func/mean": -0.00833333358168602,
|
|
"rewards/language_consistency_reward_func/std": 0.040175053477287295,
|
|
"rewards/length_ratio_reward_func/mean": -0.023032626137137413,
|
|
"rewards/length_ratio_reward_func/std": 0.053523562103509906,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 66.3,
|
|
"completions/max_terminated_length": 66.3,
|
|
"completions/mean_length": 26.44166717529297,
|
|
"completions/mean_terminated_length": 26.44166717529297,
|
|
"completions/min_length": 8.1,
|
|
"completions/min_terminated_length": 8.1,
|
|
"entropy": 0.4681344309821725,
|
|
"epoch": 0.04220183486238532,
|
|
"frac_reward_zero_std": 0.01666666716337204,
|
|
"grad_norm": 0.6896452307701111,
|
|
"kl": 0.009968540079717059,
|
|
"learning_rate": 2.5357142857142855e-07,
|
|
"loss": 0.0051,
|
|
"num_tokens": 2220694.0,
|
|
"reward": 0.3430625945329666,
|
|
"reward_std": 0.04380330629646778,
|
|
"rewards/bleu_reward_func/mean": 0.38238590955734253,
|
|
"rewards/bleu_reward_func/std": 0.19272863268852233,
|
|
"rewards/chrf_reward_func/mean": 0.6592188775539398,
|
|
"rewards/chrf_reward_func/std": 0.14674031287431716,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2568253993988037,
|
|
"rewards/cometkiwi_reward_func/std": 0.09049702882766723,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": -0.00416666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.020087526738643648,
|
|
"rewards/length_ratio_reward_func/mean": -0.022876000497490168,
|
|
"rewards/length_ratio_reward_func/std": 0.05649545229971409,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 60.8,
|
|
"completions/max_terminated_length": 60.8,
|
|
"completions/mean_length": 27.033334159851073,
|
|
"completions/mean_terminated_length": 27.033334159851073,
|
|
"completions/min_length": 9.7,
|
|
"completions/min_terminated_length": 9.7,
|
|
"entropy": 0.4581865567713976,
|
|
"epoch": 0.044036697247706424,
|
|
"frac_reward_zero_std": 0.01250000037252903,
|
|
"grad_norm": 0.39556214213371277,
|
|
"kl": 0.010138250569434603,
|
|
"learning_rate": 2.1785714285714284e-07,
|
|
"loss": 0.0012,
|
|
"num_tokens": 2318422.0,
|
|
"reward": 0.34348171949386597,
|
|
"reward_std": 0.04288452863693237,
|
|
"rewards/bleu_reward_func/mean": 0.3838361710309982,
|
|
"rewards/bleu_reward_func/std": 0.19907908290624618,
|
|
"rewards/chrf_reward_func/mean": 0.6513116240501404,
|
|
"rewards/chrf_reward_func/std": 0.15005288049578666,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2693155124783516,
|
|
"rewards/cometkiwi_reward_func/std": 0.0907361775636673,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.007178794592618942,
|
|
"rewards/language_consistency_reward_func/mean": -0.00833333358168602,
|
|
"rewards/language_consistency_reward_func/std": 0.040175053477287295,
|
|
"rewards/length_ratio_reward_func/mean": -0.015928789228200912,
|
|
"rewards/length_ratio_reward_func/std": 0.03196616154164076,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 59.1,
|
|
"completions/max_terminated_length": 59.1,
|
|
"completions/mean_length": 27.44270896911621,
|
|
"completions/mean_terminated_length": 27.44270896911621,
|
|
"completions/min_length": 9.4,
|
|
"completions/min_terminated_length": 9.4,
|
|
"entropy": 0.4524392428807914,
|
|
"epoch": 0.045871559633027525,
|
|
"frac_reward_zero_std": 0.01250000037252903,
|
|
"grad_norm": 0.3612808883190155,
|
|
"kl": 0.01004770951331011,
|
|
"learning_rate": 1.8214285714285714e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 2417571.0,
|
|
"reward": 0.3399767935276031,
|
|
"reward_std": 0.04221693202853203,
|
|
"rewards/bleu_reward_func/mean": 0.374004602432251,
|
|
"rewards/bleu_reward_func/std": 0.19991846084594728,
|
|
"rewards/chrf_reward_func/mean": 0.6473468661308288,
|
|
"rewards/chrf_reward_func/std": 0.14243985041975976,
|
|
"rewards/cometkiwi_reward_func/mean": 0.268462772667408,
|
|
"rewards/cometkiwi_reward_func/std": 0.08694111928343773,
|
|
"rewards/format_reward_func/mean": -0.003645833441987634,
|
|
"rewards/format_reward_func/std": 0.022325661033391953,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.02112814225256443,
|
|
"rewards/length_ratio_reward_func/std": 0.046202591806650165,
|
|
"step": 250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 64.4,
|
|
"completions/max_terminated_length": 64.4,
|
|
"completions/mean_length": 26.903125762939453,
|
|
"completions/mean_terminated_length": 26.903125762939453,
|
|
"completions/min_length": 8.8,
|
|
"completions/min_terminated_length": 8.8,
|
|
"entropy": 0.4570953262969851,
|
|
"epoch": 0.047706422018348627,
|
|
"frac_reward_zero_std": 0.00416666679084301,
|
|
"grad_norm": 1.076893925666809,
|
|
"kl": 0.009919063004053897,
|
|
"learning_rate": 1.4642857142857143e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 2515886.0,
|
|
"reward": 0.3473619669675827,
|
|
"reward_std": 0.04367151372134685,
|
|
"rewards/bleu_reward_func/mean": 0.3893332093954086,
|
|
"rewards/bleu_reward_func/std": 0.19217770248651506,
|
|
"rewards/chrf_reward_func/mean": 0.6591590642929077,
|
|
"rewards/chrf_reward_func/std": 0.1382906071841717,
|
|
"rewards/cometkiwi_reward_func/mean": 0.26887305080890656,
|
|
"rewards/cometkiwi_reward_func/std": 0.08954875394701958,
|
|
"rewards/format_reward_func/mean": -0.004166666697710752,
|
|
"rewards/format_reward_func/std": 0.01934540122747421,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.02154010096564889,
|
|
"rewards/length_ratio_reward_func/std": 0.048370769061148164,
|
|
"step": 260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 60.9,
|
|
"completions/max_terminated_length": 60.9,
|
|
"completions/mean_length": 26.501042366027832,
|
|
"completions/mean_terminated_length": 26.501042366027832,
|
|
"completions/min_length": 8.5,
|
|
"completions/min_terminated_length": 8.5,
|
|
"entropy": 0.4371530879288912,
|
|
"epoch": 0.04954128440366973,
|
|
"frac_reward_zero_std": 0.02083333395421505,
|
|
"grad_norm": 0.733481764793396,
|
|
"kl": 0.00983754772314569,
|
|
"learning_rate": 1.107142857142857e-07,
|
|
"loss": 0.0043,
|
|
"num_tokens": 2612283.0,
|
|
"reward": 0.35095727145671846,
|
|
"reward_std": 0.043163779005408284,
|
|
"rewards/bleu_reward_func/mean": 0.4070417106151581,
|
|
"rewards/bleu_reward_func/std": 0.2058791473507881,
|
|
"rewards/chrf_reward_func/mean": 0.6601174414157868,
|
|
"rewards/chrf_reward_func/std": 0.1547859087586403,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2658486872911453,
|
|
"rewards/cometkiwi_reward_func/std": 0.08607288226485252,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.007178794592618942,
|
|
"rewards/language_consistency_reward_func/mean": -0.00833333358168602,
|
|
"rewards/language_consistency_reward_func/std": 0.040175053477287295,
|
|
"rewards/length_ratio_reward_func/mean": -0.021412898041307925,
|
|
"rewards/length_ratio_reward_func/std": 0.049203038308769466,
|
|
"step": 270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 57.6,
|
|
"completions/max_terminated_length": 57.6,
|
|
"completions/mean_length": 26.35000057220459,
|
|
"completions/mean_terminated_length": 26.35000057220459,
|
|
"completions/min_length": 9.4,
|
|
"completions/min_terminated_length": 9.4,
|
|
"entropy": 0.47783863004297017,
|
|
"epoch": 0.05137614678899083,
|
|
"frac_reward_zero_std": 0.02083333395421505,
|
|
"grad_norm": 0.5478304624557495,
|
|
"kl": 0.009364690777147189,
|
|
"learning_rate": 7.5e-08,
|
|
"loss": -0.0066,
|
|
"num_tokens": 2708231.0,
|
|
"reward": 0.345621320605278,
|
|
"reward_std": 0.044814284145832065,
|
|
"rewards/bleu_reward_func/mean": 0.39248552918434143,
|
|
"rewards/bleu_reward_func/std": 0.20868377834558488,
|
|
"rewards/chrf_reward_func/mean": 0.6586843192577362,
|
|
"rewards/chrf_reward_func/std": 0.15629934668540954,
|
|
"rewards/cometkiwi_reward_func/mean": 0.2567432075738907,
|
|
"rewards/cometkiwi_reward_func/std": 0.08736011013388634,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.00416666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.020087526738643648,
|
|
"rewards/length_ratio_reward_func/mean": -0.02074714293703437,
|
|
"rewards/length_ratio_reward_func/std": 0.04600813277065754,
|
|
"step": 280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 57.7,
|
|
"completions/max_terminated_length": 57.7,
|
|
"completions/mean_length": 25.23229236602783,
|
|
"completions/mean_terminated_length": 25.23229236602783,
|
|
"completions/min_length": 9.3,
|
|
"completions/min_terminated_length": 9.3,
|
|
"entropy": 0.46073163617402313,
|
|
"epoch": 0.05321100917431193,
|
|
"frac_reward_zero_std": 0.02083333395421505,
|
|
"grad_norm": 1.029287576675415,
|
|
"kl": 0.010707967441703659,
|
|
"learning_rate": 3.9285714285714285e-08,
|
|
"loss": 0.006,
|
|
"num_tokens": 2801854.0,
|
|
"reward": 0.34537293910980227,
|
|
"reward_std": 0.044442853704094884,
|
|
"rewards/bleu_reward_func/mean": 0.39236741364002226,
|
|
"rewards/bleu_reward_func/std": 0.21471455693244934,
|
|
"rewards/chrf_reward_func/mean": 0.6595065057277679,
|
|
"rewards/chrf_reward_func/std": 0.14770160466432572,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25192623138427733,
|
|
"rewards/cometkiwi_reward_func/std": 0.08892452940344811,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": 0.0,
|
|
"rewards/language_consistency_reward_func/std": 0.0,
|
|
"rewards/length_ratio_reward_func/mean": -0.0191226857714355,
|
|
"rewards/length_ratio_reward_func/std": 0.043208105210214856,
|
|
"step": 290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 58.3,
|
|
"completions/max_terminated_length": 58.3,
|
|
"completions/mean_length": 25.80937557220459,
|
|
"completions/mean_terminated_length": 25.80937557220459,
|
|
"completions/min_length": 9.8,
|
|
"completions/min_terminated_length": 9.8,
|
|
"entropy": 0.4690433906391263,
|
|
"epoch": 0.05504587155963303,
|
|
"frac_reward_zero_std": 0.01250000037252903,
|
|
"grad_norm": 0.8401678204536438,
|
|
"kl": 0.009869737789267674,
|
|
"learning_rate": 3.571428571428571e-09,
|
|
"loss": 0.0073,
|
|
"num_tokens": 2896907.0,
|
|
"reward": 0.3431296080350876,
|
|
"reward_std": 0.044174302741885184,
|
|
"rewards/bleu_reward_func/mean": 0.38845895826816557,
|
|
"rewards/bleu_reward_func/std": 0.19879510253667831,
|
|
"rewards/chrf_reward_func/mean": 0.6528973937034607,
|
|
"rewards/chrf_reward_func/std": 0.14420587494969367,
|
|
"rewards/cometkiwi_reward_func/mean": 0.25864928513765334,
|
|
"rewards/cometkiwi_reward_func/std": 0.08451188653707505,
|
|
"rewards/format_reward_func/mean": -0.002083333395421505,
|
|
"rewards/format_reward_func/std": 0.014357589185237885,
|
|
"rewards/language_consistency_reward_func/mean": -0.00416666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.020087526738643648,
|
|
"rewards/length_ratio_reward_func/mean": -0.01918471036478877,
|
|
"rewards/length_ratio_reward_func/std": 0.05196060817688704,
|
|
"step": 300
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 300,
|
|
"num_input_tokens_seen": 2896907,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|