1515 lines
66 KiB
JSON
1515 lines
66 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.07339449541284404,
|
|
"eval_steps": 100,
|
|
"global_step": 400,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 100.4,
|
|
"completions/max_terminated_length": 100.4,
|
|
"completions/mean_length": 43.72604255676269,
|
|
"completions/mean_terminated_length": 43.72604255676269,
|
|
"completions/min_length": 16.0,
|
|
"completions/min_terminated_length": 16.0,
|
|
"entropy": 0.641127797588706,
|
|
"epoch": 0.001834862385321101,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.37443262338638306,
|
|
"kl": 7.754791495671043e-05,
|
|
"learning_rate": 9e-08,
|
|
"loss": 0.0226,
|
|
"num_tokens": 97877.0,
|
|
"reward": 0.25495402663946154,
|
|
"reward_std": 0.05580228827893734,
|
|
"rewards/bleu_reward_func/mean": 0.23490491956472398,
|
|
"rewards/bleu_reward_func/std": 0.17681142836809158,
|
|
"rewards/chrf_reward_func/mean": 0.5386854529380798,
|
|
"rewards/chrf_reward_func/std": 0.16048878133296968,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19412437379360198,
|
|
"rewards/cometkiwi_reward_func/std": 0.10323479250073433,
|
|
"rewards/format_reward_func/mean": -0.002083333395421505,
|
|
"rewards/format_reward_func/std": 0.014357589185237885,
|
|
"rewards/language_consistency_reward_func/mean": -0.014583333767950535,
|
|
"rewards/language_consistency_reward_func/std": 0.08296603709459305,
|
|
"rewards/length_ratio_reward_func/mean": -0.050721361115574834,
|
|
"rewards/length_ratio_reward_func/std": 0.13750547505915164,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 107.1,
|
|
"completions/max_terminated_length": 107.1,
|
|
"completions/mean_length": 45.83541755676269,
|
|
"completions/mean_terminated_length": 45.83541755676269,
|
|
"completions/min_length": 14.8,
|
|
"completions/min_terminated_length": 14.8,
|
|
"entropy": 0.6840235263109207,
|
|
"epoch": 0.003669724770642202,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5438474416732788,
|
|
"kl": 0.00013811875888798398,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"loss": 0.0222,
|
|
"num_tokens": 199527.0,
|
|
"reward": 0.2511941701173782,
|
|
"reward_std": 0.05474635511636734,
|
|
"rewards/bleu_reward_func/mean": 0.23108359873294831,
|
|
"rewards/bleu_reward_func/std": 0.16682641878724097,
|
|
"rewards/chrf_reward_func/mean": 0.5350141823291779,
|
|
"rewards/chrf_reward_func/std": 0.15942308008670808,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18695294559001924,
|
|
"rewards/cometkiwi_reward_func/std": 0.10410317853093147,
|
|
"rewards/format_reward_func/mean": -0.0005208333488553763,
|
|
"rewards/format_reward_func/std": 0.005103103071451187,
|
|
"rewards/language_consistency_reward_func/mean": -0.01770833320915699,
|
|
"rewards/language_consistency_reward_func/std": 0.08296861797571183,
|
|
"rewards/length_ratio_reward_func/mean": -0.05297327507287264,
|
|
"rewards/length_ratio_reward_func/std": 0.14240012057125567,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 101.8,
|
|
"completions/max_terminated_length": 101.8,
|
|
"completions/mean_length": 45.539584732055665,
|
|
"completions/mean_terminated_length": 45.539584732055665,
|
|
"completions/min_length": 14.3,
|
|
"completions/min_terminated_length": 14.3,
|
|
"entropy": 0.6699073918163776,
|
|
"epoch": 0.005504587155963303,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9839522242546082,
|
|
"kl": 0.00012621100945864328,
|
|
"learning_rate": 1.9624999999999997e-07,
|
|
"loss": 0.028,
|
|
"num_tokens": 300209.0,
|
|
"reward": 0.2521843656897545,
|
|
"reward_std": 0.05357031859457493,
|
|
"rewards/bleu_reward_func/mean": 0.22719674855470656,
|
|
"rewards/bleu_reward_func/std": 0.15533511862158775,
|
|
"rewards/chrf_reward_func/mean": 0.5404259949922562,
|
|
"rewards/chrf_reward_func/std": 0.15872502326965332,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19334699362516403,
|
|
"rewards/cometkiwi_reward_func/std": 0.10178580582141876,
|
|
"rewards/format_reward_func/mean": -0.0005208333488553763,
|
|
"rewards/format_reward_func/std": 0.005103103071451187,
|
|
"rewards/language_consistency_reward_func/mean": -0.02499999972060323,
|
|
"rewards/language_consistency_reward_func/std": 0.13014254122972488,
|
|
"rewards/length_ratio_reward_func/mean": -0.05719910133630037,
|
|
"rewards/length_ratio_reward_func/std": 0.16827091202139854,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 107.3,
|
|
"completions/max_terminated_length": 107.3,
|
|
"completions/mean_length": 44.40104217529297,
|
|
"completions/mean_terminated_length": 44.40104217529297,
|
|
"completions/min_length": 15.1,
|
|
"completions/min_terminated_length": 15.1,
|
|
"entropy": 0.6837788574397564,
|
|
"epoch": 0.007339449541284404,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.4425327181816101,
|
|
"kl": 0.00014035037530923944,
|
|
"learning_rate": 1.9208333333333333e-07,
|
|
"loss": 0.0267,
|
|
"num_tokens": 399202.0,
|
|
"reward": 0.2536553040146828,
|
|
"reward_std": 0.05643873885273933,
|
|
"rewards/bleu_reward_func/mean": 0.23150386214256286,
|
|
"rewards/bleu_reward_func/std": 0.1702369198203087,
|
|
"rewards/chrf_reward_func/mean": 0.5466259837150573,
|
|
"rewards/chrf_reward_func/std": 0.1535796642303467,
|
|
"rewards/cometkiwi_reward_func/mean": 0.1845856264233589,
|
|
"rewards/cometkiwi_reward_func/std": 0.09680806994438171,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": -0.021875000465661286,
|
|
"rewards/language_consistency_reward_func/std": 0.09676815867424012,
|
|
"rewards/length_ratio_reward_func/mean": -0.05667870659381151,
|
|
"rewards/length_ratio_reward_func/std": 0.15275606885552406,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 108.9,
|
|
"completions/max_terminated_length": 108.9,
|
|
"completions/mean_length": 44.86875114440918,
|
|
"completions/mean_terminated_length": 44.86875114440918,
|
|
"completions/min_length": 15.1,
|
|
"completions/min_terminated_length": 15.1,
|
|
"entropy": 0.6151675179600715,
|
|
"epoch": 0.009174311926605505,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5572280287742615,
|
|
"kl": 0.00012004884272016625,
|
|
"learning_rate": 1.8791666666666666e-07,
|
|
"loss": 0.0247,
|
|
"num_tokens": 497908.0,
|
|
"reward": 0.26471140384674074,
|
|
"reward_std": 0.06101177856326103,
|
|
"rewards/bleu_reward_func/mean": 0.24996386468410492,
|
|
"rewards/bleu_reward_func/std": 0.18194751888513566,
|
|
"rewards/chrf_reward_func/mean": 0.5592117726802825,
|
|
"rewards/chrf_reward_func/std": 0.16739351451396942,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19781259596347808,
|
|
"rewards/cometkiwi_reward_func/std": 0.09948562160134315,
|
|
"rewards/format_reward_func/mean": -0.0015625000465661286,
|
|
"rewards/format_reward_func/std": 0.015309309214353561,
|
|
"rewards/language_consistency_reward_func/mean": -0.018750000093132258,
|
|
"rewards/language_consistency_reward_func/std": 0.10600315779447556,
|
|
"rewards/length_ratio_reward_func/mean": -0.06148764304816723,
|
|
"rewards/length_ratio_reward_func/std": 0.1644450891762972,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 92.6,
|
|
"completions/max_terminated_length": 92.6,
|
|
"completions/mean_length": 43.71770973205567,
|
|
"completions/mean_terminated_length": 43.71770973205567,
|
|
"completions/min_length": 11.7,
|
|
"completions/min_terminated_length": 11.7,
|
|
"entropy": 0.6476957548409701,
|
|
"epoch": 0.011009174311926606,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6614680886268616,
|
|
"kl": 0.0001245706956858328,
|
|
"learning_rate": 1.8375e-07,
|
|
"loss": 0.0195,
|
|
"num_tokens": 596037.0,
|
|
"reward": 0.26351309269666673,
|
|
"reward_std": 0.05548482425510883,
|
|
"rewards/bleu_reward_func/mean": 0.2510980129241943,
|
|
"rewards/bleu_reward_func/std": 0.17841478884220124,
|
|
"rewards/chrf_reward_func/mean": 0.5576680362224579,
|
|
"rewards/chrf_reward_func/std": 0.16576428711414337,
|
|
"rewards/cometkiwi_reward_func/mean": 0.1909983903169632,
|
|
"rewards/cometkiwi_reward_func/std": 0.10470297560095787,
|
|
"rewards/format_reward_func/mean": -0.0005208333488553763,
|
|
"rewards/format_reward_func/std": 0.005103103071451187,
|
|
"rewards/language_consistency_reward_func/mean": -0.015625000186264514,
|
|
"rewards/language_consistency_reward_func/std": 0.10159938633441926,
|
|
"rewards/length_ratio_reward_func/mean": -0.06293866671621799,
|
|
"rewards/length_ratio_reward_func/std": 0.17350348457694054,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 104.4,
|
|
"completions/max_terminated_length": 104.4,
|
|
"completions/mean_length": 42.91770935058594,
|
|
"completions/mean_terminated_length": 42.91770935058594,
|
|
"completions/min_length": 13.4,
|
|
"completions/min_terminated_length": 13.4,
|
|
"entropy": 0.6284528793767095,
|
|
"epoch": 0.012844036697247707,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.865659236907959,
|
|
"kl": 0.00011616063855015568,
|
|
"learning_rate": 1.7958333333333334e-07,
|
|
"loss": 0.0224,
|
|
"num_tokens": 692798.0,
|
|
"reward": 0.25775592029094696,
|
|
"reward_std": 0.051965680345892905,
|
|
"rewards/bleu_reward_func/mean": 0.2382669121026993,
|
|
"rewards/bleu_reward_func/std": 0.1676160916686058,
|
|
"rewards/chrf_reward_func/mean": 0.5442251682281494,
|
|
"rewards/chrf_reward_func/std": 0.16244617253541946,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19518450796604156,
|
|
"rewards/cometkiwi_reward_func/std": 0.09833320006728172,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": -0.012500000093132258,
|
|
"rewards/language_consistency_reward_func/std": 0.07304229289293289,
|
|
"rewards/length_ratio_reward_func/mean": -0.055222210474312305,
|
|
"rewards/length_ratio_reward_func/std": 0.15710434317588806,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 98.1,
|
|
"completions/max_terminated_length": 98.1,
|
|
"completions/mean_length": 44.054167938232425,
|
|
"completions/mean_terminated_length": 44.054167938232425,
|
|
"completions/min_length": 13.4,
|
|
"completions/min_terminated_length": 13.4,
|
|
"entropy": 0.6546121316030622,
|
|
"epoch": 0.014678899082568808,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6064512729644775,
|
|
"kl": 0.00013701573012880885,
|
|
"learning_rate": 1.7541666666666665e-07,
|
|
"loss": 0.0191,
|
|
"num_tokens": 791090.0,
|
|
"reward": 0.2553087532520294,
|
|
"reward_std": 0.05587970241904259,
|
|
"rewards/bleu_reward_func/mean": 0.23882630318403245,
|
|
"rewards/bleu_reward_func/std": 0.16801564246416092,
|
|
"rewards/chrf_reward_func/mean": 0.5401974469423294,
|
|
"rewards/chrf_reward_func/std": 0.1637423127889633,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19503184258937836,
|
|
"rewards/cometkiwi_reward_func/std": 0.10646996274590492,
|
|
"rewards/format_reward_func/mean": -0.0015625000465661286,
|
|
"rewards/format_reward_func/std": 0.015309309214353561,
|
|
"rewards/language_consistency_reward_func/mean": -0.01979166707023978,
|
|
"rewards/language_consistency_reward_func/std": 0.12013245820999145,
|
|
"rewards/length_ratio_reward_func/mean": -0.06656032614409924,
|
|
"rewards/length_ratio_reward_func/std": 0.18060478195548058,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 99.2,
|
|
"completions/max_terminated_length": 99.2,
|
|
"completions/mean_length": 44.43854331970215,
|
|
"completions/mean_terminated_length": 44.43854331970215,
|
|
"completions/min_length": 11.7,
|
|
"completions/min_terminated_length": 11.7,
|
|
"entropy": 0.6622514044865966,
|
|
"epoch": 0.01651376146788991,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8367324471473694,
|
|
"kl": 0.00013917137834837945,
|
|
"learning_rate": 1.7124999999999998e-07,
|
|
"loss": 0.0316,
|
|
"num_tokens": 890063.0,
|
|
"reward": 0.2606888830661774,
|
|
"reward_std": 0.05680148750543594,
|
|
"rewards/bleu_reward_func/mean": 0.25098930299282074,
|
|
"rewards/bleu_reward_func/std": 0.17165935039520264,
|
|
"rewards/chrf_reward_func/mean": 0.5513496577739716,
|
|
"rewards/chrf_reward_func/std": 0.16711620539426802,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19178129583597184,
|
|
"rewards/cometkiwi_reward_func/std": 0.10177103579044341,
|
|
"rewards/format_reward_func/mean": -0.0015625000465661286,
|
|
"rewards/format_reward_func/std": 0.012281897664070129,
|
|
"rewards/language_consistency_reward_func/mean": -0.022916666883975267,
|
|
"rewards/language_consistency_reward_func/std": 0.1281110167503357,
|
|
"rewards/length_ratio_reward_func/mean": -0.06743388511240482,
|
|
"rewards/length_ratio_reward_func/std": 0.18414335772395135,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 95.5,
|
|
"completions/max_terminated_length": 95.5,
|
|
"completions/mean_length": 41.194792938232425,
|
|
"completions/mean_terminated_length": 41.194792938232425,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.6621533090248704,
|
|
"epoch": 0.01834862385321101,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6025453805923462,
|
|
"kl": 0.0001354588838793802,
|
|
"learning_rate": 1.6708333333333333e-07,
|
|
"loss": 0.0283,
|
|
"num_tokens": 984070.0,
|
|
"reward": 0.25523336231708527,
|
|
"reward_std": 0.06146847456693649,
|
|
"rewards/bleu_reward_func/mean": 0.24761200845241546,
|
|
"rewards/bleu_reward_func/std": 0.18527472615242005,
|
|
"rewards/chrf_reward_func/mean": 0.5412918895483017,
|
|
"rewards/chrf_reward_func/std": 0.17488922774791718,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18139185458421708,
|
|
"rewards/cometkiwi_reward_func/std": 0.10495599806308746,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.016666667070239782,
|
|
"rewards/language_consistency_reward_func/std": 0.09658703655004501,
|
|
"rewards/length_ratio_reward_func/mean": -0.073378368280828,
|
|
"rewards/length_ratio_reward_func/std": 0.19662413075566293,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 106.7,
|
|
"completions/max_terminated_length": 106.7,
|
|
"completions/mean_length": 43.871876907348636,
|
|
"completions/mean_terminated_length": 43.871876907348636,
|
|
"completions/min_length": 13.6,
|
|
"completions/min_terminated_length": 13.6,
|
|
"entropy": 0.6730291748419404,
|
|
"epoch": 0.02018348623853211,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.49704262614250183,
|
|
"kl": 0.00015640652959802992,
|
|
"learning_rate": 1.6291666666666666e-07,
|
|
"loss": 0.0209,
|
|
"num_tokens": 1081963.0,
|
|
"reward": 0.2582371696829796,
|
|
"reward_std": 0.05744688808917999,
|
|
"rewards/bleu_reward_func/mean": 0.24368569850921631,
|
|
"rewards/bleu_reward_func/std": 0.1703436091542244,
|
|
"rewards/chrf_reward_func/mean": 0.5500691384077072,
|
|
"rewards/chrf_reward_func/std": 0.15263175517320632,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18191470056772233,
|
|
"rewards/cometkiwi_reward_func/std": 0.09751566648483276,
|
|
"rewards/format_reward_func/mean": -0.0005208333488553763,
|
|
"rewards/format_reward_func/std": 0.005103103071451187,
|
|
"rewards/language_consistency_reward_func/mean": -0.009375000279396773,
|
|
"rewards/language_consistency_reward_func/std": 0.05900891125202179,
|
|
"rewards/length_ratio_reward_func/mean": -0.06196755077689886,
|
|
"rewards/length_ratio_reward_func/std": 0.1651702594012022,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0010416666977107526,
|
|
"completions/max_length": 127.8,
|
|
"completions/max_terminated_length": 110.5,
|
|
"completions/mean_length": 44.51666831970215,
|
|
"completions/mean_terminated_length": 44.29504547119141,
|
|
"completions/min_length": 12.2,
|
|
"completions/min_terminated_length": 12.2,
|
|
"entropy": 0.6618775950744749,
|
|
"epoch": 0.022018348623853212,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.20791220664978027,
|
|
"kl": 0.000156837863167425,
|
|
"learning_rate": 1.5875e-07,
|
|
"loss": 0.0372,
|
|
"num_tokens": 1181431.0,
|
|
"reward": 0.2539855733513832,
|
|
"reward_std": 0.054305503517389296,
|
|
"rewards/bleu_reward_func/mean": 0.24562408328056334,
|
|
"rewards/bleu_reward_func/std": 0.186594095826149,
|
|
"rewards/chrf_reward_func/mean": 0.5365742057561874,
|
|
"rewards/chrf_reward_func/std": 0.17232355624437332,
|
|
"rewards/cometkiwi_reward_func/mean": 0.1859613239765167,
|
|
"rewards/cometkiwi_reward_func/std": 0.0985416479408741,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.02291666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.10565307289361954,
|
|
"rewards/length_ratio_reward_func/mean": -0.0658663498237729,
|
|
"rewards/length_ratio_reward_func/std": 0.16860042922198773,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 102.3,
|
|
"completions/max_terminated_length": 102.3,
|
|
"completions/mean_length": 43.2447925567627,
|
|
"completions/mean_terminated_length": 43.2447925567627,
|
|
"completions/min_length": 14.4,
|
|
"completions/min_terminated_length": 14.4,
|
|
"entropy": 0.6541618125513196,
|
|
"epoch": 0.023853211009174313,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8312816619873047,
|
|
"kl": 0.0001629820018877126,
|
|
"learning_rate": 1.5458333333333334e-07,
|
|
"loss": 0.0296,
|
|
"num_tokens": 1278790.0,
|
|
"reward": 0.2565669357776642,
|
|
"reward_std": 0.0581198662519455,
|
|
"rewards/bleu_reward_func/mean": 0.23815875202417375,
|
|
"rewards/bleu_reward_func/std": 0.1827351361513138,
|
|
"rewards/chrf_reward_func/mean": 0.5469865381717682,
|
|
"rewards/chrf_reward_func/std": 0.16322616785764693,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18975041061639786,
|
|
"rewards/cometkiwi_reward_func/std": 0.10062215849757195,
|
|
"rewards/format_reward_func/mean": -0.002083333395421505,
|
|
"rewards/format_reward_func/std": 0.010043763369321824,
|
|
"rewards/language_consistency_reward_func/mean": -0.01875000037252903,
|
|
"rewards/language_consistency_reward_func/std": 0.09753482192754745,
|
|
"rewards/length_ratio_reward_func/mean": -0.05870959721505642,
|
|
"rewards/length_ratio_reward_func/std": 0.1714523360133171,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0010416666977107526,
|
|
"completions/max_length": 121.7,
|
|
"completions/max_terminated_length": 106.4,
|
|
"completions/mean_length": 44.59375190734863,
|
|
"completions/mean_terminated_length": 44.37647132873535,
|
|
"completions/min_length": 15.5,
|
|
"completions/min_terminated_length": 15.5,
|
|
"entropy": 0.670651138946414,
|
|
"epoch": 0.025688073394495414,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.46090251207351685,
|
|
"kl": 0.00015821605285948337,
|
|
"learning_rate": 1.5041666666666665e-07,
|
|
"loss": 0.028,
|
|
"num_tokens": 1377888.0,
|
|
"reward": 0.2539003476500511,
|
|
"reward_std": 0.05214857570827007,
|
|
"rewards/bleu_reward_func/mean": 0.23716889470815658,
|
|
"rewards/bleu_reward_func/std": 0.17232463955879213,
|
|
"rewards/chrf_reward_func/mean": 0.5400776028633117,
|
|
"rewards/chrf_reward_func/std": 0.15634452998638154,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18494710475206375,
|
|
"rewards/cometkiwi_reward_func/std": 0.09469879120588302,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": -0.014583333116024732,
|
|
"rewards/language_consistency_reward_func/std": 0.06067586839199066,
|
|
"rewards/length_ratio_reward_func/mean": -0.056841888837516306,
|
|
"rewards/length_ratio_reward_func/std": 0.16279448978602887,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 102.5,
|
|
"completions/max_terminated_length": 102.5,
|
|
"completions/mean_length": 42.3718765258789,
|
|
"completions/mean_terminated_length": 42.3718765258789,
|
|
"completions/min_length": 14.8,
|
|
"completions/min_terminated_length": 14.8,
|
|
"entropy": 0.6472949428483844,
|
|
"epoch": 0.027522935779816515,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0228127241134644,
|
|
"kl": 0.0001538705220518466,
|
|
"learning_rate": 1.4624999999999998e-07,
|
|
"loss": 0.0233,
|
|
"num_tokens": 1473973.0,
|
|
"reward": 0.2589618906378746,
|
|
"reward_std": 0.05526665598154068,
|
|
"rewards/bleu_reward_func/mean": 0.25388616770505906,
|
|
"rewards/bleu_reward_func/std": 0.17758201360702514,
|
|
"rewards/chrf_reward_func/mean": 0.5437004506587982,
|
|
"rewards/chrf_reward_func/std": 0.15495237112045288,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18219959884881973,
|
|
"rewards/cometkiwi_reward_func/std": 0.10216084495186806,
|
|
"rewards/format_reward_func/mean": -0.0015625,
|
|
"rewards/format_reward_func/std": 0.008745299279689788,
|
|
"rewards/language_consistency_reward_func/mean": -0.00833333358168602,
|
|
"rewards/language_consistency_reward_func/std": 0.0634851798415184,
|
|
"rewards/length_ratio_reward_func/mean": -0.061402783170342444,
|
|
"rewards/length_ratio_reward_func/std": 0.16093010939657687,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 102.9,
|
|
"completions/max_terminated_length": 102.9,
|
|
"completions/mean_length": 43.596875762939455,
|
|
"completions/mean_terminated_length": 43.596875762939455,
|
|
"completions/min_length": 13.2,
|
|
"completions/min_terminated_length": 13.2,
|
|
"entropy": 0.6684370420873165,
|
|
"epoch": 0.029357798165137616,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9800335764884949,
|
|
"kl": 0.00015861360882212238,
|
|
"learning_rate": 1.4208333333333333e-07,
|
|
"loss": 0.0175,
|
|
"num_tokens": 1571886.0,
|
|
"reward": 0.25178586393594743,
|
|
"reward_std": 0.05705268681049347,
|
|
"rewards/bleu_reward_func/mean": 0.23627814501523972,
|
|
"rewards/bleu_reward_func/std": 0.18475569933652877,
|
|
"rewards/chrf_reward_func/mean": 0.5330782622098923,
|
|
"rewards/chrf_reward_func/std": 0.17333336025476456,
|
|
"rewards/cometkiwi_reward_func/mean": 0.1838628187775612,
|
|
"rewards/cometkiwi_reward_func/std": 0.09760846272110939,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": -0.009375000093132257,
|
|
"rewards/language_consistency_reward_func/std": 0.05954499244689941,
|
|
"rewards/length_ratio_reward_func/mean": -0.05876133367419243,
|
|
"rewards/length_ratio_reward_func/std": 0.17772589921951293,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 95.4,
|
|
"completions/max_terminated_length": 95.4,
|
|
"completions/mean_length": 43.84791793823242,
|
|
"completions/mean_terminated_length": 43.84791793823242,
|
|
"completions/min_length": 14.3,
|
|
"completions/min_terminated_length": 14.3,
|
|
"entropy": 0.6483647493645549,
|
|
"epoch": 0.031192660550458717,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6796091198921204,
|
|
"kl": 0.00015265502576085056,
|
|
"learning_rate": 1.3791666666666666e-07,
|
|
"loss": 0.0223,
|
|
"num_tokens": 1669888.0,
|
|
"reward": 0.2596652373671532,
|
|
"reward_std": 0.05339277386665344,
|
|
"rewards/bleu_reward_func/mean": 0.2419128969311714,
|
|
"rewards/bleu_reward_func/std": 0.16896522492170335,
|
|
"rewards/chrf_reward_func/mean": 0.551539421081543,
|
|
"rewards/chrf_reward_func/std": 0.16133359223604202,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19189785718917846,
|
|
"rewards/cometkiwi_reward_func/std": 0.10103634595870972,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.018750000465661287,
|
|
"rewards/language_consistency_reward_func/std": 0.09135284274816513,
|
|
"rewards/length_ratio_reward_func/mean": -0.05558805000036955,
|
|
"rewards/length_ratio_reward_func/std": 0.15314753018319607,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 100.3,
|
|
"completions/max_terminated_length": 100.3,
|
|
"completions/mean_length": 44.29270935058594,
|
|
"completions/mean_terminated_length": 44.29270935058594,
|
|
"completions/min_length": 15.6,
|
|
"completions/min_terminated_length": 15.6,
|
|
"entropy": 0.6337203053757549,
|
|
"epoch": 0.03302752293577982,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6656765341758728,
|
|
"kl": 0.00014801357551164074,
|
|
"learning_rate": 1.3375e-07,
|
|
"loss": 0.022,
|
|
"num_tokens": 1768725.0,
|
|
"reward": 0.2622971773147583,
|
|
"reward_std": 0.055173343792557714,
|
|
"rewards/bleu_reward_func/mean": 0.2477172002196312,
|
|
"rewards/bleu_reward_func/std": 0.17861862182617189,
|
|
"rewards/chrf_reward_func/mean": 0.5499911308288574,
|
|
"rewards/chrf_reward_func/std": 0.15990156978368758,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19504035115242005,
|
|
"rewards/cometkiwi_reward_func/std": 0.10304989591240883,
|
|
"rewards/format_reward_func/mean": -0.0005208333488553763,
|
|
"rewards/format_reward_func/std": 0.005103103071451187,
|
|
"rewards/language_consistency_reward_func/mean": -0.008333333488553762,
|
|
"rewards/language_consistency_reward_func/std": 0.062466806173324584,
|
|
"rewards/length_ratio_reward_func/mean": -0.055042792297899726,
|
|
"rewards/length_ratio_reward_func/std": 0.15278466865420343,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 114.1,
|
|
"completions/max_terminated_length": 114.1,
|
|
"completions/mean_length": 44.59271011352539,
|
|
"completions/mean_terminated_length": 44.59271011352539,
|
|
"completions/min_length": 13.1,
|
|
"completions/min_terminated_length": 13.1,
|
|
"entropy": 0.673541947081685,
|
|
"epoch": 0.03486238532110092,
|
|
"frac_reward_zero_std": 0.00416666679084301,
|
|
"grad_norm": 0.4010095000267029,
|
|
"kl": 0.00018048235118577337,
|
|
"learning_rate": 1.2958333333333335e-07,
|
|
"loss": 0.0248,
|
|
"num_tokens": 1867562.0,
|
|
"reward": 0.25853306651115415,
|
|
"reward_std": 0.05505618825554848,
|
|
"rewards/bleu_reward_func/mean": 0.24781477004289626,
|
|
"rewards/bleu_reward_func/std": 0.17701459378004075,
|
|
"rewards/chrf_reward_func/mean": 0.5488846778869629,
|
|
"rewards/chrf_reward_func/std": 0.1562599830329418,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18542918264865876,
|
|
"rewards/cometkiwi_reward_func/std": 0.10153135880827904,
|
|
"rewards/format_reward_func/mean": -0.0015625000465661286,
|
|
"rewards/format_reward_func/std": 0.015309309214353561,
|
|
"rewards/language_consistency_reward_func/mean": -0.015625000186264514,
|
|
"rewards/language_consistency_reward_func/std": 0.09095140248537063,
|
|
"rewards/length_ratio_reward_func/mean": -0.06906281486153602,
|
|
"rewards/length_ratio_reward_func/std": 0.18467037454247476,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 106.1,
|
|
"completions/max_terminated_length": 106.1,
|
|
"completions/mean_length": 44.202084350585935,
|
|
"completions/mean_terminated_length": 44.202084350585935,
|
|
"completions/min_length": 14.2,
|
|
"completions/min_terminated_length": 14.2,
|
|
"entropy": 0.6593625823035836,
|
|
"epoch": 0.03669724770642202,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6619638800621033,
|
|
"kl": 0.00018427114365699994,
|
|
"learning_rate": 1.2541666666666665e-07,
|
|
"loss": 0.0292,
|
|
"num_tokens": 1966096.0,
|
|
"reward": 0.24964400380849838,
|
|
"reward_std": 0.0573192223906517,
|
|
"rewards/bleu_reward_func/mean": 0.22469656616449357,
|
|
"rewards/bleu_reward_func/std": 0.16534899920225143,
|
|
"rewards/chrf_reward_func/mean": 0.533919632434845,
|
|
"rewards/chrf_reward_func/std": 0.162567637860775,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19024772197008133,
|
|
"rewards/cometkiwi_reward_func/std": 0.10002064555883408,
|
|
"rewards/format_reward_func/mean": -0.0015625000465661286,
|
|
"rewards/format_reward_func/std": 0.012281897664070129,
|
|
"rewards/language_consistency_reward_func/mean": -0.013541666604578496,
|
|
"rewards/language_consistency_reward_func/std": 0.07526403963565827,
|
|
"rewards/length_ratio_reward_func/mean": -0.06490336656570435,
|
|
"rewards/length_ratio_reward_func/std": 0.18494871258735657,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 113.9,
|
|
"completions/max_terminated_length": 113.9,
|
|
"completions/mean_length": 43.60625114440918,
|
|
"completions/mean_terminated_length": 43.60625114440918,
|
|
"completions/min_length": 12.5,
|
|
"completions/min_terminated_length": 12.5,
|
|
"entropy": 0.6418691325932742,
|
|
"epoch": 0.03853211009174312,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.33465251326560974,
|
|
"kl": 0.00016358401161014057,
|
|
"learning_rate": 1.2124999999999998e-07,
|
|
"loss": 0.0287,
|
|
"num_tokens": 2064026.0,
|
|
"reward": 0.2586542651057243,
|
|
"reward_std": 0.05292070098221302,
|
|
"rewards/bleu_reward_func/mean": 0.24662543684244156,
|
|
"rewards/bleu_reward_func/std": 0.17654834538698197,
|
|
"rewards/chrf_reward_func/mean": 0.5452715694904328,
|
|
"rewards/chrf_reward_func/std": 0.16452710926532746,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18978739082813262,
|
|
"rewards/cometkiwi_reward_func/std": 0.1034120574593544,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.007178794592618942,
|
|
"rewards/language_consistency_reward_func/mean": -0.01875,
|
|
"rewards/language_consistency_reward_func/std": 0.08424748629331588,
|
|
"rewards/length_ratio_reward_func/mean": -0.051237869448959826,
|
|
"rewards/length_ratio_reward_func/std": 0.15183435045182705,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0010416666977107526,
|
|
"completions/max_length": 110.4,
|
|
"completions/max_terminated_length": 96.9,
|
|
"completions/mean_length": 43.67812614440918,
|
|
"completions/mean_terminated_length": 43.45090026855469,
|
|
"completions/min_length": 13.2,
|
|
"completions/min_terminated_length": 13.2,
|
|
"entropy": 0.6375635610893369,
|
|
"epoch": 0.04036697247706422,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5024920701980591,
|
|
"kl": 0.00018076241160827066,
|
|
"learning_rate": 1.1708333333333333e-07,
|
|
"loss": 0.0346,
|
|
"num_tokens": 2161933.0,
|
|
"reward": 0.25564285218715666,
|
|
"reward_std": 0.0570327490568161,
|
|
"rewards/bleu_reward_func/mean": 0.2406044825911522,
|
|
"rewards/bleu_reward_func/std": 0.17278910130262376,
|
|
"rewards/chrf_reward_func/mean": 0.5411502540111541,
|
|
"rewards/chrf_reward_func/std": 0.16558172255754472,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18919465094804763,
|
|
"rewards/cometkiwi_reward_func/std": 0.09697392955422401,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.007178794592618942,
|
|
"rewards/language_consistency_reward_func/mean": -0.01666666679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.09468427449464797,
|
|
"rewards/length_ratio_reward_func/mean": -0.05842902027070522,
|
|
"rewards/length_ratio_reward_func/std": 0.1696479631587863,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 98.3,
|
|
"completions/max_terminated_length": 98.3,
|
|
"completions/mean_length": 44.39583396911621,
|
|
"completions/mean_terminated_length": 44.39583396911621,
|
|
"completions/min_length": 12.8,
|
|
"completions/min_terminated_length": 12.8,
|
|
"entropy": 0.634475434012711,
|
|
"epoch": 0.04220183486238532,
|
|
"frac_reward_zero_std": 0.00833333358168602,
|
|
"grad_norm": 0.6701688170433044,
|
|
"kl": 0.00017704845122850087,
|
|
"learning_rate": 1.1291666666666666e-07,
|
|
"loss": 0.0122,
|
|
"num_tokens": 2260557.0,
|
|
"reward": 0.25482323318719863,
|
|
"reward_std": 0.052761589735746385,
|
|
"rewards/bleu_reward_func/mean": 0.2390165686607361,
|
|
"rewards/bleu_reward_func/std": 0.17697836458683014,
|
|
"rewards/chrf_reward_func/mean": 0.544374880194664,
|
|
"rewards/chrf_reward_func/std": 0.17005143612623214,
|
|
"rewards/cometkiwi_reward_func/mean": 0.1851179465651512,
|
|
"rewards/cometkiwi_reward_func/std": 0.10385541170835495,
|
|
"rewards/format_reward_func/mean": -0.0005208333488553763,
|
|
"rewards/format_reward_func/std": 0.005103103071451187,
|
|
"rewards/language_consistency_reward_func/mean": -0.023958333767950535,
|
|
"rewards/language_consistency_reward_func/std": 0.12709196507930756,
|
|
"rewards/length_ratio_reward_func/mean": -0.05638109408318996,
|
|
"rewards/length_ratio_reward_func/std": 0.16170145720243453,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 110.2,
|
|
"completions/max_terminated_length": 110.2,
|
|
"completions/mean_length": 44.36562614440918,
|
|
"completions/mean_terminated_length": 44.36562614440918,
|
|
"completions/min_length": 14.6,
|
|
"completions/min_terminated_length": 14.6,
|
|
"entropy": 0.6147146269679069,
|
|
"epoch": 0.044036697247706424,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5918720364570618,
|
|
"kl": 0.00018293724483555706,
|
|
"learning_rate": 1.0874999999999999e-07,
|
|
"loss": 0.0192,
|
|
"num_tokens": 2359840.0,
|
|
"reward": 0.26452887505292894,
|
|
"reward_std": 0.05549493730068207,
|
|
"rewards/bleu_reward_func/mean": 0.2504352018237114,
|
|
"rewards/bleu_reward_func/std": 0.17506983876228333,
|
|
"rewards/chrf_reward_func/mean": 0.5545308828353882,
|
|
"rewards/chrf_reward_func/std": 0.16170726865530013,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19457724541425706,
|
|
"rewards/cometkiwi_reward_func/std": 0.10237014964222908,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.012500000279396772,
|
|
"rewards/language_consistency_reward_func/std": 0.06787185817956924,
|
|
"rewards/length_ratio_reward_func/mean": -0.04209308866411447,
|
|
"rewards/length_ratio_reward_func/std": 0.12301968969404697,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 98.1,
|
|
"completions/max_terminated_length": 98.1,
|
|
"completions/mean_length": 45.49479331970215,
|
|
"completions/mean_terminated_length": 45.49479331970215,
|
|
"completions/min_length": 14.2,
|
|
"completions/min_terminated_length": 14.2,
|
|
"entropy": 0.653174646012485,
|
|
"epoch": 0.045871559633027525,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5670405030250549,
|
|
"kl": 0.00021479631867293847,
|
|
"learning_rate": 1.0458333333333333e-07,
|
|
"loss": 0.0257,
|
|
"num_tokens": 2460487.0,
|
|
"reward": 0.2563754260540009,
|
|
"reward_std": 0.05743147693574428,
|
|
"rewards/bleu_reward_func/mean": 0.24003432244062423,
|
|
"rewards/bleu_reward_func/std": 0.17627231925725936,
|
|
"rewards/chrf_reward_func/mean": 0.5448970705270767,
|
|
"rewards/chrf_reward_func/std": 0.1635767325758934,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18955157995223998,
|
|
"rewards/cometkiwi_reward_func/std": 0.10070921257138252,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.01562500009313226,
|
|
"rewards/language_consistency_reward_func/std": 0.08993302881717682,
|
|
"rewards/length_ratio_reward_func/mean": -0.06900205351412296,
|
|
"rewards/length_ratio_reward_func/std": 0.19703088849782943,
|
|
"step": 250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 105.9,
|
|
"completions/max_terminated_length": 105.9,
|
|
"completions/mean_length": 44.89270935058594,
|
|
"completions/mean_terminated_length": 44.89270935058594,
|
|
"completions/min_length": 13.5,
|
|
"completions/min_terminated_length": 13.5,
|
|
"entropy": 0.6311878031119704,
|
|
"epoch": 0.047706422018348627,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6202528476715088,
|
|
"kl": 0.00019017388852375915,
|
|
"learning_rate": 1.0041666666666666e-07,
|
|
"loss": 0.0243,
|
|
"num_tokens": 2559868.0,
|
|
"reward": 0.26012127548456193,
|
|
"reward_std": 0.051936975121498107,
|
|
"rewards/bleu_reward_func/mean": 0.24583716690540314,
|
|
"rewards/bleu_reward_func/std": 0.1697734773159027,
|
|
"rewards/chrf_reward_func/mean": 0.5507355511188508,
|
|
"rewards/chrf_reward_func/std": 0.15270370244979858,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19447257369756699,
|
|
"rewards/cometkiwi_reward_func/std": 0.10607269033789635,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.007178794592618942,
|
|
"rewards/language_consistency_reward_func/mean": -0.02395833320915699,
|
|
"rewards/language_consistency_reward_func/std": 0.11652934551239014,
|
|
"rewards/length_ratio_reward_func/mean": -0.05906417798250914,
|
|
"rewards/length_ratio_reward_func/std": 0.1636540561914444,
|
|
"step": 260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 103.9,
|
|
"completions/max_terminated_length": 103.9,
|
|
"completions/mean_length": 42.911459350585936,
|
|
"completions/mean_terminated_length": 42.911459350585936,
|
|
"completions/min_length": 13.1,
|
|
"completions/min_terminated_length": 13.1,
|
|
"entropy": 0.6228655286133289,
|
|
"epoch": 0.04954128440366973,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.3194192051887512,
|
|
"kl": 0.00017723985347544157,
|
|
"learning_rate": 9.624999999999999e-08,
|
|
"loss": 0.0209,
|
|
"num_tokens": 2656923.0,
|
|
"reward": 0.26310849040746687,
|
|
"reward_std": 0.053727029263973235,
|
|
"rewards/bleu_reward_func/mean": 0.2558119460940361,
|
|
"rewards/bleu_reward_func/std": 0.1691900834441185,
|
|
"rewards/chrf_reward_func/mean": 0.5507318347692489,
|
|
"rewards/chrf_reward_func/std": 0.15769821107387544,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19195634871721268,
|
|
"rewards/cometkiwi_reward_func/std": 0.09820595383644104,
|
|
"rewards/format_reward_func/mean": -0.0015625000465661286,
|
|
"rewards/format_reward_func/std": 0.015309309214353561,
|
|
"rewards/language_consistency_reward_func/mean": -0.01770833386108279,
|
|
"rewards/language_consistency_reward_func/std": 0.09284735769033432,
|
|
"rewards/length_ratio_reward_func/mean": -0.05056500323116779,
|
|
"rewards/length_ratio_reward_func/std": 0.13873719573020935,
|
|
"step": 270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 104.0,
|
|
"completions/max_terminated_length": 104.0,
|
|
"completions/mean_length": 43.88125152587891,
|
|
"completions/mean_terminated_length": 43.88125152587891,
|
|
"completions/min_length": 14.8,
|
|
"completions/min_terminated_length": 14.8,
|
|
"entropy": 0.6292154354974627,
|
|
"epoch": 0.05137614678899083,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5514017343521118,
|
|
"kl": 0.0001810227445076862,
|
|
"learning_rate": 9.208333333333332e-08,
|
|
"loss": 0.0179,
|
|
"num_tokens": 2755389.0,
|
|
"reward": 0.2628601312637329,
|
|
"reward_std": 0.054994525387883186,
|
|
"rewards/bleu_reward_func/mean": 0.2517073154449463,
|
|
"rewards/bleu_reward_func/std": 0.17896566540002823,
|
|
"rewards/chrf_reward_func/mean": 0.553756496310234,
|
|
"rewards/chrf_reward_func/std": 0.169968581199646,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19523608535528184,
|
|
"rewards/cometkiwi_reward_func/std": 0.10094026103615761,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.021874999813735484,
|
|
"rewards/language_consistency_reward_func/std": 0.09980844855308532,
|
|
"rewards/length_ratio_reward_func/mean": -0.06106763053685427,
|
|
"rewards/length_ratio_reward_func/std": 0.16477116979658604,
|
|
"step": 280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 99.0,
|
|
"completions/max_terminated_length": 99.0,
|
|
"completions/mean_length": 41.57396011352539,
|
|
"completions/mean_terminated_length": 41.57396011352539,
|
|
"completions/min_length": 13.5,
|
|
"completions/min_terminated_length": 13.5,
|
|
"entropy": 0.6181827073916792,
|
|
"epoch": 0.05321100917431193,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.41664525866508484,
|
|
"kl": 0.0001709121137551506,
|
|
"learning_rate": 8.791666666666666e-08,
|
|
"loss": 0.0129,
|
|
"num_tokens": 2850036.0,
|
|
"reward": 0.26171541661024095,
|
|
"reward_std": 0.05975321568548679,
|
|
"rewards/bleu_reward_func/mean": 0.25616615563631057,
|
|
"rewards/bleu_reward_func/std": 0.18920768201351165,
|
|
"rewards/chrf_reward_func/mean": 0.5557349532842636,
|
|
"rewards/chrf_reward_func/std": 0.17004363536834716,
|
|
"rewards/cometkiwi_reward_func/mean": 0.18365714699029922,
|
|
"rewards/cometkiwi_reward_func/std": 0.10184274166822434,
|
|
"rewards/format_reward_func/mean": -0.0005208333488553763,
|
|
"rewards/format_reward_func/std": 0.005103103071451187,
|
|
"rewards/language_consistency_reward_func/mean": -0.029166666604578496,
|
|
"rewards/language_consistency_reward_func/std": 0.14508650749921798,
|
|
"rewards/length_ratio_reward_func/mean": -0.056186207197606565,
|
|
"rewards/length_ratio_reward_func/std": 0.16040774881839753,
|
|
"step": 290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 92.7,
|
|
"completions/max_terminated_length": 92.7,
|
|
"completions/mean_length": 43.1041675567627,
|
|
"completions/mean_terminated_length": 43.1041675567627,
|
|
"completions/min_length": 15.7,
|
|
"completions/min_terminated_length": 15.7,
|
|
"entropy": 0.6469945680350065,
|
|
"epoch": 0.05504587155963303,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.3611624836921692,
|
|
"kl": 0.00021151780006789522,
|
|
"learning_rate": 8.374999999999999e-08,
|
|
"loss": 0.0224,
|
|
"num_tokens": 2947288.0,
|
|
"reward": 0.25626824647188184,
|
|
"reward_std": 0.059349266439676286,
|
|
"rewards/bleu_reward_func/mean": 0.24265132695436478,
|
|
"rewards/bleu_reward_func/std": 0.17555446773767472,
|
|
"rewards/chrf_reward_func/mean": 0.5456648528575897,
|
|
"rewards/chrf_reward_func/std": 0.1533607818186283,
|
|
"rewards/cometkiwi_reward_func/mean": 0.181993205845356,
|
|
"rewards/cometkiwi_reward_func/std": 0.09872750267386436,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.017708333767950536,
|
|
"rewards/language_consistency_reward_func/std": 0.12147571742534638,
|
|
"rewards/length_ratio_reward_func/mean": -0.05443737171590328,
|
|
"rewards/length_ratio_reward_func/std": 0.14040280021727086,
|
|
"step": 300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 97.4,
|
|
"completions/max_terminated_length": 97.4,
|
|
"completions/mean_length": 44.07708511352539,
|
|
"completions/mean_terminated_length": 44.07708511352539,
|
|
"completions/min_length": 13.4,
|
|
"completions/min_terminated_length": 13.4,
|
|
"entropy": 0.6438201436772942,
|
|
"epoch": 0.05688073394495413,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.3928717374801636,
|
|
"kl": 0.00019581909378345584,
|
|
"learning_rate": 7.958333333333332e-08,
|
|
"loss": 0.0168,
|
|
"num_tokens": 3046274.0,
|
|
"reward": 0.25316522270441055,
|
|
"reward_std": 0.055007413029670715,
|
|
"rewards/bleu_reward_func/mean": 0.2251873254776001,
|
|
"rewards/bleu_reward_func/std": 0.16340944916009903,
|
|
"rewards/chrf_reward_func/mean": 0.5393065989017487,
|
|
"rewards/chrf_reward_func/std": 0.15180564746260644,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19246248602867128,
|
|
"rewards/cometkiwi_reward_func/std": 0.10071525424718857,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.009375000186264516,
|
|
"rewards/language_consistency_reward_func/std": 0.0666181892156601,
|
|
"rewards/length_ratio_reward_func/mean": -0.04957201443612576,
|
|
"rewards/length_ratio_reward_func/std": 0.14855510778725148,
|
|
"step": 310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 104.0,
|
|
"completions/max_terminated_length": 104.0,
|
|
"completions/mean_length": 43.33541831970215,
|
|
"completions/mean_terminated_length": 43.33541831970215,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.6282176364213228,
|
|
"epoch": 0.05871559633027523,
|
|
"frac_reward_zero_std": 0.00416666679084301,
|
|
"grad_norm": 0.8842199444770813,
|
|
"kl": 0.0002011102048278701,
|
|
"learning_rate": 7.541666666666666e-08,
|
|
"loss": 0.0159,
|
|
"num_tokens": 3143732.0,
|
|
"reward": 0.2639510840177536,
|
|
"reward_std": 0.05788003951311112,
|
|
"rewards/bleu_reward_func/mean": 0.249883009493351,
|
|
"rewards/bleu_reward_func/std": 0.1720690682530403,
|
|
"rewards/chrf_reward_func/mean": 0.5582046627998352,
|
|
"rewards/chrf_reward_func/std": 0.16165395081043243,
|
|
"rewards/cometkiwi_reward_func/mean": 0.1860242322087288,
|
|
"rewards/cometkiwi_reward_func/std": 0.10341301634907722,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": -0.007291666883975268,
|
|
"rewards/language_consistency_reward_func/std": 0.059333796799182895,
|
|
"rewards/length_ratio_reward_func/mean": -0.04705172665417194,
|
|
"rewards/length_ratio_reward_func/std": 0.14283834099769593,
|
|
"step": 320
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 111.1,
|
|
"completions/max_terminated_length": 111.1,
|
|
"completions/mean_length": 46.58854293823242,
|
|
"completions/mean_terminated_length": 46.58854293823242,
|
|
"completions/min_length": 14.3,
|
|
"completions/min_terminated_length": 14.3,
|
|
"entropy": 0.6521254789084197,
|
|
"epoch": 0.060550458715596334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5029048323631287,
|
|
"kl": 0.00021772640041319846,
|
|
"learning_rate": 7.124999999999999e-08,
|
|
"loss": 0.0281,
|
|
"num_tokens": 3245793.0,
|
|
"reward": 0.2593665853142738,
|
|
"reward_std": 0.05643341764807701,
|
|
"rewards/bleu_reward_func/mean": 0.2419154480099678,
|
|
"rewards/bleu_reward_func/std": 0.17055073231458664,
|
|
"rewards/chrf_reward_func/mean": 0.5498581081628799,
|
|
"rewards/chrf_reward_func/std": 0.15819194316864013,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19563313871622084,
|
|
"rewards/cometkiwi_reward_func/std": 0.10352360531687736,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": -0.019791666604578494,
|
|
"rewards/language_consistency_reward_func/std": 0.09096960127353668,
|
|
"rewards/length_ratio_reward_func/mean": -0.062260305508971214,
|
|
"rewards/length_ratio_reward_func/std": 0.17141706347465516,
|
|
"step": 330
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 105.9,
|
|
"completions/max_terminated_length": 105.9,
|
|
"completions/mean_length": 45.28958396911621,
|
|
"completions/mean_terminated_length": 45.28958396911621,
|
|
"completions/min_length": 14.5,
|
|
"completions/min_terminated_length": 14.5,
|
|
"entropy": 0.627023839019239,
|
|
"epoch": 0.062385321100917435,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.649787962436676,
|
|
"kl": 0.0001784461377582147,
|
|
"learning_rate": 6.708333333333332e-08,
|
|
"loss": 0.0188,
|
|
"num_tokens": 3346143.0,
|
|
"reward": 0.2598617047071457,
|
|
"reward_std": 0.0506177619099617,
|
|
"rewards/bleu_reward_func/mean": 0.23222340941429137,
|
|
"rewards/bleu_reward_func/std": 0.1577960506081581,
|
|
"rewards/chrf_reward_func/mean": 0.554724931716919,
|
|
"rewards/chrf_reward_func/std": 0.14705884382128714,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19921476989984513,
|
|
"rewards/cometkiwi_reward_func/std": 0.1003293514251709,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.01145833358168602,
|
|
"rewards/language_consistency_reward_func/std": 0.08097577840089798,
|
|
"rewards/length_ratio_reward_func/mean": -0.06617535650730133,
|
|
"rewards/length_ratio_reward_func/std": 0.18938983008265495,
|
|
"step": 340
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 95.5,
|
|
"completions/max_terminated_length": 95.5,
|
|
"completions/mean_length": 44.056250762939456,
|
|
"completions/mean_terminated_length": 44.056250762939456,
|
|
"completions/min_length": 14.5,
|
|
"completions/min_terminated_length": 14.5,
|
|
"entropy": 0.6539943715557456,
|
|
"epoch": 0.06422018348623854,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8196911811828613,
|
|
"kl": 0.0002165580266083822,
|
|
"learning_rate": 6.291666666666666e-08,
|
|
"loss": 0.0176,
|
|
"num_tokens": 3444705.0,
|
|
"reward": 0.2607134133577347,
|
|
"reward_std": 0.05691562630236149,
|
|
"rewards/bleu_reward_func/mean": 0.24038996398448945,
|
|
"rewards/bleu_reward_func/std": 0.1754391074180603,
|
|
"rewards/chrf_reward_func/mean": 0.5483970165252685,
|
|
"rewards/chrf_reward_func/std": 0.1611342027783394,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19633589386940004,
|
|
"rewards/cometkiwi_reward_func/std": 0.09456013664603233,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.00729166679084301,
|
|
"rewards/language_consistency_reward_func/std": 0.05226060003042221,
|
|
"rewards/length_ratio_reward_func/mean": -0.04882421903312206,
|
|
"rewards/length_ratio_reward_func/std": 0.13714837282896042,
|
|
"step": 350
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 117.1,
|
|
"completions/max_terminated_length": 117.1,
|
|
"completions/mean_length": 47.09895973205566,
|
|
"completions/mean_terminated_length": 47.09895973205566,
|
|
"completions/min_length": 16.1,
|
|
"completions/min_terminated_length": 16.1,
|
|
"entropy": 0.6483545204624533,
|
|
"epoch": 0.06605504587155964,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.4362502098083496,
|
|
"kl": 0.00021835152112785748,
|
|
"learning_rate": 5.875e-08,
|
|
"loss": 0.0208,
|
|
"num_tokens": 3547684.0,
|
|
"reward": 0.2560607045888901,
|
|
"reward_std": 0.05092560723423958,
|
|
"rewards/bleu_reward_func/mean": 0.22509208470582961,
|
|
"rewards/bleu_reward_func/std": 0.15356209725141526,
|
|
"rewards/chrf_reward_func/mean": 0.5450605660676956,
|
|
"rewards/chrf_reward_func/std": 0.1497661955654621,
|
|
"rewards/cometkiwi_reward_func/mean": 0.19920842498540878,
|
|
"rewards/cometkiwi_reward_func/std": 0.10105147436261178,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": -0.009375000279396773,
|
|
"rewards/language_consistency_reward_func/std": 0.05900891125202179,
|
|
"rewards/length_ratio_reward_func/mean": -0.05061038145795464,
|
|
"rewards/length_ratio_reward_func/std": 0.14105941466987132,
|
|
"step": 360
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 98.7,
|
|
"completions/max_terminated_length": 98.7,
|
|
"completions/mean_length": 43.163542938232425,
|
|
"completions/mean_terminated_length": 43.163542938232425,
|
|
"completions/min_length": 15.6,
|
|
"completions/min_terminated_length": 15.6,
|
|
"entropy": 0.6518666887655854,
|
|
"epoch": 0.06788990825688074,
|
|
"frac_reward_zero_std": 0.00416666679084301,
|
|
"grad_norm": 0.5423976182937622,
|
|
"kl": 0.00021834072572133323,
|
|
"learning_rate": 5.458333333333332e-08,
|
|
"loss": 0.0203,
|
|
"num_tokens": 3644497.0,
|
|
"reward": 0.24656057059764863,
|
|
"reward_std": 0.05382158868014812,
|
|
"rewards/bleu_reward_func/mean": 0.22274319976568221,
|
|
"rewards/bleu_reward_func/std": 0.16228083819150924,
|
|
"rewards/chrf_reward_func/mean": 0.5328379124403,
|
|
"rewards/chrf_reward_func/std": 0.1528321087360382,
|
|
"rewards/cometkiwi_reward_func/mean": 0.17924209237098693,
|
|
"rewards/cometkiwi_reward_func/std": 0.09815422371029854,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.015625000279396773,
|
|
"rewards/language_consistency_reward_func/std": 0.08536245822906494,
|
|
"rewards/length_ratio_reward_func/mean": -0.06525074634701014,
|
|
"rewards/length_ratio_reward_func/std": 0.16153896190226077,
|
|
"step": 370
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 105.1,
|
|
"completions/max_terminated_length": 105.1,
|
|
"completions/mean_length": 44.89270935058594,
|
|
"completions/mean_terminated_length": 44.89270935058594,
|
|
"completions/min_length": 15.2,
|
|
"completions/min_terminated_length": 15.2,
|
|
"entropy": 0.6546351000666618,
|
|
"epoch": 0.06972477064220184,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.3517623841762543,
|
|
"kl": 0.00022097344894973504,
|
|
"learning_rate": 5.0416666666666665e-08,
|
|
"loss": 0.0162,
|
|
"num_tokens": 3745114.0,
|
|
"reward": 0.25862839818000793,
|
|
"reward_std": 0.05509440451860428,
|
|
"rewards/bleu_reward_func/mean": 0.23898565918207168,
|
|
"rewards/bleu_reward_func/std": 0.15826577320694923,
|
|
"rewards/chrf_reward_func/mean": 0.5495024740695953,
|
|
"rewards/chrf_reward_func/std": 0.15000368282198906,
|
|
"rewards/cometkiwi_reward_func/mean": 0.1875978320837021,
|
|
"rewards/cometkiwi_reward_func/std": 0.10115860104560852,
|
|
"rewards/format_reward_func/mean": -0.0010416666977107526,
|
|
"rewards/format_reward_func/std": 0.010206206142902375,
|
|
"rewards/language_consistency_reward_func/mean": -0.01145833358168602,
|
|
"rewards/language_consistency_reward_func/std": 0.07234812676906585,
|
|
"rewards/length_ratio_reward_func/mean": -0.04476670976728201,
|
|
"rewards/length_ratio_reward_func/std": 0.13339075222611427,
|
|
"step": 380
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 95.9,
|
|
"completions/max_terminated_length": 95.9,
|
|
"completions/mean_length": 42.595834732055664,
|
|
"completions/mean_terminated_length": 42.595834732055664,
|
|
"completions/min_length": 15.7,
|
|
"completions/min_terminated_length": 15.7,
|
|
"entropy": 0.6591628199443221,
|
|
"epoch": 0.07155963302752294,
|
|
"frac_reward_zero_std": 0.01250000037252903,
|
|
"grad_norm": 0.3945656418800354,
|
|
"kl": 0.00020127672983676347,
|
|
"learning_rate": 4.625e-08,
|
|
"loss": 0.0146,
|
|
"num_tokens": 3841090.0,
|
|
"reward": 0.24266515970230101,
|
|
"reward_std": 0.050619854405522344,
|
|
"rewards/bleu_reward_func/mean": 0.22361065298318863,
|
|
"rewards/bleu_reward_func/std": 0.16414945498108863,
|
|
"rewards/chrf_reward_func/mean": 0.519814333319664,
|
|
"rewards/chrf_reward_func/std": 0.16327179223299026,
|
|
"rewards/cometkiwi_reward_func/mean": 0.17661845088005065,
|
|
"rewards/cometkiwi_reward_func/std": 0.09794699177145957,
|
|
"rewards/format_reward_func/mean": -0.0015625,
|
|
"rewards/format_reward_func/std": 0.008745299279689788,
|
|
"rewards/language_consistency_reward_func/mean": -0.01770833330228925,
|
|
"rewards/language_consistency_reward_func/std": 0.08932361751794815,
|
|
"rewards/length_ratio_reward_func/mean": -0.051568548940122125,
|
|
"rewards/length_ratio_reward_func/std": 0.1473702261224389,
|
|
"step": 390
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 99.2,
|
|
"completions/max_terminated_length": 99.2,
|
|
"completions/mean_length": 43.300001525878905,
|
|
"completions/mean_terminated_length": 43.300001525878905,
|
|
"completions/min_length": 12.8,
|
|
"completions/min_terminated_length": 12.8,
|
|
"entropy": 0.6424567837268114,
|
|
"epoch": 0.07339449541284404,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5354830622673035,
|
|
"kl": 0.00021015736312790523,
|
|
"learning_rate": 4.208333333333333e-08,
|
|
"loss": 0.0213,
|
|
"num_tokens": 3938070.0,
|
|
"reward": 0.2554669827222824,
|
|
"reward_std": 0.05637744516134262,
|
|
"rewards/bleu_reward_func/mean": 0.23057636320590974,
|
|
"rewards/bleu_reward_func/std": 0.1640048161149025,
|
|
"rewards/chrf_reward_func/mean": 0.5491251111030578,
|
|
"rewards/chrf_reward_func/std": 0.1627779111266136,
|
|
"rewards/cometkiwi_reward_func/mean": 0.1875588119029999,
|
|
"rewards/cometkiwi_reward_func/std": 0.09957792684435844,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"rewards/language_consistency_reward_func/mean": -0.01875,
|
|
"rewards/language_consistency_reward_func/std": 0.10711222738027573,
|
|
"rewards/length_ratio_reward_func/mean": -0.05102834459394216,
|
|
"rewards/length_ratio_reward_func/std": 0.1482754446566105,
|
|
"step": 400
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 500,
|
|
"num_input_tokens_seen": 3938070,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 100,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|