1019 lines
41 KiB
JSON
1019 lines
41 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 16.571428571428573,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 50,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 114.5,
|
||
|
|
"completions/max_terminated_length": 114.5,
|
||
|
|
"completions/mean_length": 111.25,
|
||
|
|
"completions/mean_terminated_length": 111.25,
|
||
|
|
"completions/min_length": 108.0,
|
||
|
|
"completions/min_terminated_length": 108.0,
|
||
|
|
"epoch": 0.5714285714285714,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 2.666666666666666e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 2938.0,
|
||
|
|
"reward": 7.67693305015564,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 2.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5089328289031982,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5670355260372162,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 2
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 125.5,
|
||
|
|
"completions/max_terminated_length": 125.5,
|
||
|
|
"completions/mean_length": 121.66666412353516,
|
||
|
|
"completions/mean_terminated_length": 121.66666412353516,
|
||
|
|
"completions/min_length": 116.5,
|
||
|
|
"completions/min_terminated_length": 116.5,
|
||
|
|
"epoch": 1.2857142857142856,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 8e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 5954.0,
|
||
|
|
"reward": 6.134842395782471,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.4668421894311905,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5008190274238586,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 4
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 129.0,
|
||
|
|
"completions/max_terminated_length": 129.0,
|
||
|
|
"completions/mean_length": 113.91666793823242,
|
||
|
|
"completions/mean_terminated_length": 113.91666793823242,
|
||
|
|
"completions/min_length": 96.0,
|
||
|
|
"completions/min_terminated_length": 96.0,
|
||
|
|
"epoch": 1.8571428571428572,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 7.96430995261912e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 8902.0,
|
||
|
|
"reward": 6.701709508895874,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9160000085830688,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5357092022895813,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5361168682575226,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 6
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 109.5,
|
||
|
|
"completions/max_terminated_length": 109.5,
|
||
|
|
"completions/mean_length": 102.75,
|
||
|
|
"completions/mean_terminated_length": 102.75,
|
||
|
|
"completions/min_length": 96.0,
|
||
|
|
"completions/min_terminated_length": 96.0,
|
||
|
|
"epoch": 2.571428571428571,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 7.857876700217506e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 11772.0,
|
||
|
|
"reward": 6.697775840759277,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9160000085830688,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5317758321762085,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5406587421894073,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 8
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 125.5,
|
||
|
|
"completions/max_terminated_length": 125.5,
|
||
|
|
"completions/mean_length": 114.33333206176758,
|
||
|
|
"completions/mean_terminated_length": 114.33333206176758,
|
||
|
|
"completions/min_length": 104.5,
|
||
|
|
"completions/min_terminated_length": 104.5,
|
||
|
|
"epoch": 3.2857142857142856,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 7.682599546705714e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 14740.0,
|
||
|
|
"reward": 6.227731227874756,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9160000085830688,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5617313385009766,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5060690939426422,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 118.0,
|
||
|
|
"completions/max_terminated_length": 118.0,
|
||
|
|
"completions/mean_length": 111.58333206176758,
|
||
|
|
"completions/mean_terminated_length": 111.58333206176758,
|
||
|
|
"completions/min_length": 108.0,
|
||
|
|
"completions/min_terminated_length": 108.0,
|
||
|
|
"epoch": 3.857142857142857,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 7.441606317040557e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 17692.0,
|
||
|
|
"reward": 7.679747819900513,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 2.916000008583069,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5137476325035095,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5614758431911469,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 12
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 120.5,
|
||
|
|
"completions/max_terminated_length": 120.5,
|
||
|
|
"completions/mean_length": 112.5,
|
||
|
|
"completions/mean_terminated_length": 112.5,
|
||
|
|
"completions/min_length": 104.5,
|
||
|
|
"completions/min_terminated_length": 104.5,
|
||
|
|
"epoch": 4.571428571428571,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 7.139197541114644e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 20640.0,
|
||
|
|
"reward": 6.190184593200684,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9160000085830688,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5241846442222595,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5494242608547211,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 14
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 113.0,
|
||
|
|
"completions/max_terminated_length": 113.0,
|
||
|
|
"completions/mean_length": 105.83333206176758,
|
||
|
|
"completions/mean_terminated_length": 105.83333206176758,
|
||
|
|
"completions/min_length": 97.5,
|
||
|
|
"completions/min_terminated_length": 97.5,
|
||
|
|
"epoch": 5.285714285714286,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 6.780769710698569e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 23530.0,
|
||
|
|
"reward": 6.729287624359131,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5612876415252686,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5065814107656479,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 16
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 125.5,
|
||
|
|
"completions/max_terminated_length": 125.5,
|
||
|
|
"completions/mean_length": 121.16666793823242,
|
||
|
|
"completions/mean_terminated_length": 121.16666793823242,
|
||
|
|
"completions/min_length": 118.0,
|
||
|
|
"completions/min_terminated_length": 118.0,
|
||
|
|
"epoch": 5.857142857142857,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 6.37271897891742e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 26552.0,
|
||
|
|
"reward": 6.184166669845581,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9140000343322754,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5201666951179504,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5540637671947479,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 18
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 129.0,
|
||
|
|
"completions/max_terminated_length": 129.0,
|
||
|
|
"completions/mean_length": 121.0,
|
||
|
|
"completions/mean_terminated_length": 121.0,
|
||
|
|
"completions/min_length": 113.0,
|
||
|
|
"completions/min_terminated_length": 113.0,
|
||
|
|
"epoch": 6.571428571428571,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 5.922327020746735e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 29568.0,
|
||
|
|
"reward": 6.147816181182861,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9160000085830688,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.4818165749311447,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.48352810740470886,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 122.0,
|
||
|
|
"completions/max_terminated_length": 122.0,
|
||
|
|
"completions/mean_length": 117.75,
|
||
|
|
"completions/mean_terminated_length": 117.75,
|
||
|
|
"completions/min_length": 113.0,
|
||
|
|
"completions/min_terminated_length": 113.0,
|
||
|
|
"epoch": 7.285714285714286,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 5.437631091350051e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 32556.0,
|
||
|
|
"reward": 6.695430040359497,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9160000085830688,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5294299721717834,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.543367475271225,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 22
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 118.0,
|
||
|
|
"completions/max_terminated_length": 118.0,
|
||
|
|
"completions/mean_length": 111.58333206176758,
|
||
|
|
"completions/mean_terminated_length": 111.58333206176758,
|
||
|
|
"completions/min_length": 108.0,
|
||
|
|
"completions/min_terminated_length": 108.0,
|
||
|
|
"epoch": 7.857142857142857,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 4.927280601070113e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 35508.0,
|
||
|
|
"reward": 7.68305778503418,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 2.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5150575637817383,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5599632859230042,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 24
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 125.5,
|
||
|
|
"completions/max_terminated_length": 125.5,
|
||
|
|
"completions/mean_length": 111.5,
|
||
|
|
"completions/mean_terminated_length": 111.5,
|
||
|
|
"completions/min_length": 97.5,
|
||
|
|
"completions/min_terminated_length": 97.5,
|
||
|
|
"epoch": 8.571428571428571,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 4.400382766496394e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 38448.0,
|
||
|
|
"reward": 6.215600967407227,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5476008653640747,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5223855376243591,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 26
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 116.5,
|
||
|
|
"completions/max_terminated_length": 116.5,
|
||
|
|
"completions/mean_length": 107.66666793823242,
|
||
|
|
"completions/mean_terminated_length": 107.66666793823242,
|
||
|
|
"completions/min_length": 96.0,
|
||
|
|
"completions/min_terminated_length": 96.0,
|
||
|
|
"epoch": 9.285714285714286,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 3.866340091969303e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 41346.0,
|
||
|
|
"reward": 8.184330940246582,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 2.916000008583069,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 1.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 1.154700517654419,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5183308720588684,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5561836063861847,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 28
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 121.5,
|
||
|
|
"completions/max_terminated_length": 121.5,
|
||
|
|
"completions/mean_length": 114.91666793823242,
|
||
|
|
"completions/mean_terminated_length": 114.91666793823242,
|
||
|
|
"completions/min_length": 109.5,
|
||
|
|
"completions/min_terminated_length": 109.5,
|
||
|
|
"epoch": 9.857142857142858,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 3.33468258166748e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 44318.0,
|
||
|
|
"reward": 6.189554929733276,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9140000343322754,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5255548357963562,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5478420853614807,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 113.0,
|
||
|
|
"completions/max_terminated_length": 113.0,
|
||
|
|
"completions/mean_length": 104.5,
|
||
|
|
"completions/mean_terminated_length": 104.5,
|
||
|
|
"completions/min_length": 96.0,
|
||
|
|
"completions/min_terminated_length": 96.0,
|
||
|
|
"epoch": 10.571428571428571,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 2.8148976764574644e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 47202.0,
|
||
|
|
"reward": 6.747161626815796,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.579161524772644,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.48594246804714203,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 32
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 133.0,
|
||
|
|
"completions/max_terminated_length": 133.0,
|
||
|
|
"completions/mean_length": 125.58333206176758,
|
||
|
|
"completions/mean_terminated_length": 125.58333206176758,
|
||
|
|
"completions/min_length": 114.5,
|
||
|
|
"completions/min_terminated_length": 114.5,
|
||
|
|
"epoch": 11.285714285714286,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 2.3162609502866607e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 50240.0,
|
||
|
|
"reward": 6.091987371444702,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.4239872694015503,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.43548497557640076,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 34
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 118.0,
|
||
|
|
"completions/max_terminated_length": 118.0,
|
||
|
|
"completions/mean_length": 109.25,
|
||
|
|
"completions/mean_terminated_length": 109.25,
|
||
|
|
"completions/min_length": 96.0,
|
||
|
|
"completions/min_terminated_length": 96.0,
|
||
|
|
"epoch": 11.857142857142858,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 1.8476705873465095e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 53144.0,
|
||
|
|
"reward": 6.189610004425049,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9160000085830688,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5236097574234009,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5500880181789398,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 36
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 120.5,
|
||
|
|
"completions/max_terminated_length": 120.5,
|
||
|
|
"completions/mean_length": 110.75,
|
||
|
|
"completions/mean_terminated_length": 110.75,
|
||
|
|
"completions/min_length": 101.0,
|
||
|
|
"completions/min_terminated_length": 101.0,
|
||
|
|
"epoch": 12.571428571428571,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 1.41748859376479e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 56078.0,
|
||
|
|
"reward": 6.197602987289429,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5296030640602112,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5431675910949707,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 38
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 125.0,
|
||
|
|
"completions/max_terminated_length": 125.0,
|
||
|
|
"completions/mean_length": 119.16666793823242,
|
||
|
|
"completions/mean_terminated_length": 119.16666793823242,
|
||
|
|
"completions/min_length": 111.0,
|
||
|
|
"completions/min_terminated_length": 111.0,
|
||
|
|
"epoch": 13.285714285714286,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 1.0333915774081696e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 59070.0,
|
||
|
|
"reward": 6.176448345184326,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9140000343322754,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5124481320381165,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5629763901233673,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 120.5,
|
||
|
|
"completions/max_terminated_length": 120.5,
|
||
|
|
"completions/mean_length": 106.16666793823242,
|
||
|
|
"completions/mean_terminated_length": 106.16666793823242,
|
||
|
|
"completions/min_length": 96.0,
|
||
|
|
"completions/min_terminated_length": 96.0,
|
||
|
|
"epoch": 13.857142857142858,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 7.022337586329597e-08,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 61984.0,
|
||
|
|
"reward": 7.68181848526001,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 2.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5138185024261475,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.561394065618515,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 42
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 122.0,
|
||
|
|
"completions/max_terminated_length": 122.0,
|
||
|
|
"completions/mean_length": 115.0,
|
||
|
|
"completions/mean_terminated_length": 115.0,
|
||
|
|
"completions/min_length": 108.0,
|
||
|
|
"completions/min_terminated_length": 108.0,
|
||
|
|
"epoch": 14.571428571428571,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 4.299246565604755e-08,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 64952.0,
|
||
|
|
"reward": 6.64246392250061,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9160000085830688,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.47646407783031464,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.48970863223075867,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 44
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 121.5,
|
||
|
|
"completions/max_terminated_length": 121.5,
|
||
|
|
"completions/mean_length": 106.08333206176758,
|
||
|
|
"completions/mean_terminated_length": 106.08333206176758,
|
||
|
|
"completions/min_length": 97.5,
|
||
|
|
"completions/min_terminated_length": 97.5,
|
||
|
|
"epoch": 15.285714285714286,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 2.213236335683253e-08,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 67876.0,
|
||
|
|
"reward": 6.259791612625122,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.591791570186615,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.47135844826698303,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 46
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 129.0,
|
||
|
|
"completions/max_terminated_length": 129.0,
|
||
|
|
"completions/mean_length": 113.91666793823242,
|
||
|
|
"completions/mean_terminated_length": 113.91666793823242,
|
||
|
|
"completions/min_length": 96.0,
|
||
|
|
"completions/min_terminated_length": 96.0,
|
||
|
|
"epoch": 15.857142857142858,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 8.0153179853653e-09,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 70824.0,
|
||
|
|
"reward": 6.701709508895874,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9160000085830688,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.5357092022895813,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.5361168682575226,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 48
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 120.5,
|
||
|
|
"completions/max_terminated_length": 120.5,
|
||
|
|
"completions/mean_length": 108.25,
|
||
|
|
"completions/mean_terminated_length": 108.25,
|
||
|
|
"completions/min_length": 96.0,
|
||
|
|
"completions/min_terminated_length": 96.0,
|
||
|
|
"epoch": 16.571428571428573,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 8.932485507387344e-10,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 73738.0,
|
||
|
|
"reward": 6.639105796813965,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 1.9179999828338623,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 2.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.5773502588272095,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.47110553085803986,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.4958961009979248,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.5,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 1.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 16.571428571428573,
|
||
|
|
"step": 50,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.0,
|
||
|
|
"train_runtime": 1743.1391,
|
||
|
|
"train_samples_per_second": 0.115,
|
||
|
|
"train_steps_per_second": 0.029
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 2,
|
||
|
|
"max_steps": 50,
|
||
|
|
"num_input_tokens_seen": 73738,
|
||
|
|
"num_train_epochs": 17,
|
||
|
|
"save_steps": 25,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|