824 lines
33 KiB
JSON
824 lines
33 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.6666666666666665,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 20,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 152.0,
|
||
|
|
"completions/max_terminated_length": 152.0,
|
||
|
|
"completions/mean_length": 66.5,
|
||
|
|
"completions/mean_terminated_length": 66.5,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"epoch": 0.08333333333333333,
|
||
|
|
"grad_norm": 16.002685546875,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 0.0,
|
||
|
|
"loss": 0.2374,
|
||
|
|
"num_tokens": 778.0,
|
||
|
|
"reward": 0.37110745906829834,
|
||
|
|
"reward_std": 0.02719510719180107,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.12110745906829834,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.02461756393313408,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 1
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 208.0,
|
||
|
|
"completions/max_terminated_length": 208.0,
|
||
|
|
"completions/mean_length": 83.0,
|
||
|
|
"completions/mean_terminated_length": 83.0,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"epoch": 0.16666666666666666,
|
||
|
|
"grad_norm": 17.740102767944336,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.1808,
|
||
|
|
"num_tokens": 1622.0,
|
||
|
|
"reward": 0.7830941677093506,
|
||
|
|
"reward_std": 0.709587574005127,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.03309421241283417,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.022268760949373245,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 2
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 66.0,
|
||
|
|
"completions/max_terminated_length": 66.0,
|
||
|
|
"completions/mean_length": 43.25,
|
||
|
|
"completions/mean_terminated_length": 43.25,
|
||
|
|
"completions/min_length": 30.0,
|
||
|
|
"completions/min_terminated_length": 30.0,
|
||
|
|
"epoch": 0.25,
|
||
|
|
"grad_norm": 20.364530563354492,
|
||
|
|
"kl": 0.0010823954944498837,
|
||
|
|
"learning_rate": 9.931806517013612e-07,
|
||
|
|
"loss": 0.1757,
|
||
|
|
"num_tokens": 2307.0,
|
||
|
|
"reward": 0.31522244215011597,
|
||
|
|
"reward_std": 0.04115162417292595,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.06522242724895477,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.03646043315529823,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 3
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 41.0,
|
||
|
|
"completions/max_terminated_length": 41.0,
|
||
|
|
"completions/mean_length": 27.0,
|
||
|
|
"completions/mean_terminated_length": 27.0,
|
||
|
|
"completions/min_length": 21.0,
|
||
|
|
"completions/min_terminated_length": 21.0,
|
||
|
|
"epoch": 0.3333333333333333,
|
||
|
|
"grad_norm": 29.647855758666992,
|
||
|
|
"kl": 0.005260475678369403,
|
||
|
|
"learning_rate": 9.729086208503173e-07,
|
||
|
|
"loss": -0.129,
|
||
|
|
"num_tokens": 2927.0,
|
||
|
|
"reward": 0.45596686005592346,
|
||
|
|
"reward_std": 0.03609445318579674,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.20596684515476227,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.040882658213377,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 4
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 217.0,
|
||
|
|
"completions/max_terminated_length": 217.0,
|
||
|
|
"completions/mean_length": 85.5,
|
||
|
|
"completions/mean_terminated_length": 85.5,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"epoch": 0.4166666666666667,
|
||
|
|
"grad_norm": 18.206703186035156,
|
||
|
|
"kl": 0.005200653220526874,
|
||
|
|
"learning_rate": 9.397368756032444e-07,
|
||
|
|
"loss": 0.2747,
|
||
|
|
"num_tokens": 3781.0,
|
||
|
|
"reward": 0.27538827061653137,
|
||
|
|
"reward_std": 0.014613513834774494,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.025388263165950775,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.012090591713786125,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 66.0,
|
||
|
|
"completions/max_terminated_length": 66.0,
|
||
|
|
"completions/mean_length": 56.0,
|
||
|
|
"completions/mean_terminated_length": 56.0,
|
||
|
|
"completions/min_length": 50.0,
|
||
|
|
"completions/min_terminated_length": 50.0,
|
||
|
|
"epoch": 0.5,
|
||
|
|
"grad_norm": 19.714519500732422,
|
||
|
|
"kl": 0.008209641091525555,
|
||
|
|
"learning_rate": 8.945702546981968e-07,
|
||
|
|
"loss": -0.0485,
|
||
|
|
"num_tokens": 4517.0,
|
||
|
|
"reward": 0.48157644271850586,
|
||
|
|
"reward_std": 0.04048363119363785,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.23157644271850586,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.04181241616606712,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 6
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 87.0,
|
||
|
|
"completions/max_terminated_length": 87.0,
|
||
|
|
"completions/mean_length": 67.25,
|
||
|
|
"completions/mean_terminated_length": 67.25,
|
||
|
|
"completions/min_length": 59.0,
|
||
|
|
"completions/min_terminated_length": 59.0,
|
||
|
|
"epoch": 0.5833333333333334,
|
||
|
|
"grad_norm": 22.889354705810547,
|
||
|
|
"kl": 0.017146551283076406,
|
||
|
|
"learning_rate": 8.386407858128706e-07,
|
||
|
|
"loss": 0.0765,
|
||
|
|
"num_tokens": 5298.0,
|
||
|
|
"reward": 0.2642485201358795,
|
||
|
|
"reward_std": 0.009464496746659279,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.014248517341911793,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.007800546009093523,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 7
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 54.0,
|
||
|
|
"completions/max_terminated_length": 54.0,
|
||
|
|
"completions/mean_length": 47.5,
|
||
|
|
"completions/mean_terminated_length": 47.5,
|
||
|
|
"completions/min_length": 41.0,
|
||
|
|
"completions/min_terminated_length": 41.0,
|
||
|
|
"epoch": 0.6666666666666666,
|
||
|
|
"grad_norm": 16.884830474853516,
|
||
|
|
"kl": 0.01035750936716795,
|
||
|
|
"learning_rate": 7.734740790612136e-07,
|
||
|
|
"loss": -0.0007,
|
||
|
|
"num_tokens": 6000.0,
|
||
|
|
"reward": 1.334761381149292,
|
||
|
|
"reward_std": 0.0021128952503204346,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 1.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 1.154700517654419,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.08476129919290543,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.03886423632502556,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 8
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 106.0,
|
||
|
|
"completions/max_terminated_length": 106.0,
|
||
|
|
"completions/mean_length": 53.5,
|
||
|
|
"completions/mean_terminated_length": 53.5,
|
||
|
|
"completions/min_length": 22.0,
|
||
|
|
"completions/min_terminated_length": 22.0,
|
||
|
|
"epoch": 0.75,
|
||
|
|
"grad_norm": 24.254104614257812,
|
||
|
|
"kl": 0.018237160984426737,
|
||
|
|
"learning_rate": 7.008477123264847e-07,
|
||
|
|
"loss": 0.0369,
|
||
|
|
"num_tokens": 6726.0,
|
||
|
|
"reward": 0.2588782012462616,
|
||
|
|
"reward_std": 0.017027506604790688,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.017878185957670212,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.004615777172148228,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.2409999966621399,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.017999999225139618,
|
||
|
|
"step": 9
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 143.0,
|
||
|
|
"completions/max_terminated_length": 143.0,
|
||
|
|
"completions/mean_length": 75.0,
|
||
|
|
"completions/mean_terminated_length": 75.0,
|
||
|
|
"completions/min_length": 42.0,
|
||
|
|
"completions/min_terminated_length": 42.0,
|
||
|
|
"epoch": 0.8333333333333334,
|
||
|
|
"grad_norm": 14.903592109680176,
|
||
|
|
"kl": 0.00908482726663351,
|
||
|
|
"learning_rate": 6.227427435703995e-07,
|
||
|
|
"loss": 0.1415,
|
||
|
|
"num_tokens": 7538.0,
|
||
|
|
"reward": 0.27809321880340576,
|
||
|
|
"reward_std": 0.0042277188040316105,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.028093229979276657,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.004102820064872503,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 215.0,
|
||
|
|
"completions/max_terminated_length": 215.0,
|
||
|
|
"completions/mean_length": 93.5,
|
||
|
|
"completions/mean_terminated_length": 93.5,
|
||
|
|
"completions/min_length": 50.0,
|
||
|
|
"completions/min_terminated_length": 50.0,
|
||
|
|
"epoch": 0.9166666666666666,
|
||
|
|
"grad_norm": 12.293601989746094,
|
||
|
|
"kl": 0.00679829076398164,
|
||
|
|
"learning_rate": 5.412896727361662e-07,
|
||
|
|
"loss": 0.2122,
|
||
|
|
"num_tokens": 8424.0,
|
||
|
|
"reward": 0.297762393951416,
|
||
|
|
"reward_std": 0.01529813650995493,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.047762420028448105,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.022239817306399345,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 11
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 56.75,
|
||
|
|
"completions/mean_terminated_length": 56.75,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"grad_norm": 15.407873153686523,
|
||
|
|
"kl": 0.019390438217669725,
|
||
|
|
"learning_rate": 4.5871032726383385e-07,
|
||
|
|
"loss": -0.0285,
|
||
|
|
"num_tokens": 9163.0,
|
||
|
|
"reward": 0.29567059874534607,
|
||
|
|
"reward_std": 0.005479501560330391,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.045670587569475174,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.021927477791905403,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 12
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 99.0,
|
||
|
|
"completions/max_terminated_length": 99.0,
|
||
|
|
"completions/mean_length": 64.25,
|
||
|
|
"completions/mean_terminated_length": 64.25,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"epoch": 1.0833333333333333,
|
||
|
|
"grad_norm": 16.31627655029297,
|
||
|
|
"kl": 0.018778782337903976,
|
||
|
|
"learning_rate": 3.772572564296004e-07,
|
||
|
|
"loss": 0.0125,
|
||
|
|
"num_tokens": 9932.0,
|
||
|
|
"reward": 0.27973929047584534,
|
||
|
|
"reward_std": 0.007485522888600826,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.02973928302526474,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.007503707893192768,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 13
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 95.0,
|
||
|
|
"completions/max_terminated_length": 95.0,
|
||
|
|
"completions/mean_length": 62.0,
|
||
|
|
"completions/mean_terminated_length": 62.0,
|
||
|
|
"completions/min_length": 24.0,
|
||
|
|
"completions/min_terminated_length": 24.0,
|
||
|
|
"epoch": 1.1666666666666667,
|
||
|
|
"grad_norm": 19.889291763305664,
|
||
|
|
"kl": 0.02217284543439746,
|
||
|
|
"learning_rate": 2.9915228767351535e-07,
|
||
|
|
"loss": 0.0596,
|
||
|
|
"num_tokens": 10692.0,
|
||
|
|
"reward": 0.2187139242887497,
|
||
|
|
"reward_std": 0.07829191535711288,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.031213931739330292,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.013533255085349083,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.125,
|
||
|
|
"step": 14
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 101.0,
|
||
|
|
"completions/max_terminated_length": 101.0,
|
||
|
|
"completions/mean_length": 53.0,
|
||
|
|
"completions/mean_terminated_length": 53.0,
|
||
|
|
"completions/min_length": 26.0,
|
||
|
|
"completions/min_terminated_length": 26.0,
|
||
|
|
"epoch": 1.25,
|
||
|
|
"grad_norm": 21.13649559020996,
|
||
|
|
"kl": 0.03571599721908569,
|
||
|
|
"learning_rate": 2.2652592093878665e-07,
|
||
|
|
"loss": 0.058,
|
||
|
|
"num_tokens": 11416.0,
|
||
|
|
"reward": 1.2932653427124023,
|
||
|
|
"reward_std": 0.012066778726875782,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 1.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 1.154700517654419,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.043265439569950104,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.013169766403734684,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 73.0,
|
||
|
|
"completions/max_terminated_length": 73.0,
|
||
|
|
"completions/mean_length": 54.5,
|
||
|
|
"completions/mean_terminated_length": 54.5,
|
||
|
|
"completions/min_length": 30.0,
|
||
|
|
"completions/min_terminated_length": 30.0,
|
||
|
|
"epoch": 1.3333333333333333,
|
||
|
|
"grad_norm": 17.267553329467773,
|
||
|
|
"kl": 0.022342026233673096,
|
||
|
|
"learning_rate": 1.6135921418712955e-07,
|
||
|
|
"loss": -0.2058,
|
||
|
|
"num_tokens": 12146.0,
|
||
|
|
"reward": 1.2824556827545166,
|
||
|
|
"reward_std": 0.006193901412189007,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 1.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 1.154700517654419,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.03245573490858078,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.0076392344199121,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 16
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 421.0,
|
||
|
|
"completions/max_terminated_length": 421.0,
|
||
|
|
"completions/mean_length": 141.25,
|
||
|
|
"completions/mean_terminated_length": 141.25,
|
||
|
|
"completions/min_length": 45.0,
|
||
|
|
"completions/min_terminated_length": 45.0,
|
||
|
|
"epoch": 1.4166666666666667,
|
||
|
|
"grad_norm": 17.668960571289062,
|
||
|
|
"kl": 0.0332432733848691,
|
||
|
|
"learning_rate": 1.0542974530180327e-07,
|
||
|
|
"loss": 0.2508,
|
||
|
|
"num_tokens": 13223.0,
|
||
|
|
"reward": 0.1529623568058014,
|
||
|
|
"reward_std": 0.01418876089155674,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.027962369844317436,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.024242181330919266,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.125,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.14433756470680237,
|
||
|
|
"step": 17
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 61.0,
|
||
|
|
"completions/max_terminated_length": 61.0,
|
||
|
|
"completions/mean_length": 46.75,
|
||
|
|
"completions/mean_terminated_length": 46.75,
|
||
|
|
"completions/min_length": 41.0,
|
||
|
|
"completions/min_terminated_length": 41.0,
|
||
|
|
"epoch": 1.5,
|
||
|
|
"grad_norm": 22.05333137512207,
|
||
|
|
"kl": 0.029448412358760834,
|
||
|
|
"learning_rate": 6.026312439675551e-08,
|
||
|
|
"loss": 0.0745,
|
||
|
|
"num_tokens": 13922.0,
|
||
|
|
"reward": 0.43199265003204346,
|
||
|
|
"reward_std": 0.03799587860703468,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.18199265003204346,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.032286591827869415,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 18
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 67.0,
|
||
|
|
"completions/max_terminated_length": 67.0,
|
||
|
|
"completions/mean_length": 53.5,
|
||
|
|
"completions/mean_terminated_length": 53.5,
|
||
|
|
"completions/min_length": 45.0,
|
||
|
|
"completions/min_terminated_length": 45.0,
|
||
|
|
"epoch": 1.5833333333333335,
|
||
|
|
"grad_norm": 17.41017723083496,
|
||
|
|
"kl": 0.030331933870911598,
|
||
|
|
"learning_rate": 2.7091379149682682e-08,
|
||
|
|
"loss": 0.0366,
|
||
|
|
"num_tokens": 14648.0,
|
||
|
|
"reward": 0.30462953448295593,
|
||
|
|
"reward_std": 0.02412303350865841,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.05462953448295593,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.021661905571818352,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 19
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 39.5,
|
||
|
|
"completions/mean_terminated_length": 39.5,
|
||
|
|
"completions/min_length": 26.0,
|
||
|
|
"completions/min_terminated_length": 26.0,
|
||
|
|
"epoch": 1.6666666666666665,
|
||
|
|
"grad_norm": 27.531686782836914,
|
||
|
|
"kl": 0.04908224381506443,
|
||
|
|
"learning_rate": 6.819348298638839e-09,
|
||
|
|
"loss": 0.1549,
|
||
|
|
"num_tokens": 15318.0,
|
||
|
|
"reward": 0.9455468654632568,
|
||
|
|
"reward_std": 0.7136027812957764,
|
||
|
|
"rewards/concensus_correctness_reward_func/mean": 0.0,
|
||
|
|
"rewards/concensus_correctness_reward_func/std": 0.0,
|
||
|
|
"rewards/consensus_reward_func/mean": 0.0,
|
||
|
|
"rewards/consensus_reward_func/std": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/mean": 0.0,
|
||
|
|
"rewards/cumulative_reward_2/std": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func/mean": 0.5,
|
||
|
|
"rewards/final_correctness_reward_func/std": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func/mean": 0.19554688036441803,
|
||
|
|
"rewards/question_recreation_reward_func/std": 0.04141835495829582,
|
||
|
|
"rewards/soft_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/soft_format_reward_func/std": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/mean": 0.0,
|
||
|
|
"rewards/strict_format_reward_func/std": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func/mean": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func/std": 0.0,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6666666666666665,
|
||
|
|
"step": 20,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.07851207228377462,
|
||
|
|
"train_runtime": 5836.1529,
|
||
|
|
"train_samples_per_second": 0.014,
|
||
|
|
"train_steps_per_second": 0.003
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 1,
|
||
|
|
"max_steps": 20,
|
||
|
|
"num_input_tokens_seen": 15318,
|
||
|
|
"num_train_epochs": 2,
|
||
|
|
"save_steps": 25,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 2,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|