1794 lines
72 KiB
JSON
1794 lines
72 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.0022222222222222222,
|
|
"eval_steps": 500,
|
|
"global_step": 100,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 1024.0,
|
|
"completions/max_terminated_length": 872.0,
|
|
"completions/mean_length": 624.5625,
|
|
"completions/mean_terminated_length": 567.5000305175781,
|
|
"completions/min_length": 209.5,
|
|
"completions/min_terminated_length": 209.5,
|
|
"entropy": 1.33274307847023,
|
|
"epoch": 4.4444444444444447e-05,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.749985828335076,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.6666666666666665e-07,
|
|
"loss": -0.1614,
|
|
"num_tokens": 12457.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.4629100561141968,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.25,
|
|
"rewards/format_reward_func/std": 0.4629100561141968,
|
|
"sampling/importance_sampling_ratio/max": 2.5595189332962036,
|
|
"sampling/importance_sampling_ratio/mean": 0.7684620320796967,
|
|
"sampling/importance_sampling_ratio/min": 0.14499648660421371,
|
|
"sampling/sampling_logp_difference/max": 0.5778531432151794,
|
|
"sampling/sampling_logp_difference/mean": 0.026035048998892307,
|
|
"step": 2,
|
|
"step_time": 10.443448680220172
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 903.0,
|
|
"completions/max_terminated_length": 838.0,
|
|
"completions/mean_length": 584.0,
|
|
"completions/mean_terminated_length": 559.2410888671875,
|
|
"completions/min_length": 262.5,
|
|
"completions/min_terminated_length": 262.5,
|
|
"entropy": 1.2995611056685448,
|
|
"epoch": 8.888888888888889e-05,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 4.687210700851442,
|
|
"kl": 0.0005161362223589094,
|
|
"learning_rate": 5e-07,
|
|
"loss": -0.0495,
|
|
"num_tokens": 24225.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.2587745785713196,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.1875,
|
|
"rewards/format_reward_func/std": 0.25877460837364197,
|
|
"sampling/importance_sampling_ratio/max": 2.0775018334388733,
|
|
"sampling/importance_sampling_ratio/mean": 0.982254296541214,
|
|
"sampling/importance_sampling_ratio/min": 0.0867920070886612,
|
|
"sampling/sampling_logp_difference/max": 0.41237497329711914,
|
|
"sampling/sampling_logp_difference/mean": 0.025452900677919388,
|
|
"step": 4,
|
|
"step_time": 8.720533160958439
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 894.5,
|
|
"completions/max_terminated_length": 748.0,
|
|
"completions/mean_length": 544.9375,
|
|
"completions/mean_terminated_length": 512.7946472167969,
|
|
"completions/min_length": 237.5,
|
|
"completions/min_terminated_length": 237.5,
|
|
"entropy": 1.1739214062690735,
|
|
"epoch": 0.00013333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.7835863650793127,
|
|
"kl": 0.0004755087238663691,
|
|
"learning_rate": 4.994757065594279e-07,
|
|
"loss": 0.1861,
|
|
"num_tokens": 35360.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.49022960662841797,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.3125,
|
|
"rewards/format_reward_func/std": 0.49022963643074036,
|
|
"sampling/importance_sampling_ratio/max": 1.6567679643630981,
|
|
"sampling/importance_sampling_ratio/mean": 0.5457935333251953,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.5021777153015137,
|
|
"sampling/sampling_logp_difference/mean": 0.023564884439110756,
|
|
"step": 6,
|
|
"step_time": 8.810739000793546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 981.0,
|
|
"completions/max_terminated_length": 981.0,
|
|
"completions/mean_length": 519.5625,
|
|
"completions/mean_terminated_length": 519.5625,
|
|
"completions/min_length": 211.0,
|
|
"completions/min_terminated_length": 211.0,
|
|
"entropy": 1.199426531791687,
|
|
"epoch": 0.00017777777777777779,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.2845524174720118,
|
|
"kl": 0.0006594761198357446,
|
|
"learning_rate": 4.979050253066063e-07,
|
|
"loss": -0.0057,
|
|
"num_tokens": 46153.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.5345224738121033,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.5,
|
|
"rewards/format_reward_func/std": 0.5345224738121033,
|
|
"sampling/importance_sampling_ratio/max": 1.4953356981277466,
|
|
"sampling/importance_sampling_ratio/mean": 0.7003053724765778,
|
|
"sampling/importance_sampling_ratio/min": 0.10495343431830406,
|
|
"sampling/sampling_logp_difference/max": 0.3991909921169281,
|
|
"sampling/sampling_logp_difference/mean": 0.024509839713573456,
|
|
"step": 8,
|
|
"step_time": 9.232346954755485
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 806.5,
|
|
"completions/max_terminated_length": 756.0,
|
|
"completions/mean_length": 432.4375,
|
|
"completions/mean_terminated_length": 395.0089416503906,
|
|
"completions/min_length": 218.5,
|
|
"completions/min_terminated_length": 218.5,
|
|
"entropy": 1.1880374103784561,
|
|
"epoch": 0.00022222222222222223,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.640790667150943,
|
|
"kl": 0.0004793386397068389,
|
|
"learning_rate": 4.952945442245597e-07,
|
|
"loss": 0.2931,
|
|
"num_tokens": 55480.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.6208146214485168,
|
|
"rewards/equation_reward_func/mean": 0.1875,
|
|
"rewards/equation_reward_func/std": 0.408231720328331,
|
|
"rewards/format_reward_func/mean": 0.1875,
|
|
"rewards/format_reward_func/std": 0.408231720328331,
|
|
"sampling/importance_sampling_ratio/max": 1.863376498222351,
|
|
"sampling/importance_sampling_ratio/mean": 1.0063321888446808,
|
|
"sampling/importance_sampling_ratio/min": 0.1957462728023529,
|
|
"sampling/sampling_logp_difference/max": 0.37483203411102295,
|
|
"sampling/sampling_logp_difference/mean": 0.023145501501858234,
|
|
"step": 10,
|
|
"step_time": 8.811809529666789
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 640.5,
|
|
"completions/max_terminated_length": 640.5,
|
|
"completions/mean_length": 423.5625,
|
|
"completions/mean_terminated_length": 423.5625,
|
|
"completions/min_length": 180.0,
|
|
"completions/min_terminated_length": 180.0,
|
|
"entropy": 0.9937291070818901,
|
|
"epoch": 0.0002666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.702686109757061,
|
|
"kl": 0.0006009896087562083,
|
|
"learning_rate": 4.916552125781528e-07,
|
|
"loss": 0.0342,
|
|
"num_tokens": 64697.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.5518900156021118,
|
|
"rewards/equation_reward_func/mean": 0.0625,
|
|
"rewards/equation_reward_func/std": 0.1767766922712326,
|
|
"rewards/format_reward_func/mean": 0.5,
|
|
"rewards/format_reward_func/std": 0.4629100561141968,
|
|
"sampling/importance_sampling_ratio/max": 1.8338406682014465,
|
|
"sampling/importance_sampling_ratio/mean": 0.8249517679214478,
|
|
"sampling/importance_sampling_ratio/min": 0.12194083631038666,
|
|
"sampling/sampling_logp_difference/max": 0.4249380826950073,
|
|
"sampling/sampling_logp_difference/mean": 0.02203766629099846,
|
|
"step": 12,
|
|
"step_time": 8.034374623908661
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 870.5,
|
|
"completions/max_terminated_length": 682.5,
|
|
"completions/mean_length": 420.1875,
|
|
"completions/mean_terminated_length": 376.9464416503906,
|
|
"completions/min_length": 142.5,
|
|
"completions/min_terminated_length": 142.5,
|
|
"entropy": 1.1723689064383507,
|
|
"epoch": 0.0003111111111111111,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3029004457455653,
|
|
"kl": 0.0009138785835602903,
|
|
"learning_rate": 4.870022949890676e-07,
|
|
"loss": 0.0623,
|
|
"num_tokens": 73836.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.636739045381546,
|
|
"rewards/equation_reward_func/mean": 0.1875,
|
|
"rewards/equation_reward_func/std": 0.408231720328331,
|
|
"rewards/format_reward_func/mean": 0.25,
|
|
"rewards/format_reward_func/std": 0.4629100561141968,
|
|
"sampling/importance_sampling_ratio/max": 0.8718301355838776,
|
|
"sampling/importance_sampling_ratio/mean": 0.46648962795734406,
|
|
"sampling/importance_sampling_ratio/min": 0.07531017251312733,
|
|
"sampling/sampling_logp_difference/max": 0.8265839219093323,
|
|
"sampling/sampling_logp_difference/mean": 0.024566035717725754,
|
|
"step": 14,
|
|
"step_time": 8.633461503428407
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 664.0,
|
|
"completions/max_terminated_length": 664.0,
|
|
"completions/mean_length": 471.0,
|
|
"completions/mean_terminated_length": 471.0,
|
|
"completions/min_length": 241.5,
|
|
"completions/min_terminated_length": 241.5,
|
|
"entropy": 1.0899769812822342,
|
|
"epoch": 0.00035555555555555557,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0010745967865654204,
|
|
"kl": 0.000946686575844069,
|
|
"learning_rate": 4.81355307410676e-07,
|
|
"loss": 0.0858,
|
|
"num_tokens": 83804.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.2314550280570984,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.125,
|
|
"rewards/format_reward_func/std": 0.2314550280570984,
|
|
"sampling/importance_sampling_ratio/max": 1.2488415241241455,
|
|
"sampling/importance_sampling_ratio/mean": 0.6085900068283081,
|
|
"sampling/importance_sampling_ratio/min": 0.10226996056735516,
|
|
"sampling/sampling_logp_difference/max": 0.4703601598739624,
|
|
"sampling/sampling_logp_difference/mean": 0.023032911121845245,
|
|
"step": 16,
|
|
"step_time": 8.332381021697074
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 856.5,
|
|
"completions/max_terminated_length": 856.5,
|
|
"completions/mean_length": 538.625,
|
|
"completions/mean_terminated_length": 538.625,
|
|
"completions/min_length": 249.5,
|
|
"completions/min_terminated_length": 249.5,
|
|
"entropy": 1.2622679881751537,
|
|
"epoch": 0.0004,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.971699889643826,
|
|
"kl": 0.001047886358719552,
|
|
"learning_rate": 4.747379352713488e-07,
|
|
"loss": -0.0704,
|
|
"num_tokens": 94886.0,
|
|
"reward": 0.6875,
|
|
"reward_std": 0.5876962244510651,
|
|
"rewards/equation_reward_func/mean": 0.0625,
|
|
"rewards/equation_reward_func/std": 0.1767766922712326,
|
|
"rewards/format_reward_func/mean": 0.625,
|
|
"rewards/format_reward_func/std": 0.49871626496315,
|
|
"sampling/importance_sampling_ratio/max": 1.9497967958450317,
|
|
"sampling/importance_sampling_ratio/mean": 0.8915638625621796,
|
|
"sampling/importance_sampling_ratio/min": 0.22629090957343578,
|
|
"sampling/sampling_logp_difference/max": 0.42359161376953125,
|
|
"sampling/sampling_logp_difference/mean": 0.02525496855378151,
|
|
"step": 18,
|
|
"step_time": 8.536863426212221
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 723.0,
|
|
"completions/max_terminated_length": 723.0,
|
|
"completions/mean_length": 453.875,
|
|
"completions/mean_terminated_length": 453.875,
|
|
"completions/min_length": 149.0,
|
|
"completions/min_terminated_length": 149.0,
|
|
"entropy": 1.1858837865293026,
|
|
"epoch": 0.00044444444444444447,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3370582534401416,
|
|
"kl": 0.001601253719854867,
|
|
"learning_rate": 4.6717793412953776e-07,
|
|
"loss": -0.0119,
|
|
"num_tokens": 104620.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.5260358154773712,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.4375,
|
|
"rewards/format_reward_func/std": 0.5260358452796936,
|
|
"sampling/importance_sampling_ratio/max": 1.5436645448207855,
|
|
"sampling/importance_sampling_ratio/mean": 0.8180775046348572,
|
|
"sampling/importance_sampling_ratio/min": 0.04623246192932129,
|
|
"sampling/sampling_logp_difference/max": 0.39241665601730347,
|
|
"sampling/sampling_logp_difference/mean": 0.02441485971212387,
|
|
"step": 20,
|
|
"step_time": 8.402821157709695
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 803.5,
|
|
"completions/max_terminated_length": 803.5,
|
|
"completions/mean_length": 548.375,
|
|
"completions/mean_terminated_length": 548.375,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 1.3530792817473412,
|
|
"epoch": 0.0004888888888888889,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.5438846441337637,
|
|
"kl": 0.0017551442215335555,
|
|
"learning_rate": 4.5870701325731773e-07,
|
|
"loss": 0.0066,
|
|
"num_tokens": 115874.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.408231720328331,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.1875,
|
|
"rewards/format_reward_func/std": 0.408231720328331,
|
|
"sampling/importance_sampling_ratio/max": 2.0694342851638794,
|
|
"sampling/importance_sampling_ratio/mean": 0.7663778364658356,
|
|
"sampling/importance_sampling_ratio/min": 0.07287665084004402,
|
|
"sampling/sampling_logp_difference/max": 0.4795522689819336,
|
|
"sampling/sampling_logp_difference/mean": 0.026419796980917454,
|
|
"step": 22,
|
|
"step_time": 8.40855041577015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 727.0,
|
|
"completions/max_terminated_length": 727.0,
|
|
"completions/mean_length": 427.1875,
|
|
"completions/mean_terminated_length": 427.1875,
|
|
"completions/min_length": 192.5,
|
|
"completions/min_terminated_length": 192.5,
|
|
"entropy": 1.1948794089257717,
|
|
"epoch": 0.0005333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.8019870210040545,
|
|
"kl": 0.00165552277394454,
|
|
"learning_rate": 4.4936070264068016e-07,
|
|
"loss": -0.0724,
|
|
"num_tokens": 125149.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.5850084125995636,
|
|
"rewards/equation_reward_func/mean": 0.125,
|
|
"rewards/equation_reward_func/std": 0.2314550280570984,
|
|
"rewards/format_reward_func/mean": 0.375,
|
|
"rewards/format_reward_func/std": 0.49871626496315,
|
|
"sampling/importance_sampling_ratio/max": 1.5116711854934692,
|
|
"sampling/importance_sampling_ratio/mean": 0.5533610284328461,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.5177854299545288,
|
|
"sampling/sampling_logp_difference/mean": 0.025090673007071018,
|
|
"step": 24,
|
|
"step_time": 8.48335952940397
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 967.0,
|
|
"completions/max_terminated_length": 941.5,
|
|
"completions/mean_length": 545.875,
|
|
"completions/mean_terminated_length": 517.6160888671875,
|
|
"completions/min_length": 226.0,
|
|
"completions/min_terminated_length": 226.0,
|
|
"entropy": 1.184289637953043,
|
|
"epoch": 0.0005777777777777778,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.829245155965375,
|
|
"kl": 0.0043001044177799486,
|
|
"learning_rate": 4.391782039544238e-07,
|
|
"loss": 0.0465,
|
|
"num_tokens": 136355.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.5175491571426392,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.5,
|
|
"rewards/format_reward_func/std": 0.5175492167472839,
|
|
"sampling/importance_sampling_ratio/max": 1.729424774646759,
|
|
"sampling/importance_sampling_ratio/mean": 0.8748546242713928,
|
|
"sampling/importance_sampling_ratio/min": 0.19289711862802505,
|
|
"sampling/sampling_logp_difference/max": 0.533031702041626,
|
|
"sampling/sampling_logp_difference/mean": 0.023543622344732285,
|
|
"step": 26,
|
|
"step_time": 9.136462694499642
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 976.5,
|
|
"completions/max_terminated_length": 855.5,
|
|
"completions/mean_length": 524.75,
|
|
"completions/mean_terminated_length": 494.3035888671875,
|
|
"completions/min_length": 249.5,
|
|
"completions/min_terminated_length": 249.5,
|
|
"entropy": 1.2454889379441738,
|
|
"epoch": 0.0006222222222222223,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.0966414039490338,
|
|
"kl": 0.004009995092928875,
|
|
"learning_rate": 4.282022261367073e-07,
|
|
"loss": -0.0593,
|
|
"num_tokens": 147215.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.49022960662841797,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.5625,
|
|
"rewards/format_reward_func/std": 0.49022963643074036,
|
|
"sampling/importance_sampling_ratio/max": 2.0301511883735657,
|
|
"sampling/importance_sampling_ratio/mean": 0.6258620917797089,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.4906817674636841,
|
|
"sampling/sampling_logp_difference/mean": 0.025028634816408157,
|
|
"step": 28,
|
|
"step_time": 9.194792822585441
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 981.0,
|
|
"completions/max_terminated_length": 818.5,
|
|
"completions/mean_length": 529.375,
|
|
"completions/mean_terminated_length": 496.3035888671875,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 1.3521863892674446,
|
|
"epoch": 0.0006666666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.6396233251107017,
|
|
"kl": 0.002601242988021113,
|
|
"learning_rate": 4.1647880625292027e-07,
|
|
"loss": -0.2112,
|
|
"num_tokens": 158125.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.5175491571426392,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.5,
|
|
"rewards/format_reward_func/std": 0.5175492167472839,
|
|
"sampling/importance_sampling_ratio/max": 1.740391492843628,
|
|
"sampling/importance_sampling_ratio/mean": 0.60825514793396,
|
|
"sampling/importance_sampling_ratio/min": 0.1199931800365448,
|
|
"sampling/sampling_logp_difference/max": 0.7060830593109131,
|
|
"sampling/sampling_logp_difference/mean": 0.025156039744615555,
|
|
"step": 30,
|
|
"step_time": 9.511650393018499
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 1000.0,
|
|
"completions/max_terminated_length": 783.0,
|
|
"completions/mean_length": 504.625,
|
|
"completions/mean_terminated_length": 460.24107360839844,
|
|
"completions/min_length": 225.5,
|
|
"completions/min_terminated_length": 225.5,
|
|
"entropy": 1.1279298327863216,
|
|
"epoch": 0.0007111111111111111,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.3928007543799974,
|
|
"kl": 0.006199299117724877,
|
|
"learning_rate": 4.040571164002318e-07,
|
|
"loss": -0.2301,
|
|
"num_tokens": 168679.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.49022960662841797,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.5625,
|
|
"rewards/format_reward_func/std": 0.49022963643074036,
|
|
"sampling/importance_sampling_ratio/max": 2.283095419406891,
|
|
"sampling/importance_sampling_ratio/mean": 0.9702154397964478,
|
|
"sampling/importance_sampling_ratio/min": 0.26252565532922745,
|
|
"sampling/sampling_logp_difference/max": 0.8464382886886597,
|
|
"sampling/sampling_logp_difference/mean": 0.022625007666647434,
|
|
"step": 32,
|
|
"step_time": 9.397412348655052
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 902.0,
|
|
"completions/max_terminated_length": 902.0,
|
|
"completions/mean_length": 513.125,
|
|
"completions/mean_terminated_length": 513.125,
|
|
"completions/min_length": 150.0,
|
|
"completions/min_terminated_length": 150.0,
|
|
"entropy": 1.3348925039172173,
|
|
"epoch": 0.0007555555555555555,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.00057282284152,
|
|
"kl": 0.0058585846272762865,
|
|
"learning_rate": 3.909892574627266e-07,
|
|
"loss": -0.0172,
|
|
"num_tokens": 179305.0,
|
|
"reward": 0.6875,
|
|
"reward_std": 0.6123279631137848,
|
|
"rewards/equation_reward_func/mean": 0.0625,
|
|
"rewards/equation_reward_func/std": 0.1767766922712326,
|
|
"rewards/format_reward_func/mean": 0.625,
|
|
"rewards/format_reward_func/std": 0.5175492167472839,
|
|
"sampling/importance_sampling_ratio/max": 2.2137855291366577,
|
|
"sampling/importance_sampling_ratio/mean": 0.805354505777359,
|
|
"sampling/importance_sampling_ratio/min": 0.12096437066793442,
|
|
"sampling/sampling_logp_difference/max": 0.47625434398651123,
|
|
"sampling/sampling_logp_difference/mean": 0.025196850299835205,
|
|
"step": 34,
|
|
"step_time": 8.92305501177907
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 749.0,
|
|
"completions/max_terminated_length": 749.0,
|
|
"completions/mean_length": 374.1875,
|
|
"completions/mean_terminated_length": 374.1875,
|
|
"completions/min_length": 201.0,
|
|
"completions/min_terminated_length": 201.0,
|
|
"entropy": 1.0916240960359573,
|
|
"epoch": 0.0008,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.9305443524320012,
|
|
"kl": 0.005473379431350622,
|
|
"learning_rate": 3.773300405821908e-07,
|
|
"loss": 0.1427,
|
|
"num_tokens": 187748.0,
|
|
"reward": 0.6875,
|
|
"reward_std": 0.49022960662841797,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.6875,
|
|
"rewards/format_reward_func/std": 0.49022963643074036,
|
|
"sampling/importance_sampling_ratio/max": 1.9894119501113892,
|
|
"sampling/importance_sampling_ratio/mean": 0.7066371142864227,
|
|
"sampling/importance_sampling_ratio/min": 0.16429987736046314,
|
|
"sampling/sampling_logp_difference/max": 0.34800171852111816,
|
|
"sampling/sampling_logp_difference/mean": 0.02382123749703169,
|
|
"step": 36,
|
|
"step_time": 8.38340774225071
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 743.0,
|
|
"completions/max_terminated_length": 607.5,
|
|
"completions/mean_length": 432.875,
|
|
"completions/mean_terminated_length": 400.0446472167969,
|
|
"completions/min_length": 193.0,
|
|
"completions/min_terminated_length": 193.0,
|
|
"entropy": 1.1879093013703823,
|
|
"epoch": 0.0008444444444444444,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.6618913839693144,
|
|
"kl": 0.0069876342167845,
|
|
"learning_rate": 3.6313675726113475e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 197082.0,
|
|
"reward": 0.8125,
|
|
"reward_std": 0.6739883720874786,
|
|
"rewards/equation_reward_func/mean": 0.125,
|
|
"rewards/equation_reward_func/std": 0.3535533845424652,
|
|
"rewards/format_reward_func/mean": 0.6875,
|
|
"rewards/format_reward_func/std": 0.49022963643074036,
|
|
"sampling/importance_sampling_ratio/max": 1.6417380571365356,
|
|
"sampling/importance_sampling_ratio/mean": 0.73714280128479,
|
|
"sampling/importance_sampling_ratio/min": 0.035142697393894196,
|
|
"sampling/sampling_logp_difference/max": 0.5109619498252869,
|
|
"sampling/sampling_logp_difference/mean": 0.025707824155688286,
|
|
"step": 38,
|
|
"step_time": 8.638400478637777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 906.0,
|
|
"completions/max_terminated_length": 906.0,
|
|
"completions/mean_length": 495.0,
|
|
"completions/mean_terminated_length": 495.0,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 1.2622622810304165,
|
|
"epoch": 0.0008888888888888889,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.2967943028741558,
|
|
"kl": 0.005186507463804446,
|
|
"learning_rate": 3.484689390623218e-07,
|
|
"loss": -0.1358,
|
|
"num_tokens": 207442.0,
|
|
"reward": 0.75,
|
|
"reward_std": 0.4355512708425522,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.75,
|
|
"rewards/format_reward_func/std": 0.4355513006448746,
|
|
"sampling/importance_sampling_ratio/max": 2.0180424451828003,
|
|
"sampling/importance_sampling_ratio/mean": 0.8196437358856201,
|
|
"sampling/importance_sampling_ratio/min": 0.11646383255720139,
|
|
"sampling/sampling_logp_difference/max": 0.4600484371185303,
|
|
"sampling/sampling_logp_difference/mean": 0.02518580388277769,
|
|
"step": 40,
|
|
"step_time": 9.020468398462981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 748.0,
|
|
"completions/max_terminated_length": 748.0,
|
|
"completions/mean_length": 448.6875,
|
|
"completions/mean_terminated_length": 448.6875,
|
|
"completions/min_length": 135.5,
|
|
"completions/min_terminated_length": 135.5,
|
|
"entropy": 1.1781928315758705,
|
|
"epoch": 0.0009333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.2445751378816468,
|
|
"kl": 0.007875571063777898,
|
|
"learning_rate": 3.3338810791270517e-07,
|
|
"loss": -0.0195,
|
|
"num_tokens": 217037.0,
|
|
"reward": 0.75,
|
|
"reward_std": 0.6746576428413391,
|
|
"rewards/equation_reward_func/mean": 0.125,
|
|
"rewards/equation_reward_func/std": 0.2314550280570984,
|
|
"rewards/format_reward_func/mean": 0.625,
|
|
"rewards/format_reward_func/std": 0.49871626496315,
|
|
"sampling/importance_sampling_ratio/max": 2.228373646736145,
|
|
"sampling/importance_sampling_ratio/mean": 0.8085304498672485,
|
|
"sampling/importance_sampling_ratio/min": 0.05368908867239952,
|
|
"sampling/sampling_logp_difference/max": 0.5465056896209717,
|
|
"sampling/sampling_logp_difference/mean": 0.024572196416556835,
|
|
"step": 42,
|
|
"step_time": 8.777628015144728
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 691.5,
|
|
"completions/max_terminated_length": 691.5,
|
|
"completions/mean_length": 433.25,
|
|
"completions/mean_terminated_length": 433.25,
|
|
"completions/min_length": 236.5,
|
|
"completions/min_terminated_length": 236.5,
|
|
"entropy": 1.2821482792496681,
|
|
"epoch": 0.0009777777777777777,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.114077373655992,
|
|
"kl": 0.0050713238888420165,
|
|
"learning_rate": 3.179575180590857e-07,
|
|
"loss": -0.0374,
|
|
"num_tokens": 226449.0,
|
|
"reward": 0.8125,
|
|
"reward_std": 0.5518900156021118,
|
|
"rewards/equation_reward_func/mean": 0.0625,
|
|
"rewards/equation_reward_func/std": 0.1767766922712326,
|
|
"rewards/format_reward_func/mean": 0.75,
|
|
"rewards/format_reward_func/std": 0.4629100561141968,
|
|
"sampling/importance_sampling_ratio/max": 1.9372240900993347,
|
|
"sampling/importance_sampling_ratio/mean": 1.0048049092292786,
|
|
"sampling/importance_sampling_ratio/min": 0.06090385094285011,
|
|
"sampling/sampling_logp_difference/max": 0.4808278977870941,
|
|
"sampling/sampling_logp_difference/mean": 0.0246611712500453,
|
|
"step": 44,
|
|
"step_time": 8.335677679046057
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 855.5,
|
|
"completions/max_terminated_length": 802.0,
|
|
"completions/mean_length": 511.3125,
|
|
"completions/mean_terminated_length": 474.3571472167969,
|
|
"completions/min_length": 244.5,
|
|
"completions/min_terminated_length": 244.5,
|
|
"entropy": 1.1734298951923847,
|
|
"epoch": 0.0010222222222222223,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.9815566006792418,
|
|
"kl": 0.005326784594217315,
|
|
"learning_rate": 3.022418907578188e-07,
|
|
"loss": 0.4983,
|
|
"num_tokens": 237046.0,
|
|
"reward": 0.625,
|
|
"reward_std": 0.5175491571426392,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.625,
|
|
"rewards/format_reward_func/std": 0.5175492167472839,
|
|
"sampling/importance_sampling_ratio/max": 2.138944089412689,
|
|
"sampling/importance_sampling_ratio/mean": 1.1211198568344116,
|
|
"sampling/importance_sampling_ratio/min": 0.320043221116066,
|
|
"sampling/sampling_logp_difference/max": 0.4653007984161377,
|
|
"sampling/sampling_logp_difference/mean": 0.023578885942697525,
|
|
"step": 46,
|
|
"step_time": 8.643824698519893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 771.0,
|
|
"completions/max_terminated_length": 771.0,
|
|
"completions/mean_length": 439.125,
|
|
"completions/mean_terminated_length": 439.125,
|
|
"completions/min_length": 216.0,
|
|
"completions/min_terminated_length": 216.0,
|
|
"entropy": 1.2340615428984165,
|
|
"epoch": 0.0010666666666666667,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 4.172865195190822,
|
|
"kl": 0.004765985926496796,
|
|
"learning_rate": 2.863071428113726e-07,
|
|
"loss": 0.0352,
|
|
"num_tokens": 246552.0,
|
|
"reward": 0.875,
|
|
"reward_std": 0.2314550280570984,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.875,
|
|
"rewards/format_reward_func/std": 0.2314550280570984,
|
|
"sampling/importance_sampling_ratio/max": 1.7466660737991333,
|
|
"sampling/importance_sampling_ratio/mean": 0.8979056179523468,
|
|
"sampling/importance_sampling_ratio/min": 0.14472143352031708,
|
|
"sampling/sampling_logp_difference/max": 0.3502110242843628,
|
|
"sampling/sampling_logp_difference/mean": 0.024309751577675343,
|
|
"step": 48,
|
|
"step_time": 8.856163574731909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 791.5,
|
|
"completions/max_terminated_length": 791.5,
|
|
"completions/mean_length": 456.375,
|
|
"completions/mean_terminated_length": 456.375,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 1.326795369386673,
|
|
"epoch": 0.0011111111111111111,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.8629349975816751,
|
|
"kl": 0.0053678700351156294,
|
|
"learning_rate": 2.7022011009035107e-07,
|
|
"loss": -0.0543,
|
|
"num_tokens": 256334.0,
|
|
"reward": 0.8125,
|
|
"reward_std": 0.408231720328331,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.8125,
|
|
"rewards/format_reward_func/std": 0.408231720328331,
|
|
"sampling/importance_sampling_ratio/max": 1.5121912360191345,
|
|
"sampling/importance_sampling_ratio/mean": 0.6738958954811096,
|
|
"sampling/importance_sampling_ratio/min": 0.018768588081002235,
|
|
"sampling/sampling_logp_difference/max": 0.4129265546798706,
|
|
"sampling/sampling_logp_difference/mean": 0.025299090892076492,
|
|
"step": 50,
|
|
"step_time": 8.548322821385227
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 778.5,
|
|
"completions/max_terminated_length": 778.5,
|
|
"completions/mean_length": 472.5625,
|
|
"completions/mean_terminated_length": 472.5625,
|
|
"completions/min_length": 221.0,
|
|
"completions/min_terminated_length": 221.0,
|
|
"entropy": 1.2406046949326992,
|
|
"epoch": 0.0011555555555555555,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.4813630161379192,
|
|
"kl": 0.008232236992625985,
|
|
"learning_rate": 2.540482672006254e-07,
|
|
"loss": -0.0971,
|
|
"num_tokens": 266343.0,
|
|
"reward": 0.625,
|
|
"reward_std": 0.49871626496315,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.625,
|
|
"rewards/format_reward_func/std": 0.49871626496315,
|
|
"sampling/importance_sampling_ratio/max": 0.9217514991760254,
|
|
"sampling/importance_sampling_ratio/mean": 0.4532146751880646,
|
|
"sampling/importance_sampling_ratio/min": 0.004407819826155901,
|
|
"sampling/sampling_logp_difference/max": 1.6009975671768188,
|
|
"sampling/sampling_logp_difference/mean": 0.025143861770629883,
|
|
"step": 52,
|
|
"step_time": 8.712176971603185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 726.5,
|
|
"completions/max_terminated_length": 726.5,
|
|
"completions/mean_length": 438.25,
|
|
"completions/mean_terminated_length": 438.25,
|
|
"completions/min_length": 239.5,
|
|
"completions/min_terminated_length": 239.5,
|
|
"entropy": 1.098836075514555,
|
|
"epoch": 0.0012,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.003079731152243608,
|
|
"kl": 0.006043522240361199,
|
|
"learning_rate": 2.37859444471388e-07,
|
|
"loss": -0.0679,
|
|
"num_tokens": 275835.0,
|
|
"reward": 0.9375,
|
|
"reward_std": 0.1767766922712326,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.9375,
|
|
"rewards/format_reward_func/std": 0.1767766922712326,
|
|
"sampling/importance_sampling_ratio/max": 1.760864406824112,
|
|
"sampling/importance_sampling_ratio/mean": 0.669313445687294,
|
|
"sampling/importance_sampling_ratio/min": 0.037004563957452774,
|
|
"sampling/sampling_logp_difference/max": 0.4206712245941162,
|
|
"sampling/sampling_logp_difference/mean": 0.024013984948396683,
|
|
"step": 54,
|
|
"step_time": 8.665698574273847
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 827.5,
|
|
"completions/max_terminated_length": 827.5,
|
|
"completions/mean_length": 474.4375,
|
|
"completions/mean_terminated_length": 474.4375,
|
|
"completions/min_length": 179.0,
|
|
"completions/min_terminated_length": 179.0,
|
|
"entropy": 1.2204781025648117,
|
|
"epoch": 0.0012444444444444445,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.5745030120265129,
|
|
"kl": 0.007742437141132541,
|
|
"learning_rate": 2.2172154345117894e-07,
|
|
"loss": -0.1581,
|
|
"num_tokens": 285890.0,
|
|
"reward": 0.8125,
|
|
"reward_std": 0.408231720328331,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.8125,
|
|
"rewards/format_reward_func/std": 0.408231720328331,
|
|
"sampling/importance_sampling_ratio/max": 1.5822263956069946,
|
|
"sampling/importance_sampling_ratio/mean": 0.6425267159938812,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.843230128288269,
|
|
"sampling/sampling_logp_difference/mean": 0.025018153712153435,
|
|
"step": 56,
|
|
"step_time": 9.268435405334458
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 800.0,
|
|
"completions/max_terminated_length": 800.0,
|
|
"completions/mean_length": 479.5,
|
|
"completions/mean_terminated_length": 479.5,
|
|
"completions/min_length": 202.5,
|
|
"completions/min_terminated_length": 202.5,
|
|
"entropy": 1.111366856843233,
|
|
"epoch": 0.001288888888888889,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.168791141589522,
|
|
"kl": 0.008615911778178997,
|
|
"learning_rate": 2.0570225210519433e-07,
|
|
"loss": -0.0564,
|
|
"num_tokens": 296010.0,
|
|
"reward": 1.0625,
|
|
"reward_std": 0.408231720328331,
|
|
"rewards/equation_reward_func/mean": 0.125,
|
|
"rewards/equation_reward_func/std": 0.2314550280570984,
|
|
"rewards/format_reward_func/mean": 0.9375,
|
|
"rewards/format_reward_func/std": 0.1767766922712326,
|
|
"sampling/importance_sampling_ratio/max": 1.7061268091201782,
|
|
"sampling/importance_sampling_ratio/mean": 0.6005833446979523,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.441937193274498,
|
|
"sampling/sampling_logp_difference/mean": 0.022888539358973503,
|
|
"step": 58,
|
|
"step_time": 8.758348444476724
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 712.0,
|
|
"completions/max_terminated_length": 712.0,
|
|
"completions/mean_length": 428.0,
|
|
"completions/mean_terminated_length": 428.0,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"entropy": 1.1277053244411945,
|
|
"epoch": 0.0013333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.575149195352441,
|
|
"kl": 0.006991666596150026,
|
|
"learning_rate": 1.8986876090843664e-07,
|
|
"loss": 0.0453,
|
|
"num_tokens": 305266.0,
|
|
"reward": 0.8125,
|
|
"reward_std": 0.5876962244510651,
|
|
"rewards/equation_reward_func/mean": 0.125,
|
|
"rewards/equation_reward_func/std": 0.2314550280570984,
|
|
"rewards/format_reward_func/mean": 0.6875,
|
|
"rewards/format_reward_func/std": 0.44403792917728424,
|
|
"sampling/importance_sampling_ratio/max": 2.0417069792747498,
|
|
"sampling/importance_sampling_ratio/mean": 0.8298235833644867,
|
|
"sampling/importance_sampling_ratio/min": 0.06559842079877853,
|
|
"sampling/sampling_logp_difference/max": 0.3905753046274185,
|
|
"sampling/sampling_logp_difference/mean": 0.023326802998781204,
|
|
"step": 60,
|
|
"step_time": 8.25899032666348
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 975.5,
|
|
"completions/max_terminated_length": 950.0,
|
|
"completions/mean_length": 504.625,
|
|
"completions/mean_terminated_length": 469.02679443359375,
|
|
"completions/min_length": 218.0,
|
|
"completions/min_terminated_length": 218.0,
|
|
"entropy": 1.1242654994130135,
|
|
"epoch": 0.0013777777777777777,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6518752561806848,
|
|
"kl": 0.006885310140205547,
|
|
"learning_rate": 1.7428748102551234e-07,
|
|
"loss": -0.0569,
|
|
"num_tokens": 315764.0,
|
|
"reward": 0.8125,
|
|
"reward_std": 0.408231720328331,
|
|
"rewards/equation_reward_func/mean": 0.0625,
|
|
"rewards/equation_reward_func/std": 0.1767766922712326,
|
|
"rewards/format_reward_func/mean": 0.75,
|
|
"rewards/format_reward_func/std": 0.4355513006448746,
|
|
"sampling/importance_sampling_ratio/max": 1.5682951211929321,
|
|
"sampling/importance_sampling_ratio/mean": 0.6582451313734055,
|
|
"sampling/importance_sampling_ratio/min": 0.13125959038734436,
|
|
"sampling/sampling_logp_difference/max": 0.4419562816619873,
|
|
"sampling/sampling_logp_difference/mean": 0.02288186550140381,
|
|
"step": 62,
|
|
"step_time": 9.141117992810905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 890.5,
|
|
"completions/max_terminated_length": 890.5,
|
|
"completions/mean_length": 540.75,
|
|
"completions/mean_terminated_length": 540.75,
|
|
"completions/min_length": 214.0,
|
|
"completions/min_terminated_length": 214.0,
|
|
"entropy": 1.2960376776754856,
|
|
"epoch": 0.0014222222222222223,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.562892780108444,
|
|
"kl": 0.006526344732264988,
|
|
"learning_rate": 1.5902376575912814e-07,
|
|
"loss": 0.0148,
|
|
"num_tokens": 326848.0,
|
|
"reward": 0.75,
|
|
"reward_std": 0.4355512708425522,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.75,
|
|
"rewards/format_reward_func/std": 0.4355513006448746,
|
|
"sampling/importance_sampling_ratio/max": 2.478863477706909,
|
|
"sampling/importance_sampling_ratio/mean": 0.8480255007743835,
|
|
"sampling/importance_sampling_ratio/min": 0.09445519745349884,
|
|
"sampling/sampling_logp_difference/max": 0.4862937927246094,
|
|
"sampling/sampling_logp_difference/mean": 0.025070350617170334,
|
|
"step": 64,
|
|
"step_time": 8.837976877926849
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 740.0,
|
|
"completions/max_terminated_length": 740.0,
|
|
"completions/mean_length": 394.8125,
|
|
"completions/mean_terminated_length": 394.8125,
|
|
"completions/min_length": 167.5,
|
|
"completions/min_terminated_length": 167.5,
|
|
"entropy": 1.1651179119944572,
|
|
"epoch": 0.0014666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.7610832787793433,
|
|
"kl": 0.009777049403055571,
|
|
"learning_rate": 1.4414163643562753e-07,
|
|
"loss": 0.0365,
|
|
"num_tokens": 335597.0,
|
|
"reward": 0.875,
|
|
"reward_std": 0.5940381735563278,
|
|
"rewards/equation_reward_func/mean": 0.125,
|
|
"rewards/equation_reward_func/std": 0.2314550280570984,
|
|
"rewards/format_reward_func/mean": 0.75,
|
|
"rewards/format_reward_func/std": 0.4355513006448746,
|
|
"sampling/importance_sampling_ratio/max": 2.0373843908309937,
|
|
"sampling/importance_sampling_ratio/mean": 0.7534764409065247,
|
|
"sampling/importance_sampling_ratio/min": 0.07185052335262299,
|
|
"sampling/sampling_logp_difference/max": 0.6981399059295654,
|
|
"sampling/sampling_logp_difference/mean": 0.025042645633220673,
|
|
"step": 66,
|
|
"step_time": 8.30885543313343
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 798.5,
|
|
"completions/max_terminated_length": 776.5,
|
|
"completions/mean_length": 574.1875,
|
|
"completions/mean_terminated_length": 523.375,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 1.1272502169013023,
|
|
"epoch": 0.001511111111111111,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.6380871495100986,
|
|
"kl": 0.0075112473423359916,
|
|
"learning_rate": 1.2970351387729872e-07,
|
|
"loss": -0.1702,
|
|
"num_tokens": 347264.0,
|
|
"reward": 0.6875,
|
|
"reward_std": 0.49022960662841797,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.6875,
|
|
"rewards/format_reward_func/std": 0.49022963643074036,
|
|
"sampling/importance_sampling_ratio/max": 1.8942421078681946,
|
|
"sampling/importance_sampling_ratio/mean": 0.7581468820571899,
|
|
"sampling/importance_sampling_ratio/min": 0.22752907872200012,
|
|
"sampling/sampling_logp_difference/max": 0.6634873747825623,
|
|
"sampling/sampling_logp_difference/mean": 0.022335450164973736,
|
|
"step": 68,
|
|
"step_time": 8.51237611565739
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 613.5,
|
|
"completions/max_terminated_length": 613.5,
|
|
"completions/mean_length": 412.1875,
|
|
"completions/mean_terminated_length": 412.1875,
|
|
"completions/min_length": 220.5,
|
|
"completions/min_terminated_length": 220.5,
|
|
"entropy": 1.2611229680478573,
|
|
"epoch": 0.0015555555555555555,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.223649574374791,
|
|
"kl": 0.011163236122229137,
|
|
"learning_rate": 1.1576995658775404e-07,
|
|
"loss": -0.1019,
|
|
"num_tokens": 356299.0,
|
|
"reward": 0.75,
|
|
"reward_std": 0.579209566116333,
|
|
"rewards/equation_reward_func/mean": 0.0625,
|
|
"rewards/equation_reward_func/std": 0.1767766922712326,
|
|
"rewards/format_reward_func/mean": 0.6875,
|
|
"rewards/format_reward_func/std": 0.49022963643074036,
|
|
"sampling/importance_sampling_ratio/max": 2.2894433736801147,
|
|
"sampling/importance_sampling_ratio/mean": 0.8129299879074097,
|
|
"sampling/importance_sampling_ratio/min": 0.15451717376708984,
|
|
"sampling/sampling_logp_difference/max": 0.3217533826828003,
|
|
"sampling/sampling_logp_difference/mean": 0.024897821247577667,
|
|
"step": 70,
|
|
"step_time": 8.147723604924977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 785.0,
|
|
"completions/max_terminated_length": 785.0,
|
|
"completions/mean_length": 559.6875,
|
|
"completions/mean_terminated_length": 559.6875,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 1.2647213600575924,
|
|
"epoch": 0.0016,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.061023835100866,
|
|
"kl": 0.00741576035215985,
|
|
"learning_rate": 1.0239940674851941e-07,
|
|
"loss": -0.2195,
|
|
"num_tokens": 367726.0,
|
|
"reward": 0.75,
|
|
"reward_std": 0.4629100561141968,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.75,
|
|
"rewards/format_reward_func/std": 0.4629100561141968,
|
|
"sampling/importance_sampling_ratio/max": 1.8395887613296509,
|
|
"sampling/importance_sampling_ratio/mean": 0.7119902670383453,
|
|
"sampling/importance_sampling_ratio/min": 0.05588642507791519,
|
|
"sampling/sampling_logp_difference/max": 0.40247368812561035,
|
|
"sampling/sampling_logp_difference/mean": 0.023968717083334923,
|
|
"step": 72,
|
|
"step_time": 8.402685680543073
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 915.0,
|
|
"completions/max_terminated_length": 915.0,
|
|
"completions/mean_length": 527.25,
|
|
"completions/mean_terminated_length": 527.25,
|
|
"completions/min_length": 158.0,
|
|
"completions/min_terminated_length": 158.0,
|
|
"entropy": 1.1917564272880554,
|
|
"epoch": 0.0016444444444444445,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.6423976929527928,
|
|
"kl": 0.006568885612068698,
|
|
"learning_rate": 8.964794509221507e-08,
|
|
"loss": 0.2395,
|
|
"num_tokens": 378610.0,
|
|
"reward": 0.75,
|
|
"reward_std": 0.4629100561141968,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.75,
|
|
"rewards/format_reward_func/std": 0.4629100561141968,
|
|
"sampling/importance_sampling_ratio/max": 1.3353188037872314,
|
|
"sampling/importance_sampling_ratio/mean": 0.6343062818050385,
|
|
"sampling/importance_sampling_ratio/min": 0.06923050060868263,
|
|
"sampling/sampling_logp_difference/max": 0.5188714563846588,
|
|
"sampling/sampling_logp_difference/mean": 0.02377952355891466,
|
|
"step": 74,
|
|
"step_time": 8.72973417036701
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 881.0,
|
|
"completions/max_terminated_length": 881.0,
|
|
"completions/mean_length": 477.1875,
|
|
"completions/mean_terminated_length": 477.1875,
|
|
"completions/min_length": 173.5,
|
|
"completions/min_terminated_length": 173.5,
|
|
"entropy": 1.1763490214943886,
|
|
"epoch": 0.0016888888888888889,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.692320484588319,
|
|
"kl": 0.007726169380475767,
|
|
"learning_rate": 7.756905568047392e-08,
|
|
"loss": -0.0879,
|
|
"num_tokens": 388717.0,
|
|
"reward": 1.0,
|
|
"reward_std": 0.3535533845424652,
|
|
"rewards/equation_reward_func/mean": 0.125,
|
|
"rewards/equation_reward_func/std": 0.2314550280570984,
|
|
"rewards/format_reward_func/mean": 0.875,
|
|
"rewards/format_reward_func/std": 0.3535533845424652,
|
|
"sampling/importance_sampling_ratio/max": 1.2693214416503906,
|
|
"sampling/importance_sampling_ratio/mean": 0.43672314286231995,
|
|
"sampling/importance_sampling_ratio/min": 0.046458715572953224,
|
|
"sampling/sampling_logp_difference/max": 0.3335939645767212,
|
|
"sampling/sampling_logp_difference/mean": 0.023782381787896156,
|
|
"step": 76,
|
|
"step_time": 8.931666341493838
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1875,
|
|
"completions/max_length": 1024.0,
|
|
"completions/max_terminated_length": 626.0,
|
|
"completions/mean_length": 556.125,
|
|
"completions/mean_terminated_length": 446.22621154785156,
|
|
"completions/min_length": 282.0,
|
|
"completions/min_terminated_length": 282.0,
|
|
"entropy": 1.2554637528955936,
|
|
"epoch": 0.0017333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.2327740462000585,
|
|
"kl": 0.0065791915549198166,
|
|
"learning_rate": 6.621340157319996e-08,
|
|
"loss": -0.0658,
|
|
"num_tokens": 400055.0,
|
|
"reward": 0.75,
|
|
"reward_std": 0.4629100561141968,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.75,
|
|
"rewards/format_reward_func/std": 0.4629100561141968,
|
|
"sampling/importance_sampling_ratio/max": 1.4898787140846252,
|
|
"sampling/importance_sampling_ratio/mean": 0.6342976093292236,
|
|
"sampling/importance_sampling_ratio/min": 0.03369775041937828,
|
|
"sampling/sampling_logp_difference/max": 0.45903223752975464,
|
|
"sampling/sampling_logp_difference/mean": 0.024691437371075153,
|
|
"step": 78,
|
|
"step_time": 9.230151811032556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1012.5,
|
|
"completions/max_terminated_length": 1012.5,
|
|
"completions/mean_length": 508.3125,
|
|
"completions/mean_terminated_length": 508.3125,
|
|
"completions/min_length": 198.5,
|
|
"completions/min_terminated_length": 198.5,
|
|
"entropy": 1.2609276585280895,
|
|
"epoch": 0.0017777777777777779,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.13225387600314,
|
|
"kl": 0.007441339563229121,
|
|
"learning_rate": 5.5628612330087724e-08,
|
|
"loss": -0.1382,
|
|
"num_tokens": 410652.0,
|
|
"reward": 0.6875,
|
|
"reward_std": 0.49022960662841797,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.6875,
|
|
"rewards/format_reward_func/std": 0.49022963643074036,
|
|
"sampling/importance_sampling_ratio/max": 2.2199747562408447,
|
|
"sampling/importance_sampling_ratio/mean": 0.8080930113792419,
|
|
"sampling/importance_sampling_ratio/min": 0.17915555834770203,
|
|
"sampling/sampling_logp_difference/max": 0.43730688095092773,
|
|
"sampling/sampling_logp_difference/mean": 0.02470558974891901,
|
|
"step": 80,
|
|
"step_time": 9.341547849937342
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 705.0,
|
|
"completions/max_terminated_length": 705.0,
|
|
"completions/mean_length": 431.9375,
|
|
"completions/mean_terminated_length": 431.9375,
|
|
"completions/min_length": 275.5,
|
|
"completions/min_terminated_length": 275.5,
|
|
"entropy": 1.1785611994564533,
|
|
"epoch": 0.0018222222222222223,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.7533872596303937,
|
|
"kl": 0.0060285410727374256,
|
|
"learning_rate": 4.5859084235697235e-08,
|
|
"loss": -0.0697,
|
|
"num_tokens": 419979.0,
|
|
"reward": 1.0,
|
|
"reward_std": 0.6408699750900269,
|
|
"rewards/equation_reward_func/mean": 0.1875,
|
|
"rewards/equation_reward_func/std": 0.408231720328331,
|
|
"rewards/format_reward_func/mean": 0.8125,
|
|
"rewards/format_reward_func/std": 0.408231720328331,
|
|
"sampling/importance_sampling_ratio/max": 2.7441056966781616,
|
|
"sampling/importance_sampling_ratio/mean": 0.8698084950447083,
|
|
"sampling/importance_sampling_ratio/min": 0.05789299122989178,
|
|
"sampling/sampling_logp_difference/max": 0.9367153644561768,
|
|
"sampling/sampling_logp_difference/mean": 0.024671697057783604,
|
|
"step": 82,
|
|
"step_time": 8.355979559826665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 905.0,
|
|
"completions/max_terminated_length": 905.0,
|
|
"completions/mean_length": 495.5,
|
|
"completions/mean_terminated_length": 495.5,
|
|
"completions/min_length": 230.5,
|
|
"completions/min_terminated_length": 230.5,
|
|
"entropy": 1.2624935917556286,
|
|
"epoch": 0.0018666666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.7840327619724896,
|
|
"kl": 0.006831162696471438,
|
|
"learning_rate": 3.6945794086007705e-08,
|
|
"loss": -0.0282,
|
|
"num_tokens": 430347.0,
|
|
"reward": 0.8125,
|
|
"reward_std": 0.408231720328331,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.8125,
|
|
"rewards/format_reward_func/std": 0.408231720328331,
|
|
"sampling/importance_sampling_ratio/max": 1.5519986152648926,
|
|
"sampling/importance_sampling_ratio/mean": 0.5716961026191711,
|
|
"sampling/importance_sampling_ratio/min": 0.15530326962471008,
|
|
"sampling/sampling_logp_difference/max": 0.5926408171653748,
|
|
"sampling/sampling_logp_difference/mean": 0.024989251978695393,
|
|
"step": 84,
|
|
"step_time": 8.7652991093928
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 778.5,
|
|
"completions/max_terminated_length": 739.0,
|
|
"completions/mean_length": 491.0,
|
|
"completions/mean_terminated_length": 459.4821472167969,
|
|
"completions/min_length": 261.5,
|
|
"completions/min_terminated_length": 261.5,
|
|
"entropy": 1.1592534072697163,
|
|
"epoch": 0.001911111111111111,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.889869064786408,
|
|
"kl": 0.008921730826841667,
|
|
"learning_rate": 2.892612731749414e-08,
|
|
"loss": -0.0768,
|
|
"num_tokens": 440675.0,
|
|
"reward": 0.875,
|
|
"reward_std": 0.3535533845424652,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.875,
|
|
"rewards/format_reward_func/std": 0.3535533845424652,
|
|
"sampling/importance_sampling_ratio/max": 1.261430025100708,
|
|
"sampling/importance_sampling_ratio/mean": 0.5434921085834503,
|
|
"sampling/importance_sampling_ratio/min": 0.12478071637451649,
|
|
"sampling/sampling_logp_difference/max": 1.3635588884353638,
|
|
"sampling/sampling_logp_difference/mean": 0.024433997459709644,
|
|
"step": 86,
|
|
"step_time": 8.525219976552762
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 683.0,
|
|
"completions/max_terminated_length": 683.0,
|
|
"completions/mean_length": 363.5,
|
|
"completions/mean_terminated_length": 363.5,
|
|
"completions/min_length": 165.5,
|
|
"completions/min_terminated_length": 165.5,
|
|
"entropy": 1.1377994492650032,
|
|
"epoch": 0.0019555555555555554,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.134372721816042,
|
|
"kl": 0.015324803374824114,
|
|
"learning_rate": 2.183372119961499e-08,
|
|
"loss": 0.5464,
|
|
"num_tokens": 448899.0,
|
|
"reward": 0.875,
|
|
"reward_std": 0.3535533845424652,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.875,
|
|
"rewards/format_reward_func/std": 0.3535533845424652,
|
|
"sampling/importance_sampling_ratio/max": 2.269628405570984,
|
|
"sampling/importance_sampling_ratio/mean": 1.1133230924606323,
|
|
"sampling/importance_sampling_ratio/min": 0.2772098332643509,
|
|
"sampling/sampling_logp_difference/max": 0.8273682594299316,
|
|
"sampling/sampling_logp_difference/mean": 0.02418291289359331,
|
|
"step": 88,
|
|
"step_time": 8.307329173432663
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 830.0,
|
|
"completions/max_terminated_length": 830.0,
|
|
"completions/mean_length": 560.9375,
|
|
"completions/mean_terminated_length": 560.9375,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 1.2554445713758469,
|
|
"epoch": 0.002,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.8357390700930227,
|
|
"kl": 0.015233526355586946,
|
|
"learning_rate": 1.5698323748414122e-08,
|
|
"loss": -0.2772,
|
|
"num_tokens": 460354.0,
|
|
"reward": 0.8125,
|
|
"reward_std": 0.5260358154773712,
|
|
"rewards/equation_reward_func/mean": 0.0625,
|
|
"rewards/equation_reward_func/std": 0.1767766922712326,
|
|
"rewards/format_reward_func/mean": 0.75,
|
|
"rewards/format_reward_func/std": 0.4355513006448746,
|
|
"sampling/importance_sampling_ratio/max": 2.431339144706726,
|
|
"sampling/importance_sampling_ratio/mean": 0.6834549903869629,
|
|
"sampling/importance_sampling_ratio/min": 0.01039754506200552,
|
|
"sampling/sampling_logp_difference/max": 0.5135123133659363,
|
|
"sampling/sampling_logp_difference/mean": 0.02495667804032564,
|
|
"step": 90,
|
|
"step_time": 8.614867440075614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 924.0,
|
|
"completions/max_terminated_length": 772.5,
|
|
"completions/mean_length": 546.5,
|
|
"completions/mean_terminated_length": 516.4196472167969,
|
|
"completions/min_length": 266.5,
|
|
"completions/min_terminated_length": 266.5,
|
|
"entropy": 1.4302296712994576,
|
|
"epoch": 0.0020444444444444447,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.4766318095819377,
|
|
"kl": 0.006567903692484833,
|
|
"learning_rate": 1.054566895300324e-08,
|
|
"loss": -0.1519,
|
|
"num_tokens": 471546.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.49022960662841797,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.5625,
|
|
"rewards/format_reward_func/std": 0.49022963643074036,
|
|
"sampling/importance_sampling_ratio/max": 2.179307520389557,
|
|
"sampling/importance_sampling_ratio/mean": 0.7372348606586456,
|
|
"sampling/importance_sampling_ratio/min": 0.0416359007358551,
|
|
"sampling/sampling_logp_difference/max": 0.8628695011138916,
|
|
"sampling/sampling_logp_difference/mean": 0.025372425094246864,
|
|
"step": 92,
|
|
"step_time": 9.102577322046272
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 753.0,
|
|
"completions/max_terminated_length": 753.0,
|
|
"completions/mean_length": 468.8125,
|
|
"completions/mean_terminated_length": 468.8125,
|
|
"completions/min_length": 178.5,
|
|
"completions/min_terminated_length": 178.5,
|
|
"entropy": 1.2605133689939976,
|
|
"epoch": 0.002088888888888889,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.5504326025550264,
|
|
"kl": 0.00589993035828229,
|
|
"learning_rate": 6.397368838268496e-09,
|
|
"loss": -0.1529,
|
|
"num_tokens": 481495.0,
|
|
"reward": 0.875,
|
|
"reward_std": 0.3535533845424652,
|
|
"rewards/equation_reward_func/mean": 0.0,
|
|
"rewards/equation_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.875,
|
|
"rewards/format_reward_func/std": 0.3535533845424652,
|
|
"sampling/importance_sampling_ratio/max": 2.2573115825653076,
|
|
"sampling/importance_sampling_ratio/mean": 0.8763566613197327,
|
|
"sampling/importance_sampling_ratio/min": 0.26127955317497253,
|
|
"sampling/sampling_logp_difference/max": 0.3228236436843872,
|
|
"sampling/sampling_logp_difference/mean": 0.02433642279356718,
|
|
"step": 94,
|
|
"step_time": 8.598844347870909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 454.0,
|
|
"completions/max_terminated_length": 454.0,
|
|
"completions/mean_length": 321.9375,
|
|
"completions/mean_terminated_length": 321.9375,
|
|
"completions/min_length": 197.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"entropy": 1.196368370205164,
|
|
"epoch": 0.0021333333333333334,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.004081368931451629,
|
|
"kl": 0.013201179972384125,
|
|
"learning_rate": 3.2708228165273244e-09,
|
|
"loss": -0.0133,
|
|
"num_tokens": 489038.0,
|
|
"reward": 1.0625,
|
|
"reward_std": 0.3204349875450134,
|
|
"rewards/equation_reward_func/mean": 0.125,
|
|
"rewards/equation_reward_func/std": 0.2314550280570984,
|
|
"rewards/format_reward_func/mean": 0.9375,
|
|
"rewards/format_reward_func/std": 0.1767766922712326,
|
|
"sampling/importance_sampling_ratio/max": 1.5249131321907043,
|
|
"sampling/importance_sampling_ratio/mean": 0.6771150529384613,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.3968193531036377,
|
|
"sampling/sampling_logp_difference/mean": 0.02468986716121435,
|
|
"step": 96,
|
|
"step_time": 7.818771839141846
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 542.0,
|
|
"completions/max_terminated_length": 542.0,
|
|
"completions/mean_length": 361.875,
|
|
"completions/mean_terminated_length": 361.875,
|
|
"completions/min_length": 201.0,
|
|
"completions/min_terminated_length": 201.0,
|
|
"entropy": 1.1868297532200813,
|
|
"epoch": 0.0021777777777777776,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.254444002886367,
|
|
"kl": 0.011681211748509668,
|
|
"learning_rate": 1.1791447083465133e-09,
|
|
"loss": 0.4881,
|
|
"num_tokens": 497244.0,
|
|
"reward": 1.125,
|
|
"reward_std": 0.49721167981624603,
|
|
"rewards/equation_reward_func/mean": 0.1875,
|
|
"rewards/equation_reward_func/std": 0.408231720328331,
|
|
"rewards/format_reward_func/mean": 0.9375,
|
|
"rewards/format_reward_func/std": 0.1767766922712326,
|
|
"sampling/importance_sampling_ratio/max": 2.307553768157959,
|
|
"sampling/importance_sampling_ratio/mean": 0.7432051002979279,
|
|
"sampling/importance_sampling_ratio/min": 0.06528420001268387,
|
|
"sampling/sampling_logp_difference/max": 0.4255487322807312,
|
|
"sampling/sampling_logp_difference/mean": 0.025311018340289593,
|
|
"step": 98,
|
|
"step_time": 7.8635854490567
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 583.0,
|
|
"completions/max_terminated_length": 583.0,
|
|
"completions/mean_length": 371.6875,
|
|
"completions/mean_terminated_length": 371.6875,
|
|
"completions/min_length": 173.5,
|
|
"completions/min_terminated_length": 173.5,
|
|
"entropy": 1.1904011890292168,
|
|
"epoch": 0.0022222222222222222,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.048771291369092,
|
|
"kl": 0.0074510836420813575,
|
|
"learning_rate": 1.3110773862126667e-10,
|
|
"loss": 0.5377,
|
|
"num_tokens": 505607.0,
|
|
"reward": 1.0,
|
|
"reward_std": 0.49721167981624603,
|
|
"rewards/equation_reward_func/mean": 0.125,
|
|
"rewards/equation_reward_func/std": 0.2314550280570984,
|
|
"rewards/format_reward_func/mean": 0.875,
|
|
"rewards/format_reward_func/std": 0.3535533845424652,
|
|
"sampling/importance_sampling_ratio/max": 1.8672412633895874,
|
|
"sampling/importance_sampling_ratio/mean": 0.7961099743843079,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.4504559636116028,
|
|
"sampling/sampling_logp_difference/mean": 0.024699319154024124,
|
|
"step": 100,
|
|
"step_time": 8.325517300749198
|
|
},
|
|
{
|
|
"epoch": 0.0022222222222222222,
|
|
"step": 100,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.003586103413254023,
|
|
"train_runtime": 1244.4861,
|
|
"train_samples_per_second": 0.643,
|
|
"train_steps_per_second": 0.08
|
|
}
|
|
],
|
|
"logging_steps": 2,
|
|
"max_steps": 100,
|
|
"num_input_tokens_seen": 505607,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 25,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|